vern-llm 2.3.0 → 2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,4154 +1,15 @@
1
- import { randomUUID } from "crypto";
1
+ import{randomUUID as e}from"crypto";const t=new Set([`unknown_tool`,`duplicate_tool_call_id`,`tool_choice_none_violated`,`unexpected_tool_calls`]),n=new Set([`rate_limit_queue_full`,`rate_limit_queue_timeout`,`rate_limit_capacity_exceeded`]),r=new Set([`parse`,`validation`,`invalid_params`,`aborted`]);function i(e,i){return!(r.has(e)||i&&t.has(i)||i&&n.has(i))}function a(e){if(e!==void 0)try{return JSON.stringify(e),e}catch{return`[Unserializable: issues contained a circular reference]`}}function o(e){if(e!==void 0)try{return JSON.parse(JSON.stringify(e))}catch{return`[Unserializable: request body contained a circular reference]`}}const s=new Set([`authorization`,`x-api-key`,`x-goog-api-key`,`api-key`]);function c(e){if(e===void 0)return;let t={};for(let[n,r]of Object.entries(e))s.has(n.toLowerCase())||(t[n]=r);return t}const l=20;function u(e,t=0){if(e!==void 0)return t>=l?[]:e.map(e=>({...e,error:{...e.error,issues:a(e.error.issues),attempts:u(e.error.attempts,t+1)},request:e.request&&{...e.request,body:o(e.request.body),headers:c(e.request.headers)}}))}function d(e,t,n,r,i=Date.now()){return{provider:e,model:t,body:o(n),headers:c(r),startedAt:i}}var f=class extends Error{status;issues;cause;retryAfterMs;code;attempts;constructor(e,t,n={}){super(e),this.type=t,this.name=`LLMError`,this.status=n.status,this.issues=n.issues,this.cause=n.cause,this.retryAfterMs=n.retryAfterMs,this.code=n.code,this.attempts=n.attempts}get retryable(){return i(this.type,this.code)}toSnapshot(){return{message:this.message,type:this.type,status:this.status,issues:a(this.issues),retryAfterMs:this.retryAfterMs,code:this.code,retryable:this.retryable,attempts:u(this.attempts)}}toJSON(){return{name:this.name,message:this.message,type:this.type,status:this.status,issues:a(this.issues),retryAfterMs:this.retryAfterMs,code:this.code,retryable:this.retryable,attempts:u(this.attempts)}}};function p(e){return e instanceof f}function m(e,t){return e.code===t&&e.issues!==void 0}var h=class{store=new Map;constructor(e=1e3){this.maxSize=e}async get(e){let t=this.store.get(e);return t?Date.now()>=t.expiresAt?(this.store.delete(e),{hit:!1,value:null}):{hit:!0,value:t.value}:{hit:!1,value:null}}async set(e,t,n){this.cleanupExpiredEntries(),this.store.set(e,{value:t,expiresAt:Date.now()+n*1e3}),this.enforceSizeLimit()}async delete(e){this.store.delete(e)}cleanupExpiredEntries(){let e=Date.now();for(let[t,n]of this.store)e>=n.expiresAt&&this.store.delete(t)}enforceSizeLimit(){for(;this.store.size>this.maxSize;){let e=this.store.keys().next().value;if(e===void 0)break;this.store.delete(e)}}},g=class{constructor(e=new h){this.inner=e}normalize(e){return e.toLowerCase().trim().replace(/[^\p{L}\p{N}\s]/gu,` `).replace(/\s+/g,` `).trim()}async resolveKey(e){return this.normalize(e)}async get(e){return this.inner.get(this.normalize(e))}async set(e,t,n){await this.inner.set(this.normalize(e),t,n)}async delete(e){await this.inner.delete?.(this.normalize(e))}},_=class{constructor(e,t,n){this.l1=e,this.l2=t,this.l1Ttl=n}async resolveKey(e){return this.l1.resolveKey?this.l1.resolveKey(e):this.l2.resolveKey?this.l2.resolveKey(e):e}async get(e){let t=await this.l1.get(e);if(t.hit)return t;let n=await this.l2.get(e);return n.hit&&await this.l1.set(e,n.value,this.l1Ttl??60),n}async set(e,t,n){await Promise.all([this.l1.set(e,t,this.l1Ttl??n),this.l2.set(e,t,n)])}async delete(e){await Promise.all([this.l1.delete?.(e),this.l2.delete?.(e)])}};function v(e){return typeof e==`object`&&!!e&&`type`in e&&e.type===`tool_calls`&&Array.isArray(e.toolCalls)}const y=new Set([`unknown_tool`,`duplicate_tool_call_id`,`tool_choice_none_violated`,`unexpected_tool_calls`]),b=e=>e.type===`parse`||e.type===`validation`||e.type===`aborted`||e.type===`quota_exceeded`||e.code&&y.has(e.code)?`stop`:`next`;var x=class extends f{constructor(e){let t=e[e.length-1]?.error;super(`${e.length} provider${e.length===1?``:`s`} attempted and failed: ${e.map(e=>`${e.provider}(${e.error.type})`).join(` then `)}`,`fallback_exhausted`,{status:t?.status,cause:t,retryAfterMs:t?.retryAfterMs,code:`fallback_exhausted`,attempts:e}),this.attempts=e}get retryable(){let e=this.attempts[this.attempts.length-1]?.error;return e?e.retryable:super.retryable}};function S(e){return e instanceof x}async function C(e,t,n){if(!e.reserveUsage)return!1;try{return await e.reserveUsage({coalesced:t,signal:n}),!0}catch(e){throw n?.aborted?new f(`LLM request aborted`,`aborted`):new f(e instanceof Error?e.message:`Usage reservation failed`,`quota_exceeded`,{cause:e})}}function w(e,t,n,r){return async i=>{try{await e.refundUsage?.({coalesced:t,signal:n})}catch(e){r(i,e)}}}async function T(e,t,n,r,i){if(r?.aborted)throw new f(`LLM request aborted`,`aborted`);let a=await C(e,t,r),o=w(e,t,r,i);if(r?.aborted)throw a&&await o(`[VernLLM] refundUsage failed after abort`),new f(`LLM request aborted`,`aborted`);let s;try{s=await n()}catch(e){throw a&&await o(`[VernLLM] refundUsage failed`),e}if(r?.aborted)throw a&&await o(`[VernLLM] refundUsage failed after abort`),new f(`LLM request aborted`,`aborted`);return s}async function E(e,t,n,r){if(n?.aborted)throw new f(`LLM request aborted`,`aborted`);let i=await C(e,!1,n),a=w(e,!1,n,r);if(n?.aborted)throw i&&await a(`[VernLLM] refundUsage failed after abort`),new f(`LLM request aborted`,`aborted`);let o;try{o=await t()}catch(e){throw i&&await a(`[VernLLM] refundUsage failed after stream-open failure`),e}let s=o.finalResult.then(e=>e,async e=>{throw i&&await a(`[VernLLM] refundUsage failed after stream error`),e});return s.catch(()=>{}),{chunks:o.chunks,finalResult:s}}function D(e){return typeof e==`string`?e:JSON.stringify(e)??``}function ee(e,t){let n=[];if(t){let t=e;t.type===`tool_calls`?(t.toolCalls.forEach((e,t)=>{n.push({type:`tool_call_delta`,index:t,id:e.id,name:e.name,argsDelta:JSON.stringify(e.arguments??{}),complete:!0})}),t.content&&n.push({type:`text-delta`,delta:t.content})):n.push({type:`text-delta`,delta:D(t.content)})}else n.push({type:`text-delta`,delta:D(e)});return{async*[Symbol.asyncIterator](){for(let e of n)yield e}}}function te(e,t){return{async*[Symbol.asyncIterator](){let n=await e;yield*ee(n,t)}}}var ne=class{inFlight=new Map;constructor(e,t){this.cache=e,this.logger=t}async resolveCacheKey(e){return this.cache.resolveKey?await this.cache.resolveKey(e):e}async deleteCache(e){if(this.cache.delete)try{await this.cache.delete(await this.resolveCacheKey(e))}catch(e){this.logger.warn(`[VernLLM] cache delete failed: ${e instanceof Error?e.message:`unknown`}`)}}logRefundError(e,t){this.logger.error(e,{message:t instanceof Error?t.message:`unknown`})}async getCached(e){try{return await this.cache.get(e)}catch(e){return this.logger.warn(`[VernLLM] cache read failed: ${e instanceof Error?e.message:`unknown`}`),{hit:!1}}}async runCached(e){let t=await this.resolveCacheKey(e.cacheKey),n=t===e.cacheKey?e:{...e,cacheKey:t},r=await this.getCached(t);if(r.hit)return r.value;let i=this.inFlight.get(t);return i?T(n,!0,()=>i,e.signal,(e,t)=>this.logRefundError(e,t)):this.registerTrigger(n)}registerTrigger(e){let t=T(e,!1,()=>this.runAndCache(e),e.signal,(e,t)=>this.logRefundError(e,t));return this.inFlight.set(e.cacheKey,t),t.catch(()=>{}).finally(()=>{this.inFlight.delete(e.cacheKey)}),t}async runAndCache(e){let t=await e.fn();try{await this.cache.set(e.cacheKey,t,e.ttl)}catch(e){this.logger.warn(`[VernLLM] cache write failed: ${e instanceof Error?e.message:`unknown`}`)}return t}async runCachedStream(e,t){let n=await this.resolveCacheKey(e.cacheKey),r=n===e.cacheKey?e:{...e,cacheKey:n},i=await this.getCached(n);if(i.hit){let e=i.value;return{chunks:ee(e,t),finalResult:Promise.resolve(e)}}let a=this.inFlight.get(n);if(a){let n=T(r,!0,()=>a,e.signal,(e,t)=>this.logRefundError(e,t));return n.catch(()=>{}),{chunks:te(n,t),finalResult:n}}return this.registerStreamTrigger(r)}registerStreamTrigger(e){let t,n,r=new Promise((e,r)=>{t=e,n=r});this.inFlight.set(e.cacheKey,r),r.catch(()=>{}).finally(()=>{this.inFlight.delete(e.cacheKey)});let i=E(e,async()=>{let t=await e.openStream(),n=t.finalResult.then(async t=>{try{await this.cache.set(e.cacheKey,t,e.ttl)}catch(e){this.logger.warn(`[VernLLM] cache write failed: ${e instanceof Error?e.message:`unknown`}`)}return t},e=>{throw e});return{chunks:t.chunks,finalResult:n}},e.signal,(e,t)=>this.logRefundError(e,t));return i.then(e=>{e.finalResult.then(t,n)},e=>{n(e)}),i}};function re(){return{state:`closed`,consecutiveFailures:0,openedAt:0,trialInFlight:!1}}const O=``;var k=class{threshold;cooldownMs;onStateChange;isolateByModel;sharedBucket=re();bucketsByModel=new Map;constructor(e={}){this.threshold=e.threshold??5,this.cooldownMs=e.cooldownMs??3e4,this.onStateChange=e.onStateChange,this.isolateByModel=e.isolateByModel??!1}lookupBucket(e){if(!this.isolateByModel)return this.sharedBucket;let t=e??O;return this.bucketsByModel.get(t)}ensureBucketFor(e){if(!this.isolateByModel)return this.sharedBucket;let t=e??O,n=this.bucketsByModel.get(t);return n||(n=re(),this.bucketsByModel.set(t,n)),n}transition(e,t,n){if(t===e.state)return;let r=e.state;e.state=t,this.onStateChange?.(r,t,e.consecutiveFailures,n)}assertClosed(e){let t=this.ensureBucketFor(e);if(t.state!==`closed`){if(t.state===`open`){let n=Date.now()-t.openedAt;if(n<this.cooldownMs)throw new f(`Circuit open, provider has failed ${t.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs-n)/1e3)}s.`,`circuit_open`,{code:`circuit_cooling_down`});t.trialInFlight=!0,this.transition(t,`half-open`,e);return}if(t.trialInFlight)throw new f(`Circuit half-open. A trial request is already in flight. Try again shortly.`,`circuit_open`,{code:`circuit_trial_in_flight`});t.trialInFlight=!0}}recordSuccess(e){let t=this.lookupBucket(e);t&&(t.consecutiveFailures=0,t.trialInFlight=!1,this.transition(t,`closed`,e),this.isolateByModel&&t.state===`closed`&&t.consecutiveFailures===0&&this.bucketsByModel.delete(e??O))}recordFailure(e){let t=this.ensureBucketFor(e);if(t.consecutiveFailures+=1,t.trialInFlight=!1,t.state===`half-open`){t.openedAt=Date.now(),this.transition(t,`open`,e);return}t.consecutiveFailures>=this.threshold&&(t.openedAt=Date.now(),this.transition(t,`open`,e))}getState(e){return this.lookupBucket(e)?.state??`closed`}open(e){let t=this.ensureBucketFor(e);t.openedAt=Date.now(),t.trialInFlight=!1,this.transition(t,`open`,e)}close(e){let t=this.ensureBucketFor(e);t.consecutiveFailures=0,t.trialInFlight=!1,this.transition(t,`closed`,e),this.isolateByModel&&t.state===`closed`&&t.consecutiveFailures===0&&this.bucketsByModel.delete(e??O)}};function A(e,t){return n=>{if(e)try{e(n)}catch(e){t.error(`[VernLLM] onEvent failed`,{message:e instanceof Error?e.message:`unknown`})}}}function ie(e,t,n,r,i){if(!e)return;let a=typeof e==`object`?e:void 0,o=a?.onStateChange,s=A(r,i);return new k({...a,onStateChange:(e,r,a,c)=>{if(s({kind:`circuit_state`,provider:t,model:c??n,from:e,to:r,consecutiveFailures:a}),o)try{o(e,r,a,c)}catch(e){i.error(`[VernLLM] circuitBreaker.onStateChange failed`,{message:e instanceof Error?e.message:`unknown`})}}})}const j=1e4,ae=2147483647;function M(e){return!e||e<=0||e===1/0?void 0:e}function N(e){return Math.min(e,ae)}async function oe(e,t,n){let r=new AbortController,i=M(t),a=i===void 0?void 0:setTimeout(()=>{r.abort()},N(i)),o=n?AbortSignal.any([n,r.signal]):r.signal;try{return await e(o)}catch(e){throw r.signal.aborted&&!n?.aborted&&e instanceof DOMException&&e.name===`AbortError`?new f(`Request timed out`,`timeout`,{code:`request_timeout`}):e}finally{clearTimeout(a)}}function se(e,t,n,r){let i=M(t);if(i===void 0)return e();let a=!1;return new Promise((t,o)=>{let s=setTimeout(()=>{a=!0,n?.(),o(new f(`No stream chunk received for ${i}ms (idle timeout)`,`timeout`,{code:`idle_timeout`}))},N(i));e().then(e=>{if(clearTimeout(s),a){r?.debug(`[VernLLM] chunk resolved after idle timeout already fired; discarding`);return}a=!0,t(e)},e=>{if(clearTimeout(s),a){r?.debug(`[VernLLM] chunk rejection arrived after idle timeout already fired; discarding`);return}a=!0,o(e)})})}function ce(e,t=j){if(!e||typeof e!=`object`)return;let n=e,r=n.headers??n.response?.headers;if(!r||typeof r!=`object`)return;let i=r,a=typeof i.get==`function`?i.get(`Retry-After`):Object.entries(r).find(([e])=>e.toLowerCase()===`retry-after`)?.at(1);if(typeof a!=`string`||a.trim()===``)return;let o=a.trim();if(/^\d+$/.test(o))return Math.max(0,Math.min(Number(o)*1e3,t));let s=Date.parse(o);if(!Number.isNaN(s))return Math.max(0,Math.min(s-Date.now(),t))}function le(e,t,n=j){let r=Math.min(e*2**t,n);return r/2+Math.random()*(r/2)}async function ue(e,t){if(t?.aborted)throw new f(`Operation aborted`,`aborted`);await new Promise((n,r)=>{let i=()=>{clearTimeout(a),r(new f(`Operation aborted`,`aborted`))},a=setTimeout(()=>{t?.removeEventListener(`abort`,i),n()},e);t?.addEventListener(`abort`,i,{once:!0})})}function de(e){if(!e||typeof e!=`object`)return;let t=e;if(typeof t.status==`number`)return t.status;if(typeof t.statusCode==`number`)return t.statusCode;if(typeof t.$metadata?.httpStatusCode==`number`)return t.$metadata.httpStatusCode}const fe=new Set([`ECONNREFUSED`,`ENOTFOUND`,`ECONNRESET`,`ETIMEDOUT`,`EAI_AGAIN`,`EPIPE`,`ECONNABORTED`,`EHOSTUNREACH`,`ENETUNREACH`]),pe=new Set([`fetch failed`,`failed to fetch`,`load failed`,`networkerror when attempting to fetch resource.`]);function me(e){if(!e||typeof e!=`object`)return!1;let t=e;if(typeof t.code==`string`&&fe.has(t.code)||typeof t.message==`string`&&pe.has(t.message.toLowerCase()))return!0;if(t.cause&&typeof t.cause==`object`){let e=t.cause;if(typeof e.code==`string`&&fe.has(e.code))return!0}return!1}function he(e){try{return JSON.stringify(e,null,2)??String(e)}catch{try{return String(e)}catch{return`[unprintable error]`}}}function P(e){if(e&&typeof e==`object`)try{let t=e;if(t.error!==void 0)return he(t.error);if(typeof t.message==`string`)return t.message}catch{}return he(e)}function ge(e){switch(e){case 429:return`provider_rate_limited`;case 401:return`authentication`;case 403:return`authorization`;case 404:return`not_found`;case 413:return`payload_too_large`;default:return e>=500?`server_error`:void 0}}const _e=/\(no body\)/i;function ve(e){return Object.keys(e).length===0}function ye(e){if(e&&typeof e==`object`){let{error:t,message:n}=e;if(t!=null){let e=typeof t==`string`&&t.trim().length===0,n=typeof t==`object`&&ve(t);if(!e&&!n)return!1}if(typeof n==`string`){let e=n.trim();return e.length===0||_e.test(e)}return!0}return!0}function F(e,t,n){if(t?.aborted)return new f(`LLM request aborted`,`aborted`,{attempts:n});if(e instanceof f)return e.code===void 0&&e.status!==void 0&&(e.code=ge(e.status)),e.attempts===void 0&&n!==void 0&&(e.attempts=n),e;let r=de(e),i=ce(e);if(r!==void 0){let t=P(e),a=ge(r),o=a===void 0,s=ye(e)?o?`LLM request failed with status ${r} and no error detail from the provider. This usually means a field or value in the request isn't supported by the specific model (for example, a reasoning/thinking parameter the model doesn't accept), rather than a transport or auth problem.`:`LLM request failed with status ${r} and no error detail from the provider.`:`LLM request failed: ${t}`;return new f(s,`api`,{status:r,cause:e,retryAfterMs:i,code:a,attempts:n})}return me(e)?new f(`LLM request failed`,`network`,{cause:e,retryAfterMs:i,code:`connection_failed`,attempts:n}):new f(`LLM request failed`,`unknown`,{cause:e,retryAfterMs:i,attempts:n})}function be(e){try{return JSON.parse(e)}catch{return}}function xe(e){return e.map(e=>({type:`function`,function:{name:e.name,description:e.description,parameters:e.parameters}}))}function Se(e){return e.map(e=>({id:e.id,type:`function`,function:{name:e.name,arguments:JSON.stringify(e.arguments??{})}}))}function Ce(e){return e.map(e=>{let t;try{t=e.function.arguments.trim()?JSON.parse(e.function.arguments):{}}catch{throw new f(`Invalid JSON arguments for tool call "${e.function.name}"`,`parse`,{code:`tool_arguments_parse_failed`})}return{id:e.id,name:e.function.name,arguments:t}})}function we(e){return typeof e==`string`?e:JSON.stringify(e)}var Te=class{model;defaultMaxTokens;defaultTemperature;defaultReasoningEffort;defaultBudgetTokens;supportsJsonObjectMode;constructor(e){this.model=e.model,this.defaultMaxTokens=e.defaultMaxTokens,this.defaultTemperature=e.defaultTemperature,this.defaultReasoningEffort=e.defaultReasoningEffort,this.defaultBudgetTokens=e.defaultBudgetTokens,this.supportsJsonObjectMode=e.supportsJsonObjectMode}build(e){let{systemPrompt:t,userContent:n,history:r=[],maxTokens:i=this.defaultMaxTokens,model:a=this.model,jsonSchema:o,tools:s,toolChoice:c}=e,l=e.temperature===void 0?this.defaultTemperature:e.temperature,u=e.reasoningEffort===void 0?this.defaultReasoningEffort:e.reasoningEffort,d=e.budgetTokens===void 0?this.defaultBudgetTokens:e.budgetTokens;if(s&&s.length===0)throw new f("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.",`invalid_params`);if(s){let e=new Set,t=new Set;for(let n of s)e.has(n.name)&&t.add(n.name),e.add(n.name);if(t.size)throw new f(`\`tools\` has duplicate name(s): [${[...t].join(`, `)}]. Tool names must be unique.`,`invalid_params`,{code:`duplicate_tool_names`,issues:{names:[...t]}})}if(c&&!s)throw new f("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.",`invalid_params`);if(s&&typeof c==`object`&&!s.some(e=>e.name===c.name))throw new f(`toolChoice names "${c.name}", which is not in \`tools\` ([${s.map(e=>e.name).join(`, `)}]).`,`invalid_params`,{code:`unknown_tool_choice`,issues:{requested:c.name,available:s.map(e=>e.name)}});let p=e.jsonMode,m=p??!s;if(!this.supportsJsonObjectMode&&!o&&p===!0)throw new f('jsonMode: true was set explicitly, but this client does not support `response_format: "json_object"` (see LLMClient.supportsJsonObjectMode). Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output for this mode. Use `jsonSchema` instead, which maps to a real constraint on both.',`invalid_params`);if(!this.supportsJsonObjectMode&&!o&&p===void 0&&e.schema)throw new f('`schema` was provided, which requires JSON output to validate against, but this client does not support `response_format: "json_object"` (see LLMClient.supportsJsonObjectMode) and no `jsonSchema` was set. Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output without one. Use `jsonSchema` instead, which maps to a real constraint on both and still runs `schema` against its parsed result.',`invalid_params`);let h=!this.supportsJsonObjectMode&&!o&&p===void 0?!1:m,g=h||!!o;if(e.schema&&!g)throw new f(`schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.`,`invalid_params`);let _=this.buildResponseFormat(o,g);this.validateHistory(r);let v={model:a,...l===null?{}:{temperature:l},max_tokens:i,..._?{response_format:_}:{},...u?{reasoning_effort:u}:{},...d==null?{}:{budget_tokens:d},...s?{tools:xe(s)}:{},...s?{tool_choice:this.buildWireToolChoice(c)}:{},messages:[...t?[{role:`system`,content:t}]:[],...r.flatMap(e=>this.turnToWireMessages(e)),{role:`user`,content:n}]};return{useJson:g,model:a,request:v}}validateHistory(e){let t;for(let[n,r]of e.entries()){if(r.role===`tool`){if(t?.role!==`assistant`||!t.toolCalls?.length)throw new f(`history[${n}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`,`invalid_params`);if(!r.toolResults?.length)throw new f(`history[${n}] is a "tool" turn but has no toolResults`,`invalid_params`);let e=new Set(t.toolCalls.map(e=>e.id)),i=r.toolResults.map(e=>e.toolCallId),a=i.filter(t=>!e.has(t));if(a.length)throw new f(`history[${n}].toolResults references unknown toolCallId(s) [${a.join(`, `)}]`,`invalid_params`,{code:`unknown_tool_result_ids`,issues:{historyIndex:n,ids:a}});let o=new Set,s=new Set;for(let e of i)o.has(e)&&s.add(e),o.add(e);if(s.size)throw new f(`history[${n}].toolResults has duplicate toolCallId(s) [${[...s].join(`, `)}]`,`invalid_params`,{code:`duplicate_tool_result_ids`,issues:{historyIndex:n,ids:[...s]}});let c=[...e].filter(e=>!i.includes(e));if(c.length)throw new f(`history[${n}] is missing toolResults for toolCallId(s) [${c.join(`, `)}]`,`invalid_params`,{code:`missing_tool_results`,issues:{historyIndex:n,ids:c}})}else{if(r.role===t?.role)throw new f(`history must alternate user/assistant turns: consecutive "${r.role}" turns at history[${n-1}] and history[${n}]`,`invalid_params`);if(t?.role===`assistant`&&t.toolCalls?.length)throw new f(`history[${n}] follows an assistant tool request without tool results`,`invalid_params`)}t=r}if(t?.role===`assistant`&&t.toolCalls?.length)throw new f(`The last entry in history is an assistant tool request without tool results`,`invalid_params`);if(t?.role===`user`)throw new f(`The last entry in history is a "user" turn, which would collide with the current userContent turn.`,`invalid_params`)}buildWireToolChoice(e){return!e||e===`auto`?`auto`:e===`none`||e===`required`?e:{type:`function`,function:{name:e.name}}}turnToWireMessages(e){return e.role===`tool`?(e.toolResults??[]).map(e=>({role:`tool`,tool_call_id:e.toolCallId,content:typeof e.content==`string`?e.content:JSON.stringify(e.content??null),...e.isError?{is_error:!0}:{}})):e.role===`assistant`&&e.toolCalls?.length?[{role:`assistant`,...e.content===void 0?{}:{content:we(e.content)},tool_calls:Se(e.toolCalls)}]:e.role===`assistant`?[{role:`assistant`,content:we(e.content===void 0?``:e.content)}]:[{role:e.role,content:e.content??``}]}buildResponseFormat(e,t){return e?{type:`json_schema`,json_schema:{name:e.name,schema:e.schema,strict:e.strict??!0,description:e.description}}:t?{type:`json_object`}:void 0}};function Ee(e,t,n){let{requestId:r,model:i,providerName:a,isFallback:o,chunkIdleTimeoutMs:s,streamController:c,logger:l,signal:u}=n,d,f,p=new Promise((e,t)=>{d=e,f=t});p.catch(()=>{});let m=1e4,h=[],g=[],_=!1,v,y=!1,b=e=>{let t=g.shift();if(t){t.resolve({done:!1,value:e});return}h.push(e),h.length>m*2&&(y||(y=!0,l.warn(`[VernLLM] stream chunk buffer exceeded cap (${m}), evicting ${h.length-m} oldest chunk(s); buffered=${h.length}. The chunks iterable was never read (or fell far behind) for this stream.`)),h.splice(0,h.length-m))},x=()=>{_=!0;for(let e of g.splice(0))e.resolve({done:!0,value:void 0})},S=e=>{_=!0,v=e;for(let t of g.splice(0))t.reject(e)},C={[Symbol.asyncIterator](){return{next(){return h.length?Promise.resolve({done:!1,value:h.shift()}):_?v?Promise.reject(v):Promise.resolve({done:!0,value:void 0}):new Promise((e,t)=>{g.push({resolve:e,reject:t})})}}}},w=new Map,T=``,E;return(async()=>{try{let n=t;for(;!n.done;){let t=n.value;if(t.type!==`ping`){if(t.type===`text-delta`)T+=t.delta,b({type:`text-delta`,delta:t.delta});else if(t.type===`tool_call_delta`){let e=w.get(t.index)??{args:``};e.id??=t.id,e.name??=t.name,e.args+=t.argumentsDelta??``,w.set(t.index,e),b({type:`tool_call_delta`,index:t.index,id:t.id,name:t.name,argsDelta:t.argumentsDelta,complete:t.complete})}else if(t.type===`usage`){let e=t.usage.completion_tokens_details?.reasoning_tokens;E={promptTokens:t.usage.prompt_tokens??0,completionTokens:t.usage.completion_tokens??0,totalTokens:t.usage.total_tokens??0,...e===void 0?{}:{reasoningTokens:e},requestId:r,model:i,provider:a,usedFallback:o},b({type:`usage`,usage:E})}}n=await se(()=>e.next(),s,()=>c.abort(),l)}}catch(t){try{await e.return?.()}catch{}c.abort();let r=F(t,u);try{n.onStreamFailure(r,E)}catch{}S(r),f(r);return}x();try{n.onStreamSuccess(E)}catch{}try{let e=w.size?[...w.entries()].sort(([e],[t])=>e-t).map(([,e])=>({id:e.id??``,type:`function`,function:{name:e.name??``,arguments:e.args}})):void 0,t=n.finalize(T,e,E);d(t)}catch(e){f(e)}})(),{chunks:C,finalResult:p}}function De(e){return e}var Oe=class{maxRetries;timeoutMs;chunkIdleTimeoutMs;baseDelayMs;nonRetryableStatus;parseJson;logger;redact;onUsage;onUsageFailure;reportEvent;breaker;limiter;isFallback;requestBuilder;constructor(e,t,n,r){this.providerName=e,this.client=t,this.model=n,this.maxRetries=r.maxRetries,this.timeoutMs=r.timeoutMs,this.chunkIdleTimeoutMs=r.chunkIdleTimeoutMs,this.baseDelayMs=r.baseDelayMs,this.nonRetryableStatus=r.nonRetryableStatus,this.parseJson=r.parseJson??be,this.logger=r.logger,this.redact=r.redact,this.onUsage=r.onUsage,this.onUsageFailure=r.onUsageFailure,this.reportEvent=A(r.onEvent,this.logger),this.breaker=r.breaker,this.limiter=r.limiter,this.isFallback=r.isFallback??!1,this.requestBuilder=new Te({model:n,defaultMaxTokens:r.defaultMaxTokens,defaultTemperature:r.defaultTemperature,defaultReasoningEffort:r.defaultReasoningEffort,defaultBudgetTokens:r.defaultBudgetTokens,supportsJsonObjectMode:t.supportsJsonObjectMode??!0})}getCircuitState(e){return this.breaker?.getState(e)}get isolateByModel(){return this.breaker?.isolateByModel??!1}openCircuit(e){this.breaker?.open(e)}closeCircuit(e){this.breaker?.close(e)}assertBreakerClosed(e){this.breaker?.assertClosed(e??this.model)}async run(e,t,n){let r=e.model??this.model,i=[];try{return await this.retryWithBackoff((n,r)=>this.executeCall(e,t,n,r),t,r,e.signal,n,i)}catch(n){let a=F(n,e.signal,i.length>0?i:void 0);throw this.countsTowardBreaker(a)&&this.breaker?.recordFailure(r),this.logger.debug(`[VernLLM:${t}] error:\n${this.redactText(P(n))}`),a}}async runStream(e,t,n){let r=e.model??this.model,i=[];try{return await this.retryWithBackoff((n,r)=>this.executeStreamCall(e,t,n,r),t,r,e.signal,n,i)}catch(n){let a=F(n,e.signal,i.length>0?i:void 0);throw this.countsTowardBreaker(a)&&this.breaker?.recordFailure(r),this.logger.debug(`[VernLLM:${t}] stream-open error:\n${this.redactText(P(n))}`),a}}async executeCall(e,t,n,r){let{useJson:i,model:a,request:o}=this.requestBuilder.build(e);r?.(d(this.providerName,a,o,void 0,Date.now()));let s;if(this.limiter){let n=await this.limiter.acquire(this.limiter.estimate(o),e.signal);s=n.release,n.waitedMs>0&&this.reportEvent({kind:`rate_limited`,requestId:t,provider:this.providerName,model:a,waitedMs:n.waitedMs,reason:n.reason??`rpm`})}try{let r=await oe(e=>this.client.chat.completions.create(o,{signal:e}),this.timeoutMs,e.signal),c=this.extractUsage(r,t,a);s?.(this.actualTokensFor(c)),s=void 0;let l=r.choices?.[0]?.message?.content,u=r.choices?.[0]?.message?.tool_calls;return this.finalizeResponse(l,u,e,i,a,c,t,n)}finally{s?.()}}redactText(e){return this.redact?this.redact(e):e}redactedOutput(e,t){return this.redactText(e??`[${t?.length??0} tool call(s)]`)}finalizeResponse(e,t,n,r,i,a,o,s){try{let s=e?.trim();if(!s&&!t?.length)throw new f(`Empty LLM response`,`api`,{code:`empty_response`});if(this.logger.debug(`[VernLLM:${o}] output:\n${this.redactedOutput(s,t).slice(0,800)}`),t?.length){if(!n.tools)throw new f("Provider returned tool_calls but no `tools` were sent with this call.",`validation`,{code:`unexpected_tool_calls`});if(n.toolChoice===`none`)throw new f(`Provider returned tool_calls despite toolChoice: 'none'.`,`validation`,{code:`tool_choice_none_violated`});let e=Ce(t);return this.validateToolCallArguments(e,n.tools),this.breaker?.recordSuccess(i),this.reportUsage(a),{type:`tool_calls`,toolCalls:e,...s?{content:s}:{}}}let c=s??``;if(!r)return this.breaker?.recordSuccess(i),this.reportUsage(a),n.tools?{type:`content`,content:c}:c;let l=this.parseAndValidate(c,n.schema);return this.breaker?.recordSuccess(i),this.reportUsage(a),n.tools?{type:`content`,content:l}:l}catch(e){let t=F(e,n.signal);throw a&&t.type!==`aborted`&&this.reportUsageFailure(a,t,s),t}}async executeStreamCall(e,t,n,r){let{useJson:i,model:a,request:o}=this.requestBuilder.build(e);r?.(d(this.providerName,a,o,void 0,Date.now()));let s=this.client.chat.completions;if(!s.createStream)throw new f(`stream: true requires a client/adapter with createStream`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`createStream`}});let c=s.createStream.bind(s),l;if(this.limiter){let n=await this.limiter.acquire(this.limiter.estimate(o),e.signal);l=n.release,n.waitedMs>0&&this.reportEvent({kind:`rate_limited`,requestId:t,provider:this.providerName,model:a,waitedMs:n.waitedMs,reason:n.reason??`rpm`})}let u=new AbortController,p=e.signal?AbortSignal.any([e.signal,u.signal]):u.signal;try{let{iterator:r,first:s}=await oe(async e=>{let t=c(o,{signal:e})[Symbol.asyncIterator](),n=await t.next();return{iterator:t,first:n}},this.timeoutMs,p);if(s.done)throw new f(`Empty LLM response`,`api`);let d=l,m=Ee(r,s,{requestId:t,model:a,providerName:this.providerName,isFallback:this.isFallback,chunkIdleTimeoutMs:e.chunkIdleTimeoutMs??this.chunkIdleTimeoutMs,streamController:u,logger:this.logger,signal:e.signal,onStreamSuccess:e=>{this.breaker?.recordSuccess(a),d?.(this.actualTokensFor(e))},onStreamFailure:(e,t)=>{e.type===`timeout`&&this.breaker?.recordFailure(a),t&&e.type!==`aborted`&&this.reportUsageFailure(t,e,n,!0),d?.(this.actualTokensFor(t))},finalize:(r,o,s)=>this.finalizeResponse(r,o,e,i,a,s,t,n)});return l=void 0,m}finally{l?.()}}validateToolCallArguments(e,t){let n=new Map(t.map(e=>[e.name,e])),r=new Set,i=[];for(let t of e)r.has(t.id)&&i.push({name:t.name,toolCallId:t.id,code:`duplicate_tool_call_id`}),r.add(t.id),n.has(t.name)||i.push({name:t.name,toolCallId:t.id,code:`unknown_tool`});if(i.length>0){let e=i.find(e=>e.code===`unknown_tool`),t=e?`Model requested tool "${e.name}", which was not in the tools offered ([${[...n.keys()].join(`, `)}]).`:`Duplicate tool call id "${i[0].toolCallId}" in the model's response.`,r=i.length>1?`${t} (${i.length} tool call issues total, see error.issues.)`:t;throw new f(r,`validation`,{code:e?`unknown_tool`:`duplicate_tool_call_id`,issues:i})}for(let t of e){let e=n.get(t.name);if(!e?.argumentsSchema)continue;let r=e.argumentsSchema.safeParse(t.arguments);if(!r.success)throw new f(`Arguments for tool call "${t.name}" failed validation`,`validation`,{issues:r.error})}}async retryWithBackoff(e,t,n,r,i,a){let o,s;for(let c=0;c<=this.maxRetries;c++){s=void 0;try{return c>0&&await this.recoverDelay(t,n,c,o,r),i?.(),await e(c,e=>{s=e})}catch(e){o=e;let t=c<this.maxRetries&&this.shouldRetry(e,r);if(!t)break;a?.push({index:c,error:F(e,r).toSnapshot(),request:De(s)})}}throw o}extractUsage(e,t,n){if(!e.usage)return;let r=e.usage.completion_tokens_details?.reasoning_tokens;return{promptTokens:e.usage.prompt_tokens??0,completionTokens:e.usage.completion_tokens??0,totalTokens:e.usage.total_tokens??0,...r===void 0?{}:{reasoningTokens:r},requestId:t,model:n,provider:this.providerName,usedFallback:this.isFallback}}actualTokensFor(e){if(e)return e.totalTokens||e.promptTokens+e.completionTokens}reportUsage(e){if(!(!e||!this.onUsage))try{this.onUsage(e)}catch(e){this.logger.error(`[VernLLM] onUsage failed`,{message:e instanceof Error?e.message:`unknown`})}}reportUsageFailure(e,t,n,r=!1){let i=e.totalTokens||e.promptTokens+e.completionTokens,a=r?`mid-stream failure (terminal, no further attempts)`:`attempt ${n+1}/${this.maxRetries+1}`;if(this.logger.warn(`[VernLLM:${e.requestId}] usage failure, ${a}: type=${t.type} tokens=${i}`),this.onUsageFailure)try{this.onUsageFailure(e,t)}catch(e){this.logger.error(`[VernLLM] onUsageFailure failed`,{message:e instanceof Error?e.message:`unknown`})}}parseAndValidate(e,t){let n;try{n=this.parseJson(e)}catch{throw new f(`Invalid JSON response`,`parse`)}if(n==null)throw new f(`Invalid JSON response`,`parse`);if(!t)return n;let r=t.safeParse(n);if(!r.success)throw new f(`Schema validation failed`,`validation`,{issues:r.error});return r.data}async recoverDelay(e,t,n,r,i){let a=ce(r),o=a??le(this.baseDelayMs,n),s=a!==void 0;this.logger.warn(`[VernLLM:${e}] recovery attempt ${n}/${this.maxRetries}, waiting ${Math.ceil(o)}ms`+(s?` (honoring Retry-After)`:``)),this.reportEvent({kind:`retry`,requestId:e,provider:this.providerName,model:t,attempt:n,maxRetries:this.maxRetries,delayMs:o,retryAfterHonored:s,error:F(r,i)}),await ue(o,i)}shouldRetry(e,t){if(t?.aborted||e instanceof f&&!e.retryable)return!1;let n=de(e);return!(n!==void 0&&this.nonRetryableStatus.includes(n))}countsTowardBreaker(e){return e.retryable}};function ke(e){return{debug:I(e,`debug`),warn:I(e,`warn`),error:I(e,`error`)}}function I(e,t){let n=e[t].bind(e);return(...e)=>{try{je(n(...e))}catch{}}}function Ae(e){return typeof e?.then==`function`}function je(e){Ae(e)&&Promise.resolve(e).catch(()=>{})}var L=class{constructor(e){this.debugEnabled=e}debug(e){this.debugEnabled&&console.debug(e)}warn(e){console.warn(e)}error(e,t){console.error(e,t??``)}};function R(e){let t=e.messages.reduce((e,t)=>{let n=t.content;if(typeof n==`string`)return e+n.length;if(n==null)return e;try{return e+JSON.stringify(n).length}catch{return e}},0);return Math.ceil(t/4)+(e.max_tokens??0)}var z=class{available;lastRefill=Date.now();constructor(e,t){this.capacity=e,this.refillPerMs=t,this.available=e}refill(){if(this.refillPerMs===0)return;let e=Date.now(),t=e-this.lastRefill;this.available=Math.min(this.capacity,this.available+Math.max(0,t)*this.refillPerMs),this.lastRefill=e}tryTake(e){return this.refill(),this.available<e?!1:(this.available-=e,!0)}msUntilAvailable(e){return this.refill(),this.available>=e?0:this.refillPerMs===0?1/0:(e-this.available)/this.refillPerMs}give(e){this.available=Math.min(this.capacity,this.available+e)}getCapacity(){return this.capacity}};const Me=2147483647;var B=class{requests;tokens;concurrency;maxQueueMs;maxQueueSize;estimateTokensFn;queue=[];wakeTimer;constructor(e){e.requestsPerMinute&&(this.requests=new z(e.requestsPerMinute,e.requestsPerMinute/6e4)),e.tokensPerMinute&&(this.tokens=new z(e.tokensPerMinute,e.tokensPerMinute/6e4)),e.maxConcurrent&&(this.concurrency=new z(e.maxConcurrent,0)),this.maxQueueMs=e.maxQueueMs??3e4,this.maxQueueSize=e.maxQueueSize??0,this.estimateTokensFn=e.estimateTokens??R}estimate(e){return this.estimateTokensFn(e)}async acquire(e,t){if(t?.aborted)throw new f(`LLM request aborted`,`aborted`);if(!Number.isFinite(e)||e<0)throw new f(`estimatedTokens must be a finite, non-negative number, got ${String(e)}`,`invalid_params`);if(this.tokens&&e>this.tokens.getCapacity())throw new f(`estimatedTokens (${e}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`,`rate_limited`,{code:`rate_limit_capacity_exceeded`});if(this.queue.length===0){let n=this.tryAcquireBuckets(e);return n.ok?{release:this.makeRelease(e),waitedMs:0}:this.enqueue(e,n.reason,t)}if(this.maxQueueSize>0&&this.queue.length>=this.maxQueueSize)throw this.queueFullError();return this.enqueue(e,void 0,t)}queueFullError(){return new f(`Rate limit queue is full`,`rate_limited`,{code:`rate_limit_queue_full`})}enqueue(e,t,n){return new Promise((r,i)=>{let a={estimatedTokens:e,enqueuedAt:Date.now(),lastReason:t,resolve:e=>{c(),r(e)},reject:e=>{c(),this.wakeTimer&&(clearTimeout(this.wakeTimer),this.wakeTimer=void 0),this.drain(),i(e)}},o,s=()=>{a.reject(new f(`LLM request aborted`,`aborted`))},c=()=>{o&&clearTimeout(o),n?.removeEventListener(`abort`,s);let e=this.queue.indexOf(a);e!==-1&&this.queue.splice(e,1)};this.maxQueueMs>0&&(o=setTimeout(()=>{a.reject(new f(`Rate limit queue timed out before capacity was available`,`rate_limited`,{code:`rate_limit_queue_timeout`}))},this.maxQueueMs)),n?.addEventListener(`abort`,s,{once:!0}),this.queue.push(a),this.drain()})}tryAcquireBuckets(e){let t=[],n=(e,n)=>e?e.tryTake(n)?(t.push({bucket:e,amount:n}),!0):!1:!0;if(!n(this.concurrency,1))return{ok:!1,reason:`concurrency`};if(!n(this.requests,1)){for(let e of t)e.bucket.give(e.amount);return{ok:!1,reason:`rpm`}}if(!n(this.tokens,e)){for(let e of t)e.bucket.give(e.amount);return{ok:!1,reason:`tpm`}}return{ok:!0}}drain(){for(;this.queue.length>0;){let e=this.queue[0],t=this.tryAcquireBuckets(e.estimatedTokens);if(!t.ok){e.lastReason=t.reason,this.scheduleWake(t.reason,e.estimatedTokens);return}let n=Date.now()-e.enqueuedAt;e.resolve({release:this.makeRelease(e.estimatedTokens),waitedMs:n,reason:e.lastReason})}}scheduleWake(e,t){if(this.wakeTimer)return;let n=e===`rpm`?this.requests?.msUntilAvailable(1):e===`tpm`?this.tokens?.msUntilAvailable(t):void 0;if(n===void 0||!Number.isFinite(n))return;let r=Math.min(Math.max(1,Math.ceil(n)),Me);this.wakeTimer=setTimeout(()=>{this.wakeTimer=void 0,this.drain()},r)}makeRelease(e){let t=!1;return n=>{t||(t=!0,this.concurrency?.give(1),this.tokens&&n!==void 0&&Number.isFinite(n)&&this.tokens.give(e-n),this.drain())}}},Ne=class{logger;executors;fallbackOn;reportEvent;cacheOrchestrator;constructor(e){this.logger=ke(e.logger??new L(e.debug??!1));let t=e.name??`primary`;this.cacheOrchestrator=new ne(e.cache??new h,this.logger),this.fallbackOn=e.fallbackOn??b,this.reportEvent=A(e.onEvent,this.logger);let n=e.defaultTemperature===void 0?.2:e.defaultTemperature,r=e.defaultReasoningEffort,i=e.defaultBudgetTokens,a={client:e.client,model:e.model,name:t,maxRetries:e.maxRetries,timeoutMs:e.timeoutMs,chunkIdleTimeoutMs:e.chunkIdleTimeoutMs,baseDelayMs:e.baseDelayMs,defaultMaxTokens:e.defaultMaxTokens,defaultTemperature:n,defaultReasoningEffort:r,defaultBudgetTokens:i,nonRetryableStatus:e.nonRetryableStatus,circuitBreaker:e.circuitBreaker,rateLimit:e.rateLimit},o=Array.isArray(e.fallback)?e.fallback:e.fallback?[e.fallback]:[],s=[a,...o];this.executors=s.map((a,o)=>{let s=o>0,c=a.name??(s?`fallback[${o-1}]`:t),l=ie(a.circuitBreaker,c,a.model,e.onEvent,this.logger);return new Oe(c,a.client,a.model,{maxRetries:a.maxRetries??e.maxRetries??1,timeoutMs:a.timeoutMs??e.timeoutMs??25e3,chunkIdleTimeoutMs:a.chunkIdleTimeoutMs??e.chunkIdleTimeoutMs??3e4,baseDelayMs:a.baseDelayMs??e.baseDelayMs??500,defaultMaxTokens:a.defaultMaxTokens??e.defaultMaxTokens??1e3,defaultTemperature:a.defaultTemperature===void 0?n:a.defaultTemperature,defaultReasoningEffort:a.defaultReasoningEffort===void 0?r:a.defaultReasoningEffort,defaultBudgetTokens:a.defaultBudgetTokens===void 0?i:a.defaultBudgetTokens,nonRetryableStatus:a.nonRetryableStatus??e.nonRetryableStatus??[400,401,403,404,422],parseJson:e.parseJson,logger:this.logger,redact:e.redact,onUsage:e.onUsage,onUsageFailure:e.onUsageFailure,onEvent:e.onEvent,breaker:l,limiter:a.rateLimit?new B(a.rateLimit):void 0,isFallback:s})})}logRefundError(e,t){this.logger.error(e,{message:t instanceof Error?t.message:`unknown`})}async runFallbackChain(e,t,n,r=!1){let i=[];for(let a=0;a<this.executors.length;a++){let o=this.executors[a],s=Date.now(),c=0;try{a===0&&r||o.assertBreakerClosed(e.model);let t=await n(o,()=>{c+=1});return{result:t,executor:o,index:a,attemptCount:c}}catch(n){let r=F(n,e.signal);i.push({index:a-1,provider:o.providerName,model:e.model??o.model,error:r.toSnapshot()});let c=a===this.executors.length-1,l=this.fallbackOn(r,{isLastTarget:c}),u=c?`stop`:l;if(u===`stop`)throw i.length>1?new x(i):r;let d=this.executors[a+1];this.reportEvent({kind:`fallback`,requestId:t,from:o.providerName,to:d.providerName,fromIndex:a-1,toIndex:a,error:r,elapsedMs:Date.now()-s})}}throw new f(`No provider targets configured`,`invalid_params`)}async call(t){if(t.signal?.aborted)throw new f(`LLM request aborted`,`aborted`);let n=t.requestId??e(),r=this.executors.length===1;return r&&this.executors[0].assertBreakerClosed(t.model),t.stream?E(t,async()=>{let{result:e}=await this.runFallbackChain(t,n,(e,r)=>e.runStream(t,n,r),r);return e},t.signal,(e,t)=>this.logRefundError(e,t)):T(t,!1,async()=>{let{result:e,executor:i,index:a,attemptCount:o}=await this.runFallbackChain(t,n,(e,r)=>e.run(t,n,r),r);return t.meta&&(t.meta.current={provider:i.providerName,model:t.model??i.model,fallbackIndex:a-1,usedFallback:a>0,attempts:o}),e},t.signal,(e,t)=>this.logRefundError(e,t))}runCached(e){return this.cacheOrchestrator.runCached(e)}async deleteCache(e){await this.cacheOrchestrator.deleteCache(e)}async cachedCall(e){let{call:t,...n}=e,r=t;if(r.reserveUsage||r.refundUsage)throw new f("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.",`invalid_params`);if(r.stream){let e=r;return this.cacheOrchestrator.runCachedStream({...n,openStream:()=>this.call(e)},!!r.tools)}return this.runCached({...n,fn:()=>this.call(r)})}getCircuitState(e){let t=this.resolveExecutor(e?.index??0,`getCircuitState`);return this.warnIfModelUnsupported(t.isolateByModel,e?.model,`getCircuitState`),t.getCircuitState(e?.model??t.model)}getCircuitStates(e){return this.executors.map((t,n)=>({provider:t.providerName,index:n,isFallback:n>0,isolateByModel:t.isolateByModel,state:t.getCircuitState(e??t.model)}))}openCircuit(e){let t=this.resolveExecutor(e?.index??0,`openCircuit`);this.warnIfModelUnsupported(t.isolateByModel,e?.model,`openCircuit`),t.openCircuit(e?.model??t.model)}closeCircuit(e){let t=this.resolveExecutor(e?.index??0,`closeCircuit`);this.warnIfModelUnsupported(t.isolateByModel,e?.model,`closeCircuit`),t.closeCircuit(e?.model??t.model)}resolveExecutor(e,t){let n=this.executors[e];if(!n)throw RangeError(`${t}: no target at index ${e} (chain has ${this.executors.length} target${this.executors.length===1?``:`s`})`);return n}warnIfModelUnsupported(e,t,n){t!==void 0&&!e&&this.logger.warn(`[VernLLM] ${n}: \`model: '${t}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`)}};function Pe(e){return e}function Fe(e){return e}async function*V(e){let t=new TextDecoder(`utf-8`,{fatal:!0}),n=``;for await(let r of e){let e;try{e=typeof r==`string`?r:t.decode(r,{stream:!0})}catch(e){throw new f(`Invalid UTF-8 in SSE stream`,`parse`,{cause:e})}n=(n+e).replace(/\r\n/g,`
2
+ `).replace(/\r(?!$)/g,`
3
+ `);let i=n.indexOf(`
2
4
 
3
- //#region src/types/errors.ts
4
- /**
5
- * Tool contract codes: a model or provider response defect, not a
6
- * transient provider fault. Deterministic on the wire request, so
7
- * retrying can't change the outcome and it shouldn't count toward the
8
- * circuit breaker either. Shared by `LLMError.retryable` below and by
9
- * `CallExecutor`'s own retry/breaker accounting, so the two can't drift
10
- * apart.
11
- */
12
- const NON_RETRYABLE_TOOL_CONTRACT_CODES = new Set([
13
- "unknown_tool",
14
- "duplicate_tool_call_id",
15
- "tool_choice_none_violated",
16
- "unexpected_tool_calls"
17
- ]);
18
- /**
19
- * Local rate-limit codes: the call never reached the provider, so it says
20
- * nothing about the provider's health, and retrying either just requeues
21
- * behind the same limit (the two queue codes) or can never succeed at all
22
- * (`rate_limit_capacity_exceeded`). Shared for the same reason as
23
- * {@link NON_RETRYABLE_TOOL_CONTRACT_CODES}.
24
- */
25
- const LOCAL_RATE_LIMIT_CODES = new Set([
26
- "rate_limit_queue_full",
27
- "rate_limit_queue_timeout",
28
- "rate_limit_capacity_exceeded"
29
- ]);
30
- /**
31
- * Types that are never worth retrying on their own: deterministic
32
- * caller-input, model-response, or cancellation failures rather than a
33
- * transient provider fault.
34
- */
35
- const NON_RETRYABLE_TYPES = new Set([
36
- "parse",
37
- "validation",
38
- "invalid_params",
39
- "aborted"
40
- ]);
41
- /**
42
- * Shared retryability rule behind both `LLMError.retryable` and
43
- * `LLMErrorSnapshot.retryable`. Pulled out so the two can't drift apart:
44
- * a snapshot is a point-in-time copy of an error's fields, and this is
45
- * one of them, so it has to be computed the same way in both places.
46
- */
47
- function computeRetryable(type, code) {
48
- if (NON_RETRYABLE_TYPES.has(type)) return false;
49
- if (code && NON_RETRYABLE_TOOL_CONTRACT_CODES.has(code)) return false;
50
- if (code && LOCAL_RATE_LIMIT_CODES.has(code)) return false;
51
- return true;
52
- }
53
- /**
54
- * Returns `issues` unchanged when it can survive `JSON.stringify`.
55
- * Most `issues` values are VernLLM's own structured shapes (see
56
- * `LLMErrorIssuesByCode`) and always safe. The one exception is a
57
- * schema validation failure, where `issues` is a caller supplied
58
- * `SchemaLike` validator's own `error: unknown`, not controlled by
59
- * VernLLM and not guaranteed to be circular free. Rather than silently
60
- * dropping it in that case, this returns a marker string so a reader
61
- * of serialized output can tell "no issues data" apart from "issues
62
- * existed but could not be shown".
63
- */
64
- function safeIssues(issues) {
65
- if (issues === void 0) return void 0;
66
- try {
67
- JSON.stringify(issues);
68
- return issues;
69
- } catch {
70
- return "[Unserializable: issues contained a circular reference]";
71
- }
72
- }
73
- /**
74
- * Depth cap for `safeAttempts`, guarding against a pathological,
75
- * self referential `attempts` array. `attempts` is a public
76
- * `LLMErrorOptions` field, so a caller can construct one by hand; this
77
- * keeps that path bounded the same way a circular `issues` value is
78
- * bounded, rather than assuming well formed input.
79
- */
80
- const MAX_ATTEMPTS_DEPTH = 20;
81
- /**
82
- * Returns a copy of `attempts` with every nested snapshot's `issues`
83
- * re-checked through `safeIssues`, recursively through each snapshot's
84
- * own `attempts`. Needed for two reasons: `safeIssues` returns a safe
85
- * `issues` value by reference, so a shared object can be mutated into a
86
- * circular one after the snapshot was created, and `attempts` is a
87
- * public constructor option, so a caller can hand build a `RetryAttempt`
88
- * (or a whole `LLMErrorSnapshot`) with a circular `issues` and pass it
89
- * in directly, never touching `toSnapshot()` at all. Extra fields on an
90
- * attempt (e.g. `FallbackAttempt`'s `provider`/`model`) are preserved.
91
- */
92
- function safeAttempts(attempts, depth = 0) {
93
- if (attempts === void 0) return void 0;
94
- if (depth >= MAX_ATTEMPTS_DEPTH) return [];
95
- return attempts.map((attempt) => ({
96
- ...attempt,
97
- error: {
98
- ...attempt.error,
99
- issues: safeIssues(attempt.error.issues),
100
- attempts: safeAttempts(attempt.error.attempts, depth + 1)
101
- }
102
- }));
103
- }
104
- var LLMError = class extends Error {
105
- status;
106
- issues;
107
- cause;
108
- retryAfterMs;
109
- /** Stable discriminator within `type`. Absent on errors predating it. */
110
- code;
111
- /** Every attempt made before this error was thrown, in order. Absent when nothing was retried. */
112
- attempts;
113
- constructor(message, type, options = {}) {
114
- super(message);
115
- this.type = type;
116
- this.name = "LLMError";
117
- this.status = options.status;
118
- this.issues = options.issues;
119
- this.cause = options.cause;
120
- this.retryAfterMs = options.retryAfterMs;
121
- this.code = options.code;
122
- this.attempts = options.attempts;
123
- }
124
- /**
125
- * Computed purely from `type`/`code`, independent of any specific call's
126
- * `nonRetryableStatus` list. False for `parse`/`validation`/
127
- * `invalid_params`/`aborted` types (the caller's own input, the model's
128
- * own response, or intentional cancellation, none of which are the
129
- * provider being unhealthy), the tool contract codes, and the local
130
- * rate limit codes. Subclasses (see `FallbackExhaustedError`) may
131
- * override this when `type` alone carries no retry signal.
132
- */
133
- get retryable() {
134
- return computeRetryable(this.type, this.code);
135
- }
136
- /**
137
- * Copies this error's fields into an {@link LLMErrorSnapshot}, for
138
- * recording as a `RetryAttempt`/`FallbackAttempt`. `retryable` is
139
- * captured here since a snapshot has no getter of its own. `cause` is
140
- * not copied, see `LLMErrorSnapshot`'s own doc. `issues` and every
141
- * nested `attempts` entry's own `issues` go through `safeAttempts`,
142
- * since a schema validation failure's `issues` is a caller supplied
143
- * value, not controlled by VernLLM, and `attempts` is itself a public
144
- * constructor option a caller can hand build.
145
- */
146
- toSnapshot() {
147
- return {
148
- message: this.message,
149
- type: this.type,
150
- status: this.status,
151
- issues: safeIssues(this.issues),
152
- retryAfterMs: this.retryAfterMs,
153
- code: this.code,
154
- retryable: this.retryable,
155
- attempts: safeAttempts(this.attempts)
156
- };
157
- }
158
- /**
159
- * Controls what `JSON.stringify(err)` produces. Omits `cause` for the
160
- * same reason `toSnapshot()` does: `cause` is `unknown` and never
161
- * validated by VernLLM, and some SDK errors carry circular structures
162
- * `JSON.stringify` cannot serialize at all. Read `err.cause` directly
163
- * instead. `issues`, including every nested `attempts` entry's own
164
- * `issues`, goes through `safeAttempts` for the same reason: a schema
165
- * validation failure's `issues` is caller supplied and not guaranteed
166
- * circular free. Also includes `message` and `retryable`, which a
167
- * plain property walk would otherwise miss: `message` is
168
- * non-enumerable on `Error`, and `retryable` is a getter, not an own
169
- * property.
170
- */
171
- toJSON() {
172
- return {
173
- name: this.name,
174
- message: this.message,
175
- type: this.type,
176
- status: this.status,
177
- issues: safeIssues(this.issues),
178
- retryAfterMs: this.retryAfterMs,
179
- code: this.code,
180
- retryable: this.retryable,
181
- attempts: safeAttempts(this.attempts)
182
- };
183
- }
184
- };
185
- function isLLMError(err) {
186
- return err instanceof LLMError;
187
- }
188
- /**
189
- * Narrows `err.issues` to the exact shape {@link LLMErrorIssuesByCode} maps
190
- * `code` to, for any code listed there. `code` stays the only discriminator
191
- * VernLLM uses; this just gives that existing check a typed return instead
192
- * of requiring a manual cast of `issues`:
193
- *
194
- * ```ts
195
- * if (isLLMError(err) && hasIssues(err, 'duplicate_tool_names')) {
196
- * console.log(err.issues.names); // string[], no cast needed
197
- * }
198
- * ```
199
- */
200
- function hasIssues(err, code) {
201
- return err.code === code && err.issues !== void 0;
202
- }
5
+ `);for(;i!==-1;){let e=n.slice(0,i);n=n.slice(i+2);let t=G(e);if(t===H)return;t!==U&&(yield t),i=n.indexOf(`
203
6
 
204
- //#endregion
205
- //#region src/types/cache.ts
206
- /**
207
- * Trivial default so the package works out of the box with no external deps
208
- * Not shared across processes, swap in Redis/Upstash/etc for production
209
- */
210
- var InMemoryCacheAdapter = class {
211
- store = new Map();
212
- constructor(maxSize = 1e3) {
213
- this.maxSize = maxSize;
214
- }
215
- async get(key) {
216
- const entry = this.store.get(key);
217
- if (!entry) return {
218
- hit: false,
219
- value: null
220
- };
221
- if (Date.now() >= entry.expiresAt) {
222
- this.store.delete(key);
223
- return {
224
- hit: false,
225
- value: null
226
- };
227
- }
228
- return {
229
- hit: true,
230
- value: entry.value
231
- };
232
- }
233
- async set(key, value, ttl) {
234
- this.cleanupExpiredEntries();
235
- this.store.set(key, {
236
- value,
237
- expiresAt: Date.now() + ttl * 1e3
238
- });
239
- this.enforceSizeLimit();
240
- }
241
- async delete(key) {
242
- this.store.delete(key);
243
- }
244
- cleanupExpiredEntries() {
245
- const now = Date.now();
246
- for (const [key, entry] of this.store) if (now >= entry.expiresAt) this.store.delete(key);
247
- }
248
- enforceSizeLimit() {
249
- while (this.store.size > this.maxSize) {
250
- const oldestKey = this.store.keys().next().value;
251
- if (oldestKey === void 0) break;
252
- this.store.delete(oldestKey);
253
- }
254
- }
255
- };
256
- /**
257
- * Normalizes keys before caching to avoid duplicate entries from formatting differences.
258
- */
259
- var NormalizedCacheAdapter = class {
260
- constructor(inner = new InMemoryCacheAdapter()) {
261
- this.inner = inner;
262
- }
263
- normalize(key) {
264
- return key.toLowerCase().trim().replace(/[^\p{L}\p{N}\s]/gu, " ").replace(/\s+/g, " ").trim();
265
- }
266
- async resolveKey(key) {
267
- return this.normalize(key);
268
- }
269
- async get(key) {
270
- return this.inner.get(this.normalize(key));
271
- }
272
- async set(key, value, ttl) {
273
- await this.inner.set(this.normalize(key), value, ttl);
274
- }
275
- async delete(key) {
276
- await this.inner.delete?.(this.normalize(key));
277
- }
278
- };
279
- /**
280
- * Two-tier cache with fast local L1 and shared L2.
281
- * L2 hits are promoted back to L1.
282
- */
283
- var TieredCacheAdapter = class {
284
- constructor(l1, l2, l1Ttl) {
285
- this.l1 = l1;
286
- this.l2 = l2;
287
- this.l1Ttl = l1Ttl;
288
- }
289
- /**
290
- * Forwards to L1's `resolveKey` if it has one, otherwise L2's. L1 is
291
- * preferred since `get()` checks L1 first, so its notion of "the same
292
- * key" is the one that determines whether a lookup can skip L2 entirely.
293
- */
294
- async resolveKey(key) {
295
- if (this.l1.resolveKey) return this.l1.resolveKey(key);
296
- if (this.l2.resolveKey) return this.l2.resolveKey(key);
297
- return key;
298
- }
299
- async get(key) {
300
- const l1Result = await this.l1.get(key);
301
- if (l1Result.hit) return l1Result;
302
- const l2Result = await this.l2.get(key);
303
- if (l2Result.hit) await this.l1.set(key, l2Result.value, this.l1Ttl ?? 60);
304
- return l2Result;
305
- }
306
- async set(key, value, ttl) {
307
- await Promise.all([this.l1.set(key, value, this.l1Ttl ?? ttl), this.l2.set(key, value, ttl)]);
308
- }
309
- async delete(key) {
310
- await Promise.all([this.l1.delete?.(key), this.l2.delete?.(key)]);
311
- }
312
- };
7
+ `)}}try{n+=t.decode()}catch(e){throw new f(`Invalid UTF-8 in SSE stream`,`parse`,{cause:e})}n=n.replace(/\r$/,`
8
+ `);let r=n.indexOf(`
313
9
 
314
- //#endregion
315
- //#region src/types/tools.ts
316
- /**
317
- * Runtime-safe check for whether a `call()` result is a `tool_calls`
318
- * result. Prefer this over relying on TypeScript's static narrowing
319
- * whenever `params` passed to `call()` wasn't a literal with `tools`
320
- * inlined (see the "note on the overload" in `VernLLM.call`'s docs), in
321
- * that case TS may have typed the result as plain `T` even though it's
322
- * actually a `CallWithToolsResult<T>` at runtime, and this check works
323
- * either way.
324
- */
325
- function isToolCallResult(result) {
326
- return typeof result === "object" && result !== null && "type" in result && result.type === "tool_calls" && Array.isArray(result.toolCalls);
327
- }
10
+ `);for(;r!==-1;){let e=n.slice(0,r);n=n.slice(r+2);let t=G(e);if(t===H)return;t!==U&&(yield t),r=n.indexOf(`
328
11
 
329
- //#endregion
330
- //#region src/types/fallback.ts
331
- /** Tool contract failures are the model ignoring the request, not a sick provider: repeating it elsewhere can't help. */
332
- const TOOL_CONTRACT_CODES = new Set([
333
- "unknown_tool",
334
- "duplicate_tool_call_id",
335
- "tool_choice_none_violated",
336
- "unexpected_tool_calls"
337
- ]);
338
- /**
339
- * The default `fallbackOn` policy. Exported so a caller can wrap rather
340
- * than replace it, e.g. `fallbackOn: (e, ctx) => myCheck(e) ? 'stop' : defaultFallbackOn(e, ctx)`.
341
- */
342
- const defaultFallbackOn = (error) => {
343
- if (error.type === "parse" || error.type === "validation" || error.type === "aborted") return "stop";
344
- if (error.type === "quota_exceeded") return "stop";
345
- if (error.code && TOOL_CONTRACT_CODES.has(error.code)) return "stop";
346
- return "next";
347
- };
348
- /**
349
- * Thrown when the chain gives up, whether because the last target failed
350
- * or `fallbackOn` chose to stop early. Carries each attempt in order so
351
- * an outage across providers stays debuggable without reproducing it.
352
- * Extends `LLMError` so `isLLMError` and any `instanceof LLMError` check
353
- * still passes, inheriting the last failure's `type`/`status`/`retryAfterMs`
354
- * so existing type-based handling, including reading `retryAfterMs` on an
355
- * `'api'`-typed error, keeps working on a fallback-exhausted error too.
356
- */
357
- var FallbackExhaustedError = class extends LLMError {
358
- constructor(attempts) {
359
- const last = attempts[attempts.length - 1]?.error;
360
- super(`${attempts.length} provider${attempts.length === 1 ? "" : "s"} attempted and failed: ${attempts.map((a) => `${a.provider}(${a.error.type})`).join(" then ")}`, "fallback_exhausted", {
361
- status: last?.status,
362
- cause: last,
363
- retryAfterMs: last?.retryAfterMs,
364
- code: "fallback_exhausted",
365
- attempts
366
- });
367
- this.attempts = attempts;
368
- }
369
- /**
370
- * `type: 'fallback_exhausted'` by itself says nothing about whether
371
- * retrying could help; the reason the last target failed does. Defers to
372
- * that attempt's own `retryable` instead of anything about this class's
373
- * own type.
374
- */
375
- get retryable() {
376
- const last = this.attempts[this.attempts.length - 1]?.error;
377
- return last ? last.retryable : super.retryable;
378
- }
379
- };
380
- /** Narrows `err` to {@link FallbackExhaustedError}, for direct access to its `attempts` (`provider`/`model` per failed target) without a manual `instanceof` check. */
381
- function isFallbackExhaustedError(err) {
382
- return err instanceof FallbackExhaustedError;
383
- }
384
-
385
- //#endregion
386
- //#region src/internal/execution/usage.utils.ts
387
- /**
388
- * Calls `params.reserveUsage`, if present, mapping any failure to a
389
- * `quota_exceeded` LLMError (or an aborted error, if the signal fired
390
- * during reservation). Returns whether a reservation was actually made,
391
- * so callers know whether a later refund is needed. Shared by
392
- * `withReservedUsage` and `withReservedUsageForStream`, which differ only
393
- * in whether `coalesced` is caller-supplied or always `false`.
394
- */
395
- async function reserve(params, coalesced, signal) {
396
- if (!params.reserveUsage) return false;
397
- try {
398
- await params.reserveUsage({
399
- coalesced,
400
- signal
401
- });
402
- return true;
403
- } catch (error) {
404
- if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
405
- throw new LLMError(error instanceof Error ? error.message : "Usage reservation failed", "quota_exceeded", { cause: error });
406
- }
407
- }
408
- /**
409
- * Builds a `(logMessage) => Promise<void>` refund function bound to the
410
- * given hooks/coalesced/signal, reporting (instead of throwing) any error
411
- * the refund hook itself raises, so a broken refund hook never masks the
412
- * original error it was called to clean up after.
413
- */
414
- function makeRefund(params, coalesced, signal, onRefundError) {
415
- return async (logMessage) => {
416
- try {
417
- await params.refundUsage?.({
418
- coalesced,
419
- signal
420
- });
421
- } catch (refundError) {
422
- onRefundError(logMessage, refundError);
423
- }
424
- };
425
- }
426
- /**
427
- * Runs `getResult` after reserving usage, if a `reserveUsage` hook was
428
- * provided. `refundUsage` fires only if a reservation was actually made.
429
- * `onRefundError` is called (instead of throwing) whenever a refund attempt
430
- * itself fails, so a broken refund hook never masks the original error.
431
- */
432
- async function withReservedUsage(params, coalesced, getResult, signal, onRefundError) {
433
- if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
434
- const reserved = await reserve(params, coalesced, signal);
435
- const refund = makeRefund(params, coalesced, signal, onRefundError);
436
- if (signal?.aborted) {
437
- if (reserved) await refund("[VernLLM] refundUsage failed after abort");
438
- throw new LLMError("LLM request aborted", "aborted");
439
- }
440
- let result;
441
- try {
442
- result = await getResult();
443
- } catch (error) {
444
- if (reserved) await refund("[VernLLM] refundUsage failed");
445
- throw error;
446
- }
447
- if (signal?.aborted) {
448
- if (reserved) await refund("[VernLLM] refundUsage failed after abort");
449
- throw new LLMError("LLM request aborted", "aborted");
450
- }
451
- return result;
452
- }
453
- /**
454
- * Streaming counterpart to `withReservedUsage`. `withReservedUsage` assumes
455
- * `getResult()` settling *is* the operation's final outcome, awaiting it
456
- * synchronously before reserve/refund resolve. Streaming can't satisfy that:
457
- * `call()` must return `{ chunks, finalResult }` as soon as the stream
458
- * opens, well before the real outcome (validation, schema/tool-call checks)
459
- * is known.
460
- *
461
- * Reserves usage before `openStream` runs, same failure mode as the
462
- * non-streaming path if `reserveUsage` itself throws (mapped to
463
- * `quota_exceeded`, nothing opened). If `openStream` itself throws (stream
464
- * never opened), refunds synchronously and rethrows, exactly like
465
- * `withReservedUsage` does today. If it succeeds, returns `{ chunks,
466
- * finalResult }` immediately, refund/report is deferred onto
467
- * `finalResult`'s continuation, since that's the only point the real
468
- * outcome is known. This means `onUsageFailure` (and any refund) can fire
469
- * well after this function itself has returned.
470
- */
471
- async function withReservedUsageForStream(params, openStream, signal, onRefundError) {
472
- if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
473
- const reserved = await reserve(params, false, signal);
474
- const refund = makeRefund(params, false, signal, onRefundError);
475
- if (signal?.aborted) {
476
- if (reserved) await refund("[VernLLM] refundUsage failed after abort");
477
- throw new LLMError("LLM request aborted", "aborted");
478
- }
479
- let opened;
480
- try {
481
- opened = await openStream();
482
- } catch (error) {
483
- if (reserved) await refund("[VernLLM] refundUsage failed after stream-open failure");
484
- throw error;
485
- }
486
- const finalResult = opened.finalResult.then((value) => value, async (error) => {
487
- if (reserved) await refund("[VernLLM] refundUsage failed after stream error");
488
- throw error;
489
- });
490
- finalResult.catch(() => {});
491
- return {
492
- chunks: opened.chunks,
493
- finalResult
494
- };
495
- }
496
-
497
- //#endregion
498
- //#region src/internal/cache/replay.utils.ts
499
- /**
500
- * Converts an already-known cache value back into a plausible "text" form
501
- * for a one-shot replay chunk: passed through unchanged if it's already a
502
- * string (the `jsonMode: false` case), otherwise `JSON.stringify`'d (the
503
- * `jsonMode: true` case, where the cached value is the *parsed* result, not
504
- * the original raw text). This is a reasonable reconstruction, not a
505
- * byte-identical replay of whatever text the model originally streamed,
506
- * good enough for `for await (const c of chunks)` call sites that don't
507
- * branch on hit vs. miss, which is the only thing a cache-hit replay needs
508
- * to support.
509
- */
510
- function toReplayText(value) {
511
- return typeof value === "string" ? value : JSON.stringify(value) ?? "";
512
- }
513
- /**
514
- * Builds a trivially-exhausted one-shot `chunks` iterable from an
515
- * already-known value, used for a `cachedCall` cache hit, where there's no
516
- * live generation to relay (see `VernLLM.cachedCall`'s docs). No `usage`
517
- * chunk is emitted: a cache hit spent no real tokens, so there's nothing to
518
- * report, matching how non-streaming `cachedCall` never calls `onUsage` on
519
- * a hit either.
520
- *
521
- * `hasTools` must reflect whether the *original* call that produced this
522
- * cached value had `tools` set, that's what determines whether `value` is
523
- * `T` directly or a `CallWithToolsResult<T>` wrapper, and it isn't
524
- * something that can be reliably guessed from the value's shape alone
525
- * (a `schema`-validated `T` could coincidentally look like a
526
- * `CallWithToolsResult`).
527
- */
528
- function buildReplayChunks(value, hasTools) {
529
- const items = [];
530
- if (hasTools) {
531
- const result = value;
532
- if (result.type === "tool_calls") {
533
- result.toolCalls.forEach((toolCall, index) => {
534
- items.push({
535
- type: "tool_call_delta",
536
- index,
537
- id: toolCall.id,
538
- name: toolCall.name,
539
- argsDelta: JSON.stringify(toolCall.arguments ?? {}),
540
- complete: true
541
- });
542
- });
543
- if (result.content) items.push({
544
- type: "text-delta",
545
- delta: result.content
546
- });
547
- } else items.push({
548
- type: "text-delta",
549
- delta: toReplayText(result.content)
550
- });
551
- } else items.push({
552
- type: "text-delta",
553
- delta: toReplayText(value)
554
- });
555
- return { async *[Symbol.asyncIterator]() {
556
- for (const item of items) yield item;
557
- } };
558
- }
559
- /**
560
- * Streaming counterpart to `buildReplayChunks` for a `cachedCall` that
561
- * *joined* an already-in-flight call for the same key rather than
562
- * triggering one itself (see `runCachedStream`'s in-flight-coalescing
563
- * path): there's no live stream to relay (it isn't this call's stream to
564
- * relay, see the joiner-path comment in `runCachedStream`), but there's
565
- * also no value yet, only a pending promise for one. Waits for `promise`,
566
- * then delegates to `buildReplayChunks`. If `promise` rejects, iterating
567
- * `chunks` throws that same error, consistent with how a live stream's
568
- * `chunks` throws on a mid-stream failure.
569
- */
570
- function buildReplayChunksFromPromise(promise, hasTools) {
571
- return { async *[Symbol.asyncIterator]() {
572
- const value = await promise;
573
- yield* buildReplayChunks(value, hasTools);
574
- } };
575
- }
576
-
577
- //#endregion
578
- //#region src/internal/cache/cacheOrchestrator.ts
579
- /**
580
- * Owns cache key resolution, cache reads/writes, and in-flight coalescing
581
- * for concurrent misses on the same key. Doesn't know about `CallExecutor`,
582
- * retries, or providers at all: `fn`/`openStream` are opaque callbacks
583
- * (`VernLLM.cachedCall` passes `() => this.call(...)`), so this class only
584
- * needs the cache adapter and a logger. Extracted from `VernLLM` since
585
- * caching and per-target call mechanics are independent concerns that
586
- * happened to live on the same class.
587
- */
588
- var CacheOrchestrator = class {
589
- inFlight = new Map();
590
- constructor(cache, logger) {
591
- this.cache = cache;
592
- this.logger = logger;
593
- }
594
- /** Resolves a cache key through the adapter when it supports normalization. */
595
- async resolveCacheKey(key) {
596
- return this.cache.resolveKey ? await this.cache.resolveKey(key) : key;
597
- }
598
- /**
599
- * Removes a cached response by key when the configured cache adapter
600
- * supports deletion. Cache invalidation is the caller's responsibility;
601
- * only the application knows when cached data is stale.
602
- */
603
- async deleteCache(key) {
604
- if (!this.cache.delete) return;
605
- try {
606
- await this.cache.delete(await this.resolveCacheKey(key));
607
- } catch (error) {
608
- this.logger.warn(`[VernLLM] cache delete failed: ${error instanceof Error ? error.message : "unknown"}`);
609
- }
610
- }
611
- /** Logs a failed refundUsage attempt via the configured logger. */
612
- logRefundError(logMessage, error) {
613
- this.logger.error(logMessage, { message: error instanceof Error ? error.message : "unknown" });
614
- }
615
- /**
616
- * Reads from the cache, treating a failed adapter read as a miss rather
617
- * than letting it fail the call. The request still falls through to a
618
- * real provider call, but that fallback is now logged instead of silent.
619
- */
620
- async getCached(key) {
621
- try {
622
- return await this.cache.get(key);
623
- } catch (error) {
624
- this.logger.warn(`[VernLLM] cache read failed: ${error instanceof Error ? error.message : "unknown"}`);
625
- return { hit: false };
626
- }
627
- }
628
- /**
629
- * Internal cache primitive around caller-supplied logic. Concurrent misses
630
- * for the same `cacheKey` share a single in-flight call, avoiding cache
631
- * stampedes.
632
- *
633
- * Backs the public `VernLLM.cachedCall()`, which always composes this
634
- * with `call()` so cached results get the same retry/timeout/
635
- * circuit-breaker guarantees as any other LLM call.
636
- *
637
- * @param params `cacheKey`, `ttl`, `fn` (the work to run on a cache
638
- * miss, typically `() => this.call(...)`), and optional
639
- * `reserveUsage`/`refundUsage`/`signal`. See `InternalCacheParams`.
640
- * @returns The cached value on a hit, or the result of `fn()` on a miss.
641
- */
642
- async runCached(params) {
643
- const resolvedKey = await this.resolveCacheKey(params.cacheKey);
644
- const resolvedParams = resolvedKey === params.cacheKey ? params : {
645
- ...params,
646
- cacheKey: resolvedKey
647
- };
648
- const cached = await this.getCached(resolvedKey);
649
- if (cached.hit) return cached.value;
650
- const existing = this.inFlight.get(resolvedKey);
651
- if (existing) return withReservedUsage(resolvedParams, true, () => existing, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
652
- return this.registerTrigger(resolvedParams);
653
- }
654
- /** Starts the shared fn() call for a cache miss and tracks it in the in-flight map until it settles. */
655
- registerTrigger(params) {
656
- const resultPromise = withReservedUsage(params, false, () => this.runAndCache(params), params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
657
- this.inFlight.set(params.cacheKey, resultPromise);
658
- resultPromise.catch(() => {}).finally(() => {
659
- this.inFlight.delete(params.cacheKey);
660
- });
661
- return resultPromise;
662
- }
663
- /** Runs `fn` and writes its result to the cache. */
664
- async runAndCache(params) {
665
- const result = await params.fn();
666
- try {
667
- await this.cache.set(params.cacheKey, result, params.ttl);
668
- } catch (error) {
669
- this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
670
- }
671
- return result;
672
- }
673
- /**
674
- * Streaming counterpart to `runCached`. Three cases:
675
- *
676
- * - Hit: no live generation to relay. Returns immediately with
677
- * `finalResult` resolved to the cached value and a one-shot `chunks`
678
- * replay built from it, so `for await (const c of chunks)` call sites
679
- * work identically on a hit or a miss. No usage hooks fire, since
680
- * nothing was actually spent.
681
- * - Miss, nothing else in flight for this key: delegates to
682
- * `registerStreamTrigger`, which opens the stream and relays its
683
- * `chunks` live.
684
- * - Miss, but another call for the same key is already in flight: this
685
- * call has no live chunks of its own to relay, so it's treated like a
686
- * delayed hit. `finalResult` shares the trigger's in-flight promise
687
- * (the same in-flight map non-streaming `runCached` uses, so
688
- * streaming and non-streaming calls for the same key coalesce
689
- * against each other too), and `chunks` is a one-shot replay built
690
- * once that promise resolves.
691
- */
692
- async runCachedStream(params, hasTools) {
693
- const resolvedKey = await this.resolveCacheKey(params.cacheKey);
694
- const resolvedParams = resolvedKey === params.cacheKey ? params : {
695
- ...params,
696
- cacheKey: resolvedKey
697
- };
698
- const cached = await this.getCached(resolvedKey);
699
- if (cached.hit) {
700
- const value = cached.value;
701
- return {
702
- chunks: buildReplayChunks(value, hasTools),
703
- finalResult: Promise.resolve(value)
704
- };
705
- }
706
- const existing = this.inFlight.get(resolvedKey);
707
- if (existing) {
708
- const finalResult = withReservedUsage(resolvedParams, true, () => existing, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
709
- finalResult.catch(() => {});
710
- return {
711
- chunks: buildReplayChunksFromPromise(finalResult, hasTools),
712
- finalResult
713
- };
714
- }
715
- return this.registerStreamTrigger(resolvedParams);
716
- }
717
- /**
718
- * Opens the shared stream for a cache miss and tracks its settled value
719
- * in the in-flight map until it resolves or rejects. Writes to the cache
720
- * on success only, matching `runAndCache`.
721
- *
722
- * Registers the in-flight promise synchronously, before anything async
723
- * runs, so a concurrent `cachedCall` for the same key always sees it in
724
- * time to join instead of triggering its own stream. Settlement is
725
- * wired onto the whole `withReservedUsageForStream` call rather than a
726
- * line inside its callback, so any failure point (reserving usage,
727
- * opening the stream, or the stream itself) reliably settles the
728
- * in-flight entry instead of leaving it stuck.
729
- */
730
- registerStreamTrigger(params) {
731
- let resolveInFlight;
732
- let rejectInFlight;
733
- const inFlightResult = new Promise((resolve, reject) => {
734
- resolveInFlight = resolve;
735
- rejectInFlight = reject;
736
- });
737
- this.inFlight.set(params.cacheKey, inFlightResult);
738
- inFlightResult.catch(() => {}).finally(() => {
739
- this.inFlight.delete(params.cacheKey);
740
- });
741
- const streamPromise = withReservedUsageForStream(params, async () => {
742
- const opened = await params.openStream();
743
- const trackedResult = opened.finalResult.then(async (value) => {
744
- try {
745
- await this.cache.set(params.cacheKey, value, params.ttl);
746
- } catch (error) {
747
- this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
748
- }
749
- return value;
750
- }, (error) => {
751
- throw error;
752
- });
753
- return {
754
- chunks: opened.chunks,
755
- finalResult: trackedResult
756
- };
757
- }, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
758
- streamPromise.then((opened) => {
759
- opened.finalResult.then(resolveInFlight, rejectInFlight);
760
- }, (error) => {
761
- rejectInFlight(error);
762
- });
763
- return streamPromise;
764
- }
765
- };
766
-
767
- //#endregion
768
- //#region src/circuitBreaker.ts
769
- function newBucket() {
770
- return {
771
- state: "closed",
772
- consecutiveFailures: 0,
773
- openedAt: 0,
774
- trialInFlight: false
775
- };
776
- }
777
- /** Key a bucket lookup falls into when the call omitted `model` under `isolateByModel`. */
778
- const UNLABELED_MODEL = "";
779
- /**
780
- * Per retry VernLLM-instance circuit breaker. Tracks consecutive failures across
781
- * calls. Once the threshold is hit, short-circuits new calls with an
782
- * LLMError('circuit_open') instead of hitting the provider, until the
783
- * cooldown elapses and a single trial call is allowed through
784
- */
785
- var CircuitBreaker = class {
786
- threshold;
787
- cooldownMs;
788
- onStateChange;
789
- /** Whether this breaker tracks failures per model instead of one shared circuit. Read by `CallExecutor`/`VernLLM` to report per-target in `getCircuitStates`. */
790
- isolateByModel;
791
- sharedBucket = newBucket();
792
- bucketsByModel = new Map();
793
- constructor(options = {}) {
794
- this.threshold = options.threshold ?? 5;
795
- this.cooldownMs = options.cooldownMs ?? 3e4;
796
- this.onStateChange = options.onStateChange;
797
- this.isolateByModel = options.isolateByModel ?? false;
798
- }
799
- /** Returns the bucket for a model if one already exists, without allocating. */
800
- lookupBucket(model) {
801
- if (!this.isolateByModel) return this.sharedBucket;
802
- const key = model ?? UNLABELED_MODEL;
803
- return this.bucketsByModel.get(key);
804
- }
805
- /** Creates and stores a bucket for a model when the first mutation needs one. */
806
- ensureBucketFor(model) {
807
- if (!this.isolateByModel) return this.sharedBucket;
808
- const key = model ?? UNLABELED_MODEL;
809
- let bucket = this.bucketsByModel.get(key);
810
- if (!bucket) {
811
- bucket = newBucket();
812
- this.bucketsByModel.set(key, bucket);
813
- }
814
- return bucket;
815
- }
816
- /** Every state mutation routes through here, so `onStateChange` fires exactly once per real change. */
817
- transition(bucket, to, model) {
818
- if (to === bucket.state) return;
819
- const from = bucket.state;
820
- bucket.state = to;
821
- this.onStateChange?.(from, to, bucket.consecutiveFailures, model);
822
- }
823
- /**
824
- * Throws if the circuit is open and the cooldown hasn't elapsed, or if
825
- * the circuit is half-open and a trial call is already in flight.
826
- * Otherwise, if the circuit just became eligible for a trial (cooldown
827
- * elapsed, or half-open with no trial currently running), this call
828
- * becomes that trial
829
- */
830
- assertClosed(model) {
831
- const bucket = this.ensureBucketFor(model);
832
- if (bucket.state === "closed") return;
833
- if (bucket.state === "open") {
834
- const elapsed = Date.now() - bucket.openedAt;
835
- if (elapsed < this.cooldownMs) throw new LLMError(`Circuit open, provider has failed ${bucket.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open", { code: "circuit_cooling_down" });
836
- bucket.trialInFlight = true;
837
- this.transition(bucket, "half-open", model);
838
- return;
839
- }
840
- if (bucket.trialInFlight) throw new LLMError("Circuit half-open. A trial request is already in flight. Try again shortly.", "circuit_open", { code: "circuit_trial_in_flight" });
841
- bucket.trialInFlight = true;
842
- }
843
- recordSuccess(model) {
844
- const bucket = this.lookupBucket(model);
845
- if (!bucket) return;
846
- bucket.consecutiveFailures = 0;
847
- bucket.trialInFlight = false;
848
- this.transition(bucket, "closed", model);
849
- if (this.isolateByModel && bucket.state === "closed" && bucket.consecutiveFailures === 0) this.bucketsByModel.delete(model ?? UNLABELED_MODEL);
850
- }
851
- recordFailure(model) {
852
- const bucket = this.ensureBucketFor(model);
853
- bucket.consecutiveFailures += 1;
854
- bucket.trialInFlight = false;
855
- if (bucket.state === "half-open") {
856
- bucket.openedAt = Date.now();
857
- this.transition(bucket, "open", model);
858
- return;
859
- }
860
- if (bucket.consecutiveFailures >= this.threshold) {
861
- bucket.openedAt = Date.now();
862
- this.transition(bucket, "open", model);
863
- }
864
- }
865
- /**
866
- * With `isolateByModel` off (the default), `model` is ignored and the
867
- * one shared circuit's state is returned, unchanged from every version
868
- * before this option existed. With `isolateByModel` on, returns that
869
- * model's own state, `'closed'` for a model never seen yet, same as a
870
- * fresh breaker.
871
- */
872
- getState(model) {
873
- return this.lookupBucket(model)?.state ?? "closed";
874
- }
875
- /**
876
- * Manually opens the circuit, as if `threshold` consecutive failures had
877
- * just happened, e.g. to pull a provider out of rotation ahead of known
878
- * maintenance. Resets the cooldown window from now, same as a real
879
- * threshold-crossing failure would, and clears any in-flight half-open
880
- * trial since it no longer applies once the circuit is (re)opened.
881
- */
882
- open(model) {
883
- const bucket = this.ensureBucketFor(model);
884
- bucket.openedAt = Date.now();
885
- bucket.trialInFlight = false;
886
- this.transition(bucket, "open", model);
887
- }
888
- /**
889
- * Manually closes the circuit and resets its failure count, e.g. once a
890
- * provider is confirmed healthy again without waiting out the cooldown.
891
- * Mirrors `recordSuccess`'s bookkeeping (including dropping the
892
- * per-model bucket under `isolateByModel`, once idle) but without
893
- * requiring an actual successful call first.
894
- */
895
- close(model) {
896
- const bucket = this.ensureBucketFor(model);
897
- bucket.consecutiveFailures = 0;
898
- bucket.trialInFlight = false;
899
- this.transition(bucket, "closed", model);
900
- if (this.isolateByModel && bucket.state === "closed" && bucket.consecutiveFailures === 0) this.bucketsByModel.delete(model ?? UNLABELED_MODEL);
901
- }
902
- };
903
-
904
- //#endregion
905
- //#region src/internal/circuitBreaker.utils.ts
906
- /**
907
- * Builds a `(event) => void` reporter that no-ops when `onEvent` is unset,
908
- * and otherwise calls it, swallowing and logging any error the handler
909
- * throws so a broken `onEvent` can't break the call that triggered it.
910
- * Shared by `buildCircuitBreaker` (which needs to report before any
911
- * executor exists) and `CallExecutor.reportEvent`, kept independent of the
912
- * executor for that reason.
913
- */
914
- function makeEventReporter(onEvent, logger) {
915
- return (event) => {
916
- if (!onEvent) return;
917
- try {
918
- onEvent(event);
919
- } catch (error) {
920
- logger.error("[VernLLM] onEvent failed", { message: error instanceof Error ? error.message : "unknown" });
921
- }
922
- };
923
- }
924
- /**
925
- * Builds the optional circuit breaker for one provider target, wiring its
926
- * `onStateChange` to emit a `circuit_state` event and chain any
927
- * caller-supplied `onStateChange`. Returns `undefined` when
928
- * `circuitBreakerOption` is falsy, matching the option's own semantics.
929
- *
930
- * Lives outside `CallExecutor` (and outside `VernLLM`, once this were
931
- * inlined) because the breaker has to exist *before* the executor it's
932
- * passed into, so its construction can't be an executor concern.
933
- * `onEvent` is called directly rather than through the executor for the
934
- * same reason: nothing executor-shaped exists yet at this point.
935
- *
936
- * Takes the specific fields it needs (rather than a full `VernLLMOptions`)
937
- * so it works identically for the primary target and for each fallback
938
- * target, which carry their own `circuitBreaker` override alongside the
939
- * shared `onEvent`.
940
- */
941
- function buildCircuitBreaker(circuitBreakerOption, providerName, defaultModel, onEvent, logger) {
942
- if (!circuitBreakerOption) return void 0;
943
- const breakerOptions = typeof circuitBreakerOption === "object" ? circuitBreakerOption : void 0;
944
- const userOnStateChange = breakerOptions?.onStateChange;
945
- const reportEvent = makeEventReporter(onEvent, logger);
946
- return new CircuitBreaker({
947
- ...breakerOptions,
948
- onStateChange: (from, to, consecutiveFailures, model) => {
949
- reportEvent({
950
- kind: "circuit_state",
951
- provider: providerName,
952
- model: model ?? defaultModel,
953
- from,
954
- to,
955
- consecutiveFailures
956
- });
957
- if (!userOnStateChange) return;
958
- try {
959
- userOnStateChange(from, to, consecutiveFailures, model);
960
- } catch (error) {
961
- logger.error("[VernLLM] circuitBreaker.onStateChange failed", { message: error instanceof Error ? error.message : "unknown" });
962
- }
963
- }
964
- });
965
- }
966
-
967
- //#endregion
968
- //#region src/internal/execution/retry.utils.ts
969
- /**
970
- * Default cap (ms) for both exponential backoff and honored Retry-After
971
- * values, so a misbehaving/adversarial Retry-After can't stall a caller
972
- * indefinitely
973
- */
974
- const DEFAULT_MAX_DELAY_MS = 1e4;
975
- /**
976
- * `setTimeout` silently clamps any delay above this (~24.8 days) or
977
- * `Infinity` down to ~1ms instead of erroring, so a caller passing
978
- * `Infinity` as "no timeout" gets the opposite of what they asked for.
979
- * Both timeout helpers below guard against this explicitly.
980
- */
981
- const MAX_SETTIMEOUT_MS = 2147483647;
982
- /**
983
- * Resolves a timeout value to the number `setTimeout` should actually use,
984
- * or `undefined` when the timeout should be treated as disabled (0,
985
- * negative, or `Infinity`). Returning the resolved value directly, rather
986
- * than a boolean, lets callers narrow `number | undefined` to `number`
987
- * without an `as number` cast.
988
- */
989
- function resolveActiveTimeoutMs(ms) {
990
- return !ms || ms <= 0 || ms === Infinity ? void 0 : ms;
991
- }
992
- /** Caps a timeout at the largest delay `setTimeout` actually honors. */
993
- function clampTimeoutMs(ms) {
994
- return Math.min(ms, MAX_SETTIMEOUT_MS);
995
- }
996
- /**
997
- * Runs an async function and cancels it if it takes longer than the given
998
- * timeout. Creates an internal abort controller that fires after the
999
- * timeout elapses, and combines it with any external signal the caller
1000
- * passed in so either one can cancel the underlying call. If the internal
1001
- * timeout triggers and the underlying operation aborts, the error is
1002
- * converted into an LLMError with type "timeout". External cancellations
1003
- * continue to propagate as aborted errors. The internal timer is always
1004
- * cleared afterward, whether the function succeeds, fails, or is aborted,
1005
- * so nothing is left running in the background.
1006
- *
1007
- * `timeoutMs` of `Infinity` (or any value beyond what `setTimeout` can
1008
- * represent) disables the timeout rather than firing almost immediately.
1009
- */
1010
- async function withTimeout(fn, timeoutMs, externalSignal) {
1011
- const controller = new AbortController();
1012
- const activeTimeoutMs = resolveActiveTimeoutMs(timeoutMs);
1013
- const timer = activeTimeoutMs === void 0 ? void 0 : setTimeout(() => {
1014
- controller.abort();
1015
- }, clampTimeoutMs(activeTimeoutMs));
1016
- const signal = externalSignal ? AbortSignal.any([externalSignal, controller.signal]) : controller.signal;
1017
- try {
1018
- return await fn(signal);
1019
- } catch (err) {
1020
- if (controller.signal.aborted && !externalSignal?.aborted && err instanceof DOMException && err.name === "AbortError") throw new LLMError("Request timed out", "timeout", { code: "request_timeout" });
1021
- throw err;
1022
- } finally {
1023
- clearTimeout(timer);
1024
- }
1025
- }
1026
- /**
1027
- * Races one `iterator.next()` call against a per-call idle timer, to
1028
- * bound the gap *between* chunks (unlike `withTimeout`, which only bounds
1029
- * opening the stream and its first chunk). Without this, a connection
1030
- * that streams one chunk then hangs would never fail.
1031
- *
1032
- * `timeoutMs` of 0/undefined/`Infinity` disables the check. Otherwise
1033
- * rejects with `LLMError('timeout')` if `next()` doesn't settle in time.
1034
- * The clock resets on every call, so the window is measured from the most
1035
- * recent chunk, not from stream start.
1036
- *
1037
- * `onIdle`, if given, is called the moment the timer fires (before the
1038
- * rejection), so callers can abort the underlying transport instead of
1039
- * just walking away from an unread promise. `logger`, if given, records a
1040
- * debug line if `next()` still settles *after* the idle timeout already
1041
- * rejected. `resolve`/`reject` on an already-settled promise is otherwise
1042
- * a silent no-op, so without this the late chunk (possibly the final
1043
- * usage chunk) would vanish with no trace.
1044
- */
1045
- function withChunkIdleTimeout(next, timeoutMs, onIdle, logger) {
1046
- const activeTimeoutMs = resolveActiveTimeoutMs(timeoutMs);
1047
- if (activeTimeoutMs === void 0) return next();
1048
- let settled = false;
1049
- return new Promise((resolve, reject) => {
1050
- const timer = setTimeout(() => {
1051
- settled = true;
1052
- onIdle?.();
1053
- reject(new LLMError(`No stream chunk received for ${activeTimeoutMs}ms (idle timeout)`, "timeout", { code: "idle_timeout" }));
1054
- }, clampTimeoutMs(activeTimeoutMs));
1055
- next().then((result) => {
1056
- clearTimeout(timer);
1057
- if (settled) {
1058
- logger?.debug("[VernLLM] chunk resolved after idle timeout already fired; discarding");
1059
- return;
1060
- }
1061
- settled = true;
1062
- resolve(result);
1063
- }, (error) => {
1064
- clearTimeout(timer);
1065
- if (settled) {
1066
- logger?.debug("[VernLLM] chunk rejection arrived after idle timeout already fired; discarding");
1067
- return;
1068
- }
1069
- settled = true;
1070
- reject(error);
1071
- });
1072
- });
1073
- }
1074
- /**
1075
- * Looks inside an unknown error value for a Retry-After header and
1076
- * converts it to milliseconds. Checks `.headers` first (fetch-style,
1077
- * Headers-like with `.get()`), then `.response.headers` (axios-style,
1078
- * plain object) since different client libraries surface headers
1079
- * differently. Supports both the delta-seconds form ("30") and the
1080
- * HTTP-date form ("Wed, 21 Oct 2015 07:28:00 GMT"). The result is capped
1081
- * at maxDelayMs. Returns undefined when no usable Retry-After is present
1082
- */
1083
- function extractRetryAfterMs(err, maxDelayMs = DEFAULT_MAX_DELAY_MS) {
1084
- if (!err || typeof err !== "object") return void 0;
1085
- const error = err;
1086
- const headers = error.headers ?? error.response?.headers;
1087
- if (!headers || typeof headers !== "object") return void 0;
1088
- const getter = headers;
1089
- const raw = typeof getter.get === "function" ? getter.get("Retry-After") : Object.entries(headers).find(([name]) => name.toLowerCase() === "retry-after")?.at(1);
1090
- if (typeof raw !== "string" || raw.trim() === "") return void 0;
1091
- const trimmed = raw.trim();
1092
- if (/^\d+$/.test(trimmed)) return Math.max(0, Math.min(Number(trimmed) * 1e3, maxDelayMs));
1093
- const dateMs = Date.parse(trimmed);
1094
- if (!Number.isNaN(dateMs)) return Math.max(0, Math.min(dateMs - Date.now(), maxDelayMs));
1095
- return void 0;
1096
- }
1097
- /**
1098
- * Exponential backoff with jitter, capped at maxDelayMs.
1099
- * Jitter avoids thundering-herd retries when many callers back off in lockstep,
1100
- * the cap prevents unbounded delays when maxRetries is high
1101
- */
1102
- function getBackoffDelay(baseDelayMs, attempt, maxDelayMs = DEFAULT_MAX_DELAY_MS) {
1103
- const exp = Math.min(baseDelayMs * 2 ** attempt, maxDelayMs);
1104
- return exp / 2 + Math.random() * (exp / 2);
1105
- }
1106
- /**
1107
- * Pauses execution for the given delay before a retry attempt. If an
1108
- * abort signal is provided and it fires while waiting, the pending
1109
- * timer is cancelled immediately and the wait rejects right away with
1110
- * an aborted error instead of continuing to sit idle until the delay
1111
- * would have finished on its own
1112
- */
1113
- async function waitForRetry(delay, signal) {
1114
- if (signal?.aborted) throw new LLMError("Operation aborted", "aborted");
1115
- await new Promise((resolve, reject) => {
1116
- const onAbort = () => {
1117
- clearTimeout(timer);
1118
- reject(new LLMError("Operation aborted", "aborted"));
1119
- };
1120
- const timer = setTimeout(() => {
1121
- signal?.removeEventListener("abort", onAbort);
1122
- resolve();
1123
- }, delay);
1124
- signal?.addEventListener("abort", onAbort, { once: true });
1125
- });
1126
- }
1127
-
1128
- //#endregion
1129
- //#region src/internal/execution/errors.utils.ts
1130
- /**
1131
- * Looks inside an unknown error value and pulls out an http status code
1132
- * if one is present. Checks the status field first then the status code
1133
- * field since different client libraries use different names for this,
1134
- * falling back to AWS SDK v3's `$metadata.httpStatusCode` (e.g. Bedrock's
1135
- * `ThrottlingException`), which doesn't set either of the other two.
1136
- * Returns undefined when the error is not an object or carries no status
1137
- */
1138
- function extractStatus(err) {
1139
- if (!err || typeof err !== "object") return void 0;
1140
- const error = err;
1141
- if (typeof error.status === "number") return error.status;
1142
- if (typeof error.statusCode === "number") return error.statusCode;
1143
- if (typeof error.$metadata?.httpStatusCode === "number") return error.$metadata.httpStatusCode;
1144
- return void 0;
1145
- }
1146
- /**
1147
- * POSIX/libuv error codes libuv (and so Node's `fetch`/undici) attaches to
1148
- * genuine transport-level failures: connection refused, DNS lookup
1149
- * failure, connection reset mid-request, a connect that never completed,
1150
- * DNS server unreachable, broken pipe, or host/network unreachable.
1151
- * Deliberately narrow: only codes that can only mean "the connection
1152
- * itself failed," not anything that could also indicate an application
1153
- * error.
1154
- */
1155
- const NETWORK_ERROR_CODES = new Set([
1156
- "ECONNREFUSED",
1157
- "ENOTFOUND",
1158
- "ECONNRESET",
1159
- "ETIMEDOUT",
1160
- "EAI_AGAIN",
1161
- "EPIPE",
1162
- "ECONNABORTED",
1163
- "EHOSTUNREACH",
1164
- "ENETUNREACH"
1165
- ]);
1166
- /** `fetch`'s own wording for a transport-level failure, across runtimes/browsers. */
1167
- const NETWORK_ERROR_MESSAGES = new Set([
1168
- "fetch failed",
1169
- "failed to fetch",
1170
- "load failed",
1171
- "networkerror when attempting to fetch resource."
1172
- ]);
1173
- /**
1174
- * Whether `error` is, with reasonable confidence, a transport-level
1175
- * failure (never reached the provider, as opposed to the provider itself
1176
- * responding with an error) rather than some other unexpected exception.
1177
- * Checked via explicit, well-known signals only, so a genuinely unknown
1178
- * error never gets misclassified as a connection failure just because it
1179
- * also lacked an HTTP status.
1180
- */
1181
- function isNetworkError(error) {
1182
- if (!error || typeof error !== "object") return false;
1183
- const err = error;
1184
- if (typeof err.code === "string" && NETWORK_ERROR_CODES.has(err.code)) return true;
1185
- if (typeof err.message === "string" && NETWORK_ERROR_MESSAGES.has(err.message.toLowerCase())) return true;
1186
- if (err.cause && typeof err.cause === "object") {
1187
- const cause = err.cause;
1188
- if (typeof cause.code === "string" && NETWORK_ERROR_CODES.has(cause.code)) return true;
1189
- }
1190
- return false;
1191
- }
1192
- function formatSafely(value) {
1193
- try {
1194
- return JSON.stringify(value, null, 2) ?? String(value);
1195
- } catch {
1196
- try {
1197
- return String(value);
1198
- } catch {
1199
- return "[unprintable error]";
1200
- }
1201
- }
1202
- }
1203
- /**
1204
- * Looks inside an unknown thrown value and pulls out a human-readable
1205
- * description of it. Checks the `error` field first (the provider's raw
1206
- * rejection body, JSON-stringified if possible) then falls back to the
1207
- * message` field. Always returns a safe string, even when the thrown value
1208
- * has hostile properties or cannot be serialized normally.
1209
- */
1210
- function describeError(err) {
1211
- if (err && typeof err === "object") try {
1212
- const error = err;
1213
- if (error.error !== void 0) return formatSafely(error.error);
1214
- if (typeof error.message === "string") return error.message;
1215
- } catch {}
1216
- return formatSafely(err);
1217
- }
1218
- /**
1219
- * Maps an HTTP status to its corresponding `LLMErrorCode`, derived purely
1220
- * from the status itself so it applies the same way regardless of which
1221
- * adapter or client raised the error. Used both when building a fresh
1222
- * `LLMError` and when filling in a `code` on an already-normalized one
1223
- * that doesn't have one yet, so the two paths can't drift apart.
1224
- */
1225
- function codeForStatus(status) {
1226
- switch (status) {
1227
- case 429: return "provider_rate_limited";
1228
- case 401: return "authentication";
1229
- case 403: return "authorization";
1230
- case 404: return "not_found";
1231
- case 413: return "payload_too_large";
1232
- default: return status >= 500 ? "server_error" : void 0;
1233
- }
1234
- }
1235
- /**
1236
- * Converts any thrown value into a well-typed LLMError. `attempts`, when
1237
- * given, is the accumulated record of every attempt made before `error`
1238
- * was thrown; it's passed straight into the constructed error's options
1239
- * rather than assigned onto the error afterward, so `attempts` is always
1240
- * settled once, through the constructor, like every other field on
1241
- * `LLMError`.
1242
- */
1243
- function normalizeError(error, signal, attempts) {
1244
- if (signal?.aborted) return new LLMError("LLM request aborted", "aborted", { attempts });
1245
- if (error instanceof LLMError) {
1246
- if (error.code === void 0 && error.status !== void 0) error.code = codeForStatus(error.status);
1247
- if (error.attempts === void 0 && attempts !== void 0) error.attempts = attempts;
1248
- return error;
1249
- }
1250
- const status = extractStatus(error);
1251
- const retryAfterMs = extractRetryAfterMs(error);
1252
- if (status !== void 0) return new LLMError("LLM request failed", "api", {
1253
- status,
1254
- cause: error,
1255
- retryAfterMs,
1256
- code: codeForStatus(status),
1257
- attempts
1258
- });
1259
- if (isNetworkError(error)) return new LLMError("LLM request failed", "network", {
1260
- cause: error,
1261
- retryAfterMs,
1262
- code: "connection_failed",
1263
- attempts
1264
- });
1265
- return new LLMError("LLM request failed", "unknown", {
1266
- cause: error,
1267
- retryAfterMs,
1268
- attempts
1269
- });
1270
- }
1271
-
1272
- //#endregion
1273
- //#region src/internal/execution/parse.utils.ts
1274
- /** Default `parseJson`: `JSON.parse` wrapped in try/catch, returning `undefined` on failure. */
1275
- function defaultParseJson(content) {
1276
- try {
1277
- return JSON.parse(content);
1278
- } catch {
1279
- return void 0;
1280
- }
1281
- }
1282
-
1283
- //#endregion
1284
- //#region src/internal/execution/wire.utils.ts
1285
- /** Translates app-facing `ToolDefinition[]` into the OpenAI-shaped wire tools array. */
1286
- function toWireTools(tools) {
1287
- return tools.map((tool) => ({
1288
- type: "function",
1289
- function: {
1290
- name: tool.name,
1291
- description: tool.description,
1292
- parameters: tool.parameters
1293
- }
1294
- }));
1295
- }
1296
- /** Translates app-facing `ToolCall[]` (e.g. from a replayed assistant turn) into wire tool_calls. */
1297
- function toWireToolCalls(toolCalls) {
1298
- return toolCalls.map((tc) => ({
1299
- id: tc.id,
1300
- type: "function",
1301
- function: {
1302
- name: tc.name,
1303
- arguments: JSON.stringify(tc.arguments ?? {})
1304
- }
1305
- }));
1306
- }
1307
- /**
1308
- * Parses the provider's wire-shaped `tool_calls` back into VernLLM's
1309
- * `ToolCall[]`. Malformed argument JSON is a `'parse'` error, same
1310
- * convention as malformed JSON response bodies elsewhere in VernLLM.
1311
- */
1312
- function parseWireToolCalls(wireToolCalls) {
1313
- return wireToolCalls.map((wc) => {
1314
- let parsedArgs;
1315
- try {
1316
- parsedArgs = wc.function.arguments.trim() ? JSON.parse(wc.function.arguments) : {};
1317
- } catch {
1318
- throw new LLMError(`Invalid JSON arguments for tool call "${wc.function.name}"`, "parse", { code: "tool_arguments_parse_failed" });
1319
- }
1320
- return {
1321
- id: wc.id,
1322
- name: wc.function.name,
1323
- arguments: parsedArgs
1324
- };
1325
- });
1326
- }
1327
-
1328
- //#endregion
1329
- //#region src/internal/execution/requestBuilder.ts
1330
- /**
1331
- * Builds the wire request object for one call, applying per-instance
1332
- * defaults (model, max tokens, temperature) and per-call overrides.
1333
- * Owns every check that depends only on the caller's own input shape, not
1334
- * on execution: history alternation, duplicate/empty tool lists,
1335
- * `toolChoice` naming a real tool. All deterministic on the call site's
1336
- * own input and never touch the network, so every throw here is
1337
- * `type: 'invalid_params'`, not `'validation'` (which is reserved for the
1338
- * model/provider's own response failing a contract check). Has no
1339
- * knowledge of retry, timeouts, or the breaker, only
1340
- * the three defaults a `FallbackTarget` can override per-target (see the
1341
- * `defaultMaxTokens`/`defaultTemperature` overrides in the fallback
1342
- * design), which is what keeps it separable from `CallExecutor`.
1343
- */
1344
- var RequestBuilder = class {
1345
- model;
1346
- defaultMaxTokens;
1347
- defaultTemperature;
1348
- constructor(options) {
1349
- this.model = options.model;
1350
- this.defaultMaxTokens = options.defaultMaxTokens;
1351
- this.defaultTemperature = options.defaultTemperature;
1352
- }
1353
- /** Applies per-call defaults and shapes params into the client's request object. */
1354
- build(params) {
1355
- const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model, reasoningEffort, jsonSchema, tools, toolChoice } = params;
1356
- const temperature = params.temperature === void 0 ? this.defaultTemperature : params.temperature;
1357
- if (tools && tools.length === 0) throw new LLMError("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.", "invalid_params");
1358
- if (tools) {
1359
- const seen = new Set();
1360
- const duplicates = new Set();
1361
- for (const tool of tools) {
1362
- if (seen.has(tool.name)) duplicates.add(tool.name);
1363
- seen.add(tool.name);
1364
- }
1365
- if (duplicates.size) throw new LLMError(`\`tools\` has duplicate name(s): [${[...duplicates].join(", ")}]. Tool names must be unique.`, "invalid_params", {
1366
- code: "duplicate_tool_names",
1367
- issues: { names: [...duplicates] }
1368
- });
1369
- }
1370
- if (toolChoice && !tools) throw new LLMError("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.", "invalid_params");
1371
- if (tools && typeof toolChoice === "object" && !tools.some((t) => t.name === toolChoice.name)) throw new LLMError(`toolChoice names "${toolChoice.name}", which is not in \`tools\` ([${tools.map((t) => t.name).join(", ")}]).`, "invalid_params", {
1372
- code: "unknown_tool_choice",
1373
- issues: {
1374
- requested: toolChoice.name,
1375
- available: tools.map((t) => t.name)
1376
- }
1377
- });
1378
- const jsonMode = params.jsonMode ?? (tools ? false : true);
1379
- const useJson = jsonMode || Boolean(jsonSchema);
1380
- if (params.schema && !useJson) throw new LLMError("schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.", "invalid_params");
1381
- const responseFormat = this.buildResponseFormat(jsonSchema, useJson);
1382
- this.validateHistory(history);
1383
- const request = {
1384
- model,
1385
- ...temperature !== null ? { temperature } : {},
1386
- max_tokens: maxTokens,
1387
- ...responseFormat ? { response_format: responseFormat } : {},
1388
- ...reasoningEffort ? { reasoning_effort: reasoningEffort } : {},
1389
- ...tools ? { tools: toWireTools(tools) } : {},
1390
- ...tools ? { tool_choice: this.buildWireToolChoice(toolChoice) } : {},
1391
- messages: [
1392
- ...systemPrompt ? [{
1393
- role: "system",
1394
- content: systemPrompt
1395
- }] : [],
1396
- ...history.flatMap((turn) => this.turnToWireMessages(turn)),
1397
- {
1398
- role: "user",
1399
- content: userContent
1400
- }
1401
- ]
1402
- };
1403
- return {
1404
- useJson,
1405
- model,
1406
- request
1407
- };
1408
- }
1409
- /**
1410
- * Validates `history` alternates user/assistant turns, since providers
1411
- * like Anthropic/Gemini reject or mishandle consecutive same-role turns.
1412
- */
1413
- validateHistory(history) {
1414
- let previousTurn;
1415
- for (const [index, turn] of history.entries()) {
1416
- if (turn.role === "tool") {
1417
- if (previousTurn?.role !== "assistant" || !previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`, "invalid_params");
1418
- if (!turn.toolResults?.length) throw new LLMError(`history[${index}] is a "tool" turn but has no toolResults`, "invalid_params");
1419
- const requestedIds = new Set(previousTurn.toolCalls.map((tc) => tc.id));
1420
- const resultIds = turn.toolResults.map((tr) => tr.toolCallId);
1421
- const unknownIds = resultIds.filter((id) => !requestedIds.has(id));
1422
- if (unknownIds.length) throw new LLMError(`history[${index}].toolResults references unknown toolCallId(s) [${unknownIds.join(", ")}]`, "invalid_params", {
1423
- code: "unknown_tool_result_ids",
1424
- issues: {
1425
- historyIndex: index,
1426
- ids: unknownIds
1427
- }
1428
- });
1429
- const seenIds = new Set();
1430
- const duplicateIds = new Set();
1431
- for (const id of resultIds) {
1432
- if (seenIds.has(id)) duplicateIds.add(id);
1433
- seenIds.add(id);
1434
- }
1435
- if (duplicateIds.size) throw new LLMError(`history[${index}].toolResults has duplicate toolCallId(s) [${[...duplicateIds].join(", ")}]`, "invalid_params", {
1436
- code: "duplicate_tool_result_ids",
1437
- issues: {
1438
- historyIndex: index,
1439
- ids: [...duplicateIds]
1440
- }
1441
- });
1442
- const missingIds = [...requestedIds].filter((id) => !resultIds.includes(id));
1443
- if (missingIds.length) throw new LLMError(`history[${index}] is missing toolResults for toolCallId(s) [${missingIds.join(", ")}]`, "invalid_params", {
1444
- code: "missing_tool_results",
1445
- issues: {
1446
- historyIndex: index,
1447
- ids: missingIds
1448
- }
1449
- });
1450
- } else {
1451
- if (turn.role === previousTurn?.role) throw new LLMError(`history must alternate user/assistant turns: consecutive "${turn.role}" turns at history[${index - 1}] and history[${index}]`, "invalid_params");
1452
- if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] follows an assistant tool request without tool results`, "invalid_params");
1453
- }
1454
- previousTurn = turn;
1455
- }
1456
- if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError("The last entry in history is an assistant tool request without tool results", "invalid_params");
1457
- if (previousTurn?.role === "user") throw new LLMError("The last entry in history is a \"user\" turn, which would collide with the current userContent turn.", "invalid_params");
1458
- }
1459
- /** Maps VernLLM's app-facing `ToolChoice` onto the OpenAI-shaped wire `tool_choice`. */
1460
- buildWireToolChoice(toolChoice) {
1461
- if (!toolChoice || toolChoice === "auto") return "auto";
1462
- if (toolChoice === "none" || toolChoice === "required") return toolChoice;
1463
- return {
1464
- type: "function",
1465
- function: { name: toolChoice.name }
1466
- };
1467
- }
1468
- /**
1469
- * Expands one `ConversationTurn` into one or more wire messages. Plain
1470
- * user/assistant turns map 1:1. An assistant turn with `toolCalls` maps
1471
- * to an assistant message carrying wire-shaped `tool_calls`. A `'tool'`
1472
- * turn expands into one wire `tool` message per `toolResult`, since
1473
- * OpenAI-shaped wire format wants one message per tool_call_id.
1474
- */
1475
- turnToWireMessages(turn) {
1476
- if (turn.role === "tool") return (turn.toolResults ?? []).map((tr) => ({
1477
- role: "tool",
1478
- tool_call_id: tr.toolCallId,
1479
- content: typeof tr.content === "string" ? tr.content : JSON.stringify(tr.content ?? null),
1480
- ...tr.isError ? { is_error: true } : {}
1481
- }));
1482
- if (turn.role === "assistant" && turn.toolCalls?.length) return [{
1483
- role: "assistant",
1484
- ...turn.content ? { content: turn.content } : {},
1485
- tool_calls: toWireToolCalls(turn.toolCalls)
1486
- }];
1487
- return [{
1488
- role: turn.role,
1489
- content: turn.content ?? ""
1490
- }];
1491
- }
1492
- /**
1493
- * Chooses the response format: a provider-native `jsonSchema` takes
1494
- * priority when supplied (constrains generation directly), otherwise
1495
- * falls back to the looser `json_object` mode when JSON output is
1496
- * requested, or no format at all for plain text responses.
1497
- */
1498
- buildResponseFormat(jsonSchema, useJson) {
1499
- if (jsonSchema) return {
1500
- type: "json_schema",
1501
- json_schema: {
1502
- name: jsonSchema.name,
1503
- schema: jsonSchema.schema,
1504
- strict: jsonSchema.strict ?? true,
1505
- description: jsonSchema.description
1506
- }
1507
- };
1508
- return useJson ? { type: "json_object" } : void 0;
1509
- }
1510
- };
1511
-
1512
- //#endregion
1513
- //#region src/internal/execution/streamAccumulator.ts
1514
- /**
1515
- * The streaming accumulator: wraps the raw `WireStreamChunk` iterator in
1516
- * an async generator that yields translated `StreamChunk`s to the caller
1517
- * live, as they arrive, with no per-chunk timeout and no bound on total
1518
- * duration, and accumulates text/tool-call deltas internally so that
1519
- * `finalize` can produce `finalResult` once the stream completes.
1520
- *
1521
- * Two separate try/catches: the iteration loop's catch handles errors
1522
- * the transport itself throws, which aren't normalized yet, so that
1523
- * happens here, alongside the one `onStreamFailure` call for them. The
1524
- * second catch, around `finalize`, does not re-normalize or re-report,
1525
- * since `finalize`'s caller (`finalizeResponse`) already does both
1526
- * internally.
1527
- */
1528
- function buildStreamResult(iterator, first, options) {
1529
- const { requestId, model, providerName, isFallback, chunkIdleTimeoutMs, streamController, logger, signal } = options;
1530
- let resolveFinal;
1531
- let rejectFinal;
1532
- const finalResult = new Promise((resolve, reject) => {
1533
- resolveFinal = resolve;
1534
- rejectFinal = reject;
1535
- });
1536
- finalResult.catch(() => {});
1537
- const MAX_BUFFERED_CHUNKS = 1e4;
1538
- const buffered = [];
1539
- const pending = [];
1540
- let streamDone = false;
1541
- let streamError;
1542
- let hasLoggedEviction = false;
1543
- const push = (chunk) => {
1544
- const waiter = pending.shift();
1545
- if (waiter) {
1546
- waiter.resolve({
1547
- done: false,
1548
- value: chunk
1549
- });
1550
- return;
1551
- }
1552
- buffered.push(chunk);
1553
- if (buffered.length > MAX_BUFFERED_CHUNKS * 2) {
1554
- if (!hasLoggedEviction) {
1555
- hasLoggedEviction = true;
1556
- logger.warn(`[VernLLM] stream chunk buffer exceeded cap (${MAX_BUFFERED_CHUNKS}), evicting ${buffered.length - MAX_BUFFERED_CHUNKS} oldest chunk(s); buffered=${buffered.length}. The chunks iterable was never read (or fell far behind) for this stream.`);
1557
- }
1558
- buffered.splice(0, buffered.length - MAX_BUFFERED_CHUNKS);
1559
- }
1560
- };
1561
- const finish = () => {
1562
- streamDone = true;
1563
- for (const waiter of pending.splice(0)) waiter.resolve({
1564
- done: true,
1565
- value: void 0
1566
- });
1567
- };
1568
- const fail = (error) => {
1569
- streamDone = true;
1570
- streamError = error;
1571
- for (const waiter of pending.splice(0)) waiter.reject(error);
1572
- };
1573
- const chunks = { [Symbol.asyncIterator]() {
1574
- return { next() {
1575
- if (buffered.length) return Promise.resolve({
1576
- done: false,
1577
- value: buffered.shift()
1578
- });
1579
- if (streamDone) return streamError ? Promise.reject(streamError) : Promise.resolve({
1580
- done: true,
1581
- value: void 0
1582
- });
1583
- return new Promise((resolve, reject) => {
1584
- pending.push({
1585
- resolve,
1586
- reject
1587
- });
1588
- });
1589
- } };
1590
- } };
1591
- const toolCallAcc = new Map();
1592
- let textAcc = "";
1593
- let usage;
1594
- (async () => {
1595
- try {
1596
- let result = first;
1597
- while (!result.done) {
1598
- const wireChunk = result.value;
1599
- if (wireChunk.type === "ping") {} else if (wireChunk.type === "text-delta") {
1600
- textAcc += wireChunk.delta;
1601
- push({
1602
- type: "text-delta",
1603
- delta: wireChunk.delta
1604
- });
1605
- } else if (wireChunk.type === "tool_call_delta") {
1606
- const entry = toolCallAcc.get(wireChunk.index) ?? { args: "" };
1607
- entry.id ??= wireChunk.id;
1608
- entry.name ??= wireChunk.name;
1609
- entry.args += wireChunk.argumentsDelta ?? "";
1610
- toolCallAcc.set(wireChunk.index, entry);
1611
- push({
1612
- type: "tool_call_delta",
1613
- index: wireChunk.index,
1614
- id: wireChunk.id,
1615
- name: wireChunk.name,
1616
- argsDelta: wireChunk.argumentsDelta,
1617
- complete: wireChunk.complete
1618
- });
1619
- } else if (wireChunk.type === "usage") {
1620
- usage = {
1621
- promptTokens: wireChunk.usage.prompt_tokens ?? 0,
1622
- completionTokens: wireChunk.usage.completion_tokens ?? 0,
1623
- totalTokens: wireChunk.usage.total_tokens ?? 0,
1624
- requestId,
1625
- model,
1626
- provider: providerName,
1627
- usedFallback: isFallback
1628
- };
1629
- push({
1630
- type: "usage",
1631
- usage
1632
- });
1633
- }
1634
- result = await withChunkIdleTimeout(() => iterator.next(), chunkIdleTimeoutMs, () => streamController.abort(), logger);
1635
- }
1636
- } catch (error) {
1637
- try {
1638
- await iterator.return?.();
1639
- } catch {}
1640
- streamController.abort();
1641
- const normalized = normalizeError(error, signal);
1642
- try {
1643
- options.onStreamFailure(normalized, usage);
1644
- } catch {}
1645
- fail(normalized);
1646
- rejectFinal(normalized);
1647
- return;
1648
- }
1649
- finish();
1650
- try {
1651
- options.onStreamSuccess(usage);
1652
- } catch {}
1653
- try {
1654
- const wireToolCalls = toolCallAcc.size ? [...toolCallAcc.entries()].sort(([indexA], [indexB]) => indexA - indexB).map(([, entry]) => ({
1655
- id: entry.id ?? "",
1656
- type: "function",
1657
- function: {
1658
- name: entry.name ?? "",
1659
- arguments: entry.args
1660
- }
1661
- })) : void 0;
1662
- const finalized = options.finalize(textAcc, wireToolCalls, usage);
1663
- resolveFinal(finalized);
1664
- } catch (error) {
1665
- rejectFinal(error);
1666
- }
1667
- })();
1668
- return {
1669
- chunks,
1670
- finalResult
1671
- };
1672
- }
1673
-
1674
- //#endregion
1675
- //#region src/internal/execution/callExecutor.ts
1676
- /**
1677
- * Everything one provider target needs to attempt a call: request
1678
- * building, retry with backoff, the per-target breaker, the per-target
1679
- * limiter. Never exported publicly. `VernLLM` holds one per target and
1680
- * owns the fallback loop and caching on top.
1681
- */
1682
- var CallExecutor = class {
1683
- maxRetries;
1684
- timeoutMs;
1685
- chunkIdleTimeoutMs;
1686
- baseDelayMs;
1687
- nonRetryableStatus;
1688
- parseJson;
1689
- logger;
1690
- redact;
1691
- onUsage;
1692
- onUsageFailure;
1693
- reportEvent;
1694
- breaker;
1695
- limiter;
1696
- isFallback;
1697
- requestBuilder;
1698
- constructor(providerName, client, model, options) {
1699
- this.providerName = providerName;
1700
- this.client = client;
1701
- this.model = model;
1702
- this.maxRetries = options.maxRetries;
1703
- this.timeoutMs = options.timeoutMs;
1704
- this.chunkIdleTimeoutMs = options.chunkIdleTimeoutMs;
1705
- this.baseDelayMs = options.baseDelayMs;
1706
- this.nonRetryableStatus = options.nonRetryableStatus;
1707
- this.parseJson = options.parseJson ?? defaultParseJson;
1708
- this.logger = options.logger;
1709
- this.redact = options.redact;
1710
- this.onUsage = options.onUsage;
1711
- this.onUsageFailure = options.onUsageFailure;
1712
- this.reportEvent = makeEventReporter(options.onEvent, this.logger);
1713
- this.breaker = options.breaker;
1714
- this.limiter = options.limiter;
1715
- this.isFallback = options.isFallback ?? false;
1716
- this.requestBuilder = new RequestBuilder({
1717
- model,
1718
- defaultMaxTokens: options.defaultMaxTokens,
1719
- defaultTemperature: options.defaultTemperature
1720
- });
1721
- }
1722
- getCircuitState(model) {
1723
- return this.breaker?.getState(model);
1724
- }
1725
- /** Whether this target's breaker tracks failures per model. `false` if no breaker is configured. */
1726
- get isolateByModel() {
1727
- return this.breaker?.isolateByModel ?? false;
1728
- }
1729
- /** Manually opens this target's circuit breaker, if one is configured. No-op otherwise. */
1730
- openCircuit(model) {
1731
- this.breaker?.open(model);
1732
- }
1733
- /** Manually closes this target's circuit breaker, if one is configured. No-op otherwise. */
1734
- closeCircuit(model) {
1735
- this.breaker?.close(model);
1736
- }
1737
- /**
1738
- * Throws if the breaker is open for this target/model, exactly like the
1739
- * check `run`/`runStream` used to make internally. Exposed so `VernLLM`
1740
- * can gate on it before reserving usage, avoiding a reserve-then-refund
1741
- * round trip on a call that was never going to be attempted. `assertClosed`
1742
- * has a stateful side effect (claiming a half-open trial slot), so it must
1743
- * run exactly once per logical call: `run`/`runStream` no longer call it
1744
- * themselves, this is now the only call site.
1745
- */
1746
- assertBreakerClosed(model) {
1747
- this.breaker?.assertClosed(model ?? this.model);
1748
- }
1749
- /**
1750
- * Runs a single logical call against this target: retry with backoff,
1751
- * normalized error on exhaustion. Mirrors the old `VernLLM.call`'s
1752
- * non-streaming branch, minus cache/usage-reservation and the breaker
1753
- * check, which stay one layer up since they aren't per-target concerns
1754
- * (see `assertBreakerClosed`).
1755
- */
1756
- async run(params, requestId, onAttempt) {
1757
- const model = params.model ?? this.model;
1758
- const attempts = [];
1759
- try {
1760
- return await this.retryWithBackoff((attempt) => this.executeCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
1761
- } catch (error) {
1762
- const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
1763
- if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
1764
- this.logger.debug(`[VernLLM:${requestId}] error:\n${this.redactText(describeError(error))}`);
1765
- throw normalized;
1766
- }
1767
- }
1768
- /** Streaming counterpart to `run`. Mirrors the old streaming branch of `VernLLM.call`. */
1769
- async runStream(params, requestId, onAttempt) {
1770
- const model = params.model ?? this.model;
1771
- const attempts = [];
1772
- try {
1773
- return await this.retryWithBackoff((attempt) => this.executeStreamCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
1774
- } catch (error) {
1775
- const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
1776
- if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
1777
- this.logger.debug(`[VernLLM:${requestId}] stream-open error:\n${this.redactText(describeError(error))}`);
1778
- throw normalized;
1779
- }
1780
- }
1781
- /**
1782
- * Performs a single attempt: builds the request (translating `tools` to
1783
- * wire shape when present), dispatches it with a timeout, and shapes the
1784
- * response into `T` or a `CallWithToolsResult<T>` when `params.tools` was
1785
- * set. Throws on an empty response (no text and no tool_calls) so the
1786
- * retry loop treats it like any other transient failure.
1787
- */
1788
- async executeCall(params, requestId, attempt) {
1789
- const { useJson, model, request } = this.requestBuilder.build(params);
1790
- let release;
1791
- if (this.limiter) {
1792
- const acquired = await this.limiter.acquire(this.limiter.estimate(request), params.signal);
1793
- release = acquired.release;
1794
- if (acquired.waitedMs > 0) this.reportEvent({
1795
- kind: "rate_limited",
1796
- requestId,
1797
- provider: this.providerName,
1798
- model,
1799
- waitedMs: acquired.waitedMs,
1800
- reason: acquired.reason ?? "rpm"
1801
- });
1802
- }
1803
- try {
1804
- const response = await withTimeout((attemptSignal) => this.client.chat.completions.create(request, { signal: attemptSignal }), this.timeoutMs, params.signal);
1805
- const usage = this.extractUsage(response, requestId, model);
1806
- release?.(this.actualTokensFor(usage));
1807
- release = void 0;
1808
- const rawContent = response.choices?.[0]?.message?.content;
1809
- const wireToolCalls = response.choices?.[0]?.message?.tool_calls;
1810
- return this.finalizeResponse(rawContent, wireToolCalls, params, useJson, model, usage, requestId, attempt);
1811
- } finally {
1812
- release?.();
1813
- }
1814
- }
1815
- /** Applies `redact` (if configured); otherwise returns `text` unchanged. */
1816
- redactText(text) {
1817
- return this.redact ? this.redact(text) : text;
1818
- }
1819
- /**
1820
- * Applies `redact` (if configured) to whatever the debug log is about
1821
- * to show: real content when there is any, otherwise the tool-call
1822
- * placeholder, which carries no user data and passes through
1823
- * `redact` unchanged in practice but is included for a caller whose
1824
- * `redact` does something structural (e.g. adding a marker) rather
1825
- * than just scrubbing PII.
1826
- */
1827
- redactedOutput(content, wireToolCalls) {
1828
- return this.redactText(content ?? `[${wireToolCalls?.length ?? 0} tool call(s)]`);
1829
- }
1830
- /**
1831
- * Shapes a fully-arrived response (content and/or tool_calls, already
1832
- * extracted from the provider's payload) into `T` or a
1833
- * `CallWithToolsResult<T>`. Reused by the streaming path once it has
1834
- * buffered the full text/tool-call deltas, so there's no separate
1835
- * parsing/validation logic for streaming.
1836
- *
1837
- * Normalizes and reports usage failure on error itself, so every caller
1838
- * gets identical error handling without duplicating it.
1839
- */
1840
- finalizeResponse(rawContent, wireToolCalls, params, useJson, model, usage, requestId, attempt) {
1841
- try {
1842
- const content = rawContent?.trim();
1843
- if (!content && !wireToolCalls?.length) throw new LLMError("Empty LLM response", "api", { code: "empty_response" });
1844
- this.logger.debug(`[VernLLM:${requestId}] output:\n${this.redactedOutput(content, wireToolCalls).slice(0, 800)}`);
1845
- if (wireToolCalls?.length) {
1846
- if (!params.tools) throw new LLMError("Provider returned tool_calls but no `tools` were sent with this call.", "validation", { code: "unexpected_tool_calls" });
1847
- if (params.toolChoice === "none") throw new LLMError("Provider returned tool_calls despite toolChoice: 'none'.", "validation", { code: "tool_choice_none_violated" });
1848
- const toolCalls = parseWireToolCalls(wireToolCalls);
1849
- this.validateToolCallArguments(toolCalls, params.tools);
1850
- this.breaker?.recordSuccess(model);
1851
- this.reportUsage(usage);
1852
- return {
1853
- type: "tool_calls",
1854
- toolCalls,
1855
- ...content ? { content } : {}
1856
- };
1857
- }
1858
- const textContent = content ?? "";
1859
- if (!useJson) {
1860
- this.breaker?.recordSuccess(model);
1861
- this.reportUsage(usage);
1862
- return params.tools ? {
1863
- type: "content",
1864
- content: textContent
1865
- } : textContent;
1866
- }
1867
- const result = this.parseAndValidate(textContent, params.schema);
1868
- this.breaker?.recordSuccess(model);
1869
- this.reportUsage(usage);
1870
- return params.tools ? {
1871
- type: "content",
1872
- content: result
1873
- } : result;
1874
- } catch (error) {
1875
- const normalized = normalizeError(error, params.signal);
1876
- if (usage && normalized.type !== "aborted") this.reportUsageFailure(usage, normalized, attempt);
1877
- throw normalized;
1878
- }
1879
- }
1880
- /**
1881
- * Opens a stream for a single attempt: builds the request exactly like
1882
- * `executeCall`, then requires `createStream` on the client (a clear
1883
- * `validation` error if the adapter doesn't support it). The timeout
1884
- * wraps stream construction and the first `.next()` together, not just
1885
- * construction: calling an `async function*` returns an iterator
1886
- * synchronously without running its body until `.next()` is first
1887
- * invoked, so timing only construction would time an operation that's
1888
- * always instant, not the actual connection. Both are folded into a
1889
- * single `withTimeout` so the same abort signal reaches whatever the
1890
- * adapter's `createStream` uses internally for its first network
1891
- * round-trip.
1892
- *
1893
- * Circuit-breaker success is recorded once the stream fully completes,
1894
- * not on the first chunk arriving, so a connection that opens but then
1895
- * dies mid-stream isn't masked as a success (see `buildStreamResult`).
1896
- */
1897
- async executeStreamCall(params, requestId, attempt) {
1898
- const { useJson, model, request } = this.requestBuilder.build(params);
1899
- const completions = this.client.chat.completions;
1900
- if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "invalid_params", {
1901
- code: "unsupported_capability",
1902
- issues: { capability: "createStream" }
1903
- });
1904
- const createStream = completions.createStream.bind(completions);
1905
- let release;
1906
- if (this.limiter) {
1907
- const acquired = await this.limiter.acquire(this.limiter.estimate(request), params.signal);
1908
- release = acquired.release;
1909
- if (acquired.waitedMs > 0) this.reportEvent({
1910
- kind: "rate_limited",
1911
- requestId,
1912
- provider: this.providerName,
1913
- model,
1914
- waitedMs: acquired.waitedMs,
1915
- reason: acquired.reason ?? "rpm"
1916
- });
1917
- }
1918
- const streamController = new AbortController();
1919
- const combinedExternal = params.signal ? AbortSignal.any([params.signal, streamController.signal]) : streamController.signal;
1920
- try {
1921
- const { iterator, first } = await withTimeout(async (attemptSignal) => {
1922
- const streamIterator = createStream(request, { signal: attemptSignal })[Symbol.asyncIterator]();
1923
- const firstResult = await streamIterator.next();
1924
- return {
1925
- iterator: streamIterator,
1926
- first: firstResult
1927
- };
1928
- }, this.timeoutMs, combinedExternal);
1929
- if (first.done) throw new LLMError("Empty LLM response", "api");
1930
- const releaseAtOpen = release;
1931
- const result = buildStreamResult(iterator, first, {
1932
- requestId,
1933
- model,
1934
- providerName: this.providerName,
1935
- isFallback: this.isFallback,
1936
- chunkIdleTimeoutMs: params.chunkIdleTimeoutMs ?? this.chunkIdleTimeoutMs,
1937
- streamController,
1938
- logger: this.logger,
1939
- signal: params.signal,
1940
- onStreamSuccess: (usage) => {
1941
- this.breaker?.recordSuccess(model);
1942
- releaseAtOpen?.(this.actualTokensFor(usage));
1943
- },
1944
- onStreamFailure: (normalized, usage) => {
1945
- if (normalized.type === "timeout") this.breaker?.recordFailure(model);
1946
- if (usage && normalized.type !== "aborted") this.reportUsageFailure(usage, normalized, attempt, true);
1947
- releaseAtOpen?.(this.actualTokensFor(usage));
1948
- },
1949
- finalize: (textAcc, wireToolCalls, usage) => this.finalizeResponse(textAcc, wireToolCalls, params, useJson, model, usage, requestId, attempt)
1950
- });
1951
- release = void 0;
1952
- return result;
1953
- } finally {
1954
- release?.();
1955
- }
1956
- }
1957
- /**
1958
- * Checks every `ToolCall` against the `tools` that were offered, catching
1959
- * a hallucinated tool name and a duplicate call id before either reaches
1960
- * the application's dispatch table, then runs each tool's
1961
- * `argumentsSchema`, if present.
1962
- *
1963
- * Contract failures (unknown name, duplicate id) are collected across
1964
- * every call and thrown together as one `type: 'validation'` error with
1965
- * `issues: ToolIssue[]`, since retrying a request that already has these
1966
- * errors cannot help (excluded from retry by `type`) and a caller fixing
1967
- * them wants to see every one, not just the first. Schema failures keep
1968
- * the original single-error, `type: 'validation'` shape rather than being
1969
- * folded into the aggregate, since they're a distinct failure kind from
1970
- * the contract failures above.
1971
- */
1972
- validateToolCallArguments(toolCalls, tools) {
1973
- const known = new Map(tools.map((t) => [t.name, t]));
1974
- const seenIds = new Set();
1975
- const toolIssues = [];
1976
- for (const call of toolCalls) {
1977
- if (seenIds.has(call.id)) toolIssues.push({
1978
- name: call.name,
1979
- toolCallId: call.id,
1980
- code: "duplicate_tool_call_id"
1981
- });
1982
- seenIds.add(call.id);
1983
- if (!known.has(call.name)) toolIssues.push({
1984
- name: call.name,
1985
- toolCallId: call.id,
1986
- code: "unknown_tool"
1987
- });
1988
- }
1989
- if (toolIssues.length > 0) {
1990
- const unknownTool = toolIssues.find((i) => i.code === "unknown_tool");
1991
- const primary = unknownTool ? `Model requested tool "${unknownTool.name}", which was not in the tools offered ([${[...known.keys()].join(", ")}]).` : `Duplicate tool call id "${toolIssues[0].toolCallId}" in the model's response.`;
1992
- const message = toolIssues.length > 1 ? `${primary} (${toolIssues.length} tool call issues total, see error.issues.)` : primary;
1993
- throw new LLMError(message, "validation", {
1994
- code: unknownTool ? "unknown_tool" : "duplicate_tool_call_id",
1995
- issues: toolIssues
1996
- });
1997
- }
1998
- for (const call of toolCalls) {
1999
- const definition = known.get(call.name);
2000
- if (!definition?.argumentsSchema) continue;
2001
- const result = definition.argumentsSchema.safeParse(call.arguments);
2002
- if (!result.success) throw new LLMError(`Arguments for tool call "${call.name}" failed validation`, "validation", { issues: result.error });
2003
- }
2004
- }
2005
- /**
2006
- * Runs `fn`, retrying with backoff according to `shouldRetry`. When
2007
- * `attempts` is given, every failed attempt that is actually followed by
2008
- * a retry is recorded, in order. This mirrors `LLMError.attempts`'s
2009
- * contract: every attempt made before this error was thrown. The
2010
- * terminal failure is never pushed since it isn't a prior attempt, it
2011
- * is the error being thrown. `attempts` stays empty when nothing was
2012
- * retried, so no separate bookkeeping is needed at the call sites.
2013
- * Each failure is recorded as a snapshot (`LLMError.toSnapshot()`),
2014
- * not the live `LLMError`, per `RetryAttempt`'s contract.
2015
- */
2016
- async retryWithBackoff(fn, requestId, model, signal, onAttempt, attempts) {
2017
- let lastError;
2018
- for (let attempt = 0; attempt <= this.maxRetries; attempt++) try {
2019
- if (attempt > 0) await this.recoverDelay(requestId, model, attempt, lastError, signal);
2020
- onAttempt?.();
2021
- return await fn(attempt);
2022
- } catch (error) {
2023
- lastError = error;
2024
- const willRetry = attempt < this.maxRetries && this.shouldRetry(error, signal);
2025
- if (!willRetry) break;
2026
- attempts?.push({
2027
- index: attempt,
2028
- error: normalizeError(error, signal).toSnapshot()
2029
- });
2030
- }
2031
- throw lastError;
2032
- }
2033
- /**
2034
- * Pulls `TokenUsage` out of a raw response, if the provider reported it.
2035
- * Extraction doesn't depend on what happens to the response afterward, so
2036
- * a malformed body can still yield usage if the provider's usage block
2037
- * itself came through intact.
2038
- */
2039
- extractUsage(response, requestId, model) {
2040
- if (!response.usage) return void 0;
2041
- return {
2042
- promptTokens: response.usage.prompt_tokens ?? 0,
2043
- completionTokens: response.usage.completion_tokens ?? 0,
2044
- totalTokens: response.usage.total_tokens ?? 0,
2045
- requestId,
2046
- model,
2047
- provider: this.providerName,
2048
- usedFallback: this.isFallback
2049
- };
2050
- }
2051
- /**
2052
- * The token count to reconcile the rate limiter against for a finished
2053
- * attempt: `totalTokens` when reported, otherwise the sum of prompt and
2054
- * completion tokens, matching `reportUsageFailure`'s own fallback below
2055
- * for a hand-rolled client that reports the parts but omits the total.
2056
- */
2057
- actualTokensFor(usage) {
2058
- if (!usage) return void 0;
2059
- return usage.totalTokens || usage.promptTokens + usage.completionTokens;
2060
- }
2061
- /** Reports token usage for a successful call, swallowing and logging any error `onUsage` throws. */
2062
- reportUsage(usage) {
2063
- if (!usage || !this.onUsage) return;
2064
- try {
2065
- this.onUsage(usage);
2066
- } catch (error) {
2067
- this.logger.error("[VernLLM] onUsage failed", { message: error instanceof Error ? error.message : "unknown" });
2068
- }
2069
- }
2070
- /**
2071
- * Reports token usage spent on an attempt that then failed, so it isn't
2072
- * dropped alongside the error. Covers any error thrown after usage
2073
- * extraction, since all of them happen only after a response (real
2074
- * spend) already arrived. Swallows and logs any error `onUsageFailure`
2075
- * itself throws.
2076
- */
2077
- reportUsageFailure(usage, error, attempt, terminal = false) {
2078
- const displayTokens = usage.totalTokens || usage.promptTokens + usage.completionTokens;
2079
- const attemptText = terminal ? "mid-stream failure (terminal, no further attempts)" : `attempt ${attempt + 1}/${this.maxRetries + 1}`;
2080
- this.logger.warn(`[VernLLM:${usage.requestId}] usage failure, ${attemptText}: type=${error.type} tokens=${displayTokens}`);
2081
- if (!this.onUsageFailure) return;
2082
- try {
2083
- this.onUsageFailure(usage, error);
2084
- } catch (hookError) {
2085
- this.logger.error("[VernLLM] onUsageFailure failed", { message: hookError instanceof Error ? hookError.message : "unknown" });
2086
- }
2087
- }
2088
- /** Parses response content as JSON and validates it against `schema` when supplied. */
2089
- parseAndValidate(content, schema) {
2090
- let parsed;
2091
- try {
2092
- parsed = this.parseJson(content);
2093
- } catch {
2094
- throw new LLMError("Invalid JSON response", "parse");
2095
- }
2096
- if (parsed === null || parsed === void 0) throw new LLMError("Invalid JSON response", "parse");
2097
- if (!schema) return parsed;
2098
- const result = schema.safeParse(parsed);
2099
- if (!result.success) throw new LLMError("Schema validation failed", "validation", { issues: result.error });
2100
- return result.data;
2101
- }
2102
- /**
2103
- * Waits out the backoff delay for a retry attempt, honoring a
2104
- * Retry-After header on the failed attempt's error when present.
2105
- * Both Retry-After and plain exponential backoff are capped at the same
2106
- * max delay (see `DEFAULT_MAX_DELAY_MS` in `retry.utils.ts`).
2107
- */
2108
- async recoverDelay(requestId, model, attempt, error, signal) {
2109
- const retryAfterMs = extractRetryAfterMs(error);
2110
- const delay = retryAfterMs ?? getBackoffDelay(this.baseDelayMs, attempt);
2111
- const retryAfterHonored = retryAfterMs !== void 0;
2112
- this.logger.warn(`[VernLLM:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${Math.ceil(delay)}ms` + (retryAfterHonored ? " (honoring Retry-After)" : ""));
2113
- this.reportEvent({
2114
- kind: "retry",
2115
- requestId,
2116
- provider: this.providerName,
2117
- model,
2118
- attempt,
2119
- maxRetries: this.maxRetries,
2120
- delayMs: delay,
2121
- retryAfterHonored,
2122
- error: normalizeError(error, signal)
2123
- });
2124
- await waitForRetry(delay, signal);
2125
- }
2126
- /** Decides whether a failed attempt is worth retrying. */
2127
- shouldRetry(error, signal) {
2128
- if (signal?.aborted) return false;
2129
- if (error instanceof LLMError && !error.retryable) return false;
2130
- const status = extractStatus(error);
2131
- return !(status !== void 0 && this.nonRetryableStatus.includes(status));
2132
- }
2133
- /**
2134
- * Decides whether a failed attempt should count toward the circuit
2135
- * breaker's failure threshold. A model hallucinating a tool name,
2136
- * reusing a call id, or a provider ignoring `toolChoice: 'none'` isn't
2137
- * the provider being unhealthy, it's a model/provider response defect
2138
- * that will very likely recur regardless of provider health, so it
2139
- * shouldn't push a healthy provider's circuit toward opening. Same for
2140
- * a caller-input bug or a local rate-limit rejection: neither ever
2141
- * reached the provider at all. This is exactly what `LLMError.retryable`
2142
- * already excludes, so this defers to it directly.
2143
- */
2144
- countsTowardBreaker(error) {
2145
- return error.retryable;
2146
- }
2147
- };
2148
-
2149
- //#endregion
2150
- //#region src/internal/logger.utils.ts
2151
- /**
2152
- * Wraps a `Logger` so a throwing implementation can never break the call
2153
- * it's trying to describe. `logger` is user-supplied (`VernLLMOptions.logger`),
2154
- * so a custom logger that ships to a file, Datadog, etc. can throw for
2155
- * reasons unrelated to VernLLM. Wrap once at construction so every
2156
- * downstream `this.logger.warn(...)` call stays as-is and is safe by
2157
- * construction, instead of guarding each call site individually.
2158
- */
2159
- function createSafeLogger(logger) {
2160
- return {
2161
- debug: safe(logger, "debug"),
2162
- warn: safe(logger, "warn"),
2163
- error: safe(logger, "error")
2164
- };
2165
- }
2166
- function safe(logger, method) {
2167
- const fn = logger[method].bind(logger);
2168
- return (...args) => {
2169
- try {
2170
- swallowRejection(fn(...args));
2171
- } catch {}
2172
- };
2173
- }
2174
- function isPromiseLike(value) {
2175
- return typeof value?.then === "function";
2176
- }
2177
- function swallowRejection(result) {
2178
- if (isPromiseLike(result)) Promise.resolve(result).catch(() => {});
2179
- }
2180
-
2181
- //#endregion
2182
- //#region src/logger.ts
2183
- /**
2184
- * Default logger. `debug` is gated by the `debug` option on VernLLM
2185
- * warn/error always fire since they indicate real problems (retries, cache failures)
2186
- */
2187
- var ConsoleLogger = class {
2188
- constructor(debugEnabled) {
2189
- this.debugEnabled = debugEnabled;
2190
- }
2191
- debug(message) {
2192
- if (this.debugEnabled) console.debug(message);
2193
- }
2194
- warn(message) {
2195
- console.warn(message);
2196
- }
2197
- error(message, meta) {
2198
- console.error(message, meta ?? "");
2199
- }
2200
- };
2201
-
2202
- //#endregion
2203
- //#region src/rateLimit.ts
2204
- /** Default `estimateTokens`: chars/4 over every message's content, plus the requested `max_tokens`. */
2205
- function defaultEstimateTokens(request) {
2206
- const messagesChars = request.messages.reduce((sum, message) => {
2207
- const content = message.content;
2208
- if (typeof content === "string") return sum + content.length;
2209
- if (content === void 0 || content === null) return sum;
2210
- try {
2211
- return sum + JSON.stringify(content).length;
2212
- } catch {
2213
- return sum;
2214
- }
2215
- }, 0);
2216
- return Math.ceil(messagesChars / 4) + (request.max_tokens ?? 0);
2217
- }
2218
- /**
2219
- * A capacity that refills continuously. Used for requests per minute and
2220
- * tokens per minute, where `refillPerMs` is `capacity / 60000`, and for
2221
- * concurrency, where `refillPerMs` is 0 and every release calls
2222
- * `give(1)` instead of relying on the clock.
2223
- */
2224
- var TokenBucket = class {
2225
- available;
2226
- lastRefill = Date.now();
2227
- constructor(capacity, refillPerMs) {
2228
- this.capacity = capacity;
2229
- this.refillPerMs = refillPerMs;
2230
- this.available = capacity;
2231
- }
2232
- refill() {
2233
- if (this.refillPerMs === 0) return;
2234
- const now = Date.now();
2235
- const elapsedMs = now - this.lastRefill;
2236
- this.available = Math.min(this.capacity, this.available + Math.max(0, elapsedMs) * this.refillPerMs);
2237
- this.lastRefill = now;
2238
- }
2239
- /** Refills, then takes `amount` if available. Leaves the bucket untouched if it can't. */
2240
- tryTake(amount) {
2241
- this.refill();
2242
- if (this.available < amount) return false;
2243
- this.available -= amount;
2244
- return true;
2245
- }
2246
- /**
2247
- * Refills, then reports how many ms until this bucket could supply
2248
- * `amount`, assuming nothing else takes from it meanwhile. Returns 0 if
2249
- * it already can, `Infinity` if it never will on its own (a
2250
- * concurrency bucket, `refillPerMs === 0`, only frees via `give`).
2251
- */
2252
- msUntilAvailable(amount) {
2253
- this.refill();
2254
- if (this.available >= amount) return 0;
2255
- if (this.refillPerMs === 0) return Infinity;
2256
- return (amount - this.available) / this.refillPerMs;
2257
- }
2258
- /**
2259
- * Gives capacity back. Not floored at 0: a bad token-usage estimate can
2260
- * push `available` negative, and it self-corrects on the next refill
2261
- * rather than being clamped away immediately. Only ceilinged at
2262
- * `capacity`, so a give can never overfill the bucket.
2263
- */
2264
- give(amount) {
2265
- this.available = Math.min(this.capacity, this.available + amount);
2266
- }
2267
- /** The bucket's ceiling, e.g. so a request that could never fit can fail fast instead of queueing forever. */
2268
- getCapacity() {
2269
- return this.capacity;
2270
- }
2271
- };
2272
- /**
2273
- * `setTimeout` silently clamps any delay above this (~24.8 days) instead
2274
- * of erroring, so an uncapped delay derived from a very small
2275
- * `requestsPerMinute`/`tokensPerMinute` could wrap around to firing
2276
- * almost immediately instead of waiting. Mirrors the same guard in
2277
- * `withTimeout`/`withChunkIdleTimeout`.
2278
- */
2279
- const MAX_WAKE_DELAY_MS = 2147483647;
2280
- /**
2281
- * Per-target rate limiter. Up to three buckets (requests/min, tokens/min,
2282
- * concurrency) behind one FIFO queue, so a large call isn't starved by a
2283
- * stream of small ones. Any bucket omitted from `options` has infinite
2284
- * capacity and never blocks.
2285
- */
2286
- var RateLimiter = class {
2287
- requests;
2288
- tokens;
2289
- concurrency;
2290
- maxQueueMs;
2291
- maxQueueSize;
2292
- estimateTokensFn;
2293
- queue = [];
2294
- /**
2295
- * A single scheduled re-check for the head of the queue when it's
2296
- * blocked on a bucket that refills on its own clock (rpm/tpm), so a
2297
- * queue that nobody calls `acquire`/`release` on again isn't stuck
2298
- * forever waiting for an external trigger to re-drain it. Not needed
2299
- * for a concurrency block, which only clears via `release`.
2300
- */
2301
- wakeTimer;
2302
- constructor(options) {
2303
- if (options.requestsPerMinute) this.requests = new TokenBucket(options.requestsPerMinute, options.requestsPerMinute / 6e4);
2304
- if (options.tokensPerMinute) this.tokens = new TokenBucket(options.tokensPerMinute, options.tokensPerMinute / 6e4);
2305
- if (options.maxConcurrent) this.concurrency = new TokenBucket(options.maxConcurrent, 0);
2306
- this.maxQueueMs = options.maxQueueMs ?? 3e4;
2307
- this.maxQueueSize = options.maxQueueSize ?? 0;
2308
- this.estimateTokensFn = options.estimateTokens ?? defaultEstimateTokens;
2309
- }
2310
- /** Pre-flight token estimate for a request, per the configured (or default) heuristic. */
2311
- estimate(request) {
2312
- return this.estimateTokensFn(request);
2313
- }
2314
- /**
2315
- * Waits for capacity in every configured bucket, then takes from each.
2316
- * The returned `release` gives the concurrency slot back and reconciles
2317
- * the token bucket against real usage; it must run in a `finally` block.
2318
- */
2319
- async acquire(estimatedTokens, signal) {
2320
- if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
2321
- if (!Number.isFinite(estimatedTokens) || estimatedTokens < 0) throw new LLMError(`estimatedTokens must be a finite, non-negative number, got ${String(estimatedTokens)}`, "invalid_params");
2322
- if (this.tokens && estimatedTokens > this.tokens.getCapacity()) throw new LLMError(`estimatedTokens (${estimatedTokens}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`, "rate_limited", { code: "rate_limit_capacity_exceeded" });
2323
- if (this.queue.length === 0) {
2324
- const attempt = this.tryAcquireBuckets(estimatedTokens);
2325
- if (attempt.ok) return {
2326
- release: this.makeRelease(estimatedTokens),
2327
- waitedMs: 0
2328
- };
2329
- return this.enqueue(estimatedTokens, attempt.reason, signal);
2330
- }
2331
- if (this.maxQueueSize > 0 && this.queue.length >= this.maxQueueSize) throw this.queueFullError();
2332
- return this.enqueue(estimatedTokens, void 0, signal);
2333
- }
2334
- queueFullError() {
2335
- return new LLMError("Rate limit queue is full", "rate_limited", { code: "rate_limit_queue_full" });
2336
- }
2337
- enqueue(estimatedTokens, initialReason, signal) {
2338
- return new Promise((resolvePromise, rejectPromise) => {
2339
- const waiter = {
2340
- estimatedTokens,
2341
- enqueuedAt: Date.now(),
2342
- lastReason: initialReason,
2343
- resolve: (result) => {
2344
- cleanup();
2345
- resolvePromise(result);
2346
- },
2347
- reject: (error) => {
2348
- cleanup();
2349
- if (this.wakeTimer) {
2350
- clearTimeout(this.wakeTimer);
2351
- this.wakeTimer = void 0;
2352
- }
2353
- this.drain();
2354
- rejectPromise(error);
2355
- }
2356
- };
2357
- let queueTimer;
2358
- const onAbort = () => {
2359
- waiter.reject(new LLMError("LLM request aborted", "aborted"));
2360
- };
2361
- const cleanup = () => {
2362
- if (queueTimer) clearTimeout(queueTimer);
2363
- signal?.removeEventListener("abort", onAbort);
2364
- const index = this.queue.indexOf(waiter);
2365
- if (index !== -1) this.queue.splice(index, 1);
2366
- };
2367
- if (this.maxQueueMs > 0) queueTimer = setTimeout(() => {
2368
- waiter.reject(new LLMError("Rate limit queue timed out before capacity was available", "rate_limited", { code: "rate_limit_queue_timeout" }));
2369
- }, this.maxQueueMs);
2370
- signal?.addEventListener("abort", onAbort, { once: true });
2371
- this.queue.push(waiter);
2372
- this.drain();
2373
- });
2374
- }
2375
- /**
2376
- * Checks and takes from every configured bucket as one atomic unit: if
2377
- * any bucket lacks capacity, whatever was already taken from the
2378
- * earlier ones in this attempt is rolled back before reporting which
2379
- * bucket blocked.
2380
- */
2381
- tryAcquireBuckets(estimatedTokens) {
2382
- const taken = [];
2383
- const take = (bucket, amount) => {
2384
- if (!bucket) return true;
2385
- if (!bucket.tryTake(amount)) return false;
2386
- taken.push({
2387
- bucket,
2388
- amount
2389
- });
2390
- return true;
2391
- };
2392
- if (!take(this.concurrency, 1)) return {
2393
- ok: false,
2394
- reason: "concurrency"
2395
- };
2396
- if (!take(this.requests, 1)) {
2397
- for (const entry of taken) entry.bucket.give(entry.amount);
2398
- return {
2399
- ok: false,
2400
- reason: "rpm"
2401
- };
2402
- }
2403
- if (!take(this.tokens, estimatedTokens)) {
2404
- for (const entry of taken) entry.bucket.give(entry.amount);
2405
- return {
2406
- ok: false,
2407
- reason: "tpm"
2408
- };
2409
- }
2410
- return { ok: true };
2411
- }
2412
- /** Drains the queue head first. Stops at the first waiter that still can't proceed, so no one is starved out of turn. */
2413
- drain() {
2414
- while (this.queue.length > 0) {
2415
- const waiter = this.queue[0];
2416
- const attempt = this.tryAcquireBuckets(waiter.estimatedTokens);
2417
- if (!attempt.ok) {
2418
- waiter.lastReason = attempt.reason;
2419
- this.scheduleWake(attempt.reason, waiter.estimatedTokens);
2420
- return;
2421
- }
2422
- const waitedMs = Date.now() - waiter.enqueuedAt;
2423
- waiter.resolve({
2424
- release: this.makeRelease(waiter.estimatedTokens),
2425
- waitedMs,
2426
- reason: waiter.lastReason
2427
- });
2428
- }
2429
- }
2430
- /**
2431
- * Schedules a one-shot re-check of the queue for whenever the bucket
2432
- * that's currently blocking the head waiter should next have enough
2433
- * capacity. A no-op for a concurrency block (only `release` can clear
2434
- * that) or while a wake is already pending.
2435
- */
2436
- scheduleWake(reason, estimatedTokens) {
2437
- if (this.wakeTimer) return;
2438
- const ms = reason === "rpm" ? this.requests?.msUntilAvailable(1) : reason === "tpm" ? this.tokens?.msUntilAvailable(estimatedTokens) : void 0;
2439
- if (ms === void 0 || !Number.isFinite(ms)) return;
2440
- const delay = Math.min(Math.max(1, Math.ceil(ms)), MAX_WAKE_DELAY_MS);
2441
- this.wakeTimer = setTimeout(() => {
2442
- this.wakeTimer = void 0;
2443
- this.drain();
2444
- }, delay);
2445
- }
2446
- /**
2447
- * Builds the one-shot release closure for an acquired slot. Only the
2448
- * concurrency bucket is given back on release; the requests-per-minute
2449
- * bucket is a real spend that only recovers via its own refill, and the
2450
- * tokens bucket is reconciled against `actualTokens` rather than fully
2451
- * refunded, since real tokens really were spent.
2452
- */
2453
- makeRelease(estimatedTokens) {
2454
- let released = false;
2455
- return (actualTokens) => {
2456
- if (released) return;
2457
- released = true;
2458
- this.concurrency?.give(1);
2459
- if (this.tokens && actualTokens !== void 0 && Number.isFinite(actualTokens)) this.tokens.give(estimatedTokens - actualTokens);
2460
- this.drain();
2461
- };
2462
- }
2463
- };
2464
-
2465
- //#endregion
2466
- //#region src/vernLLM.ts
2467
- /**
2468
- * A resilient layer around an LLM chat completions client. This is VernLLM!
2469
- *
2470
- * Adds retry with backoff and jitter, per-attempt timeouts, an optional
2471
- * circuit breaker, JSON parsing with optional schema validation, usage
2472
- * tracking, and an optional response cache. All configurable, all opt-in
2473
- * beyond sensible defaults.
2474
- */
2475
- var VernLLM = class {
2476
- logger;
2477
- /**
2478
- * One `CallExecutor` per provider target: index 0 is the primary,
2479
- * everything after it is a `fallback` target, in the order declared.
2480
- * Each owns its own request building, retry/timeout, circuit breaker,
2481
- * and rate limiter. `call()` walks this array in `runFallbackChain`,
2482
- * moving to the next entry only when `fallbackOn` says to.
2483
- */
2484
- executors;
2485
- /** Decides whether a failed target is followed by the next one or the chain stops. See `VernLLMOptions['fallbackOn']`. */
2486
- fallbackOn;
2487
- /** Reports a `'fallback'` event when the chain moves to the next target. Shared `onEvent` plumbing, same as every executor's. */
2488
- reportEvent;
2489
- /**
2490
- * Owns cache key resolution, cache reads/writes, and in-flight
2491
- * coalescing for `cachedCall()`. Independent of `executor`: it only
2492
- * ever calls back into `this.call()` as an opaque function.
2493
- */
2494
- cacheOrchestrator;
2495
- /**
2496
- * @param options Client, model, and tunables. Defaults: `maxRetries` 1,
2497
- * `timeoutMs` 25000, `baseDelayMs` 500, `defaultMaxTokens` 1000,
2498
- * `defaultTemperature` 0.2, `cache` an in-memory adapter,
2499
- * `nonRetryableStatus` `[400, 401, 403, 404, 422]`, `debug` false.
2500
- */
2501
- constructor(options) {
2502
- this.logger = createSafeLogger(options.logger ?? new ConsoleLogger(options.debug ?? false));
2503
- const providerName = options.name ?? "primary";
2504
- this.cacheOrchestrator = new CacheOrchestrator(options.cache ?? new InMemoryCacheAdapter(), this.logger);
2505
- this.fallbackOn = options.fallbackOn ?? defaultFallbackOn;
2506
- this.reportEvent = makeEventReporter(options.onEvent, this.logger);
2507
- const primaryDefaultTemperature = options.defaultTemperature === void 0 ? .2 : options.defaultTemperature;
2508
- const primaryTarget = {
2509
- client: options.client,
2510
- model: options.model,
2511
- name: providerName,
2512
- maxRetries: options.maxRetries,
2513
- timeoutMs: options.timeoutMs,
2514
- chunkIdleTimeoutMs: options.chunkIdleTimeoutMs,
2515
- baseDelayMs: options.baseDelayMs,
2516
- defaultMaxTokens: options.defaultMaxTokens,
2517
- defaultTemperature: primaryDefaultTemperature,
2518
- nonRetryableStatus: options.nonRetryableStatus,
2519
- circuitBreaker: options.circuitBreaker,
2520
- rateLimit: options.rateLimit
2521
- };
2522
- const declaredFallbacks = Array.isArray(options.fallback) ? options.fallback : options.fallback ? [options.fallback] : [];
2523
- const targets = [primaryTarget, ...declaredFallbacks];
2524
- this.executors = targets.map((target, i) => {
2525
- const isFallback = i > 0;
2526
- const name = target.name ?? (isFallback ? `fallback[${i - 1}]` : providerName);
2527
- const breaker = buildCircuitBreaker(target.circuitBreaker, name, target.model, options.onEvent, this.logger);
2528
- return new CallExecutor(name, target.client, target.model, {
2529
- maxRetries: target.maxRetries ?? options.maxRetries ?? 1,
2530
- timeoutMs: target.timeoutMs ?? options.timeoutMs ?? 25e3,
2531
- chunkIdleTimeoutMs: target.chunkIdleTimeoutMs ?? options.chunkIdleTimeoutMs ?? 3e4,
2532
- baseDelayMs: target.baseDelayMs ?? options.baseDelayMs ?? 500,
2533
- defaultMaxTokens: target.defaultMaxTokens ?? options.defaultMaxTokens ?? 1e3,
2534
- defaultTemperature: target.defaultTemperature === void 0 ? primaryDefaultTemperature : target.defaultTemperature,
2535
- nonRetryableStatus: target.nonRetryableStatus ?? options.nonRetryableStatus ?? [
2536
- 400,
2537
- 401,
2538
- 403,
2539
- 404,
2540
- 422
2541
- ],
2542
- parseJson: options.parseJson,
2543
- logger: this.logger,
2544
- redact: options.redact,
2545
- onUsage: options.onUsage,
2546
- onUsageFailure: options.onUsageFailure,
2547
- onEvent: options.onEvent,
2548
- breaker,
2549
- limiter: target.rateLimit ? new RateLimiter(target.rateLimit) : void 0,
2550
- isFallback
2551
- });
2552
- });
2553
- }
2554
- /** Logs a failed refundUsage attempt via the configured logger. */
2555
- logRefundError(logMessage, error) {
2556
- this.logger.error(logMessage, { message: error instanceof Error ? error.message : "unknown" });
2557
- }
2558
- /**
2559
- * Walks `this.executors` in order, running `attempt` against each until
2560
- * one succeeds or every target has failed. `run` on a lone target
2561
- * (no `fallback` configured) throws exactly what it throws today: the
2562
- * loop's single iteration path is unchanged from pre-fallback behavior.
2563
- *
2564
- * For streaming, `attempt` is `executor.runStream`, whose own retries
2565
- * only cover *opening* the stream (see `CallExecutor.runStream`). A
2566
- * mid-stream failure surfaces through `finalResult` after this function
2567
- * has already returned, so it's never seen here and never falls over,
2568
- * per the streaming limitation: splicing a second model's output into a
2569
- * response the consumer has already partially rendered would corrupt
2570
- * it.
2571
- */
2572
- async runFallbackChain(params, requestId, attempt, skipBreakerCheckForFirst = false) {
2573
- const attempts = [];
2574
- for (let i = 0; i < this.executors.length; i++) {
2575
- const executor = this.executors[i];
2576
- const startedAt = Date.now();
2577
- let attemptCount = 0;
2578
- try {
2579
- if (!(i === 0 && skipBreakerCheckForFirst)) executor.assertBreakerClosed(params.model);
2580
- const result = await attempt(executor, () => {
2581
- attemptCount += 1;
2582
- });
2583
- return {
2584
- result,
2585
- executor,
2586
- index: i,
2587
- attemptCount
2588
- };
2589
- } catch (error) {
2590
- const normalized = normalizeError(error, params.signal);
2591
- attempts.push({
2592
- index: i - 1,
2593
- provider: executor.providerName,
2594
- model: params.model ?? executor.model,
2595
- error: normalized.toSnapshot()
2596
- });
2597
- const isLast = i === this.executors.length - 1;
2598
- const policyDecision = this.fallbackOn(normalized, { isLastTarget: isLast });
2599
- const decision = isLast ? "stop" : policyDecision;
2600
- if (decision === "stop") throw attempts.length > 1 ? new FallbackExhaustedError(attempts) : normalized;
2601
- const next = this.executors[i + 1];
2602
- this.reportEvent({
2603
- kind: "fallback",
2604
- requestId,
2605
- from: executor.providerName,
2606
- to: next.providerName,
2607
- fromIndex: i - 1,
2608
- toIndex: i,
2609
- error: normalized,
2610
- elapsedMs: Date.now() - startedAt
2611
- });
2612
- }
2613
- }
2614
- throw new LLMError("No provider targets configured", "invalid_params");
2615
- }
2616
- async call(params) {
2617
- if (params.signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
2618
- const requestId = params.requestId ?? randomUUID();
2619
- const soleTarget = this.executors.length === 1;
2620
- if (soleTarget) this.executors[0].assertBreakerClosed(params.model);
2621
- if (params.stream) return withReservedUsageForStream(params, async () => {
2622
- const { result } = await this.runFallbackChain(params, requestId, (executor, onAttempt) => executor.runStream(params, requestId, onAttempt), soleTarget);
2623
- return result;
2624
- }, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
2625
- return withReservedUsage(params, false, async () => {
2626
- const { result, executor, index, attemptCount } = await this.runFallbackChain(params, requestId, (target, onAttempt) => target.run(params, requestId, onAttempt), soleTarget);
2627
- if (params.meta) params.meta.current = {
2628
- provider: executor.providerName,
2629
- model: params.model ?? executor.model,
2630
- fallbackIndex: index - 1,
2631
- usedFallback: index > 0,
2632
- attempts: attemptCount
2633
- };
2634
- return result;
2635
- }, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
2636
- }
2637
- /**
2638
- * Thin delegator kept private on `VernLLM` (rather than only existing on
2639
- * `CacheOrchestrator`) since it's the one caching primitive exercised
2640
- * directly by white-box tests, independent of the public `cachedCall()`
2641
- * surface.
2642
- */
2643
- runCached(params) {
2644
- return this.cacheOrchestrator.runCached(params);
2645
- }
2646
- /**
2647
- * Removes a cached response by key when the configured cache adapter
2648
- * supports deletion. Cache invalidation is the caller's responsibility;
2649
- * only the application knows when cached data is stale.
2650
- *
2651
- * @param key The raw cache key (resolved through the adapter's
2652
- * `resolveKey`, if any, before deletion).
2653
- */
2654
- async deleteCache(key) {
2655
- await this.cacheOrchestrator.deleteCache(key);
2656
- }
2657
- async cachedCall(params) {
2658
- const { call: callParams,...cacheParams } = params;
2659
- const restCallParams = callParams;
2660
- if (restCallParams.reserveUsage || restCallParams.refundUsage) throw new LLMError("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.", "invalid_params");
2661
- if (restCallParams.stream) {
2662
- const streamParams = restCallParams;
2663
- return this.cacheOrchestrator.runCachedStream({
2664
- ...cacheParams,
2665
- openStream: () => this.call(streamParams)
2666
- }, Boolean(restCallParams.tools));
2667
- }
2668
- return this.runCached({
2669
- ...cacheParams,
2670
- fn: () => this.call(restCallParams)
2671
- });
2672
- }
2673
- /**
2674
- * @param target.index Which target to read. Defaults to the primary.
2675
- * @param target.model Which model bucket to read, if the target isolates by model.
2676
- * @returns The breaker state, or `undefined` if that target has no breaker.
2677
- * @throws {RangeError} If `target.index` names no target. Lets a real
2678
- * target with no breaker (`undefined`) stay distinguishable from a
2679
- * target that doesn't exist.
2680
- */
2681
- getCircuitState(target) {
2682
- const executor = this.resolveExecutor(target?.index ?? 0, "getCircuitState");
2683
- this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "getCircuitState");
2684
- return executor.getCircuitState(target?.model ?? executor.model);
2685
- }
2686
- /**
2687
- * @param model Which model bucket to read, for targets that isolate by model.
2688
- * @returns Every target's state, in chain order.
2689
- */
2690
- getCircuitStates(model) {
2691
- return this.executors.map((executor, index) => ({
2692
- provider: executor.providerName,
2693
- index,
2694
- isFallback: index > 0,
2695
- isolateByModel: executor.isolateByModel,
2696
- state: executor.getCircuitState(model ?? executor.model)
2697
- }));
2698
- }
2699
- /**
2700
- * Manually opens a target's breaker, e.g. to pull a provider out of
2701
- * rotation ahead of known maintenance instead of waiting for it to fail.
2702
- *
2703
- * @param target.index Which target to open. Defaults to the primary.
2704
- * @param target.model Which model bucket to open, if the target isolates by model.
2705
- * @throws {RangeError} If `target.index` names no target.
2706
- */
2707
- openCircuit(target) {
2708
- const executor = this.resolveExecutor(target?.index ?? 0, "openCircuit");
2709
- this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "openCircuit");
2710
- executor.openCircuit(target?.model ?? executor.model);
2711
- }
2712
- /**
2713
- * Manually closes a target's breaker, e.g. once a provider is confirmed
2714
- * healthy again without waiting out the cooldown.
2715
- *
2716
- * @param target.index Which target to close. Defaults to the primary.
2717
- * @param target.model Which model bucket to close, if the target isolates by model.
2718
- * @throws {RangeError} If `target.index` names no target.
2719
- */
2720
- closeCircuit(target) {
2721
- const executor = this.resolveExecutor(target?.index ?? 0, "closeCircuit");
2722
- this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "closeCircuit");
2723
- executor.closeCircuit(target?.model ?? executor.model);
2724
- }
2725
- /** Resolves a target index so every circuit-breaker method agrees on what counts as valid. */
2726
- resolveExecutor(index, caller) {
2727
- const executor = this.executors[index];
2728
- if (!executor) throw new RangeError(`${caller}: no target at index ${index} (chain has ${this.executors.length} target${this.executors.length === 1 ? "" : "s"})`);
2729
- return executor;
2730
- }
2731
- /** Warns when `model` can't do anything on this target, so it's never silently ignored. */
2732
- warnIfModelUnsupported(isolateByModel, model, caller) {
2733
- if (model !== void 0 && !isolateByModel) this.logger.warn(`[VernLLM] ${caller}: \`model: '${model}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`);
2734
- }
2735
- };
2736
-
2737
- //#endregion
2738
- //#region src/adapters/internal/sse.ts
2739
- /**
2740
- * Parses a Server-Sent-Events byte/text stream into the JSON payload of
2741
- * each `data:` frame, in arrival order. Generic over transport: works with
2742
- * anything that hands back progressively-arriving `Uint8Array` or `string`
2743
- * chunks via async iteration: native `fetch`'s `response.body` (wrapped
2744
- * to be iterable, see `webStreamToAsyncIterable` in `fetch.ts`), axios's
2745
- * Node `Readable` (already async-iterable, no wrapping needed), etc, so
2746
- * this framing layer doesn't care which transport produced the bytes.
2747
- *
2748
- * Follows the SSE spec's frame-delimiting rules closely enough for LLM
2749
- * streaming responses: frames are separated by a blank line, each frame
2750
- * may carry one or more `data:` lines (joined with `\n` per spec when
2751
- * there's more than one), `:`-prefixed lines are comments and ignored, and
2752
- * other SSE fields (`event:`, `id:`, `retry:`) are ignored since VernLLM
2753
- * only needs the payload. A frame whose data is exactly `[DONE]` (the
2754
- * sentinel several providers, notably OpenAI, send to mark stream end)
2755
- * ends iteration without yielding it.
2756
- *
2757
- * Line endings: `\r\n` and bare `\r` (both legal per the SSE spec, alongside `\n`) are normalized
2758
- * to `\n` before frame splitting. A `\r` at the very end of the currently-buffered text is left
2759
- * alone until either more text arrives (in case it's the first half of a split `\r\n` pair) or the
2760
- * stream ends, so a `\r\n` pair split across two transport chunks is never misread as two blank
2761
- * lines.
2762
- *
2763
- * Malformed JSON in a frame throws `LLMError('parse')`, consistent with
2764
- * how malformed JSON is handled elsewhere in VernLLM.
2765
- */
2766
- async function* parseSseStream(source) {
2767
- const decoder = new TextDecoder("utf-8", { fatal: true });
2768
- let buffer = "";
2769
- for await (const chunk of source) {
2770
- let text;
2771
- try {
2772
- text = typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true });
2773
- } catch (cause) {
2774
- throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
2775
- }
2776
- buffer = (buffer + text).replace(/\r\n/g, "\n").replace(/\r(?!$)/g, "\n");
2777
- let boundary$1 = buffer.indexOf("\n\n");
2778
- while (boundary$1 !== -1) {
2779
- const frame = buffer.slice(0, boundary$1);
2780
- buffer = buffer.slice(boundary$1 + 2);
2781
- const event = parseSseFrame(frame);
2782
- if (event === DONE) return;
2783
- if (event !== NO_DATA) yield event;
2784
- boundary$1 = buffer.indexOf("\n\n");
2785
- }
2786
- }
2787
- try {
2788
- buffer += decoder.decode();
2789
- } catch (cause) {
2790
- throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
2791
- }
2792
- buffer = buffer.replace(/\r$/, "\n");
2793
- let boundary = buffer.indexOf("\n\n");
2794
- while (boundary !== -1) {
2795
- const frame = buffer.slice(0, boundary);
2796
- buffer = buffer.slice(boundary + 2);
2797
- const event = parseSseFrame(frame);
2798
- if (event === DONE) return;
2799
- if (event !== NO_DATA) yield event;
2800
- boundary = buffer.indexOf("\n\n");
2801
- }
2802
- const trailing = buffer.trim();
2803
- if (trailing) {
2804
- const event = parseSseFrame(trailing);
2805
- if (event !== DONE && event !== NO_DATA) yield event;
2806
- }
2807
- }
2808
- const DONE = Symbol("sse-stream-done");
2809
- const NO_DATA = Symbol("sse-frame-no-data");
2810
- /**
2811
- * Sentinel yielded by `parseSseStream` for a comment-only frame (no
2812
- * `data:` payload), the mechanism providers use for SSE keep-alive
2813
- * pings. Exported so a consumer (e.g. `fromFetch`) can react to "still
2814
- * alive" separately from a genuinely empty frame (`NO_DATA`, kept internal).
2815
- */
2816
- const SSE_PING = Symbol("sse-frame-ping");
2817
- /** Extracts and JSON-parses the `data:` payload of one SSE frame (the text between two blank lines). */
2818
- function parseSseFrame(frame) {
2819
- const dataLines = [];
2820
- let sawComment = false;
2821
- for (const line of frame.split("\n")) {
2822
- if (line.startsWith(":")) {
2823
- sawComment = true;
2824
- continue;
2825
- }
2826
- if (!line.startsWith("data:")) continue;
2827
- dataLines.push(line.startsWith("data: ") ? line.slice(6) : line.slice(5));
2828
- }
2829
- if (!dataLines.length) return sawComment ? SSE_PING : NO_DATA;
2830
- const data = dataLines.join("\n");
2831
- if (data === "[DONE]") return DONE;
2832
- try {
2833
- return JSON.parse(data);
2834
- } catch (cause) {
2835
- throw new LLMError(`Invalid JSON in SSE frame: ${data.slice(0, 200)}`, "parse", {
2836
- cause,
2837
- code: "stream_frame_invalid"
2838
- });
2839
- }
2840
- }
2841
-
2842
- //#endregion
2843
- //#region src/adapters/internal/imageFormat.ts
2844
- /**
2845
- * MIME types accepted for `ImageBlock.mimeType` across all adapters. This is
2846
- * the intersection of what Anthropic, Gemini, OpenAI-compatible, and Bedrock
2847
- * Converse all natively support, so a `ContentBlock[]` that validates for
2848
- * one provider validates for all of them.
2849
- */
2850
- const SUPPORTED_IMAGE_MIME_TYPES = [
2851
- "image/png",
2852
- "image/jpeg",
2853
- "image/gif",
2854
- "image/webp"
2855
- ];
2856
- /**
2857
- * Validates an `ImageBlock.mimeType` against the shared supported set.
2858
- * Throws a non-retryable `LLMError('invalid_params')`, since an unsupported
2859
- * mimeType is a bug in the caller's own input, deterministic before any
2860
- * request is built, the same class of failure as every other check in
2861
- * `RequestBuilder`.
2862
- */
2863
- function assertSupportedImageMimeType(mimeType) {
2864
- if (SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType)) return mimeType;
2865
- throw new LLMError(`Unsupported image mimeType "${mimeType}": expected one of ${SUPPORTED_IMAGE_MIME_TYPES.join(", ")}`, "invalid_params");
2866
- }
2867
-
2868
- //#endregion
2869
- //#region src/adapters/internal/nativeStructuredOutput.ts
2870
- /** Resolves whether `model` is covered by a caller-supplied allow-list/predicate. */
2871
- function supportsNativeStructuredOutput(model, override) {
2872
- if (!override) return false;
2873
- return Array.isArray(override) ? override.includes(model) : override(model);
2874
- }
2875
-
2876
- //#endregion
2877
- //#region src/adapters/anthropic.ts
2878
- /**
2879
- * Translates a VernLLM `ContentBlock[]` (our provider-agnostic multimodal
2880
- * shape) into Anthropic's native content-block array: text blocks pass
2881
- * through as-is, image blocks become `{ type: 'image', source: { type:
2882
- * 'base64', media_type, data } }`.
2883
- */
2884
- function toAnthropicContent(blocks) {
2885
- return blocks.map((block) => block.type === "image" ? {
2886
- type: "image",
2887
- source: {
2888
- type: "base64",
2889
- media_type: assertSupportedImageMimeType(block.mimeType),
2890
- data: block.data
2891
- }
2892
- } : {
2893
- type: "text",
2894
- text: block.text
2895
- });
2896
- }
2897
- /**
2898
- * Asserts a caller-supplied JSON Schema is an object schema before it's
2899
- * used as Anthropic's `Tool.input_schema`, which (like every other
2900
- * provider's function-calling API) requires `type: 'object'`. VernLLM's own
2901
- * public `tools`/`jsonSchema` APIs accept freeform `Record<string,
2902
- * unknown>` JSON Schema, so nothing upstream guarantees this at compile
2903
- * time; this is the runtime check that stands in for that, so a schema
2904
- * missing (or mistyping) `type: 'object'` fails loudly and immediately
2905
- * instead of being silently forwarded to Anthropic malformed.
2906
- */
2907
- function assertObjectSchema(schema, toolName) {
2908
- if (schema.type !== "object") throw new LLMError(`Tool "${toolName}"'s schema must have "type": "object" (Anthropic requires object-shaped tool parameters).`, "validation");
2909
- return schema;
2910
- }
2911
- /**
2912
- * Translates VernLLM's OpenAI-shaped wire `tool_choice` into Anthropic's
2913
- * `{ type: 'auto' | 'any' | 'none' | 'tool', name? }` shape. `'required'`
2914
- * maps to `'any'` (Anthropic's "must call some tool" equivalent).
2915
- */
2916
- function toAnthropicToolChoice(toolChoice) {
2917
- if (!toolChoice || toolChoice === "auto") return { type: "auto" };
2918
- if (toolChoice === "none") return { type: "none" };
2919
- if (toolChoice === "required") return { type: "any" };
2920
- return {
2921
- type: "tool",
2922
- name: toolChoice.function.name
2923
- };
2924
- }
2925
- /**
2926
- * Maps VernLLM's OpenAI-shaped wire `tools`/`tool_choice` into Anthropic's
2927
- * `tools`/`tool_choice` shape. Shared by the two call sites that build real
2928
- * (non-schema-forced) tool definitions: the plain tools-only branch, and
2929
- * the native-structured-output branch, which sends real tools alongside
2930
- * `output_config` rather than instead of it.
2931
- */
2932
- function buildAnthropicTools(tools, toolChoiceParam) {
2933
- return {
2934
- tools: tools.map((t) => ({
2935
- name: t.function.name,
2936
- description: t.function.description,
2937
- input_schema: assertObjectSchema(t.function.parameters, t.function.name)
2938
- })),
2939
- toolChoice: toAnthropicToolChoice(toolChoiceParam)
2940
- };
2941
- }
2942
- /**
2943
- * Builds the Anthropic-shaped request body from VernLLM's wire params,
2944
- * shared between `create` and `createStream` so both go through identical
2945
- * translation (system prompt, message shaping, and the jsonSchema →
2946
- * forced-single-tool mapping all happen exactly once, not once per entry
2947
- * point).
2948
- *
2949
- * Returns `toolName` alongside the body: when set, the model was forced to
2950
- * call a single synthetic tool standing in for `jsonSchema` output (the
2951
- * legacy path, for models without native structured-output support), and
2952
- * both `create` and `createStream` need to know this so they can unwrap
2953
- * that tool call back into plain text content instead of treating it like
2954
- * a real tool call. On the native path (model supports `output_config`),
2955
- * `toolName` is `undefined`: the schema-conforming JSON already arrives as
2956
- * ordinary text content, nothing to unwrap, and any real tool calls in
2957
- * `params.tools` are left for the normal, non-forced tool-call handling
2958
- * both `create` and `createStream` already do when `toolName` is unset.
2959
- */
2960
- function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
2961
- const systemMessage = params.messages.find((m) => m.role === "system");
2962
- const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
2963
- const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
2964
- const schemaName = jsonSchema?.name.trim();
2965
- if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
2966
- const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
2967
- if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Anthropic model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there, which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromAnthropic's \`nativeStructuredOutputModels\` option once you've confirmed it supports Anthropic's \`output_config.format\`.`, "validation");
2968
- let toolName;
2969
- let jsonInstruction;
2970
- let outputFormat;
2971
- let tools;
2972
- let toolChoice;
2973
- if (jsonSchema && isNative) {
2974
- outputFormat = {
2975
- type: "json_schema",
2976
- schema: jsonSchema.schema
2977
- };
2978
- if (params.tools?.length) ({tools, toolChoice} = buildAnthropicTools(params.tools, params.tool_choice));
2979
- } else if (jsonSchema && schemaName) {
2980
- const { schema, description, strict } = jsonSchema;
2981
- toolName = schemaName;
2982
- tools = [{
2983
- name: toolName,
2984
- description,
2985
- input_schema: assertObjectSchema(schema, toolName),
2986
- strict
2987
- }];
2988
- toolChoice = {
2989
- type: "tool",
2990
- name: toolName
2991
- };
2992
- } else if (params.response_format?.type === "json_object") jsonInstruction = "Respond with valid JSON only, no prose or markdown fences.";
2993
- if (!jsonSchema && params.tools?.length) ({tools, toolChoice} = buildAnthropicTools(params.tools, params.tool_choice));
2994
- const system = [systemMessage?.content, jsonInstruction].filter(Boolean).join("\n\n");
2995
- const body = {
2996
- model: params.model,
2997
- max_tokens: params.max_tokens,
2998
- ...params.temperature !== void 0 ? { temperature: params.temperature } : {},
2999
- system: system || void 0,
3000
- messages: mergeConsecutiveToolResults$1(conversationMessages.map((m) => toAnthropicMessage(m))),
3001
- ...tools ? {
3002
- tools,
3003
- tool_choice: toolChoice
3004
- } : {},
3005
- ...outputFormat ? { output_config: { format: outputFormat } } : {}
3006
- };
3007
- return {
3008
- body,
3009
- toolName
3010
- };
3011
- }
3012
- /**
3013
- * Wraps an Anthropic SDK client so it satisfies the same `LLMClient`
3014
- * interface VernLLM uses for OpenAI/Groq.
3015
- *
3016
- * `response_format: json_schema`, on a model covered by
3017
- * `options.nativeStructuredOutputModels`, is sent as `output_config.format`,
3018
- * its own request field, independent of `tools`/`tool_choice`, so it can be
3019
- * combined with real, caller-supplied `tools` in the same request. Only
3020
- * `type` and `schema` are sent on this path, the real Anthropic API's
3021
- * `output_config.format` has no `name`/`description`/`strict` fields.
3022
- *
3023
- * On any other model (the default, since `nativeStructuredOutputModels` is
3024
- * opt-in), `response_format: json_schema` is mapped to Anthropic's forced
3025
- * tool-use instead: a single tool is defined with `input_schema` set to
3026
- * the caller's schema, `description` forwarded when provided, and `strict`
3027
- * forwarded when set, and `tool_choice` forces the model to call it. This
3028
- * legacy path cannot be combined with real `tools` (both would need the
3029
- * same `tools`/`tool_choice` field), and a call that tries throws
3030
- * `LLMError('validation')` before reaching the API. Provider-constrained
3031
- * schema matching applies only when `strict: true` is forwarded and
3032
- * supported.
3033
- *
3034
- * `response_format: json_object` (no schema to build a tool from) falls
3035
- * back to a system-prompt instruction, since there's nothing to constrain
3036
- * generation against. Unlike `jsonSchema`, this combines with real `tools`
3037
- * freely on every model: it's a prompt nudge, not a request field, so
3038
- * there's nothing for it to collide with.
3039
- */
3040
- function fromAnthropic(anthropicClient, options) {
3041
- const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
3042
- const rawMessagesCreate = anthropicClient.messages.create.bind(anthropicClient.messages);
3043
- return { chat: { completions: {
3044
- async create(params, options$1) {
3045
- const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels);
3046
- const response = await anthropicClient.messages.create(body, options$1);
3047
- let text;
3048
- let wireToolCalls;
3049
- if (toolName) {
3050
- const toolUse = response.content.find((block) => block.type === "tool_use" && block.name === toolName);
3051
- if (!toolUse) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
3052
- if (!toolUse.input || typeof toolUse.input !== "object" || Array.isArray(toolUse.input)) throw new LLMError(`Anthropic returned invalid structured output for tool "${toolName}". Expected an object.`, "validation");
3053
- text = JSON.stringify(toolUse.input);
3054
- } else {
3055
- text = response.content.filter((block) => block.type === "text").map((block) => block.text ?? "").join("");
3056
- const toolUses = response.content.filter((block) => block.type === "tool_use");
3057
- if (toolUses.length) wireToolCalls = toolUses.map((block) => ({
3058
- id: block.id,
3059
- type: "function",
3060
- function: {
3061
- name: block.name,
3062
- arguments: JSON.stringify(block.input ?? {})
3063
- }
3064
- }));
3065
- }
3066
- return {
3067
- choices: [{ message: {
3068
- content: text,
3069
- ...wireToolCalls ? { tool_calls: wireToolCalls } : {}
3070
- } }],
3071
- usage: {
3072
- prompt_tokens: response.usage?.input_tokens,
3073
- completion_tokens: response.usage?.output_tokens,
3074
- total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0)
3075
- }
3076
- };
3077
- },
3078
- async *createStream(params, options$1) {
3079
- const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels);
3080
- const stream = await rawMessagesCreate({
3081
- ...body,
3082
- stream: true
3083
- }, options$1);
3084
- const blockKinds = new Map();
3085
- let inputTokens = 0;
3086
- let sawJsonTool = false;
3087
- for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
3088
- else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
3089
- const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
3090
- blockKinds.set(event.index, kind);
3091
- if (kind === "json-tool") sawJsonTool = true;
3092
- else if (!toolName) yield {
3093
- type: "tool_call_delta",
3094
- index: event.index,
3095
- id: event.content_block.id,
3096
- name: event.content_block.name
3097
- };
3098
- } else blockKinds.set(event.index, "text");
3099
- else if (event.type === "content_block_delta") {
3100
- if (event.delta.type === "text_delta") {
3101
- if (!toolName) yield {
3102
- type: "text-delta",
3103
- delta: event.delta.text
3104
- };
3105
- } else if (event.delta.type === "input_json_delta") {
3106
- const kind = blockKinds.get(event.index);
3107
- if (kind === "json-tool") yield {
3108
- type: "text-delta",
3109
- delta: event.delta.partial_json
3110
- };
3111
- else if (!toolName) yield {
3112
- type: "tool_call_delta",
3113
- index: event.index,
3114
- argumentsDelta: event.delta.partial_json
3115
- };
3116
- }
3117
- } else if (event.type === "message_delta") {
3118
- const outputTokens = event.usage?.output_tokens ?? 0;
3119
- yield {
3120
- type: "usage",
3121
- usage: {
3122
- prompt_tokens: inputTokens,
3123
- completion_tokens: outputTokens,
3124
- total_tokens: inputTokens + outputTokens
3125
- }
3126
- };
3127
- } else if (event.type === "ping") yield { type: "ping" };
3128
- if (toolName && !sawJsonTool) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
3129
- }
3130
- } } };
3131
- }
3132
- /**
3133
- * Anthropic requires strict role alternation, so the per-wire-message
3134
- * mapping above (one `{role:'user', content:[tool_result]}` per VernLLM
3135
- * wire tool message) needs merging back together when an assistant turn
3136
- * requested more than one tool: multiple consecutive user turns would
3137
- * violate that alternation, and Anthropic's API rejects it outright. This
3138
- * merges any run of tool-result-only user messages into one, with all
3139
- * their tool_result blocks combined, the shape Anthropic expects for "here
3140
- * are the results of everything you just asked for."
3141
- */
3142
- function mergeConsecutiveToolResults$1(messages) {
3143
- const isToolResultOnly = (m) => m.role === "user" && Array.isArray(m.content) && m.content.length > 0 && m.content.every((b) => b.type === "tool_result");
3144
- const merged = [];
3145
- for (const m of messages) {
3146
- const prev = merged.at(-1);
3147
- if (isToolResultOnly(m) && prev && isToolResultOnly(prev)) prev.content.push(...m.content);
3148
- else merged.push(m);
3149
- }
3150
- return merged;
3151
- }
3152
- /**
3153
- * Translates one VernLLM wire message (OpenAI-shaped: plain user/assistant
3154
- * turns, an assistant turn with `tool_calls`, or a `tool` turn) into
3155
- * Anthropic's `{ role: 'user' | 'assistant', content }` shape.
3156
- */
3157
- function toAnthropicMessage(m) {
3158
- if (m.role === "tool") return {
3159
- role: "user",
3160
- content: [{
3161
- type: "tool_result",
3162
- tool_use_id: m.tool_call_id,
3163
- content: m.content,
3164
- ...m.is_error ? { is_error: true } : {}
3165
- }]
3166
- };
3167
- if (m.role === "assistant" && m.tool_calls?.length) {
3168
- const blocks = [];
3169
- if (m.content) blocks.push({
3170
- type: "text",
3171
- text: m.content
3172
- });
3173
- for (const tc of m.tool_calls) {
3174
- let input;
3175
- try {
3176
- input = tc.function.arguments.trim() ? JSON.parse(tc.function.arguments) : {};
3177
- } catch (cause) {
3178
- throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
3179
- }
3180
- if (input === null || Array.isArray(input) || typeof input !== "object") throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) arguments must be a JSON object.`, "validation");
3181
- blocks.push({
3182
- type: "tool_use",
3183
- id: tc.id,
3184
- name: tc.function.name,
3185
- input
3186
- });
3187
- }
3188
- return {
3189
- role: "assistant",
3190
- content: blocks
3191
- };
3192
- }
3193
- return {
3194
- role: m.role,
3195
- content: Array.isArray(m.content) ? toAnthropicContent(m.content) : m.content ?? ""
3196
- };
3197
- }
3198
-
3199
- //#endregion
3200
- //#region src/adapters/gemini.ts
3201
- /**
3202
- * Translates a VernLLM `ContentBlock[]` into Gemini's native `parts` array:
3203
- * text blocks become `{ text }`, image blocks become inline data parts
3204
- * (`{ inlineData: { mimeType, data } }`), Gemini's shape for embedding raw
3205
- * base64 image bytes directly in the request.
3206
- */
3207
- function toGeminiParts(blocks) {
3208
- return blocks.map((block) => block.type === "image" ? { inlineData: {
3209
- mimeType: assertSupportedImageMimeType(block.mimeType),
3210
- data: block.data
3211
- } } : { text: block.text });
3212
- }
3213
- /** Maps VernLLM's OpenAI-shaped wire `tool_choice` onto Gemini's `functionCallingConfig`. */
3214
- function toGeminiToolConfig(toolChoice) {
3215
- if (!toolChoice || toolChoice === "auto") return { functionCallingConfig: { mode: "AUTO" } };
3216
- if (toolChoice === "none") return { functionCallingConfig: { mode: "NONE" } };
3217
- if (toolChoice === "required") return { functionCallingConfig: { mode: "ANY" } };
3218
- return { functionCallingConfig: {
3219
- mode: "ANY",
3220
- allowedFunctionNames: [toolChoice.function.name]
3221
- } };
3222
- }
3223
- /**
3224
- * Translates one VernLLM wire message into a Gemini `contents` entry.
3225
- * Gemini has no separate 'tool' role: a prior assistant tool request
3226
- * becomes a `'model'` turn with `functionCall` parts, and its result
3227
- * becomes a `'user'` turn with `functionResponse` parts.
3228
- */
3229
- function toGeminiContent(m) {
3230
- if (m.role === "tool") return {
3231
- role: "user",
3232
- parts: [{ functionResponse: {
3233
- name: m.tool_call_id,
3234
- response: parseToolResult(m.content)
3235
- } }]
3236
- };
3237
- if (m.role === "assistant" && m.tool_calls?.length) {
3238
- const parts = [];
3239
- if (typeof m.content === "string" && m.content) parts.push({ text: m.content });
3240
- parts.push(...m.tool_calls.map((tc) => ({ functionCall: {
3241
- name: tc.function.name,
3242
- args: parseToolArguments(tc.function.arguments, tc.function.name)
3243
- } })));
3244
- return {
3245
- role: "model",
3246
- parts
3247
- };
3248
- }
3249
- return {
3250
- role: m.role === "assistant" ? "model" : "user",
3251
- parts: Array.isArray(m.content) ? toGeminiParts(m.content) : [{ text: m.content ?? "" }]
3252
- };
3253
- }
3254
- function parseToolArguments(text, toolName) {
3255
- let parsed;
3256
- try {
3257
- parsed = text.trim() ? JSON.parse(text) : {};
3258
- } catch (cause) {
3259
- throw new LLMError(`Tool call "${toolName}" arguments are not valid JSON.`, "parse", {
3260
- cause,
3261
- code: "tool_arguments_parse_failed"
3262
- });
3263
- }
3264
- if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") throw new LLMError(`Tool call "${toolName}" arguments must be a JSON object.`, "validation");
3265
- return parsed;
3266
- }
3267
- function parseToolResult(text) {
3268
- try {
3269
- return text.trim() ? JSON.parse(text) : "";
3270
- } catch {
3271
- return text;
3272
- }
3273
- }
3274
- /**
3275
- * Gemini expects the results of everything the model asked for in one turn
3276
- * to arrive together as multiple `functionResponse` parts on a single
3277
- * `'user'` entry, not as separate consecutive `'user'` entries. The
3278
- * per-wire-message mapping above produces one `'user'` entry per VernLLM
3279
- * wire tool message, so when an assistant turn requested more than one
3280
- * tool, this merges the resulting run of functionResponse-only `'user'`
3281
- * entries back into one.
3282
- */
3283
- function mergeConsecutiveFunctionResponses(contents) {
3284
- const isFunctionResponseOnly = (c) => c.role === "user" && c.parts.length > 0 && c.parts.every((p) => "functionResponse" in p);
3285
- const merged = [];
3286
- for (const c of contents) {
3287
- const prev = merged.at(-1);
3288
- if (isFunctionResponseOnly(c) && prev && isFunctionResponseOnly(prev)) prev.parts.push(...c.parts);
3289
- else merged.push(c);
3290
- }
3291
- return merged;
3292
- }
3293
- /**
3294
- * Builds the Gemini-shaped request from VernLLM's wire params, shared
3295
- * between `create` and `createStream` so both go through identical
3296
- * translation (contents shaping, `responseSchema`/`responseMimeType`
3297
- * mapping, and tool/toolConfig translation all happen exactly once).
3298
- * `abortSignal` is folded into `config` by the caller (`create`/
3299
- * `createStream`), once the request options are available.
3300
- */
3301
- function buildGeminiRequest(params) {
3302
- const systemMessage = params.messages.find((m) => m.role === "system");
3303
- const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
3304
- const wantsJson = Boolean(params.response_format);
3305
- const config = {
3306
- ...params.temperature !== void 0 ? { temperature: params.temperature } : {},
3307
- maxOutputTokens: params.max_tokens,
3308
- ...systemMessage ? { systemInstruction: { parts: [{ text: systemMessage.content }] } } : {}
3309
- };
3310
- if (wantsJson) config.responseMimeType = "application/json";
3311
- if (params.response_format?.type === "json_schema") {
3312
- const { schema, description } = params.response_format.json_schema;
3313
- config.responseSchema = {
3314
- ...schema,
3315
- ...description ? { description } : {}
3316
- };
3317
- }
3318
- if (params.tools?.length) {
3319
- config.tools = [{ functionDeclarations: params.tools.map((t) => ({
3320
- name: t.function.name,
3321
- description: t.function.description,
3322
- parameters: t.function.parameters
3323
- })) }];
3324
- config.toolConfig = toGeminiToolConfig(params.tool_choice);
3325
- }
3326
- return {
3327
- model: params.model,
3328
- contents: mergeConsecutiveFunctionResponses(conversationMessages.map((m) => toGeminiContent(m))),
3329
- config
3330
- };
3331
- }
3332
- /**
3333
- * Wraps a Gemini client so it satisfies the `LLMClient` interface VernLLM
3334
- * uses for OpenAI-compatible APIs. Gemini's shape differs on nearly every
3335
- * axis: a `contents` array instead of `messages`, a separate
3336
- * `systemInstruction` field instead of a `system` role message,
3337
- * `generationConfig` instead of top-level `temperature`/`max_tokens`, and
3338
- * native JSON Schema support via `responseMimeType: 'application/json'` +
3339
- * `responseSchema`. `reasoning_effort` has no equivalent. Gemini's thinking
3340
- * models use a token budget, not an effort tier, so it's dropped, same as
3341
- * Anthropic.
3342
- *
3343
- * `tools` maps to Gemini's native `functionDeclarations`/`functionCall`;
3344
- * `tool_choice` maps to `toolConfig.functionCallingConfig`. Gemini accepts
3345
- * `responseSchema` and `tools` in the same request natively, so both are
3346
- * set independently here and no special-casing is needed for the
3347
- * combination, unlike `fromAnthropic`/`fromBedrock`.
3348
- *
3349
- * `createStream` calls `generateContentStream` (optional on `GeminiClient`
3350
- *, required only if the caller sets `stream: true`) and translates each
3351
- * partial response into `WireStreamChunk`s. Unlike OpenAI/Anthropic,
3352
- * Gemini's own function-calling API doesn't stream tool-call arguments
3353
- * incrementally: a `functionCall` part always arrives whole in one chunk,
3354
- * so each one is emitted as a single, complete `tool_call_delta` (a
3355
- * one-shot "delta" containing the full arguments) rather than accumulated
3356
- * fragments, that's a real difference in the underlying API, not
3357
- * something this adapter can smooth over. `usageMetadata` is (per Gemini's
3358
- * own behavior) only reliably present on the last chunk, so the `usage`
3359
- * `WireStreamChunk` is emitted once, after the stream completes, from
3360
- * whichever chunk's `usageMetadata` was seen last.
3361
- */
3362
- function fromGemini(geminiClient) {
3363
- return { chat: { completions: {
3364
- async create(params, options) {
3365
- const request = buildGeminiRequest(params);
3366
- request.config = {
3367
- ...request.config,
3368
- abortSignal: options.signal
3369
- };
3370
- const response = await geminiClient.generateContent(request);
3371
- const parts = response.candidates?.[0]?.content?.parts ?? [];
3372
- const text = parts.map((p) => p.text ?? "").join("");
3373
- const functionCalls = parts.filter((p) => p.functionCall);
3374
- let wireToolCalls;
3375
- if (functionCalls.length) wireToolCalls = functionCalls.map((p) => ({
3376
- id: p.functionCall.name,
3377
- type: "function",
3378
- function: {
3379
- name: p.functionCall.name,
3380
- arguments: JSON.stringify(p.functionCall.args ?? {})
3381
- }
3382
- }));
3383
- return {
3384
- choices: [{ message: {
3385
- content: text,
3386
- ...wireToolCalls ? { tool_calls: wireToolCalls } : {}
3387
- } }],
3388
- usage: {
3389
- prompt_tokens: response.usageMetadata?.promptTokenCount,
3390
- completion_tokens: response.usageMetadata?.candidatesTokenCount,
3391
- total_tokens: response.usageMetadata?.totalTokenCount
3392
- }
3393
- };
3394
- },
3395
- async *createStream(params, options) {
3396
- if (!geminiClient.generateContentStream) throw new LLMError("stream: true requires a Gemini client with generateContentStream", "invalid_params", {
3397
- code: "unsupported_capability",
3398
- issues: { capability: "generateContentStream" }
3399
- });
3400
- const request = buildGeminiRequest(params);
3401
- request.config = {
3402
- ...request.config,
3403
- abortSignal: options.signal
3404
- };
3405
- const stream = await geminiClient.generateContentStream(request);
3406
- let toolCallIndex = 0;
3407
- let lastUsage;
3408
- for await (const chunk of stream) {
3409
- const parts = chunk.candidates?.[0]?.content?.parts ?? [];
3410
- for (const part of parts) {
3411
- if (part.text) yield {
3412
- type: "text-delta",
3413
- delta: part.text
3414
- };
3415
- if (part.functionCall) {
3416
- yield {
3417
- type: "tool_call_delta",
3418
- index: toolCallIndex,
3419
- id: part.functionCall.name,
3420
- name: part.functionCall.name,
3421
- argumentsDelta: JSON.stringify(part.functionCall.args ?? {}),
3422
- complete: true
3423
- };
3424
- toolCallIndex++;
3425
- }
3426
- }
3427
- if (chunk.usageMetadata) lastUsage = chunk.usageMetadata;
3428
- }
3429
- if (lastUsage) yield {
3430
- type: "usage",
3431
- usage: {
3432
- prompt_tokens: lastUsage.promptTokenCount,
3433
- completion_tokens: lastUsage.candidatesTokenCount,
3434
- total_tokens: lastUsage.totalTokenCount
3435
- }
3436
- };
3437
- }
3438
- } } };
3439
- }
3440
-
3441
- //#endregion
3442
- //#region src/adapters/bedrock.ts
3443
- /** Maps a `ContentBlock` image MIME type, already validated, to Converse's `format` enum. */
3444
- function toBedrockImageFormat(mimeType) {
3445
- switch (assertSupportedImageMimeType(mimeType)) {
3446
- case "image/png": return "png";
3447
- case "image/jpeg": return "jpeg";
3448
- case "image/gif": return "gif";
3449
- case "image/webp": return "webp";
3450
- }
3451
- }
3452
- /**
3453
- * Decodes base64 image data into the raw `Uint8Array` bytes Converse's
3454
- * `image.source.bytes` expects (unlike Anthropic/Gemini/OpenAI, which all
3455
- * take base64 strings directly). Uses `Buffer`, since this adapter, like
3456
- * the rest of the package, targets Node.
3457
- */
3458
- function decodeBase64(data) {
3459
- return new Uint8Array(Buffer.from(data, "base64"));
3460
- }
3461
- /**
3462
- * Translates a VernLLM `ContentBlock[]` into Converse's native content-block
3463
- * array: text blocks pass through as `{ text }`, image blocks become
3464
- * `{ image: { format, source: { bytes } } }` with the base64 payload decoded
3465
- * to raw bytes, since Converse doesn't accept base64 strings directly.
3466
- */
3467
- function toBedrockContent(blocks) {
3468
- return blocks.map((block) => block.type === "image" ? { image: {
3469
- format: toBedrockImageFormat(block.mimeType),
3470
- source: { bytes: decodeBase64(block.data) }
3471
- } } : { text: block.text });
3472
- }
3473
- /**
3474
- * Maps VernLLM's OpenAI-shaped wire `tools`/`tool_choice` into Converse's
3475
- * `toolConfig` shape. Shared by the two call sites that build real
3476
- * (non-schema-forced) tool definitions: the plain tools-only branch, and
3477
- * the native-structured-output branch, which sends real tools alongside
3478
- * `outputConfig` rather than instead of it.
3479
- */
3480
- function buildBedrockToolConfig(tools, toolChoiceParam) {
3481
- return {
3482
- tools: tools.map((t) => ({ toolSpec: {
3483
- name: t.function.name,
3484
- description: t.function.description,
3485
- inputSchema: { json: t.function.parameters }
3486
- } })),
3487
- toolChoice: toBedrockToolChoice(toolChoiceParam)
3488
- };
3489
- }
3490
- /**
3491
- * Builds the Converse-shaped request from VernLLM's wire params, shared
3492
- * between `create` and `createStream` so both go through identical
3493
- * translation (system prompt, message shaping, the jsonSchema →
3494
- * forced-single-tool mapping, and the `toolUseSupportedModels` preflight
3495
- * check all happen exactly once).
3496
- *
3497
- * Returns `toolName` alongside the request: when set, the model was forced
3498
- * to call a single synthetic tool standing in for `jsonSchema` output (the
3499
- * legacy path, for models not covered by `nativeStructuredOutputModels`),
3500
- * and both `create` and `createStream` need to know this so they can
3501
- * unwrap that tool call back into plain text content instead of treating
3502
- * it like a real tool call. On the native path (model covered by
3503
- * `nativeStructuredOutputModels`), `toolName` is `undefined`: the
3504
- * schema-conforming JSON already arrives as ordinary text content, nothing
3505
- * to unwrap, and any real tool calls in `params.tools` are left for the
3506
- * normal, non-forced tool-call handling both `create` and `createStream`
3507
- * already do when `toolName` is unset.
3508
- */
3509
- function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels) {
3510
- const systemMessage = params.messages.find((m) => m.role === "system");
3511
- const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
3512
- const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
3513
- const schemaName = jsonSchema?.name.trim();
3514
- if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
3515
- const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
3516
- if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Bedrock model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there (via \`toolConfig\`), which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromBedrock's \`nativeStructuredOutputModels\` option once you've confirmed it supports Converse's \`outputConfig.textFormat\`.`, "validation");
3517
- let toolName;
3518
- let jsonInstruction;
3519
- let toolConfig;
3520
- let outputConfig;
3521
- if (jsonSchema && isNative) {
3522
- const { schema, description } = jsonSchema;
3523
- outputConfig = { textFormat: {
3524
- type: "json_schema",
3525
- structure: { jsonSchema: {
3526
- schema: JSON.stringify(schema),
3527
- name: schemaName,
3528
- description
3529
- } }
3530
- } };
3531
- } else if (jsonSchema && schemaName) {
3532
- const { schema, description, strict } = jsonSchema;
3533
- toolName = schemaName;
3534
- toolConfig = {
3535
- tools: [{ toolSpec: {
3536
- name: toolName,
3537
- description,
3538
- inputSchema: { json: schema },
3539
- strict
3540
- } }],
3541
- toolChoice: { tool: { name: toolName } }
3542
- };
3543
- } else if (params.response_format?.type === "json_object") jsonInstruction = "Respond with valid JSON only, no prose or markdown fences.";
3544
- if (params.tools?.length && !toolName) toolConfig = buildBedrockToolConfig(params.tools, params.tool_choice);
3545
- if (jsonSchema && toolConfig && toolUseSupportedModels) {
3546
- const isSupported = Array.isArray(toolUseSupportedModels) ? toolUseSupportedModels.includes(params.model) : toolUseSupportedModels(params.model);
3547
- if (!isSupported) throw new LLMError(`Bedrock model "${params.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`, "invalid_params", {
3548
- code: "unsupported_capability",
3549
- issues: { capability: "toolUseSupportedModels" }
3550
- });
3551
- }
3552
- const systemParts = [systemMessage?.content, jsonInstruction].filter((s) => Boolean(s));
3553
- const request = {
3554
- modelId: params.model,
3555
- messages: mergeConsecutiveToolResults(conversationMessages.map((m) => toBedrockMessage(m))),
3556
- system: systemParts.length ? systemParts.map((text) => ({ text })) : void 0,
3557
- inferenceConfig: {
3558
- ...params.temperature !== void 0 ? { temperature: params.temperature } : {},
3559
- maxTokens: params.max_tokens
3560
- },
3561
- ...toolConfig ? { toolConfig } : {},
3562
- ...outputConfig ? { outputConfig } : {}
3563
- };
3564
- return {
3565
- request,
3566
- toolName
3567
- };
3568
- }
3569
- /**
3570
- * Wraps a Bedrock Converse-API client so it satisfies the `LLMClient`
3571
- * interface VernLLM uses for OpenAI/Groq. The Converse API is unified
3572
- * across Bedrock's model families (Anthropic, Titan, Llama, Mistral, etc.),
3573
- * so unlike raw per-model Bedrock invocation, this one adapter works
3574
- * regardless of which underlying model `modelId` points at, as long as
3575
- * that model supports Converse (most current-generation ones do)
3576
- *
3577
- * `response_format: json_schema`, on a model covered by
3578
- * `options.nativeStructuredOutputModels` (opt-in, unset by default), is
3579
- * sent as `outputConfig.textFormat`, its own request field, independent of
3580
- * `toolConfig`, so it can be combined with real, caller-supplied `tools`
3581
- * in the same request. Matches the real Converse API's shape exactly: the
3582
- * schema is nested under `structure.jsonSchema` and JSON-encoded as a
3583
- * string, not the parsed object `toolConfig`'s tool schemas use, and there
3584
- * is no `strict` field on this path.
3585
- *
3586
- * On any other model (the default), `response_format: json_schema` is
3587
- * mapped to Converse's `toolConfig` instead: a single tool is defined from
3588
- * the schema, description, and strictness settings, and `toolChoice`
3589
- * forces the model to call it. This legacy path cannot be combined with
3590
- * real `tools` (both would need the same `toolConfig`), and a call that
3591
- * tries throws `LLMError('validation')` before reaching the API.
3592
- * Provider-constrained schema matching applies only when `strict: true` is
3593
- * forwarded and supported. Native tool support varies by model family;
3594
- * pass `toolUseSupportedModels` to preflight-check it (see
3595
- * `BedrockAdapterOptions`), otherwise a `jsonSchema` call to an
3596
- * unsupported model surfaces Bedrock's raw error unchanged.
3597
- *
3598
- * `response_format: json_object` (no schema to build a tool from) and
3599
- * `reasoning_effort` (no Converse equivalent) fall back to a system-prompt
3600
- * instruction and are dropped respectively. Unlike `jsonSchema`,
3601
- * `json_object` combines with real `tools` freely on every model: it's a
3602
- * prompt nudge, not a request field, so there's nothing for it to collide
3603
- * with.
3604
- *
3605
- * `tools` alone maps to Converse's native `toolConfig`/`toolUse`/
3606
- * `toolResult`; `tool_choice` maps to `toolConfig.toolChoice`.
3607
- *
3608
- * `createStream` calls `converseStream` (optional on `BedrockConverseClient`
3609
- *, required only if the caller sets `stream: true`) and translates its
3610
- * `contentBlockStart`/`contentBlockDelta`/`metadata` events into
3611
- * `WireStreamChunk`s. Content blocks are tracked by `contentBlockIndex`,
3612
- * same as `fromAnthropic`'s block-index tracking (Converse's streaming
3613
- * shape is structurally close to Anthropic's own, both being tool-use-aware
3614
- * content-block streams), including the same `json-tool` unwrapping: a
3615
- * `jsonSchema`-forced tool's `toolUse.input` deltas are re-emitted as
3616
- * `text-delta`, not `tool_call_delta`, so the accumulated result lands in
3617
- * `finalizeResponse`'s `content` path exactly like the non-streaming
3618
- * `create` branch above unwraps it.
3619
- */
3620
- function fromBedrock(bedrockClient, options) {
3621
- const toolUseSupportedModels = options?.toolUseSupportedModels;
3622
- const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
3623
- return { chat: { completions: {
3624
- async create(params, requestOptions) {
3625
- const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels);
3626
- const response = await bedrockClient.converse(request, requestOptions);
3627
- let text;
3628
- let wireToolCalls;
3629
- if (toolName) {
3630
- const toolUseBlock = response.output?.message?.content?.find((block) => block.toolUse?.name === toolName);
3631
- text = toolUseBlock?.toolUse ? JSON.stringify(toolUseBlock.toolUse.input) : "";
3632
- } else {
3633
- const blocks = response.output?.message?.content ?? [];
3634
- text = blocks.map((c) => c.text ?? "").join("");
3635
- const toolUses = blocks.filter((block) => Boolean(block.toolUse));
3636
- if (toolUses.length) wireToolCalls = toolUses.map((block, i) => {
3637
- const toolUse = block.toolUse;
3638
- if (!toolUse.name) throw new LLMError(`Bedrock returned a toolUse block without a name at index ${i}.`, "validation");
3639
- return {
3640
- id: toolUse.toolUseId ?? `${toolUse.name}_${i}`,
3641
- type: "function",
3642
- function: {
3643
- name: toolUse.name,
3644
- arguments: JSON.stringify(toolUse.input ?? {})
3645
- }
3646
- };
3647
- });
3648
- }
3649
- return {
3650
- choices: [{ message: {
3651
- content: text,
3652
- ...wireToolCalls ? { tool_calls: wireToolCalls } : {}
3653
- } }],
3654
- usage: {
3655
- prompt_tokens: response.usage?.inputTokens,
3656
- completion_tokens: response.usage?.outputTokens,
3657
- total_tokens: response.usage?.totalTokens
3658
- }
3659
- };
3660
- },
3661
- async *createStream(params, requestOptions) {
3662
- if (!bedrockClient.converseStream) throw new LLMError("stream: true requires a Bedrock client with converseStream", "invalid_params", {
3663
- code: "unsupported_capability",
3664
- issues: { capability: "converseStream" }
3665
- });
3666
- const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels);
3667
- const { stream } = await bedrockClient.converseStream(request, requestOptions);
3668
- const blockKinds = new Map();
3669
- for await (const event of stream) if ("contentBlockStart" in event) {
3670
- const { contentBlockIndex, start } = event.contentBlockStart;
3671
- if (start?.toolUse) {
3672
- const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
3673
- blockKinds.set(contentBlockIndex, kind);
3674
- if (kind === "tool_use" && !toolName) yield {
3675
- type: "tool_call_delta",
3676
- index: contentBlockIndex,
3677
- id: start.toolUse.toolUseId,
3678
- name: start.toolUse.name
3679
- };
3680
- } else blockKinds.set(contentBlockIndex, "text");
3681
- } else if ("contentBlockDelta" in event) {
3682
- const { contentBlockIndex, delta } = event.contentBlockDelta;
3683
- if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
3684
- type: "text-delta",
3685
- delta: delta.text
3686
- };
3687
- else if (delta && "toolUse" in delta && delta.toolUse?.input !== void 0) {
3688
- const kind = blockKinds.get(contentBlockIndex);
3689
- if (kind === "json-tool") yield {
3690
- type: "text-delta",
3691
- delta: delta.toolUse.input
3692
- };
3693
- else if (!toolName) yield {
3694
- type: "tool_call_delta",
3695
- index: contentBlockIndex,
3696
- argumentsDelta: delta.toolUse.input
3697
- };
3698
- }
3699
- } else if ("metadata" in event && event.metadata.usage) yield {
3700
- type: "usage",
3701
- usage: {
3702
- prompt_tokens: event.metadata.usage.inputTokens,
3703
- completion_tokens: event.metadata.usage.outputTokens,
3704
- total_tokens: event.metadata.usage.totalTokens
3705
- }
3706
- };
3707
- else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api", {
3708
- status: 429,
3709
- code: "provider_rate_limited"
3710
- });
3711
- else if ("validationException" in event) throw new LLMError(event.validationException.message ?? "Bedrock rejected the request mid-stream", "validation");
3712
- else if ("internalServerException" in event || "serviceUnavailableException" in event || "modelStreamErrorException" in event) {
3713
- const detail = "internalServerException" in event && event.internalServerException.message || "serviceUnavailableException" in event && event.serviceUnavailableException.message || "modelStreamErrorException" in event && event.modelStreamErrorException.message || "Bedrock reported a mid-stream error";
3714
- const status = "modelStreamErrorException" in event && event.modelStreamErrorException.originalStatusCode || "serviceUnavailableException" in event && 503 || 500;
3715
- throw new LLMError(detail, "api", {
3716
- status,
3717
- code: status >= 500 ? "server_error" : void 0
3718
- });
3719
- }
3720
- }
3721
- } } };
3722
- }
3723
- /** Maps VernLLM's OpenAI-shaped wire `tool_choice` onto Converse's `toolChoice`. */
3724
- function toBedrockToolChoice(toolChoice) {
3725
- if (!toolChoice || toolChoice === "auto") return { auto: {} };
3726
- if (toolChoice === "required") return { any: {} };
3727
- if (toolChoice === "none") throw new LLMError("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.", "invalid_params", {
3728
- code: "unsupported_capability",
3729
- issues: { capability: "toolChoice: 'none'" }
3730
- });
3731
- return { tool: { name: toolChoice.function.name } };
3732
- }
3733
- /**
3734
- * Translates one VernLLM wire message into Converse's
3735
- * `{ role: 'user' | 'assistant', content }` shape.
3736
- */
3737
- function toBedrockMessage(m) {
3738
- if (m.role === "tool") return {
3739
- role: "user",
3740
- content: [{ toolResult: {
3741
- toolUseId: m.tool_call_id,
3742
- content: [{ text: m.content }],
3743
- status: m.is_error ? "error" : "success"
3744
- } }]
3745
- };
3746
- if (m.role === "assistant" && m.tool_calls?.length) {
3747
- const blocks = [];
3748
- if (m.content) blocks.push({ text: m.content });
3749
- for (const tc of m.tool_calls) {
3750
- let input;
3751
- if (!tc.function.arguments.trim()) input = {};
3752
- else try {
3753
- input = JSON.parse(tc.function.arguments);
3754
- } catch (cause) {
3755
- throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
3756
- }
3757
- blocks.push({ toolUse: {
3758
- toolUseId: tc.id,
3759
- name: tc.function.name,
3760
- input
3761
- } });
3762
- }
3763
- return {
3764
- role: "assistant",
3765
- content: blocks
3766
- };
3767
- }
3768
- return {
3769
- role: m.role,
3770
- content: Array.isArray(m.content) ? toBedrockContent(m.content) : [{ text: m.content ?? "" }]
3771
- };
3772
- }
3773
- /**
3774
- * Converse expects the results of everything the model asked for in one
3775
- * turn to arrive together as multiple `toolResult` content blocks on a
3776
- * single `'user'` message, not as separate consecutive `'user'` messages.
3777
- * The per-wire-message mapping above produces one `'user'` message per
3778
- * VernLLM wire tool message, so when an assistant turn requested more than
3779
- * one tool, this merges the resulting run of toolResult-only `'user'`
3780
- * messages back into one.
3781
- */
3782
- function mergeConsecutiveToolResults(messages) {
3783
- const isToolResultOnly = (m) => m.role === "user" && m.content.length > 0 && m.content.every((b) => "toolResult" in b);
3784
- const merged = [];
3785
- for (const m of messages) {
3786
- const prev = merged.at(-1);
3787
- if (isToolResultOnly(m) && prev && isToolResultOnly(prev)) prev.content.push(...m.content);
3788
- else merged.push(m);
3789
- }
3790
- return merged;
3791
- }
3792
-
3793
- //#endregion
3794
- //#region src/adapters/fetch.ts
3795
- /**
3796
- * Wraps a WHATWG `ReadableStream` (what `response.body` is) so it can be
3797
- * consumed with `for await`. Implemented via `getReader()` rather than
3798
- * relying on `ReadableStream` having a native `Symbol.asyncIterator`,
3799
- * that support varies across runtimes/versions, and this works everywhere
3800
- * a `ReadableStream` does.
3801
- */
3802
- async function* webStreamToAsyncIterable(stream) {
3803
- const reader = stream.getReader();
3804
- try {
3805
- for (;;) {
3806
- const { done, value } = await reader.read();
3807
- if (done) return;
3808
- if (value) yield value;
3809
- }
3810
- } finally {
3811
- try {
3812
- await reader.cancel();
3813
- } catch {}
3814
- reader.releaseLock();
3815
- }
3816
- }
3817
- /** Default `requestStream`: native `fetch`, with the same error/`.status` contract non-streaming errors get. */
3818
- async function defaultRequestStream(url, init) {
3819
- const res = await fetch(url, init);
3820
- if (!res.ok) {
3821
- const body = await res.text().catch(() => "");
3822
- const err = new Error(`Fetch adapter stream request failed (${res.status}): ${body.slice(0, 500)}`);
3823
- err.status = res.status;
3824
- err.headers = res.headers;
3825
- throw err;
3826
- }
3827
- if (!res.body) throw new Error("Fetch adapter stream request received a response with no body.");
3828
- return webStreamToAsyncIterable(res.body);
3829
- }
3830
- /** Builds the shared `{ method, headers, body? }` request-init for both `create` and `createStream`. */
3831
- async function buildRequestInit(config, params, requestBody) {
3832
- const url = typeof config.url === "function" ? config.url(params) : config.url;
3833
- const headers = typeof config.headers === "function" ? await config.headers() : config.headers;
3834
- const method = config.method ?? "POST";
3835
- const supportsBody = !["GET", "HEAD"].includes(method.toUpperCase());
3836
- return {
3837
- url,
3838
- method,
3839
- headers: supportsBody ? {
3840
- "Content-Type": "application/json",
3841
- ...headers
3842
- } : { ...headers },
3843
- ...supportsBody ? { body: JSON.stringify(requestBody) } : {}
3844
- };
3845
- }
3846
- /**
3847
- * A fetch-based escape hatch for providers with no SDK, or where pulling one
3848
- * in isnt worth it. You supply the URL, headers, and two small mapping
3849
- * functions; this handles the HTTP call and slots the result into the same
3850
- * `LLMClient` shape every other adapter produces, so retries, timeouts,
3851
- * the circuit breaker, and JSON/schema handling all still work unmodified
3852
- *
3853
- * Non-2xx responses throw an error with `.status` set to the HTTP status
3854
- * code, so VernLLMs `nonRetryableStatus` handling (e.g. failing fast on
3855
- * 401/403) applies here too
3856
- *
3857
- * Tool calling works the same way as every other adapter: `mapRequest`
3858
- * receives the full `ChatRequest`, including `tools`/`toolChoice`, so it can
3859
- * translate them into whatever shape the provider's wire format expects
3860
- * (typically an OpenAI-`function`-wrapped `tools` array plus a `tool_choice`
3861
- * field). On the way back, `mapResponse` may return a `toolCalls` array
3862
- * (id/name/JSON-encoded-arguments-string per call) alongside or instead of
3863
- * `content`; VernLLM parses and (if `argumentsSchema` was set) validates
3864
- * those arguments the same way it does for every other adapter. For
3865
- * `stream: true`, tool-call deltas go through the existing
3866
- * `mapStreamEvent` seam via `WireStreamChunk`'s `tool_call_delta` variant,
3867
- * no separate config is needed for streaming vs non-streaming tool calls.
3868
- *
3869
-
3870
- * `createStream` requires `mapStreamEvent` (there's no non-streaming
3871
- * response to fall back on, unlike the other three optional streaming
3872
- * seams). It opens the request via `requestStream` (defaults to native
3873
- * `fetch`), splits the raw bytes into individual events via
3874
- * `parseStreamFrames` (defaults to SSE framing, see `parseSseStream`),
3875
- * and translates each event into `WireStreamChunk`(s) via
3876
- * `mapStreamEvent`. Both seams are overridable per-config for providers
3877
- * that don't fit the SSE-over-fetch default. If a custom `request`
3878
- * transport is configured, `requestStream` must be configured too,
3879
- * `requestStream` never silently falls back to `request` (see
3880
- * `createStream`'s own comment for why), so a `stream: true` call with
3881
- * `request` set but no `requestStream` throws a clear
3882
- * `LLMError('validation')` instead of quietly using unrelated native
3883
- * `fetch`.
3884
- */
3885
- function fromFetch(config) {
3886
- return { chat: { completions: {
3887
- async create(params, options) {
3888
- const { url, method, headers, body } = await buildRequestInit(config, params, config.mapRequest(params));
3889
- const request = config.request ?? fetch;
3890
- const res = await request(url, {
3891
- method,
3892
- headers,
3893
- body,
3894
- signal: options.signal
3895
- });
3896
- if (!res.ok) {
3897
- const responseBody = await res.text().catch(() => "");
3898
- const err = new Error(`Fetch adapter request failed (${res.status}): ${responseBody.slice(0, 500)}`);
3899
- err.status = res.status;
3900
- err.headers = res.headers;
3901
- throw err;
3902
- }
3903
- const json = await res.json();
3904
- const { content, usage, toolCalls } = config.mapResponse(json);
3905
- const wireToolCalls = toolCalls?.length ? toolCalls.map((tc) => ({
3906
- id: tc.id,
3907
- type: "function",
3908
- function: {
3909
- name: tc.name,
3910
- arguments: tc.arguments
3911
- }
3912
- })) : void 0;
3913
- return {
3914
- choices: [{ message: {
3915
- content,
3916
- ...wireToolCalls ? { tool_calls: wireToolCalls } : {}
3917
- } }],
3918
- usage: usage ? {
3919
- prompt_tokens: usage.promptTokens,
3920
- completion_tokens: usage.completionTokens,
3921
- total_tokens: usage.totalTokens
3922
- } : void 0
3923
- };
3924
- },
3925
- async *createStream(params, options) {
3926
- if (!config.mapStreamEvent) throw new LLMError("stream: true requires mapStreamEvent to be configured on fromFetch", "invalid_params", {
3927
- code: "unsupported_capability",
3928
- issues: { capability: "mapStreamEvent" }
3929
- });
3930
- if (config.request && !config.requestStream) throw new LLMError("`stream: true` requires `requestStream` to be configured on fromFetch when a custom `request` transport is set. `requestStream` does not fall back to `request` (it needs an async-iterable byte stream, which `RequestLike`'s buffered `ResponseLike` has no way to provide), without it, `stream: true` would silently use plain native `fetch` instead of your configured transport. Add a `requestStream` that opens the same connection your `request` does, or omit `request` if native `fetch` is fine for both.", "invalid_params", {
3931
- code: "unsupported_capability",
3932
- issues: { capability: "requestStream" }
3933
- });
3934
- const { url, method, headers, body } = await buildRequestInit(config, params, config.mapRequest(params));
3935
- const requestStream = config.requestStream ?? defaultRequestStream;
3936
- const parseFrames = config.parseStreamFrames ?? parseSseStream;
3937
- const byteStream = await requestStream(url, {
3938
- method,
3939
- headers,
3940
- body,
3941
- signal: options.signal
3942
- });
3943
- for await (const event of parseFrames(byteStream)) {
3944
- if (event === SSE_PING) {
3945
- yield { type: "ping" };
3946
- continue;
3947
- }
3948
- const wireChunks = config.mapStreamEvent(event);
3949
- if (!wireChunks) continue;
3950
- if (Array.isArray(wireChunks)) yield* wireChunks;
3951
- else yield wireChunks;
3952
- }
3953
- }
3954
- } } };
3955
- }
3956
-
3957
- //#endregion
3958
- //#region src/adapters/openaiCompatible.ts
3959
- /**
3960
- * Translates a VernLLM `ContentBlock[]` into OpenAI's wire-level content
3961
- * array. Text blocks become `{ type: 'text', text }`; image blocks become
3962
- * `{ type: 'image_url', image_url: { url } }` with the base64 payload
3963
- * inlined as a `data:` URL, since our `ContentBlock` shape (`{ type:
3964
- * 'image', data, mimeType }`) is provider-agnostic and doesn't itself match
3965
- * OpenAI's wire format.
3966
- */
3967
- function toOpenAIContent(blocks) {
3968
- return blocks.map((block) => block.type === "image" ? {
3969
- type: "image_url",
3970
- image_url: { url: `data:${assertSupportedImageMimeType(block.mimeType)};base64,${block.data}` }
3971
- } : {
3972
- type: "text",
3973
- text: block.text
3974
- });
3975
- }
3976
- /**
3977
- * Translates VernLLM's provider-agnostic `messages` (the one part of a
3978
- * request that isn't a pure passthrough for OpenAI-compatible clients) into
3979
- * OpenAI's native wire shape. Shared between `create` and `createStream` so
3980
- * both go through identical message translation.
3981
- */
3982
- function toOpenAIMessages(params) {
3983
- return params.messages.map((m) => {
3984
- if (m.role === "user" && Array.isArray(m.content)) return {
3985
- ...m,
3986
- content: toOpenAIContent(m.content)
3987
- };
3988
- if (m.role === "tool") {
3989
- const { is_error: _isError,...openAIToolMessage } = m;
3990
- return openAIToolMessage;
3991
- }
3992
- return m;
3993
- });
3994
- }
3995
- /**
3996
- * Translates one OpenAI-shaped SSE chunk into zero or more `WireStreamChunk`s.
3997
- * A single chunk can carry a text delta, one or more tool-call argument
3998
- * deltas (each keyed by `index`, OpenAI's own convention for streaming
3999
- * parallel tool calls, mirrored directly by VernLLM's `tool_call_delta`
4000
- * shape so accumulation composes without translation), and/or a final
4001
- * usage block (present only when `stream_options.include_usage` is set,
4002
- * which this adapter always sets).
4003
- */
4004
- function* toWireStreamChunks(chunk) {
4005
- const delta = chunk.choices?.[0]?.delta;
4006
- if (delta?.content) yield {
4007
- type: "text-delta",
4008
- delta: delta.content
4009
- };
4010
- if (delta?.tool_calls?.length) for (const toolCall of delta.tool_calls) yield {
4011
- type: "tool_call_delta",
4012
- index: toolCall.index,
4013
- id: toolCall.id,
4014
- name: toolCall.function?.name,
4015
- argumentsDelta: toolCall.function?.arguments
4016
- };
4017
- if (chunk.usage) yield {
4018
- type: "usage",
4019
- usage: chunk.usage
4020
- };
4021
- }
4022
- function fromOpenAICompatible(client, options = {}) {
4023
- const raw = client;
4024
- const { supportsStreamUsage = true } = options;
4025
- const rawCreate = raw.chat.completions.create.bind(raw.chat.completions);
4026
- return { chat: { completions: {
4027
- async create(params, options$1) {
4028
- const messages = toOpenAIMessages(params);
4029
- return raw.chat.completions.create({
4030
- ...params,
4031
- messages
4032
- }, options$1);
4033
- },
4034
- async *createStream(params, options$1) {
4035
- const messages = toOpenAIMessages(params);
4036
- const stream = await rawCreate({
4037
- ...params,
4038
- messages,
4039
- stream: true,
4040
- ...supportsStreamUsage ? { stream_options: { include_usage: true } } : {}
4041
- }, options$1);
4042
- for await (const chunk of stream) yield* toWireStreamChunks(chunk);
4043
- }
4044
- } } };
4045
- }
4046
- /**
4047
- * Named alias for the OpenAI SDK itself. A raw `new OpenAI(...)` instance
4048
- * structurally matches most of `LLMClient`, but newer `openai` SDK major
4049
- * versions have widened `ChatCompletionContentPart` (e.g. adding a `file`
4050
- * variant) in ways that no longer structurally satisfy VernLLM's
4051
- * provider-agnostic `ContentBlock[]` on `userContent`, so passing the SDK
4052
- * instance directly can fail to typecheck depending on the installed
4053
- * `openai` version. Wrapping with `fromOpenAI()` (a plain alias of
4054
- * `fromOpenAICompatible()`) sidesteps that by translating through
4055
- * `unknown` at the boundary, and also picks up multimodal image
4056
- * translation and `createStream` wiring that a raw client doesn't have.
4057
- * See Migration Notes for details.
4058
- */
4059
- const fromOpenAI = fromOpenAICompatible;
4060
- /** Groqs SDK matches the OpenAI wire format */
4061
- const fromGroq = fromOpenAICompatible;
4062
- /**
4063
- * Mistrals `chat.completions`-shaped client (or their OpenAI-compat
4064
- * endpoint). Mistral supports `stream_options.include_usage` (added after
4065
- * an earlier period where it returned a 422 for unrecognized fields, per
4066
- * Mistral's changelog and streaming docs), so this is a plain alias like
4067
- * the others, `supportsStreamUsage` defaults to `true`.
4068
- */
4069
- const fromMistral = fromOpenAICompatible;
4070
- /** DeepSeeks API is OpenAI-compatible */
4071
- const fromDeepSeek = fromOpenAICompatible;
4072
- /** Cerebras inference API is OpenAI-compatible */
4073
- const fromCerebras = fromOpenAICompatible;
4074
- /** Together AIs API is OpenAI-compatible */
4075
- const fromTogether = fromOpenAICompatible;
4076
- /** Fireworks AIs API is OpenAI-compatible */
4077
- const fromFireworks = fromOpenAICompatible;
4078
- /**
4079
- * Ollama exposes an OpenAI-compatible endpoint at `/v1/chat/completions`
4080
- * (as opposed to its native `/api/chat` format, which differs). Point an
4081
- * OpenAI SDK instances `baseURL` at your Ollama server and pass it here:
4082
- * this does not talk to Ollamas native API directly.
4083
- */
4084
- const fromOllama = fromOpenAICompatible;
4085
- /** OpenRouter's API is OpenAI-compatible */
4086
- const fromOpenRouter = fromOpenAICompatible;
4087
- /** Perplexity's API is OpenAI-compatible */
4088
- const fromPerplexity = fromOpenAICompatible;
4089
- /** DeepInfra's API is OpenAI-compatible */
4090
- const fromDeepInfra = fromOpenAICompatible;
4091
- /** Novita's API is OpenAI-compatible */
4092
- const fromNovita = fromOpenAICompatible;
4093
- /** Hyperbolic's API is OpenAI-compatible */
4094
- const fromHyperbolic = fromOpenAICompatible;
4095
- /** Moonshot's (Kimi) API is OpenAI-compatible */
4096
- const fromMoonshot = fromOpenAICompatible;
4097
- /** Zhipu's (GLM) API is OpenAI-compatible */
4098
- const fromZhipu = fromOpenAICompatible;
4099
- /**
4100
- * LM Studio exposes an OpenAI-compatible endpoint at `/v1/chat/completions`.
4101
- * Point an OpenAI SDK instance's `baseURL` at your local LM Studio server.
4102
- */
4103
- const fromLMStudio = fromOpenAICompatible;
4104
- /**
4105
- * vLLM's OpenAI-compatible server mode exposes `/v1/chat/completions`.
4106
- * Point an OpenAI SDK instance's `baseURL` at your vLLM server.
4107
- */
4108
- const fromVLLM = fromOpenAICompatible;
4109
- /** xAI's Grok API is OpenAI-compatible */
4110
- const fromXAI = fromOpenAICompatible;
4111
- /** NVIDIA NIM's hosted and self-hosted endpoints are OpenAI-compatible */
4112
- const fromNvidiaNIM = fromOpenAICompatible;
4113
- /** Vercel AI Gateway is OpenAI-compatible */
4114
- const fromVercelAIGateway = fromOpenAICompatible;
4115
- /** Cloudflare Workers AI exposes an OpenAI-compatible endpoint */
4116
- const fromCloudflareWorkersAI = fromOpenAICompatible;
4117
- /** Nebius AI Studio is OpenAI-compatible */
4118
- const fromNebius = fromOpenAICompatible;
4119
- /** SambaNova Cloud's API is OpenAI-compatible */
4120
- const fromSambaNova = fromOpenAICompatible;
4121
- /** Baseten's model hosting exposes an OpenAI-compatible endpoint */
4122
- const fromBaseten = fromOpenAICompatible;
4123
- /** Featherless AI's API is OpenAI-compatible */
4124
- const fromFeatherless = fromOpenAICompatible;
4125
- /** Friendli AI's serving endpoint is OpenAI-compatible */
4126
- const fromFriendli = fromOpenAICompatible;
4127
- /** SiliconFlow's API is OpenAI-compatible */
4128
- const fromSiliconFlow = fromOpenAICompatible;
4129
- /** Parasail's inference API is OpenAI-compatible */
4130
- const fromParasail = fromOpenAICompatible;
4131
- /** StepFun's API is OpenAI-compatible */
4132
- const fromStepFun = fromOpenAICompatible;
4133
- /** MiniMax's API is OpenAI-compatible */
4134
- const fromMiniMax = fromOpenAICompatible;
4135
- /** Lambda Labs' Inference API is OpenAI-compatible */
4136
- const fromLambdaLabs = fromOpenAICompatible;
4137
- /** Snowflake Cortex's LLM endpoint is OpenAI-compatible */
4138
- const fromSnowflakeCortex = fromOpenAICompatible;
4139
- /** Anyscale Endpoints' API is OpenAI-compatible */
4140
- const fromAnyscale = fromOpenAICompatible;
4141
- /** Lepton AI's inference API is OpenAI-compatible */
4142
- const fromLepton = fromOpenAICompatible;
4143
- /** Inference.net's API is OpenAI-compatible */
4144
- const fromInferenceNet = fromOpenAICompatible;
4145
- /** Infermatic's API is OpenAI-compatible */
4146
- const fromInfermatic = fromOpenAICompatible;
4147
- /** AtlasCloud's inference API is OpenAI-compatible */
4148
- const fromAtlasCloud = fromOpenAICompatible;
4149
- /** 01.AI's (Yi models) API is OpenAI-compatible */
4150
- const from01AI = fromOpenAICompatible;
4151
-
4152
- //#endregion
4153
- export { CircuitBreaker, ConsoleLogger, FallbackExhaustedError, InMemoryCacheAdapter, LLMError, NormalizedCacheAdapter, RateLimiter, SSE_PING, TieredCacheAdapter, VernLLM, defaultEstimateTokens, defaultFallbackOn, from01AI, fromAnthropic, fromAnyscale, fromAtlasCloud, fromBaseten, fromBedrock, fromCerebras, fromCloudflareWorkersAI, fromDeepInfra, fromDeepSeek, fromFeatherless, fromFetch, fromFireworks, fromFriendli, fromGemini, fromGroq, fromHyperbolic, fromInferenceNet, fromInfermatic, fromLMStudio, fromLambdaLabs, fromLepton, fromMiniMax, fromMistral, fromMoonshot, fromNebius, fromNovita, fromNvidiaNIM, fromOllama, fromOpenAI, fromOpenAICompatible, fromOpenRouter, fromParasail, fromPerplexity, fromSambaNova, fromSiliconFlow, fromSnowflakeCortex, fromStepFun, fromTogether, fromVLLM, fromVercelAIGateway, fromXAI, fromZhipu, hasIssues, isFallbackExhaustedError, isLLMError, isToolCallResult, parseSseStream };
12
+ `)}let i=n.trim();if(i){let e=G(i);e!==H&&e!==U&&(yield e)}}const H=Symbol(`sse-stream-done`),U=Symbol(`sse-frame-no-data`),W=Symbol(`sse-frame-ping`);function G(e){let t=[],n=!1;for(let r of e.split(`
13
+ `)){if(r.startsWith(`:`)){n=!0;continue}if(!r.startsWith(`data:`))continue;t.push(r.startsWith(`data: `)?r.slice(6):r.slice(5))}if(!t.length)return n?W:U;let r=t.join(`
14
+ `);if(r===`[DONE]`)return H;try{return JSON.parse(r)}catch(e){throw new f(`Invalid JSON in SSE frame: ${r.slice(0,200)}`,`parse`,{cause:e,code:`stream_frame_invalid`})}}const K=[`image/png`,`image/jpeg`,`image/gif`,`image/webp`];function q(e){if(K.includes(e))return e;throw new f(`Unsupported image mimeType "${e}": expected one of ${K.join(`, `)}`,`invalid_params`)}function Ie(e,t){return t?Array.isArray(t)?t.includes(e):t(e):!1}const J={minimal:1024,low:4096,medium:16e3,high:32e3};function Y(e){if(!e)return J;let t={...J,...e};if(!(t.minimal<t.low&&t.low<t.medium&&t.medium<t.high))throw new f(`reasoningEffortTokens must keep tiers in strictly ascending order (minimal < low < medium < high), got ${JSON.stringify(t)}. An out-of-order override doesn't just misrank tiers, it can make some of them unreachable.`,`invalid_params`);return t}function X(e,t=J){return t[e]}function Z(e,t=J){return e<=t.minimal?`minimal`:e<=t.low?`low`:e<=t.medium?`medium`:`high`}function Le(e){let t=/opus-(\d+)(?:-(\d+))?/.exec(e);if(!t)return null;let n=t[2],r=n===void 0||n.length>=8?0:Number(n);return[Number(t[1]),r]}function Re(e){let t=Le(e);if(t){let[e,n]=t;return e>4||e===4&&n>=7}return[`sonnet-5`,`fable-5`,`mythos`].some(t=>e.includes(t))}function ze(e,t){return Re(e)?!0:t?Array.isArray(t)?t.includes(e):t(e):!1}function Be(e,t){return!ze(e,t)}function Ve(e,t){if(e<1024)throw new f(`budgetTokens (${e}) is below Anthropic's minimum of 1024. Raise budgetTokens, or use a reasoningEffort tier of 'low' or above with the default conversion table.`,`invalid_params`);if(e>=t)throw new f(`budgetTokens (${e}) must be less than maxTokens (${t}); the thinking budget and the reply share the same max_tokens ceiling on Anthropic. Raise maxTokens, or lower budgetTokens/reasoningEffort.`,`invalid_params`)}function He(e){if(e)throw new f(`budgetTokens/reasoningEffort was set alongside ${e}. Anthropic rejects thinking combined with a tool_choice that forces tool use, the model has to be able to reply with plain text for thinking to run. Use toolChoice: 'auto' (or omit toolChoice) for this call, or drop budgetTokens/reasoningEffort for it.`,`invalid_params`)}function Ue(e){return e===`minimal`?`low`:e}function We(e,t){return Ke(t,e.toUpperCase())}function Ge(e){let t=/gemini-\d+\.(\d+)/.exec(e);return t?Number(t[1]):0}function Ke(e,t){if(!e.includes(`pro`))return t;let n=qe(e);if(n===null||n<3)return t;let r=Ge(e);return r===0?t===`HIGH`?`HIGH`:`LOW`:t===`MINIMAL`?`LOW`:t}function qe(e){let t=/gemini-(\d+)/.exec(e);return t?Number(t[1]):null}function Je(e){let t=qe(e);return t!==null&&t>=3}function Ye(e,t){return Je(e)?!0:t?Array.isArray(t)?t.includes(e):t(e):!1}function Xe(e){return e.map(e=>e.type===`image`?{type:`image`,source:{type:`base64`,media_type:q(e.mimeType),data:e.data}}:{type:`text`,text:e.text})}function Ze(e,t){if(e.type!==`object`)throw new f(`Tool "${t}"'s schema must have "type": "object" (Anthropic requires object-shaped tool parameters).`,`validation`);return e}function Qe(e){return!e||e===`auto`?{type:`auto`}:e===`none`?{type:`none`}:e===`required`?{type:`any`}:{type:`tool`,name:e.function.name}}function Q(e,t){return{tools:e.map(e=>({name:e.function.name,description:e.function.description,input_schema:Ze(e.function.parameters,e.function.name)})),toolChoice:Qe(t)}}function $e(e,t,n,r){let i=e.messages.find(e=>e.role===`system`),a=e.messages.filter(e=>e.role===`user`||e.role===`assistant`||e.role===`tool`),o=e.response_format?.type===`json_schema`?e.response_format.json_schema:void 0,s=o?.name.trim();if(o&&!s)throw new f(`json_schema.name must not be empty.`,`validation`);let c=!!o&&Ie(e.model,t);if(o&&e.tools?.length&&!c)throw new f(`Anthropic model "${e.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there, which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromAnthropic's \`nativeStructuredOutputModels\` option once you've confirmed it supports Anthropic's \`output_config.format\`.`,`validation`);if(e.response_format?.type===`json_object`)throw new f('response_format: "json_object" is not supported on Anthropic. Unlike OpenAI, Anthropic has no API-level field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a "respond with JSON only" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real API-level constraint (Anthropic\'s native output_config.format on covered models, or a forced single tool call otherwise).',`validation`);let l,u,d,p;if(o&&c)u={type:`json_schema`,schema:o.schema},e.tools?.length&&({tools:d,toolChoice:p}=Q(e.tools,e.tool_choice));else if(o&&s){let{schema:e,description:t,strict:n}=o;l=s,d=[{name:l,description:t,input_schema:Ze(e,l),strict:n}],p={type:`tool`,name:l}}!o&&e.tools?.length&&({tools:d,toolChoice:p}=Q(e.tools,e.tool_choice));let m,h;if(e.budget_tokens!==void 0||e.reasoning_effort!==void 0)if(He(p?.type===`tool`?`toolChoice forcing the "${p.name}" tool`:p?.type===`any`?`toolChoice: 'required' (Anthropic's "any" tool_choice)`:void 0),Be(e.model,r)){let t=e.budget_tokens??X(e.reasoning_effort,n);Ve(t,e.max_tokens),m={type:`enabled`,budget_tokens:t}}else{let t=e.reasoning_effort??Z(e.budget_tokens,n);m={type:`adaptive`},h=Ue(t)}let g=i?.content,_=m?void 0:e.temperature,v={model:e.model,max_tokens:e.max_tokens,..._===void 0?{}:{temperature:_},system:g||void 0,messages:tt(a.map(e=>nt(e))),...d?{tools:d,tool_choice:p}:{},...u||h?{output_config:{...u?{format:u}:{},...h?{effort:h}:{}}}:{},...m?{thinking:m}:{}};return{body:v,toolName:l}}function et(e,t){let n=t?.nativeStructuredOutputModels,r=Y(t?.reasoningEffortTokens),i=t?.adaptiveOnlyModels,a=e.messages.create.bind(e.messages);return{supportsJsonObjectMode:!1,chat:{completions:{async create(t,a){let{body:o,toolName:s}=$e(t,n,r,i),c=await e.messages.create(o,a),l,u;if(s){let e=c.content.find(e=>e.type===`tool_use`&&e.name===s);if(!e)throw new f(`Anthropic did not return the required structured output tool "${s}".`,`validation`);if(!e.input||typeof e.input!=`object`||Array.isArray(e.input))throw new f(`Anthropic returned invalid structured output for tool "${s}". Expected an object.`,`validation`);l=JSON.stringify(e.input)}else{l=c.content.filter(e=>e.type===`text`).map(e=>e.text??``).join(``);let e=c.content.filter(e=>e.type===`tool_use`);e.length&&(u=e.map(e=>({id:e.id,type:`function`,function:{name:e.name,arguments:JSON.stringify(e.input??{})}})))}return{choices:[{message:{content:l,...u?{tool_calls:u}:{}}}],usage:{prompt_tokens:c.usage?.input_tokens,completion_tokens:c.usage?.output_tokens,total_tokens:(c.usage?.input_tokens??0)+(c.usage?.output_tokens??0),...c.usage?.output_tokens_details?.thinking_tokens===void 0?{}:{completion_tokens_details:{reasoning_tokens:c.usage.output_tokens_details.thinking_tokens}}}}},async*createStream(e,t){let{body:o,toolName:s}=$e(e,n,r,i),c=await a({...o,stream:!0},t),l=new Map,u=0,d=!1;for await(let e of c)if(e.type===`message_start`)u=e.message.usage?.input_tokens??0;else if(e.type===`content_block_start`)if(e.content_block.type===`tool_use`){let t=e.content_block.name===s?`json-tool`:`tool_use`;l.set(e.index,t),t===`json-tool`?d=!0:s||(yield{type:`tool_call_delta`,index:e.index,id:e.content_block.id,name:e.content_block.name})}else l.set(e.index,`text`);else if(e.type===`content_block_delta`){if(e.delta.type===`text_delta`)s||(yield{type:`text-delta`,delta:e.delta.text});else if(e.delta.type===`input_json_delta`){let t=l.get(e.index);t===`json-tool`?yield{type:`text-delta`,delta:e.delta.partial_json}:s||(yield{type:`tool_call_delta`,index:e.index,argumentsDelta:e.delta.partial_json})}}else if(e.type===`message_delta`){let t=e.usage?.output_tokens??0,n=e.usage?.output_tokens_details?.thinking_tokens;yield{type:`usage`,usage:{prompt_tokens:u,completion_tokens:t,total_tokens:u+t,...n===void 0?{}:{completion_tokens_details:{reasoning_tokens:n}}}}}else e.type===`ping`&&(yield{type:`ping`});if(s&&!d)throw new f(`Anthropic did not return the required structured output tool "${s}".`,`validation`)}}}}}function tt(e){let t=e=>e.role===`user`&&Array.isArray(e.content)&&e.content.length>0&&e.content.every(e=>e.type===`tool_result`),n=[];for(let r of e){let e=n.at(-1);t(r)&&e&&t(e)?e.content.push(...r.content):n.push(r)}return n}function nt(e){if(e.role===`tool`)return{role:`user`,content:[{type:`tool_result`,tool_use_id:e.tool_call_id,content:e.content,...e.is_error?{is_error:!0}:{}}]};if(e.role===`assistant`&&e.tool_calls?.length){let t=[];e.content&&t.push({type:`text`,text:e.content});for(let n of e.tool_calls){let e;try{e=n.function.arguments.trim()?JSON.parse(n.function.arguments):{}}catch(e){throw new f(`Assistant tool call "${n.function.name}" (${n.id}) has arguments that are not valid JSON.`,`validation`,{cause:e})}if(e===null||Array.isArray(e)||typeof e!=`object`)throw new f(`Assistant tool call "${n.function.name}" (${n.id}) arguments must be a JSON object.`,`validation`);t.push({type:`tool_use`,id:n.id,name:n.function.name,input:e})}return{role:`assistant`,content:t}}return{role:e.role,content:Array.isArray(e.content)?Xe(e.content):e.content??``}}function rt(e){return e.map(e=>e.type===`image`?{inlineData:{mimeType:q(e.mimeType),data:e.data}}:{text:e.text})}function it(e){return!e||e===`auto`?{functionCallingConfig:{mode:`AUTO`}}:e===`none`?{functionCallingConfig:{mode:`NONE`}}:e===`required`?{functionCallingConfig:{mode:`ANY`}}:{functionCallingConfig:{mode:`ANY`,allowedFunctionNames:[e.function.name]}}}function at(e){if(e.role===`tool`)return{role:`user`,parts:[{functionResponse:{name:e.tool_call_id,response:st(e.content)}}]};if(e.role===`assistant`&&e.tool_calls?.length){let t=[];return typeof e.content==`string`&&e.content&&t.push({text:e.content}),t.push(...e.tool_calls.map(e=>({functionCall:{name:e.function.name,args:ot(e.function.arguments,e.function.name)}}))),{role:`model`,parts:t}}return{role:e.role===`assistant`?`model`:`user`,parts:Array.isArray(e.content)?rt(e.content):[{text:e.content??``}]}}function ot(e,t){let n;try{n=e.trim()?JSON.parse(e):{}}catch(e){throw new f(`Tool call "${t}" arguments are not valid JSON.`,`parse`,{cause:e,code:`tool_arguments_parse_failed`})}if(!n||Array.isArray(n)||typeof n!=`object`)throw new f(`Tool call "${t}" arguments must be a JSON object.`,`validation`);return n}function st(e){let t;try{t=e.trim()?JSON.parse(e):``}catch{t=e}return t&&!Array.isArray(t)&&typeof t==`object`?t:{output:t}}function ct(e){let t=e=>e.role===`user`&&e.parts.length>0&&e.parts.every(e=>`functionResponse`in e),n=[];for(let r of e){let e=n.at(-1);t(r)&&e&&t(e)?e.parts.push(...r.parts):n.push(r)}return n}function lt(e,t,n){let r=e.messages.find(e=>e.role===`system`),i=e.messages.filter(e=>e.role===`user`||e.role===`assistant`||e.role===`tool`),a=!!e.response_format,o={...e.temperature===void 0?{}:{temperature:e.temperature},maxOutputTokens:e.max_tokens,...r?{systemInstruction:{parts:[{text:r.content}]}}:{}};if(a&&(o.responseMimeType=`application/json`),e.response_format?.type===`json_schema`){let{schema:t,description:n}=e.response_format.json_schema;o.responseSchema={...t,...n?{description:n}:{}}}if(e.tools?.length&&(o.tools=[{functionDeclarations:e.tools.map(e=>({name:e.function.name,description:e.function.description,parameters:e.function.parameters}))}],o.toolConfig=it(e.tool_choice)),Ye(e.model,n)){let n=e.reasoning_effort??(e.budget_tokens===void 0?void 0:Z(e.budget_tokens,t));n!==void 0&&(o.thinkingConfig={thinkingLevel:We(n,e.model)})}else{let n=e.budget_tokens??(e.reasoning_effort?X(e.reasoning_effort,t):void 0);n!==void 0&&(o.thinkingConfig={thinkingBudget:n})}return{model:e.model,contents:ct(i.map(e=>at(e))),config:o}}function ut(e,t){let n=Y(t?.reasoningEffortTokens),r=t?.thinkingLevelModels,i=e.models??e;if(typeof i.generateContent!=`function`)throw new f(`fromGemini requires a client with generateContent: pass ai.models, or the whole ai client (fromGemini(ai)).`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`generateContent`}});let a=i.generateContent.bind(i),o=typeof i.generateContentStream==`function`?i.generateContentStream.bind(i):void 0;return{chat:{completions:{async create(e,t){let i=lt(e,n,r);i.config={...i.config,abortSignal:t.signal};let o=await a(i),s=o.candidates?.[0]?.content?.parts??[],c=s.map(e=>e.text??``).join(``),l=s.filter(e=>e.functionCall),u;return l.length&&(u=l.map(e=>({id:e.functionCall.name,type:`function`,function:{name:e.functionCall.name,arguments:JSON.stringify(e.functionCall.args??{})}}))),{choices:[{message:{content:c,...u?{tool_calls:u}:{}}}],usage:{prompt_tokens:o.usageMetadata?.promptTokenCount,completion_tokens:o.usageMetadata?.candidatesTokenCount,total_tokens:o.usageMetadata?.totalTokenCount,...o.usageMetadata?.thoughtsTokenCount===void 0?{}:{completion_tokens_details:{reasoning_tokens:o.usageMetadata.thoughtsTokenCount}}}}},async*createStream(e,t){if(!o)throw new f(`stream: true requires a Gemini client with generateContentStream`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`generateContentStream`}});let i=lt(e,n,r);i.config={...i.config,abortSignal:t.signal};let a=await o(i),s=0,c;for await(let e of a){let t=e.candidates?.[0]?.content?.parts??[];for(let e of t)e.text&&(yield{type:`text-delta`,delta:e.text}),e.functionCall&&(yield{type:`tool_call_delta`,index:s,id:e.functionCall.name,name:e.functionCall.name,argumentsDelta:JSON.stringify(e.functionCall.args??{}),complete:!0},s++);e.usageMetadata&&(c=e.usageMetadata)}c&&(yield{type:`usage`,usage:{prompt_tokens:c.promptTokenCount,completion_tokens:c.candidatesTokenCount,total_tokens:c.totalTokenCount,...c.thoughtsTokenCount===void 0?{}:{completion_tokens_details:{reasoning_tokens:c.thoughtsTokenCount}}}})}}}}}function dt(e){return e.includes(`claude`)}function ft(e){switch(q(e)){case`image/png`:return`png`;case`image/jpeg`:return`jpeg`;case`image/gif`:return`gif`;case`image/webp`:return`webp`}}function pt(e){return new Uint8Array(Buffer.from(e,`base64`))}function mt(e){return e.map(e=>e.type===`image`?{image:{format:ft(e.mimeType),source:{bytes:pt(e.data)}}}:{text:e.text})}function ht(e,t){return{tools:e.map(e=>({toolSpec:{name:e.function.name,description:e.function.description,inputSchema:{json:e.function.parameters}}})),toolChoice:St(t)}}function gt(e,t,n,r,i){let a=e.messages.find(e=>e.role===`system`),o=e.messages.filter(e=>e.role===`user`||e.role===`assistant`||e.role===`tool`),s=e.response_format?.type===`json_schema`?e.response_format.json_schema:void 0,c=s?.name.trim();if(s&&!c)throw new f(`json_schema.name must not be empty.`,`validation`);let l=!!s&&Ie(e.model,n);if(s&&e.tools?.length&&!l)throw new f(`Bedrock model "${e.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there (via \`toolConfig\`), which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromBedrock's \`nativeStructuredOutputModels\` option once you've confirmed it supports Converse's \`outputConfig.textFormat\`.`,`validation`);if(e.response_format?.type===`json_object`)throw new f('response_format: "json_object" is not supported on Bedrock. Converse has no field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a "respond with JSON only" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real constraint (Converse\'s native outputConfig.textFormat on covered models, or a forced tool call otherwise).',`validation`);let u,d,p;if(s&&l){let{schema:e,description:t}=s;p={textFormat:{type:`json_schema`,structure:{jsonSchema:{schema:JSON.stringify(e),name:c,description:t}}}}}else if(s&&c){let{schema:e,description:t,strict:n}=s;u=c,d={tools:[{toolSpec:{name:u,description:t,inputSchema:{json:e},strict:n}}],toolChoice:{tool:{name:u}}}}if(e.tools?.length&&!u&&(d=ht(e.tools,e.tool_choice)),s&&d&&t){let n=Array.isArray(t)?t.includes(e.model):t(e.model);if(!n)throw new f(`Bedrock model "${e.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`toolUseSupportedModels`}})}let m,h;if(dt(e.model)&&(e.budget_tokens!==void 0||e.reasoning_effort!==void 0)){let t=d?.toolChoice;if(He(t&&`tool`in t?`toolChoice forcing the "${t.tool?.name}" tool`:t&&`any`in t?`toolChoice: 'required' (Converse's "any" tool_choice)`:void 0),Be(e.model,i)){let t=e.budget_tokens??X(e.reasoning_effort,r);Ve(t,e.max_tokens),m={thinking:{type:`enabled`,budget_tokens:t}}}else{let t=e.reasoning_effort??Z(e.budget_tokens,r);m={thinking:{type:`adaptive`}},h=Ue(t)}}let g=m?void 0:e.temperature,_={modelId:e.model,messages:wt(o.map(e=>Ct(e))),system:a?.content?[{text:a.content}]:void 0,inferenceConfig:{...g===void 0?{}:{temperature:g},maxTokens:e.max_tokens},...d?{toolConfig:d}:{},...p||h?{outputConfig:{...p??{},...h?{effort:h}:{}}}:{},...m?{additionalModelRequestFields:m}:{}};return{request:_,toolName:u}}function _t(e){return typeof e.send==`function`}function vt(e){if(`messageStart`in e)return{messageStart:e.messageStart};if(`contentBlockStart`in e)return{contentBlockStart:e.contentBlockStart};if(`contentBlockDelta`in e)return{contentBlockDelta:e.contentBlockDelta};if(`contentBlockStop`in e)return{contentBlockStop:e.contentBlockStop};if(`messageStop`in e)return{messageStop:e.messageStop};if(`metadata`in e)return{metadata:e.metadata};if(`internalServerException`in e)return{internalServerException:e.internalServerException};if(`modelStreamErrorException`in e)return{modelStreamErrorException:e.modelStreamErrorException};if(`validationException`in e)return{validationException:e.validationException};if(`throttlingException`in e)return{throttlingException:e.throttlingException};if(`serviceUnavailableException`in e)return{serviceUnavailableException:e.serviceUnavailableException}}async function*yt(e){for await(let t of e){let e=vt(t);e&&(yield e)}}function bt(e){let t;function n(){return t??=import(`@aws-sdk/client-bedrock-runtime`).then(e=>e,e=>{throw t=void 0,new f(`fromBedrock requires "@aws-sdk/client-bedrock-runtime" to be installed to use a raw AWS SDK client (it is not a dependency of vern-llm itself). Install it, or pass your own object with .converse()/.converseStream() methods instead.`,`validation`,{cause:e})}),t}return{converse:async(t,r)=>{let{ConverseCommand:i}=await n();return e.send(new i(t),{abortSignal:r.signal})},converseStream:async(t,r)=>{let{ConverseStreamCommand:i}=await n(),a=await e.send(new i(t),{abortSignal:r.signal});if(!a.stream)throw new f(`Bedrock ConverseStreamCommand response did not include a stream. This can happen if the request or the model doesn't actually support Converse streaming.`,`api`,{code:`server_error`});return{stream:yt(a.stream)}}}}function xt(e,t){let n=_t(e)?bt(e):e,r=t?.toolUseSupportedModels,i=t?.nativeStructuredOutputModels,a=Y(t?.reasoningEffortTokens),o=t?.adaptiveOnlyModels;return{supportsJsonObjectMode:!1,chat:{completions:{async create(e,t){let{request:s,toolName:c}=gt(e,r,i,a,o),l=await n.converse(s,t),u,d;if(c){let e=l.output?.message?.content?.find(e=>e.toolUse?.name===c);u=e?.toolUse?JSON.stringify(e.toolUse.input):``}else{let e=l.output?.message?.content??[];u=e.map(e=>e.text??``).join(``);let t=e.filter(e=>!!e.toolUse);t.length&&(d=t.map((e,t)=>{let n=e.toolUse;if(!n.name)throw new f(`Bedrock returned a toolUse block without a name at index ${t}.`,`validation`);return{id:n.toolUseId??`${n.name}_${t}`,type:`function`,function:{name:n.name,arguments:JSON.stringify(n.input??{})}}}))}return{choices:[{message:{content:u,...d?{tool_calls:d}:{}}}],usage:{prompt_tokens:l.usage?.inputTokens,completion_tokens:l.usage?.outputTokens,total_tokens:l.usage?.totalTokens}}},async*createStream(e,t){if(!n.converseStream)throw new f(`stream: true requires a Bedrock client with converseStream`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`converseStream`}});let{request:s,toolName:c}=gt(e,r,i,a,o),{stream:l}=await n.converseStream(s,t),u=new Map;for await(let e of l)if(`contentBlockStart`in e){let{contentBlockIndex:t,start:n}=e.contentBlockStart;if(n?.toolUse){let e=n.toolUse.name===c?`json-tool`:`tool_use`;u.set(t,e),e===`tool_use`&&!c&&(yield{type:`tool_call_delta`,index:t,id:n.toolUse.toolUseId,name:n.toolUse.name})}else u.set(t,`text`)}else if(`contentBlockDelta`in e){let{contentBlockIndex:t,delta:n}=e.contentBlockDelta;if(n&&`text`in n&&n.text!==void 0&&!c)yield{type:`text-delta`,delta:n.text};else if(n&&`toolUse`in n&&n.toolUse?.input!==void 0){let e=u.get(t);e===`json-tool`?yield{type:`text-delta`,delta:n.toolUse.input}:c||(yield{type:`tool_call_delta`,index:t,argumentsDelta:n.toolUse.input})}}else if(`metadata`in e&&e.metadata.usage)yield{type:`usage`,usage:{prompt_tokens:e.metadata.usage.inputTokens,completion_tokens:e.metadata.usage.outputTokens,total_tokens:e.metadata.usage.totalTokens}};else if(`throttlingException`in e)throw new f(e.throttlingException.message??`Bedrock throttled the request mid-stream`,`api`,{status:429,code:`provider_rate_limited`});else if(`validationException`in e)throw new f(e.validationException.message??`Bedrock rejected the request mid-stream`,`validation`);else if(`internalServerException`in e||`serviceUnavailableException`in e||`modelStreamErrorException`in e){let t=`internalServerException`in e&&e.internalServerException.message||`serviceUnavailableException`in e&&e.serviceUnavailableException.message||`modelStreamErrorException`in e&&e.modelStreamErrorException.message||`Bedrock reported a mid-stream error`,n=`modelStreamErrorException`in e&&e.modelStreamErrorException.originalStatusCode||`serviceUnavailableException`in e&&503||500;throw new f(t,`api`,{status:n,code:n>=500?`server_error`:void 0})}}}}}}function St(e){if(!e||e===`auto`)return{auto:{}};if(e===`required`)return{any:{}};if(e===`none`)throw new f("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.",`invalid_params`,{code:`unsupported_capability`,issues:{capability:`toolChoice: 'none'`}});return{tool:{name:e.function.name}}}function Ct(e){if(e.role===`tool`)return{role:`user`,content:[{toolResult:{toolUseId:e.tool_call_id,content:[{text:e.content}],status:e.is_error?`error`:`success`}}]};if(e.role===`assistant`&&e.tool_calls?.length){let t=[];e.content&&t.push({text:e.content});for(let n of e.tool_calls){let e;if(!n.function.arguments.trim())e={};else try{e=JSON.parse(n.function.arguments)}catch(e){throw new f(`Assistant tool call "${n.function.name}" (${n.id}) has arguments that are not valid JSON.`,`validation`,{cause:e})}t.push({toolUse:{toolUseId:n.id,name:n.function.name,input:e}})}return{role:`assistant`,content:t}}return{role:e.role,content:Array.isArray(e.content)?mt(e.content):[{text:e.content??``}]}}function wt(e){let t=e=>e.role===`user`&&e.content.length>0&&e.content.every(e=>`toolResult`in e),n=[];for(let r of e){let e=n.at(-1);t(r)&&e&&t(e)?e.content.push(...r.content):n.push(r)}return n}async function*Tt(e){let t=e.getReader();try{for(;;){let{done:e,value:n}=await t.read();if(e)return;n&&(yield n)}}finally{try{await t.cancel()}catch{}t.releaseLock()}}async function Et(e,t){let n=await fetch(e,t);if(!n.ok){let e=await n.text().catch(()=>``),t=Error(`Fetch adapter stream request failed (${n.status}): ${e.slice(0,500)}`);throw t.status=n.status,t.headers=n.headers,t}if(!n.body)throw Error(`Fetch adapter stream request received a response with no body.`);return Tt(n.body)}async function Dt(e,t,n){let r=typeof e.url==`function`?e.url(t):e.url,i=typeof e.headers==`function`?await e.headers():e.headers,a=e.method??`POST`,o=![`GET`,`HEAD`].includes(a.toUpperCase());return{url:r,method:a,headers:o?{"Content-Type":`application/json`,...i}:{...i},...o?{body:JSON.stringify(n)}:{}}}function Ot(e){return{chat:{completions:{async create(t,n){let{url:r,method:i,headers:a,body:o}=await Dt(e,t,e.mapRequest(t)),s=e.request??fetch,c=await s(r,{method:i,headers:a,body:o,signal:n.signal});if(!c.ok){let e=await c.text().catch(()=>``),t=Error(`Fetch adapter request failed (${c.status}): ${e.slice(0,500)}`);throw t.status=c.status,t.headers=c.headers,t}let l=await c.json(),{content:u,usage:d,toolCalls:f}=e.mapResponse(l),p=f?.length?f.map(e=>({id:e.id,type:`function`,function:{name:e.name,arguments:e.arguments}})):void 0;return{choices:[{message:{content:u,...p?{tool_calls:p}:{}}}],usage:d?{prompt_tokens:d.promptTokens,completion_tokens:d.completionTokens,total_tokens:d.totalTokens}:void 0}},async*createStream(t,n){if(!e.mapStreamEvent)throw new f(`stream: true requires mapStreamEvent to be configured on fromFetch`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`mapStreamEvent`}});if(e.request&&!e.requestStream)throw new f("`stream: true` requires `requestStream` to be configured on fromFetch when a custom `request` transport is set. `requestStream` does not fall back to `request` (it needs an async-iterable byte stream, which `RequestLike`'s buffered `ResponseLike` has no way to provide), without it, `stream: true` would silently use plain native `fetch` instead of your configured transport. Add a `requestStream` that opens the same connection your `request` does, or omit `request` if native `fetch` is fine for both.",`invalid_params`,{code:`unsupported_capability`,issues:{capability:`requestStream`}});let{url:r,method:i,headers:a,body:o}=await Dt(e,t,e.mapRequest(t)),s=e.requestStream??Et,c=e.parseStreamFrames??V,l=await s(r,{method:i,headers:a,body:o,signal:n.signal});for await(let t of c(l)){if(t===W){yield{type:`ping`};continue}let n=e.mapStreamEvent(t);if(!n)continue;Array.isArray(n)?yield*n:yield n}}}}}}function kt(e,t){if(e.budget_tokens===void 0)return e;let{budget_tokens:n,...r}=e;return r.reasoning_effort===void 0?{...r,reasoning_effort:Z(n,t)}:r}function At(e){return e.map(e=>e.type===`image`?{type:`image_url`,image_url:{url:`data:${q(e.mimeType)};base64,${e.data}`}}:{type:`text`,text:e.text})}function jt(e){return e.messages.map(e=>{if(e.role===`user`&&Array.isArray(e.content))return{...e,content:At(e.content)};if(e.role===`tool`){let{is_error:t,...n}=e;return n}return e})}function*Mt(e){let t=e.choices?.[0]?.delta;if(t?.content&&(yield{type:`text-delta`,delta:t.content}),t?.tool_calls?.length)for(let e of t.tool_calls)yield{type:`tool_call_delta`,index:e.index,id:e.id,name:e.function?.name,argumentsDelta:e.function?.arguments};e.usage&&(yield{type:`usage`,usage:e.usage})}function $(e,t={}){let n=e,{supportsStreamUsage:r=!0}=t,i=Y(t.reasoningEffortTokens),a=n.chat.completions.create.bind(n.chat.completions);return{chat:{completions:{async create(e,t){let r=jt(e);return n.chat.completions.create(kt({...e,messages:r},i),t)},async*createStream(e,t){let n=jt(e),o=await a(kt({...e,messages:n,stream:!0,...r?{stream_options:{include_usage:!0}}:{}},i),t);for await(let e of o)yield*Mt(e)}}}}}const Nt=$,Pt=$,Ft=$,It=$,Lt=$,Rt=$,zt=$,Bt=$,Vt=$,Ht=$,Ut=$,Wt=$,Gt=$,Kt=$,qt=$,Jt=$,Yt=$,Xt=$,Zt=$,Qt=$,$t=$,en=$,tn=$,nn=$,rn=$,an=$,on=$,sn=$,cn=$,ln=$,un=$,dn=$,fn=$,pn=$,mn=$,hn=$,gn=$,_n=$;export{k as CircuitBreaker,L as ConsoleLogger,x as FallbackExhaustedError,h as InMemoryCacheAdapter,f as LLMError,g as NormalizedCacheAdapter,B as RateLimiter,W as SSE_PING,_ as TieredCacheAdapter,Ne as VernLLM,R as defaultEstimateTokens,b as defaultFallbackOn,Fe as defineCachedCallParams,Pe as defineCallParams,_n as from01AI,et as fromAnthropic,fn as fromAnyscale,gn as fromAtlasCloud,nn as fromBaseten,xt as fromBedrock,Lt as fromCerebras,$t as fromCloudflareWorkersAI,Ut as fromDeepInfra,It as fromDeepSeek,rn as fromFeatherless,Ot as fromFetch,zt as fromFireworks,an as fromFriendli,ut as fromGemini,Pt as fromGroq,Gt as fromHyperbolic,mn as fromInferenceNet,hn as fromInfermatic,Jt as fromLMStudio,un as fromLambdaLabs,pn as fromLepton,ln as fromMiniMax,Ft as fromMistral,Kt as fromMoonshot,en as fromNebius,Wt as fromNovita,Zt as fromNvidiaNIM,Bt as fromOllama,Nt as fromOpenAI,$ as fromOpenAICompatible,Vt as fromOpenRouter,sn as fromParasail,Ht as fromPerplexity,tn as fromSambaNova,on as fromSiliconFlow,dn as fromSnowflakeCortex,cn as fromStepFun,Rt as fromTogether,Yt as fromVLLM,Qt as fromVercelAIGateway,Xt as fromXAI,qt as fromZhipu,m as hasIssues,S as isFallbackExhaustedError,p as isLLMError,v as isToolCallResult,V as parseSseStream};
4154
15
  //# sourceMappingURL=index.js.map