vern-llm 2.3.0 → 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.cjs +10 -4232
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +550 -105
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.ts +550 -105
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -4149
- package/dist/index.js.map +1 -1
- package/package.json +14 -6
package/dist/index.cjs
CHANGED
|
@@ -1,4237 +1,15 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
var __defProp = Object.defineProperty;
|
|
5
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
-
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
-
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
-
var __copyProps = (to, from, except, desc) => {
|
|
10
|
-
if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
|
|
11
|
-
key = keys[i];
|
|
12
|
-
if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
|
|
13
|
-
get: ((k) => from[k]).bind(null, key),
|
|
14
|
-
enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
|
|
15
|
-
});
|
|
16
|
-
}
|
|
17
|
-
return to;
|
|
18
|
-
};
|
|
19
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
|
|
20
|
-
value: mod,
|
|
21
|
-
enumerable: true
|
|
22
|
-
}) : target, mod));
|
|
1
|
+
"use strict";var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},s=(n,r,a)=>(a=n==null?{}:e(i(n)),o(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n));const c=s(require(`crypto`)),l=new Set([`unknown_tool`,`duplicate_tool_call_id`,`tool_choice_none_violated`,`unexpected_tool_calls`]),u=new Set([`rate_limit_queue_full`,`rate_limit_queue_timeout`,`rate_limit_capacity_exceeded`]),d=new Set([`parse`,`validation`,`invalid_params`,`aborted`]);function f(e,t){return!(d.has(e)||t&&l.has(t)||t&&u.has(t))}function p(e){if(e!==void 0)try{return JSON.stringify(e),e}catch{return`[Unserializable: issues contained a circular reference]`}}function m(e){if(e!==void 0)try{return JSON.parse(JSON.stringify(e))}catch{return`[Unserializable: request body contained a circular reference]`}}const h=new Set([`authorization`,`x-api-key`,`x-goog-api-key`,`api-key`]);function g(e){if(e===void 0)return;let t={};for(let[n,r]of Object.entries(e))h.has(n.toLowerCase())||(t[n]=r);return t}const _=20;function v(e,t=0){if(e!==void 0)return t>=_?[]:e.map(e=>({...e,error:{...e.error,issues:p(e.error.issues),attempts:v(e.error.attempts,t+1)},request:e.request&&{...e.request,body:m(e.request.body),headers:g(e.request.headers)}}))}function y(e,t,n,r,i=Date.now()){return{provider:e,model:t,body:m(n),headers:g(r),startedAt:i}}var b=class extends Error{status;issues;cause;retryAfterMs;code;attempts;constructor(e,t,n={}){super(e),this.type=t,this.name=`LLMError`,this.status=n.status,this.issues=n.issues,this.cause=n.cause,this.retryAfterMs=n.retryAfterMs,this.code=n.code,this.attempts=n.attempts}get retryable(){return f(this.type,this.code)}toSnapshot(){return{message:this.message,type:this.type,status:this.status,issues:p(this.issues),retryAfterMs:this.retryAfterMs,code:this.code,retryable:this.retryable,attempts:v(this.attempts)}}toJSON(){return{name:this.name,message:this.message,type:this.type,status:this.status,issues:p(this.issues),retryAfterMs:this.retryAfterMs,code:this.code,retryable:this.retryable,attempts:v(this.attempts)}}};function x(e){return e instanceof b}function ee(e,t){return e.code===t&&e.issues!==void 0}var S=class{store=new Map;constructor(e=1e3){this.maxSize=e}async get(e){let t=this.store.get(e);return t?Date.now()>=t.expiresAt?(this.store.delete(e),{hit:!1,value:null}):{hit:!0,value:t.value}:{hit:!1,value:null}}async set(e,t,n){this.cleanupExpiredEntries(),this.store.set(e,{value:t,expiresAt:Date.now()+n*1e3}),this.enforceSizeLimit()}async delete(e){this.store.delete(e)}cleanupExpiredEntries(){let e=Date.now();for(let[t,n]of this.store)e>=n.expiresAt&&this.store.delete(t)}enforceSizeLimit(){for(;this.store.size>this.maxSize;){let e=this.store.keys().next().value;if(e===void 0)break;this.store.delete(e)}}},C=class{constructor(e=new S){this.inner=e}normalize(e){return e.toLowerCase().trim().replace(/[^\p{L}\p{N}\s]/gu,` `).replace(/\s+/g,` `).trim()}async resolveKey(e){return this.normalize(e)}async get(e){return this.inner.get(this.normalize(e))}async set(e,t,n){await this.inner.set(this.normalize(e),t,n)}async delete(e){await this.inner.delete?.(this.normalize(e))}},w=class{constructor(e,t,n){this.l1=e,this.l2=t,this.l1Ttl=n}async resolveKey(e){return this.l1.resolveKey?this.l1.resolveKey(e):this.l2.resolveKey?this.l2.resolveKey(e):e}async get(e){let t=await this.l1.get(e);if(t.hit)return t;let n=await this.l2.get(e);return n.hit&&await this.l1.set(e,n.value,this.l1Ttl??60),n}async set(e,t,n){await Promise.all([this.l1.set(e,t,this.l1Ttl??n),this.l2.set(e,t,n)])}async delete(e){await Promise.all([this.l1.delete?.(e),this.l2.delete?.(e)])}};function T(e){return typeof e==`object`&&!!e&&`type`in e&&e.type===`tool_calls`&&Array.isArray(e.toolCalls)}const te=new Set([`unknown_tool`,`duplicate_tool_call_id`,`tool_choice_none_violated`,`unexpected_tool_calls`]),ne=e=>e.type===`parse`||e.type===`validation`||e.type===`aborted`||e.type===`quota_exceeded`||e.code&&te.has(e.code)?`stop`:`next`;var E=class extends b{constructor(e){let t=e[e.length-1]?.error;super(`${e.length} provider${e.length===1?``:`s`} attempted and failed: ${e.map(e=>`${e.provider}(${e.error.type})`).join(` then `)}`,`fallback_exhausted`,{status:t?.status,cause:t,retryAfterMs:t?.retryAfterMs,code:`fallback_exhausted`,attempts:e}),this.attempts=e}get retryable(){let e=this.attempts[this.attempts.length-1]?.error;return e?e.retryable:super.retryable}};function re(e){return e instanceof E}async function ie(e,t,n){if(!e.reserveUsage)return!1;try{return await e.reserveUsage({coalesced:t,signal:n}),!0}catch(e){throw n?.aborted?new b(`LLM request aborted`,`aborted`):new b(e instanceof Error?e.message:`Usage reservation failed`,`quota_exceeded`,{cause:e})}}function ae(e,t,n,r){return async i=>{try{await e.refundUsage?.({coalesced:t,signal:n})}catch(e){r(i,e)}}}async function D(e,t,n,r,i){if(r?.aborted)throw new b(`LLM request aborted`,`aborted`);let a=await ie(e,t,r),o=ae(e,t,r,i);if(r?.aborted)throw a&&await o(`[VernLLM] refundUsage failed after abort`),new b(`LLM request aborted`,`aborted`);let s;try{s=await n()}catch(e){throw a&&await o(`[VernLLM] refundUsage failed`),e}if(r?.aborted)throw a&&await o(`[VernLLM] refundUsage failed after abort`),new b(`LLM request aborted`,`aborted`);return s}async function oe(e,t,n,r){if(n?.aborted)throw new b(`LLM request aborted`,`aborted`);let i=await ie(e,!1,n),a=ae(e,!1,n,r);if(n?.aborted)throw i&&await a(`[VernLLM] refundUsage failed after abort`),new b(`LLM request aborted`,`aborted`);let o;try{o=await t()}catch(e){throw i&&await a(`[VernLLM] refundUsage failed after stream-open failure`),e}let s=o.finalResult.then(e=>e,async e=>{throw i&&await a(`[VernLLM] refundUsage failed after stream error`),e});return s.catch(()=>{}),{chunks:o.chunks,finalResult:s}}function se(e){return typeof e==`string`?e:JSON.stringify(e)??``}function ce(e,t){let n=[];if(t){let t=e;t.type===`tool_calls`?(t.toolCalls.forEach((e,t)=>{n.push({type:`tool_call_delta`,index:t,id:e.id,name:e.name,argsDelta:JSON.stringify(e.arguments??{}),complete:!0})}),t.content&&n.push({type:`text-delta`,delta:t.content})):n.push({type:`text-delta`,delta:se(t.content)})}else n.push({type:`text-delta`,delta:se(e)});return{async*[Symbol.asyncIterator](){for(let e of n)yield e}}}function le(e,t){return{async*[Symbol.asyncIterator](){let n=await e;yield*ce(n,t)}}}var ue=class{inFlight=new Map;constructor(e,t){this.cache=e,this.logger=t}async resolveCacheKey(e){return this.cache.resolveKey?await this.cache.resolveKey(e):e}async deleteCache(e){if(this.cache.delete)try{await this.cache.delete(await this.resolveCacheKey(e))}catch(e){this.logger.warn(`[VernLLM] cache delete failed: ${e instanceof Error?e.message:`unknown`}`)}}logRefundError(e,t){this.logger.error(e,{message:t instanceof Error?t.message:`unknown`})}async getCached(e){try{return await this.cache.get(e)}catch(e){return this.logger.warn(`[VernLLM] cache read failed: ${e instanceof Error?e.message:`unknown`}`),{hit:!1}}}async runCached(e){let t=await this.resolveCacheKey(e.cacheKey),n=t===e.cacheKey?e:{...e,cacheKey:t},r=await this.getCached(t);if(r.hit)return r.value;let i=this.inFlight.get(t);return i?D(n,!0,()=>i,e.signal,(e,t)=>this.logRefundError(e,t)):this.registerTrigger(n)}registerTrigger(e){let t=D(e,!1,()=>this.runAndCache(e),e.signal,(e,t)=>this.logRefundError(e,t));return this.inFlight.set(e.cacheKey,t),t.catch(()=>{}).finally(()=>{this.inFlight.delete(e.cacheKey)}),t}async runAndCache(e){let t=await e.fn();try{await this.cache.set(e.cacheKey,t,e.ttl)}catch(e){this.logger.warn(`[VernLLM] cache write failed: ${e instanceof Error?e.message:`unknown`}`)}return t}async runCachedStream(e,t){let n=await this.resolveCacheKey(e.cacheKey),r=n===e.cacheKey?e:{...e,cacheKey:n},i=await this.getCached(n);if(i.hit){let e=i.value;return{chunks:ce(e,t),finalResult:Promise.resolve(e)}}let a=this.inFlight.get(n);if(a){let n=D(r,!0,()=>a,e.signal,(e,t)=>this.logRefundError(e,t));return n.catch(()=>{}),{chunks:le(n,t),finalResult:n}}return this.registerStreamTrigger(r)}registerStreamTrigger(e){let t,n,r=new Promise((e,r)=>{t=e,n=r});this.inFlight.set(e.cacheKey,r),r.catch(()=>{}).finally(()=>{this.inFlight.delete(e.cacheKey)});let i=oe(e,async()=>{let t=await e.openStream(),n=t.finalResult.then(async t=>{try{await this.cache.set(e.cacheKey,t,e.ttl)}catch(e){this.logger.warn(`[VernLLM] cache write failed: ${e instanceof Error?e.message:`unknown`}`)}return t},e=>{throw e});return{chunks:t.chunks,finalResult:n}},e.signal,(e,t)=>this.logRefundError(e,t));return i.then(e=>{e.finalResult.then(t,n)},e=>{n(e)}),i}};function de(){return{state:`closed`,consecutiveFailures:0,openedAt:0,trialInFlight:!1}}const O=``;var fe=class{threshold;cooldownMs;onStateChange;isolateByModel;sharedBucket=de();bucketsByModel=new Map;constructor(e={}){this.threshold=e.threshold??5,this.cooldownMs=e.cooldownMs??3e4,this.onStateChange=e.onStateChange,this.isolateByModel=e.isolateByModel??!1}lookupBucket(e){if(!this.isolateByModel)return this.sharedBucket;let t=e??O;return this.bucketsByModel.get(t)}ensureBucketFor(e){if(!this.isolateByModel)return this.sharedBucket;let t=e??O,n=this.bucketsByModel.get(t);return n||(n=de(),this.bucketsByModel.set(t,n)),n}transition(e,t,n){if(t===e.state)return;let r=e.state;e.state=t,this.onStateChange?.(r,t,e.consecutiveFailures,n)}assertClosed(e){let t=this.ensureBucketFor(e);if(t.state!==`closed`){if(t.state===`open`){let n=Date.now()-t.openedAt;if(n<this.cooldownMs)throw new b(`Circuit open, provider has failed ${t.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs-n)/1e3)}s.`,`circuit_open`,{code:`circuit_cooling_down`});t.trialInFlight=!0,this.transition(t,`half-open`,e);return}if(t.trialInFlight)throw new b(`Circuit half-open. A trial request is already in flight. Try again shortly.`,`circuit_open`,{code:`circuit_trial_in_flight`});t.trialInFlight=!0}}recordSuccess(e){let t=this.lookupBucket(e);t&&(t.consecutiveFailures=0,t.trialInFlight=!1,this.transition(t,`closed`,e),this.isolateByModel&&t.state===`closed`&&t.consecutiveFailures===0&&this.bucketsByModel.delete(e??O))}recordFailure(e){let t=this.ensureBucketFor(e);if(t.consecutiveFailures+=1,t.trialInFlight=!1,t.state===`half-open`){t.openedAt=Date.now(),this.transition(t,`open`,e);return}t.consecutiveFailures>=this.threshold&&(t.openedAt=Date.now(),this.transition(t,`open`,e))}getState(e){return this.lookupBucket(e)?.state??`closed`}open(e){let t=this.ensureBucketFor(e);t.openedAt=Date.now(),t.trialInFlight=!1,this.transition(t,`open`,e)}close(e){let t=this.ensureBucketFor(e);t.consecutiveFailures=0,t.trialInFlight=!1,this.transition(t,`closed`,e),this.isolateByModel&&t.state===`closed`&&t.consecutiveFailures===0&&this.bucketsByModel.delete(e??O)}};function k(e,t){return n=>{if(e)try{e(n)}catch(e){t.error(`[VernLLM] onEvent failed`,{message:e instanceof Error?e.message:`unknown`})}}}function pe(e,t,n,r,i){if(!e)return;let a=typeof e==`object`?e:void 0,o=a?.onStateChange,s=k(r,i);return new fe({...a,onStateChange:(e,r,a,c)=>{if(s({kind:`circuit_state`,provider:t,model:c??n,from:e,to:r,consecutiveFailures:a}),o)try{o(e,r,a,c)}catch(e){i.error(`[VernLLM] circuitBreaker.onStateChange failed`,{message:e instanceof Error?e.message:`unknown`})}}})}const me=1e4,he=2147483647;function A(e){return!e||e<=0||e===1/0?void 0:e}function j(e){return Math.min(e,he)}async function M(e,t,n){let r=new AbortController,i=A(t),a=i===void 0?void 0:setTimeout(()=>{r.abort()},j(i)),o=n?AbortSignal.any([n,r.signal]):r.signal;try{return await e(o)}catch(e){throw r.signal.aborted&&!n?.aborted&&e instanceof DOMException&&e.name===`AbortError`?new b(`Request timed out`,`timeout`,{code:`request_timeout`}):e}finally{clearTimeout(a)}}function ge(e,t,n,r){let i=A(t);if(i===void 0)return e();let a=!1;return new Promise((t,o)=>{let s=setTimeout(()=>{a=!0,n?.(),o(new b(`No stream chunk received for ${i}ms (idle timeout)`,`timeout`,{code:`idle_timeout`}))},j(i));e().then(e=>{if(clearTimeout(s),a){r?.debug(`[VernLLM] chunk resolved after idle timeout already fired; discarding`);return}a=!0,t(e)},e=>{if(clearTimeout(s),a){r?.debug(`[VernLLM] chunk rejection arrived after idle timeout already fired; discarding`);return}a=!0,o(e)})})}function N(e,t=me){if(!e||typeof e!=`object`)return;let n=e,r=n.headers??n.response?.headers;if(!r||typeof r!=`object`)return;let i=r,a=typeof i.get==`function`?i.get(`Retry-After`):Object.entries(r).find(([e])=>e.toLowerCase()===`retry-after`)?.at(1);if(typeof a!=`string`||a.trim()===``)return;let o=a.trim();if(/^\d+$/.test(o))return Math.max(0,Math.min(Number(o)*1e3,t));let s=Date.parse(o);if(!Number.isNaN(s))return Math.max(0,Math.min(s-Date.now(),t))}function _e(e,t,n=me){let r=Math.min(e*2**t,n);return r/2+Math.random()*(r/2)}async function ve(e,t){if(t?.aborted)throw new b(`Operation aborted`,`aborted`);await new Promise((n,r)=>{let i=()=>{clearTimeout(a),r(new b(`Operation aborted`,`aborted`))},a=setTimeout(()=>{t?.removeEventListener(`abort`,i),n()},e);t?.addEventListener(`abort`,i,{once:!0})})}function P(e){if(!e||typeof e!=`object`)return;let t=e;if(typeof t.status==`number`)return t.status;if(typeof t.statusCode==`number`)return t.statusCode;if(typeof t.$metadata?.httpStatusCode==`number`)return t.$metadata.httpStatusCode}const F=new Set([`ECONNREFUSED`,`ENOTFOUND`,`ECONNRESET`,`ETIMEDOUT`,`EAI_AGAIN`,`EPIPE`,`ECONNABORTED`,`EHOSTUNREACH`,`ENETUNREACH`]),ye=new Set([`fetch failed`,`failed to fetch`,`load failed`,`networkerror when attempting to fetch resource.`]);function be(e){if(!e||typeof e!=`object`)return!1;let t=e;if(typeof t.code==`string`&&F.has(t.code)||typeof t.message==`string`&&ye.has(t.message.toLowerCase()))return!0;if(t.cause&&typeof t.cause==`object`){let e=t.cause;if(typeof e.code==`string`&&F.has(e.code))return!0}return!1}function I(e){try{return JSON.stringify(e,null,2)??String(e)}catch{try{return String(e)}catch{return`[unprintable error]`}}}function L(e){if(e&&typeof e==`object`)try{let t=e;if(t.error!==void 0)return I(t.error);if(typeof t.message==`string`)return t.message}catch{}return I(e)}function R(e){switch(e){case 429:return`provider_rate_limited`;case 401:return`authentication`;case 403:return`authorization`;case 404:return`not_found`;case 413:return`payload_too_large`;default:return e>=500?`server_error`:void 0}}const xe=/\(no body\)/i;function Se(e){return Object.keys(e).length===0}function Ce(e){if(e&&typeof e==`object`){let{error:t,message:n}=e;if(t!=null){let e=typeof t==`string`&&t.trim().length===0,n=typeof t==`object`&&Se(t);if(!e&&!n)return!1}if(typeof n==`string`){let e=n.trim();return e.length===0||xe.test(e)}return!0}return!0}function z(e,t,n){if(t?.aborted)return new b(`LLM request aborted`,`aborted`,{attempts:n});if(e instanceof b)return e.code===void 0&&e.status!==void 0&&(e.code=R(e.status)),e.attempts===void 0&&n!==void 0&&(e.attempts=n),e;let r=P(e),i=N(e);if(r!==void 0){let t=L(e),a=R(r),o=a===void 0,s=Ce(e)?o?`LLM request failed with status ${r} and no error detail from the provider. This usually means a field or value in the request isn't supported by the specific model (for example, a reasoning/thinking parameter the model doesn't accept), rather than a transport or auth problem.`:`LLM request failed with status ${r} and no error detail from the provider.`:`LLM request failed: ${t}`;return new b(s,`api`,{status:r,cause:e,retryAfterMs:i,code:a,attempts:n})}return be(e)?new b(`LLM request failed`,`network`,{cause:e,retryAfterMs:i,code:`connection_failed`,attempts:n}):new b(`LLM request failed`,`unknown`,{cause:e,retryAfterMs:i,attempts:n})}function we(e){try{return JSON.parse(e)}catch{return}}function Te(e){return e.map(e=>({type:`function`,function:{name:e.name,description:e.description,parameters:e.parameters}}))}function Ee(e){return e.map(e=>({id:e.id,type:`function`,function:{name:e.name,arguments:JSON.stringify(e.arguments??{})}}))}function De(e){return e.map(e=>{let t;try{t=e.function.arguments.trim()?JSON.parse(e.function.arguments):{}}catch{throw new b(`Invalid JSON arguments for tool call "${e.function.name}"`,`parse`,{code:`tool_arguments_parse_failed`})}return{id:e.id,name:e.function.name,arguments:t}})}function B(e){return typeof e==`string`?e:JSON.stringify(e)}var Oe=class{model;defaultMaxTokens;defaultTemperature;defaultReasoningEffort;defaultBudgetTokens;supportsJsonObjectMode;constructor(e){this.model=e.model,this.defaultMaxTokens=e.defaultMaxTokens,this.defaultTemperature=e.defaultTemperature,this.defaultReasoningEffort=e.defaultReasoningEffort,this.defaultBudgetTokens=e.defaultBudgetTokens,this.supportsJsonObjectMode=e.supportsJsonObjectMode}build(e){let{systemPrompt:t,userContent:n,history:r=[],maxTokens:i=this.defaultMaxTokens,model:a=this.model,jsonSchema:o,tools:s,toolChoice:c}=e,l=e.temperature===void 0?this.defaultTemperature:e.temperature,u=e.reasoningEffort===void 0?this.defaultReasoningEffort:e.reasoningEffort,d=e.budgetTokens===void 0?this.defaultBudgetTokens:e.budgetTokens;if(s&&s.length===0)throw new b("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.",`invalid_params`);if(s){let e=new Set,t=new Set;for(let n of s)e.has(n.name)&&t.add(n.name),e.add(n.name);if(t.size)throw new b(`\`tools\` has duplicate name(s): [${[...t].join(`, `)}]. Tool names must be unique.`,`invalid_params`,{code:`duplicate_tool_names`,issues:{names:[...t]}})}if(c&&!s)throw new b("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.",`invalid_params`);if(s&&typeof c==`object`&&!s.some(e=>e.name===c.name))throw new b(`toolChoice names "${c.name}", which is not in \`tools\` ([${s.map(e=>e.name).join(`, `)}]).`,`invalid_params`,{code:`unknown_tool_choice`,issues:{requested:c.name,available:s.map(e=>e.name)}});let f=e.jsonMode,p=f??!s;if(!this.supportsJsonObjectMode&&!o&&f===!0)throw new b('jsonMode: true was set explicitly, but this client does not support `response_format: "json_object"` (see LLMClient.supportsJsonObjectMode). Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output for this mode. Use `jsonSchema` instead, which maps to a real constraint on both.',`invalid_params`);if(!this.supportsJsonObjectMode&&!o&&f===void 0&&e.schema)throw new b('`schema` was provided, which requires JSON output to validate against, but this client does not support `response_format: "json_object"` (see LLMClient.supportsJsonObjectMode) and no `jsonSchema` was set. Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output without one. Use `jsonSchema` instead, which maps to a real constraint on both and still runs `schema` against its parsed result.',`invalid_params`);let m=!this.supportsJsonObjectMode&&!o&&f===void 0?!1:p,h=m||!!o;if(e.schema&&!h)throw new b(`schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.`,`invalid_params`);let g=this.buildResponseFormat(o,h);this.validateHistory(r);let _={model:a,...l===null?{}:{temperature:l},max_tokens:i,...g?{response_format:g}:{},...u?{reasoning_effort:u}:{},...d==null?{}:{budget_tokens:d},...s?{tools:Te(s)}:{},...s?{tool_choice:this.buildWireToolChoice(c)}:{},messages:[...t?[{role:`system`,content:t}]:[],...r.flatMap(e=>this.turnToWireMessages(e)),{role:`user`,content:n}]};return{useJson:h,model:a,request:_}}validateHistory(e){let t;for(let[n,r]of e.entries()){if(r.role===`tool`){if(t?.role!==`assistant`||!t.toolCalls?.length)throw new b(`history[${n}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`,`invalid_params`);if(!r.toolResults?.length)throw new b(`history[${n}] is a "tool" turn but has no toolResults`,`invalid_params`);let e=new Set(t.toolCalls.map(e=>e.id)),i=r.toolResults.map(e=>e.toolCallId),a=i.filter(t=>!e.has(t));if(a.length)throw new b(`history[${n}].toolResults references unknown toolCallId(s) [${a.join(`, `)}]`,`invalid_params`,{code:`unknown_tool_result_ids`,issues:{historyIndex:n,ids:a}});let o=new Set,s=new Set;for(let e of i)o.has(e)&&s.add(e),o.add(e);if(s.size)throw new b(`history[${n}].toolResults has duplicate toolCallId(s) [${[...s].join(`, `)}]`,`invalid_params`,{code:`duplicate_tool_result_ids`,issues:{historyIndex:n,ids:[...s]}});let c=[...e].filter(e=>!i.includes(e));if(c.length)throw new b(`history[${n}] is missing toolResults for toolCallId(s) [${c.join(`, `)}]`,`invalid_params`,{code:`missing_tool_results`,issues:{historyIndex:n,ids:c}})}else{if(r.role===t?.role)throw new b(`history must alternate user/assistant turns: consecutive "${r.role}" turns at history[${n-1}] and history[${n}]`,`invalid_params`);if(t?.role===`assistant`&&t.toolCalls?.length)throw new b(`history[${n}] follows an assistant tool request without tool results`,`invalid_params`)}t=r}if(t?.role===`assistant`&&t.toolCalls?.length)throw new b(`The last entry in history is an assistant tool request without tool results`,`invalid_params`);if(t?.role===`user`)throw new b(`The last entry in history is a "user" turn, which would collide with the current userContent turn.`,`invalid_params`)}buildWireToolChoice(e){return!e||e===`auto`?`auto`:e===`none`||e===`required`?e:{type:`function`,function:{name:e.name}}}turnToWireMessages(e){return e.role===`tool`?(e.toolResults??[]).map(e=>({role:`tool`,tool_call_id:e.toolCallId,content:typeof e.content==`string`?e.content:JSON.stringify(e.content??null),...e.isError?{is_error:!0}:{}})):e.role===`assistant`&&e.toolCalls?.length?[{role:`assistant`,...e.content===void 0?{}:{content:B(e.content)},tool_calls:Ee(e.toolCalls)}]:e.role===`assistant`?[{role:`assistant`,content:B(e.content===void 0?``:e.content)}]:[{role:e.role,content:e.content??``}]}buildResponseFormat(e,t){return e?{type:`json_schema`,json_schema:{name:e.name,schema:e.schema,strict:e.strict??!0,description:e.description}}:t?{type:`json_object`}:void 0}};function ke(e,t,n){let{requestId:r,model:i,providerName:a,isFallback:o,chunkIdleTimeoutMs:s,streamController:c,logger:l,signal:u}=n,d,f,p=new Promise((e,t)=>{d=e,f=t});p.catch(()=>{});let m=1e4,h=[],g=[],_=!1,v,y=!1,b=e=>{let t=g.shift();if(t){t.resolve({done:!1,value:e});return}h.push(e),h.length>m*2&&(y||(y=!0,l.warn(`[VernLLM] stream chunk buffer exceeded cap (${m}), evicting ${h.length-m} oldest chunk(s); buffered=${h.length}. The chunks iterable was never read (or fell far behind) for this stream.`)),h.splice(0,h.length-m))},x=()=>{_=!0;for(let e of g.splice(0))e.resolve({done:!0,value:void 0})},ee=e=>{_=!0,v=e;for(let t of g.splice(0))t.reject(e)},S={[Symbol.asyncIterator](){return{next(){return h.length?Promise.resolve({done:!1,value:h.shift()}):_?v?Promise.reject(v):Promise.resolve({done:!0,value:void 0}):new Promise((e,t)=>{g.push({resolve:e,reject:t})})}}}},C=new Map,w=``,T;return(async()=>{try{let n=t;for(;!n.done;){let t=n.value;if(t.type!==`ping`){if(t.type===`text-delta`)w+=t.delta,b({type:`text-delta`,delta:t.delta});else if(t.type===`tool_call_delta`){let e=C.get(t.index)??{args:``};e.id??=t.id,e.name??=t.name,e.args+=t.argumentsDelta??``,C.set(t.index,e),b({type:`tool_call_delta`,index:t.index,id:t.id,name:t.name,argsDelta:t.argumentsDelta,complete:t.complete})}else if(t.type===`usage`){let e=t.usage.completion_tokens_details?.reasoning_tokens;T={promptTokens:t.usage.prompt_tokens??0,completionTokens:t.usage.completion_tokens??0,totalTokens:t.usage.total_tokens??0,...e===void 0?{}:{reasoningTokens:e},requestId:r,model:i,provider:a,usedFallback:o},b({type:`usage`,usage:T})}}n=await ge(()=>e.next(),s,()=>c.abort(),l)}}catch(t){try{await e.return?.()}catch{}c.abort();let r=z(t,u);try{n.onStreamFailure(r,T)}catch{}ee(r),f(r);return}x();try{n.onStreamSuccess(T)}catch{}try{let e=C.size?[...C.entries()].sort(([e],[t])=>e-t).map(([,e])=>({id:e.id??``,type:`function`,function:{name:e.name??``,arguments:e.args}})):void 0,t=n.finalize(w,e,T);d(t)}catch(e){f(e)}})(),{chunks:S,finalResult:p}}function Ae(e){return e}var je=class{maxRetries;timeoutMs;chunkIdleTimeoutMs;baseDelayMs;nonRetryableStatus;parseJson;logger;redact;onUsage;onUsageFailure;reportEvent;breaker;limiter;isFallback;requestBuilder;constructor(e,t,n,r){this.providerName=e,this.client=t,this.model=n,this.maxRetries=r.maxRetries,this.timeoutMs=r.timeoutMs,this.chunkIdleTimeoutMs=r.chunkIdleTimeoutMs,this.baseDelayMs=r.baseDelayMs,this.nonRetryableStatus=r.nonRetryableStatus,this.parseJson=r.parseJson??we,this.logger=r.logger,this.redact=r.redact,this.onUsage=r.onUsage,this.onUsageFailure=r.onUsageFailure,this.reportEvent=k(r.onEvent,this.logger),this.breaker=r.breaker,this.limiter=r.limiter,this.isFallback=r.isFallback??!1,this.requestBuilder=new Oe({model:n,defaultMaxTokens:r.defaultMaxTokens,defaultTemperature:r.defaultTemperature,defaultReasoningEffort:r.defaultReasoningEffort,defaultBudgetTokens:r.defaultBudgetTokens,supportsJsonObjectMode:t.supportsJsonObjectMode??!0})}getCircuitState(e){return this.breaker?.getState(e)}get isolateByModel(){return this.breaker?.isolateByModel??!1}openCircuit(e){this.breaker?.open(e)}closeCircuit(e){this.breaker?.close(e)}assertBreakerClosed(e){this.breaker?.assertClosed(e??this.model)}async run(e,t,n){let r=e.model??this.model,i=[];try{return await this.retryWithBackoff((n,r)=>this.executeCall(e,t,n,r),t,r,e.signal,n,i)}catch(n){let a=z(n,e.signal,i.length>0?i:void 0);throw this.countsTowardBreaker(a)&&this.breaker?.recordFailure(r),this.logger.debug(`[VernLLM:${t}] error:\n${this.redactText(L(n))}`),a}}async runStream(e,t,n){let r=e.model??this.model,i=[];try{return await this.retryWithBackoff((n,r)=>this.executeStreamCall(e,t,n,r),t,r,e.signal,n,i)}catch(n){let a=z(n,e.signal,i.length>0?i:void 0);throw this.countsTowardBreaker(a)&&this.breaker?.recordFailure(r),this.logger.debug(`[VernLLM:${t}] stream-open error:\n${this.redactText(L(n))}`),a}}async executeCall(e,t,n,r){let{useJson:i,model:a,request:o}=this.requestBuilder.build(e);r?.(y(this.providerName,a,o,void 0,Date.now()));let s;if(this.limiter){let n=await this.limiter.acquire(this.limiter.estimate(o),e.signal);s=n.release,n.waitedMs>0&&this.reportEvent({kind:`rate_limited`,requestId:t,provider:this.providerName,model:a,waitedMs:n.waitedMs,reason:n.reason??`rpm`})}try{let r=await M(e=>this.client.chat.completions.create(o,{signal:e}),this.timeoutMs,e.signal),c=this.extractUsage(r,t,a);s?.(this.actualTokensFor(c)),s=void 0;let l=r.choices?.[0]?.message?.content,u=r.choices?.[0]?.message?.tool_calls;return this.finalizeResponse(l,u,e,i,a,c,t,n)}finally{s?.()}}redactText(e){return this.redact?this.redact(e):e}redactedOutput(e,t){return this.redactText(e??`[${t?.length??0} tool call(s)]`)}finalizeResponse(e,t,n,r,i,a,o,s){try{let s=e?.trim();if(!s&&!t?.length)throw new b(`Empty LLM response`,`api`,{code:`empty_response`});if(this.logger.debug(`[VernLLM:${o}] output:\n${this.redactedOutput(s,t).slice(0,800)}`),t?.length){if(!n.tools)throw new b("Provider returned tool_calls but no `tools` were sent with this call.",`validation`,{code:`unexpected_tool_calls`});if(n.toolChoice===`none`)throw new b(`Provider returned tool_calls despite toolChoice: 'none'.`,`validation`,{code:`tool_choice_none_violated`});let e=De(t);return this.validateToolCallArguments(e,n.tools),this.breaker?.recordSuccess(i),this.reportUsage(a),{type:`tool_calls`,toolCalls:e,...s?{content:s}:{}}}let c=s??``;if(!r)return this.breaker?.recordSuccess(i),this.reportUsage(a),n.tools?{type:`content`,content:c}:c;let l=this.parseAndValidate(c,n.schema);return this.breaker?.recordSuccess(i),this.reportUsage(a),n.tools?{type:`content`,content:l}:l}catch(e){let t=z(e,n.signal);throw a&&t.type!==`aborted`&&this.reportUsageFailure(a,t,s),t}}async executeStreamCall(e,t,n,r){let{useJson:i,model:a,request:o}=this.requestBuilder.build(e);r?.(y(this.providerName,a,o,void 0,Date.now()));let s=this.client.chat.completions;if(!s.createStream)throw new b(`stream: true requires a client/adapter with createStream`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`createStream`}});let c=s.createStream.bind(s),l;if(this.limiter){let n=await this.limiter.acquire(this.limiter.estimate(o),e.signal);l=n.release,n.waitedMs>0&&this.reportEvent({kind:`rate_limited`,requestId:t,provider:this.providerName,model:a,waitedMs:n.waitedMs,reason:n.reason??`rpm`})}let u=new AbortController,d=e.signal?AbortSignal.any([e.signal,u.signal]):u.signal;try{let{iterator:r,first:s}=await M(async e=>{let t=c(o,{signal:e})[Symbol.asyncIterator](),n=await t.next();return{iterator:t,first:n}},this.timeoutMs,d);if(s.done)throw new b(`Empty LLM response`,`api`);let f=l,p=ke(r,s,{requestId:t,model:a,providerName:this.providerName,isFallback:this.isFallback,chunkIdleTimeoutMs:e.chunkIdleTimeoutMs??this.chunkIdleTimeoutMs,streamController:u,logger:this.logger,signal:e.signal,onStreamSuccess:e=>{this.breaker?.recordSuccess(a),f?.(this.actualTokensFor(e))},onStreamFailure:(e,t)=>{e.type===`timeout`&&this.breaker?.recordFailure(a),t&&e.type!==`aborted`&&this.reportUsageFailure(t,e,n,!0),f?.(this.actualTokensFor(t))},finalize:(r,o,s)=>this.finalizeResponse(r,o,e,i,a,s,t,n)});return l=void 0,p}finally{l?.()}}validateToolCallArguments(e,t){let n=new Map(t.map(e=>[e.name,e])),r=new Set,i=[];for(let t of e)r.has(t.id)&&i.push({name:t.name,toolCallId:t.id,code:`duplicate_tool_call_id`}),r.add(t.id),n.has(t.name)||i.push({name:t.name,toolCallId:t.id,code:`unknown_tool`});if(i.length>0){let e=i.find(e=>e.code===`unknown_tool`),t=e?`Model requested tool "${e.name}", which was not in the tools offered ([${[...n.keys()].join(`, `)}]).`:`Duplicate tool call id "${i[0].toolCallId}" in the model's response.`,r=i.length>1?`${t} (${i.length} tool call issues total, see error.issues.)`:t;throw new b(r,`validation`,{code:e?`unknown_tool`:`duplicate_tool_call_id`,issues:i})}for(let t of e){let e=n.get(t.name);if(!e?.argumentsSchema)continue;let r=e.argumentsSchema.safeParse(t.arguments);if(!r.success)throw new b(`Arguments for tool call "${t.name}" failed validation`,`validation`,{issues:r.error})}}async retryWithBackoff(e,t,n,r,i,a){let o,s;for(let c=0;c<=this.maxRetries;c++){s=void 0;try{return c>0&&await this.recoverDelay(t,n,c,o,r),i?.(),await e(c,e=>{s=e})}catch(e){o=e;let t=c<this.maxRetries&&this.shouldRetry(e,r);if(!t)break;a?.push({index:c,error:z(e,r).toSnapshot(),request:Ae(s)})}}throw o}extractUsage(e,t,n){if(!e.usage)return;let r=e.usage.completion_tokens_details?.reasoning_tokens;return{promptTokens:e.usage.prompt_tokens??0,completionTokens:e.usage.completion_tokens??0,totalTokens:e.usage.total_tokens??0,...r===void 0?{}:{reasoningTokens:r},requestId:t,model:n,provider:this.providerName,usedFallback:this.isFallback}}actualTokensFor(e){if(e)return e.totalTokens||e.promptTokens+e.completionTokens}reportUsage(e){if(!(!e||!this.onUsage))try{this.onUsage(e)}catch(e){this.logger.error(`[VernLLM] onUsage failed`,{message:e instanceof Error?e.message:`unknown`})}}reportUsageFailure(e,t,n,r=!1){let i=e.totalTokens||e.promptTokens+e.completionTokens,a=r?`mid-stream failure (terminal, no further attempts)`:`attempt ${n+1}/${this.maxRetries+1}`;if(this.logger.warn(`[VernLLM:${e.requestId}] usage failure, ${a}: type=${t.type} tokens=${i}`),this.onUsageFailure)try{this.onUsageFailure(e,t)}catch(e){this.logger.error(`[VernLLM] onUsageFailure failed`,{message:e instanceof Error?e.message:`unknown`})}}parseAndValidate(e,t){let n;try{n=this.parseJson(e)}catch{throw new b(`Invalid JSON response`,`parse`)}if(n==null)throw new b(`Invalid JSON response`,`parse`);if(!t)return n;let r=t.safeParse(n);if(!r.success)throw new b(`Schema validation failed`,`validation`,{issues:r.error});return r.data}async recoverDelay(e,t,n,r,i){let a=N(r),o=a??_e(this.baseDelayMs,n),s=a!==void 0;this.logger.warn(`[VernLLM:${e}] recovery attempt ${n}/${this.maxRetries}, waiting ${Math.ceil(o)}ms`+(s?` (honoring Retry-After)`:``)),this.reportEvent({kind:`retry`,requestId:e,provider:this.providerName,model:t,attempt:n,maxRetries:this.maxRetries,delayMs:o,retryAfterHonored:s,error:z(r,i)}),await ve(o,i)}shouldRetry(e,t){if(t?.aborted||e instanceof b&&!e.retryable)return!1;let n=P(e);return!(n!==void 0&&this.nonRetryableStatus.includes(n))}countsTowardBreaker(e){return e.retryable}};function Me(e){return{debug:V(e,`debug`),warn:V(e,`warn`),error:V(e,`error`)}}function V(e,t){let n=e[t].bind(e);return(...e)=>{try{Pe(n(...e))}catch{}}}function Ne(e){return typeof e?.then==`function`}function Pe(e){Ne(e)&&Promise.resolve(e).catch(()=>{})}var Fe=class{constructor(e){this.debugEnabled=e}debug(e){this.debugEnabled&&console.debug(e)}warn(e){console.warn(e)}error(e,t){console.error(e,t??``)}};function Ie(e){let t=e.messages.reduce((e,t)=>{let n=t.content;if(typeof n==`string`)return e+n.length;if(n==null)return e;try{return e+JSON.stringify(n).length}catch{return e}},0);return Math.ceil(t/4)+(e.max_tokens??0)}var H=class{available;lastRefill=Date.now();constructor(e,t){this.capacity=e,this.refillPerMs=t,this.available=e}refill(){if(this.refillPerMs===0)return;let e=Date.now(),t=e-this.lastRefill;this.available=Math.min(this.capacity,this.available+Math.max(0,t)*this.refillPerMs),this.lastRefill=e}tryTake(e){return this.refill(),this.available<e?!1:(this.available-=e,!0)}msUntilAvailable(e){return this.refill(),this.available>=e?0:this.refillPerMs===0?1/0:(e-this.available)/this.refillPerMs}give(e){this.available=Math.min(this.capacity,this.available+e)}getCapacity(){return this.capacity}};const Le=2147483647;var Re=class{requests;tokens;concurrency;maxQueueMs;maxQueueSize;estimateTokensFn;queue=[];wakeTimer;constructor(e){e.requestsPerMinute&&(this.requests=new H(e.requestsPerMinute,e.requestsPerMinute/6e4)),e.tokensPerMinute&&(this.tokens=new H(e.tokensPerMinute,e.tokensPerMinute/6e4)),e.maxConcurrent&&(this.concurrency=new H(e.maxConcurrent,0)),this.maxQueueMs=e.maxQueueMs??3e4,this.maxQueueSize=e.maxQueueSize??0,this.estimateTokensFn=e.estimateTokens??Ie}estimate(e){return this.estimateTokensFn(e)}async acquire(e,t){if(t?.aborted)throw new b(`LLM request aborted`,`aborted`);if(!Number.isFinite(e)||e<0)throw new b(`estimatedTokens must be a finite, non-negative number, got ${String(e)}`,`invalid_params`);if(this.tokens&&e>this.tokens.getCapacity())throw new b(`estimatedTokens (${e}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`,`rate_limited`,{code:`rate_limit_capacity_exceeded`});if(this.queue.length===0){let n=this.tryAcquireBuckets(e);return n.ok?{release:this.makeRelease(e),waitedMs:0}:this.enqueue(e,n.reason,t)}if(this.maxQueueSize>0&&this.queue.length>=this.maxQueueSize)throw this.queueFullError();return this.enqueue(e,void 0,t)}queueFullError(){return new b(`Rate limit queue is full`,`rate_limited`,{code:`rate_limit_queue_full`})}enqueue(e,t,n){return new Promise((r,i)=>{let a={estimatedTokens:e,enqueuedAt:Date.now(),lastReason:t,resolve:e=>{c(),r(e)},reject:e=>{c(),this.wakeTimer&&(clearTimeout(this.wakeTimer),this.wakeTimer=void 0),this.drain(),i(e)}},o,s=()=>{a.reject(new b(`LLM request aborted`,`aborted`))},c=()=>{o&&clearTimeout(o),n?.removeEventListener(`abort`,s);let e=this.queue.indexOf(a);e!==-1&&this.queue.splice(e,1)};this.maxQueueMs>0&&(o=setTimeout(()=>{a.reject(new b(`Rate limit queue timed out before capacity was available`,`rate_limited`,{code:`rate_limit_queue_timeout`}))},this.maxQueueMs)),n?.addEventListener(`abort`,s,{once:!0}),this.queue.push(a),this.drain()})}tryAcquireBuckets(e){let t=[],n=(e,n)=>e?e.tryTake(n)?(t.push({bucket:e,amount:n}),!0):!1:!0;if(!n(this.concurrency,1))return{ok:!1,reason:`concurrency`};if(!n(this.requests,1)){for(let e of t)e.bucket.give(e.amount);return{ok:!1,reason:`rpm`}}if(!n(this.tokens,e)){for(let e of t)e.bucket.give(e.amount);return{ok:!1,reason:`tpm`}}return{ok:!0}}drain(){for(;this.queue.length>0;){let e=this.queue[0],t=this.tryAcquireBuckets(e.estimatedTokens);if(!t.ok){e.lastReason=t.reason,this.scheduleWake(t.reason,e.estimatedTokens);return}let n=Date.now()-e.enqueuedAt;e.resolve({release:this.makeRelease(e.estimatedTokens),waitedMs:n,reason:e.lastReason})}}scheduleWake(e,t){if(this.wakeTimer)return;let n=e===`rpm`?this.requests?.msUntilAvailable(1):e===`tpm`?this.tokens?.msUntilAvailable(t):void 0;if(n===void 0||!Number.isFinite(n))return;let r=Math.min(Math.max(1,Math.ceil(n)),Le);this.wakeTimer=setTimeout(()=>{this.wakeTimer=void 0,this.drain()},r)}makeRelease(e){let t=!1;return n=>{t||(t=!0,this.concurrency?.give(1),this.tokens&&n!==void 0&&Number.isFinite(n)&&this.tokens.give(e-n),this.drain())}}},ze=class{logger;executors;fallbackOn;reportEvent;cacheOrchestrator;constructor(e){this.logger=Me(e.logger??new Fe(e.debug??!1));let t=e.name??`primary`;this.cacheOrchestrator=new ue(e.cache??new S,this.logger),this.fallbackOn=e.fallbackOn??ne,this.reportEvent=k(e.onEvent,this.logger);let n=e.defaultTemperature===void 0?.2:e.defaultTemperature,r=e.defaultReasoningEffort,i=e.defaultBudgetTokens,a={client:e.client,model:e.model,name:t,maxRetries:e.maxRetries,timeoutMs:e.timeoutMs,chunkIdleTimeoutMs:e.chunkIdleTimeoutMs,baseDelayMs:e.baseDelayMs,defaultMaxTokens:e.defaultMaxTokens,defaultTemperature:n,defaultReasoningEffort:r,defaultBudgetTokens:i,nonRetryableStatus:e.nonRetryableStatus,circuitBreaker:e.circuitBreaker,rateLimit:e.rateLimit},o=Array.isArray(e.fallback)?e.fallback:e.fallback?[e.fallback]:[],s=[a,...o];this.executors=s.map((a,o)=>{let s=o>0,c=a.name??(s?`fallback[${o-1}]`:t),l=pe(a.circuitBreaker,c,a.model,e.onEvent,this.logger);return new je(c,a.client,a.model,{maxRetries:a.maxRetries??e.maxRetries??1,timeoutMs:a.timeoutMs??e.timeoutMs??25e3,chunkIdleTimeoutMs:a.chunkIdleTimeoutMs??e.chunkIdleTimeoutMs??3e4,baseDelayMs:a.baseDelayMs??e.baseDelayMs??500,defaultMaxTokens:a.defaultMaxTokens??e.defaultMaxTokens??1e3,defaultTemperature:a.defaultTemperature===void 0?n:a.defaultTemperature,defaultReasoningEffort:a.defaultReasoningEffort===void 0?r:a.defaultReasoningEffort,defaultBudgetTokens:a.defaultBudgetTokens===void 0?i:a.defaultBudgetTokens,nonRetryableStatus:a.nonRetryableStatus??e.nonRetryableStatus??[400,401,403,404,422],parseJson:e.parseJson,logger:this.logger,redact:e.redact,onUsage:e.onUsage,onUsageFailure:e.onUsageFailure,onEvent:e.onEvent,breaker:l,limiter:a.rateLimit?new Re(a.rateLimit):void 0,isFallback:s})})}logRefundError(e,t){this.logger.error(e,{message:t instanceof Error?t.message:`unknown`})}async runFallbackChain(e,t,n,r=!1){let i=[];for(let a=0;a<this.executors.length;a++){let o=this.executors[a],s=Date.now(),c=0;try{a===0&&r||o.assertBreakerClosed(e.model);let t=await n(o,()=>{c+=1});return{result:t,executor:o,index:a,attemptCount:c}}catch(n){let r=z(n,e.signal);i.push({index:a-1,provider:o.providerName,model:e.model??o.model,error:r.toSnapshot()});let c=a===this.executors.length-1,l=this.fallbackOn(r,{isLastTarget:c}),u=c?`stop`:l;if(u===`stop`)throw i.length>1?new E(i):r;let d=this.executors[a+1];this.reportEvent({kind:`fallback`,requestId:t,from:o.providerName,to:d.providerName,fromIndex:a-1,toIndex:a,error:r,elapsedMs:Date.now()-s})}}throw new b(`No provider targets configured`,`invalid_params`)}async call(e){if(e.signal?.aborted)throw new b(`LLM request aborted`,`aborted`);let t=e.requestId??(0,c.randomUUID)(),n=this.executors.length===1;return n&&this.executors[0].assertBreakerClosed(e.model),e.stream?oe(e,async()=>{let{result:r}=await this.runFallbackChain(e,t,(n,r)=>n.runStream(e,t,r),n);return r},e.signal,(e,t)=>this.logRefundError(e,t)):D(e,!1,async()=>{let{result:r,executor:i,index:a,attemptCount:o}=await this.runFallbackChain(e,t,(n,r)=>n.run(e,t,r),n);return e.meta&&(e.meta.current={provider:i.providerName,model:e.model??i.model,fallbackIndex:a-1,usedFallback:a>0,attempts:o}),r},e.signal,(e,t)=>this.logRefundError(e,t))}runCached(e){return this.cacheOrchestrator.runCached(e)}async deleteCache(e){await this.cacheOrchestrator.deleteCache(e)}async cachedCall(e){let{call:t,...n}=e,r=t;if(r.reserveUsage||r.refundUsage)throw new b("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.",`invalid_params`);if(r.stream){let e=r;return this.cacheOrchestrator.runCachedStream({...n,openStream:()=>this.call(e)},!!r.tools)}return this.runCached({...n,fn:()=>this.call(r)})}getCircuitState(e){let t=this.resolveExecutor(e?.index??0,`getCircuitState`);return this.warnIfModelUnsupported(t.isolateByModel,e?.model,`getCircuitState`),t.getCircuitState(e?.model??t.model)}getCircuitStates(e){return this.executors.map((t,n)=>({provider:t.providerName,index:n,isFallback:n>0,isolateByModel:t.isolateByModel,state:t.getCircuitState(e??t.model)}))}openCircuit(e){let t=this.resolveExecutor(e?.index??0,`openCircuit`);this.warnIfModelUnsupported(t.isolateByModel,e?.model,`openCircuit`),t.openCircuit(e?.model??t.model)}closeCircuit(e){let t=this.resolveExecutor(e?.index??0,`closeCircuit`);this.warnIfModelUnsupported(t.isolateByModel,e?.model,`closeCircuit`),t.closeCircuit(e?.model??t.model)}resolveExecutor(e,t){let n=this.executors[e];if(!n)throw RangeError(`${t}: no target at index ${e} (chain has ${this.executors.length} target${this.executors.length===1?``:`s`})`);return n}warnIfModelUnsupported(e,t,n){t!==void 0&&!e&&this.logger.warn(`[VernLLM] ${n}: \`model: '${t}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`)}};function Be(e){return e}function Ve(e){return e}async function*He(e){let t=new TextDecoder(`utf-8`,{fatal:!0}),n=``;for await(let r of e){let e;try{e=typeof r==`string`?r:t.decode(r,{stream:!0})}catch(e){throw new b(`Invalid UTF-8 in SSE stream`,`parse`,{cause:e})}n=(n+e).replace(/\r\n/g,`
|
|
2
|
+
`).replace(/\r(?!$)/g,`
|
|
3
|
+
`);let i=n.indexOf(`
|
|
23
4
|
|
|
24
|
-
|
|
25
|
-
const crypto = __toESM(require("crypto"));
|
|
5
|
+
`);for(;i!==-1;){let e=n.slice(0,i);n=n.slice(i+2);let t=K(e);if(t===U)return;t!==W&&(yield t),i=n.indexOf(`
|
|
26
6
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
* Tool contract codes: a model or provider response defect, not a
|
|
30
|
-
* transient provider fault. Deterministic on the wire request, so
|
|
31
|
-
* retrying can't change the outcome and it shouldn't count toward the
|
|
32
|
-
* circuit breaker either. Shared by `LLMError.retryable` below and by
|
|
33
|
-
* `CallExecutor`'s own retry/breaker accounting, so the two can't drift
|
|
34
|
-
* apart.
|
|
35
|
-
*/
|
|
36
|
-
const NON_RETRYABLE_TOOL_CONTRACT_CODES = new Set([
|
|
37
|
-
"unknown_tool",
|
|
38
|
-
"duplicate_tool_call_id",
|
|
39
|
-
"tool_choice_none_violated",
|
|
40
|
-
"unexpected_tool_calls"
|
|
41
|
-
]);
|
|
42
|
-
/**
|
|
43
|
-
* Local rate-limit codes: the call never reached the provider, so it says
|
|
44
|
-
* nothing about the provider's health, and retrying either just requeues
|
|
45
|
-
* behind the same limit (the two queue codes) or can never succeed at all
|
|
46
|
-
* (`rate_limit_capacity_exceeded`). Shared for the same reason as
|
|
47
|
-
* {@link NON_RETRYABLE_TOOL_CONTRACT_CODES}.
|
|
48
|
-
*/
|
|
49
|
-
const LOCAL_RATE_LIMIT_CODES = new Set([
|
|
50
|
-
"rate_limit_queue_full",
|
|
51
|
-
"rate_limit_queue_timeout",
|
|
52
|
-
"rate_limit_capacity_exceeded"
|
|
53
|
-
]);
|
|
54
|
-
/**
|
|
55
|
-
* Types that are never worth retrying on their own: deterministic
|
|
56
|
-
* caller-input, model-response, or cancellation failures rather than a
|
|
57
|
-
* transient provider fault.
|
|
58
|
-
*/
|
|
59
|
-
const NON_RETRYABLE_TYPES = new Set([
|
|
60
|
-
"parse",
|
|
61
|
-
"validation",
|
|
62
|
-
"invalid_params",
|
|
63
|
-
"aborted"
|
|
64
|
-
]);
|
|
65
|
-
/**
|
|
66
|
-
* Shared retryability rule behind both `LLMError.retryable` and
|
|
67
|
-
* `LLMErrorSnapshot.retryable`. Pulled out so the two can't drift apart:
|
|
68
|
-
* a snapshot is a point-in-time copy of an error's fields, and this is
|
|
69
|
-
* one of them, so it has to be computed the same way in both places.
|
|
70
|
-
*/
|
|
71
|
-
function computeRetryable(type, code) {
|
|
72
|
-
if (NON_RETRYABLE_TYPES.has(type)) return false;
|
|
73
|
-
if (code && NON_RETRYABLE_TOOL_CONTRACT_CODES.has(code)) return false;
|
|
74
|
-
if (code && LOCAL_RATE_LIMIT_CODES.has(code)) return false;
|
|
75
|
-
return true;
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* Returns `issues` unchanged when it can survive `JSON.stringify`.
|
|
79
|
-
* Most `issues` values are VernLLM's own structured shapes (see
|
|
80
|
-
* `LLMErrorIssuesByCode`) and always safe. The one exception is a
|
|
81
|
-
* schema validation failure, where `issues` is a caller supplied
|
|
82
|
-
* `SchemaLike` validator's own `error: unknown`, not controlled by
|
|
83
|
-
* VernLLM and not guaranteed to be circular free. Rather than silently
|
|
84
|
-
* dropping it in that case, this returns a marker string so a reader
|
|
85
|
-
* of serialized output can tell "no issues data" apart from "issues
|
|
86
|
-
* existed but could not be shown".
|
|
87
|
-
*/
|
|
88
|
-
function safeIssues(issues) {
|
|
89
|
-
if (issues === void 0) return void 0;
|
|
90
|
-
try {
|
|
91
|
-
JSON.stringify(issues);
|
|
92
|
-
return issues;
|
|
93
|
-
} catch {
|
|
94
|
-
return "[Unserializable: issues contained a circular reference]";
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
/**
|
|
98
|
-
* Depth cap for `safeAttempts`, guarding against a pathological,
|
|
99
|
-
* self referential `attempts` array. `attempts` is a public
|
|
100
|
-
* `LLMErrorOptions` field, so a caller can construct one by hand; this
|
|
101
|
-
* keeps that path bounded the same way a circular `issues` value is
|
|
102
|
-
* bounded, rather than assuming well formed input.
|
|
103
|
-
*/
|
|
104
|
-
const MAX_ATTEMPTS_DEPTH = 20;
|
|
105
|
-
/**
|
|
106
|
-
* Returns a copy of `attempts` with every nested snapshot's `issues`
|
|
107
|
-
* re-checked through `safeIssues`, recursively through each snapshot's
|
|
108
|
-
* own `attempts`. Needed for two reasons: `safeIssues` returns a safe
|
|
109
|
-
* `issues` value by reference, so a shared object can be mutated into a
|
|
110
|
-
* circular one after the snapshot was created, and `attempts` is a
|
|
111
|
-
* public constructor option, so a caller can hand build a `RetryAttempt`
|
|
112
|
-
* (or a whole `LLMErrorSnapshot`) with a circular `issues` and pass it
|
|
113
|
-
* in directly, never touching `toSnapshot()` at all. Extra fields on an
|
|
114
|
-
* attempt (e.g. `FallbackAttempt`'s `provider`/`model`) are preserved.
|
|
115
|
-
*/
|
|
116
|
-
function safeAttempts(attempts, depth = 0) {
|
|
117
|
-
if (attempts === void 0) return void 0;
|
|
118
|
-
if (depth >= MAX_ATTEMPTS_DEPTH) return [];
|
|
119
|
-
return attempts.map((attempt) => ({
|
|
120
|
-
...attempt,
|
|
121
|
-
error: {
|
|
122
|
-
...attempt.error,
|
|
123
|
-
issues: safeIssues(attempt.error.issues),
|
|
124
|
-
attempts: safeAttempts(attempt.error.attempts, depth + 1)
|
|
125
|
-
}
|
|
126
|
-
}));
|
|
127
|
-
}
|
|
128
|
-
var LLMError = class extends Error {
|
|
129
|
-
status;
|
|
130
|
-
issues;
|
|
131
|
-
cause;
|
|
132
|
-
retryAfterMs;
|
|
133
|
-
/** Stable discriminator within `type`. Absent on errors predating it. */
|
|
134
|
-
code;
|
|
135
|
-
/** Every attempt made before this error was thrown, in order. Absent when nothing was retried. */
|
|
136
|
-
attempts;
|
|
137
|
-
constructor(message, type, options = {}) {
|
|
138
|
-
super(message);
|
|
139
|
-
this.type = type;
|
|
140
|
-
this.name = "LLMError";
|
|
141
|
-
this.status = options.status;
|
|
142
|
-
this.issues = options.issues;
|
|
143
|
-
this.cause = options.cause;
|
|
144
|
-
this.retryAfterMs = options.retryAfterMs;
|
|
145
|
-
this.code = options.code;
|
|
146
|
-
this.attempts = options.attempts;
|
|
147
|
-
}
|
|
148
|
-
/**
|
|
149
|
-
* Computed purely from `type`/`code`, independent of any specific call's
|
|
150
|
-
* `nonRetryableStatus` list. False for `parse`/`validation`/
|
|
151
|
-
* `invalid_params`/`aborted` types (the caller's own input, the model's
|
|
152
|
-
* own response, or intentional cancellation, none of which are the
|
|
153
|
-
* provider being unhealthy), the tool contract codes, and the local
|
|
154
|
-
* rate limit codes. Subclasses (see `FallbackExhaustedError`) may
|
|
155
|
-
* override this when `type` alone carries no retry signal.
|
|
156
|
-
*/
|
|
157
|
-
get retryable() {
|
|
158
|
-
return computeRetryable(this.type, this.code);
|
|
159
|
-
}
|
|
160
|
-
/**
|
|
161
|
-
* Copies this error's fields into an {@link LLMErrorSnapshot}, for
|
|
162
|
-
* recording as a `RetryAttempt`/`FallbackAttempt`. `retryable` is
|
|
163
|
-
* captured here since a snapshot has no getter of its own. `cause` is
|
|
164
|
-
* not copied, see `LLMErrorSnapshot`'s own doc. `issues` and every
|
|
165
|
-
* nested `attempts` entry's own `issues` go through `safeAttempts`,
|
|
166
|
-
* since a schema validation failure's `issues` is a caller supplied
|
|
167
|
-
* value, not controlled by VernLLM, and `attempts` is itself a public
|
|
168
|
-
* constructor option a caller can hand build.
|
|
169
|
-
*/
|
|
170
|
-
toSnapshot() {
|
|
171
|
-
return {
|
|
172
|
-
message: this.message,
|
|
173
|
-
type: this.type,
|
|
174
|
-
status: this.status,
|
|
175
|
-
issues: safeIssues(this.issues),
|
|
176
|
-
retryAfterMs: this.retryAfterMs,
|
|
177
|
-
code: this.code,
|
|
178
|
-
retryable: this.retryable,
|
|
179
|
-
attempts: safeAttempts(this.attempts)
|
|
180
|
-
};
|
|
181
|
-
}
|
|
182
|
-
/**
|
|
183
|
-
* Controls what `JSON.stringify(err)` produces. Omits `cause` for the
|
|
184
|
-
* same reason `toSnapshot()` does: `cause` is `unknown` and never
|
|
185
|
-
* validated by VernLLM, and some SDK errors carry circular structures
|
|
186
|
-
* `JSON.stringify` cannot serialize at all. Read `err.cause` directly
|
|
187
|
-
* instead. `issues`, including every nested `attempts` entry's own
|
|
188
|
-
* `issues`, goes through `safeAttempts` for the same reason: a schema
|
|
189
|
-
* validation failure's `issues` is caller supplied and not guaranteed
|
|
190
|
-
* circular free. Also includes `message` and `retryable`, which a
|
|
191
|
-
* plain property walk would otherwise miss: `message` is
|
|
192
|
-
* non-enumerable on `Error`, and `retryable` is a getter, not an own
|
|
193
|
-
* property.
|
|
194
|
-
*/
|
|
195
|
-
toJSON() {
|
|
196
|
-
return {
|
|
197
|
-
name: this.name,
|
|
198
|
-
message: this.message,
|
|
199
|
-
type: this.type,
|
|
200
|
-
status: this.status,
|
|
201
|
-
issues: safeIssues(this.issues),
|
|
202
|
-
retryAfterMs: this.retryAfterMs,
|
|
203
|
-
code: this.code,
|
|
204
|
-
retryable: this.retryable,
|
|
205
|
-
attempts: safeAttempts(this.attempts)
|
|
206
|
-
};
|
|
207
|
-
}
|
|
208
|
-
};
|
|
209
|
-
function isLLMError(err) {
|
|
210
|
-
return err instanceof LLMError;
|
|
211
|
-
}
|
|
212
|
-
/**
|
|
213
|
-
* Narrows `err.issues` to the exact shape {@link LLMErrorIssuesByCode} maps
|
|
214
|
-
* `code` to, for any code listed there. `code` stays the only discriminator
|
|
215
|
-
* VernLLM uses; this just gives that existing check a typed return instead
|
|
216
|
-
* of requiring a manual cast of `issues`:
|
|
217
|
-
*
|
|
218
|
-
* ```ts
|
|
219
|
-
* if (isLLMError(err) && hasIssues(err, 'duplicate_tool_names')) {
|
|
220
|
-
* console.log(err.issues.names); // string[], no cast needed
|
|
221
|
-
* }
|
|
222
|
-
* ```
|
|
223
|
-
*/
|
|
224
|
-
function hasIssues(err, code) {
|
|
225
|
-
return err.code === code && err.issues !== void 0;
|
|
226
|
-
}
|
|
7
|
+
`)}}try{n+=t.decode()}catch(e){throw new b(`Invalid UTF-8 in SSE stream`,`parse`,{cause:e})}n=n.replace(/\r$/,`
|
|
8
|
+
`);let r=n.indexOf(`
|
|
227
9
|
|
|
228
|
-
|
|
229
|
-
//#region src/types/cache.ts
|
|
230
|
-
/**
|
|
231
|
-
* Trivial default so the package works out of the box with no external deps
|
|
232
|
-
* Not shared across processes, swap in Redis/Upstash/etc for production
|
|
233
|
-
*/
|
|
234
|
-
var InMemoryCacheAdapter = class {
|
|
235
|
-
store = new Map();
|
|
236
|
-
constructor(maxSize = 1e3) {
|
|
237
|
-
this.maxSize = maxSize;
|
|
238
|
-
}
|
|
239
|
-
async get(key) {
|
|
240
|
-
const entry = this.store.get(key);
|
|
241
|
-
if (!entry) return {
|
|
242
|
-
hit: false,
|
|
243
|
-
value: null
|
|
244
|
-
};
|
|
245
|
-
if (Date.now() >= entry.expiresAt) {
|
|
246
|
-
this.store.delete(key);
|
|
247
|
-
return {
|
|
248
|
-
hit: false,
|
|
249
|
-
value: null
|
|
250
|
-
};
|
|
251
|
-
}
|
|
252
|
-
return {
|
|
253
|
-
hit: true,
|
|
254
|
-
value: entry.value
|
|
255
|
-
};
|
|
256
|
-
}
|
|
257
|
-
async set(key, value, ttl) {
|
|
258
|
-
this.cleanupExpiredEntries();
|
|
259
|
-
this.store.set(key, {
|
|
260
|
-
value,
|
|
261
|
-
expiresAt: Date.now() + ttl * 1e3
|
|
262
|
-
});
|
|
263
|
-
this.enforceSizeLimit();
|
|
264
|
-
}
|
|
265
|
-
async delete(key) {
|
|
266
|
-
this.store.delete(key);
|
|
267
|
-
}
|
|
268
|
-
cleanupExpiredEntries() {
|
|
269
|
-
const now = Date.now();
|
|
270
|
-
for (const [key, entry] of this.store) if (now >= entry.expiresAt) this.store.delete(key);
|
|
271
|
-
}
|
|
272
|
-
enforceSizeLimit() {
|
|
273
|
-
while (this.store.size > this.maxSize) {
|
|
274
|
-
const oldestKey = this.store.keys().next().value;
|
|
275
|
-
if (oldestKey === void 0) break;
|
|
276
|
-
this.store.delete(oldestKey);
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
};
|
|
280
|
-
/**
|
|
281
|
-
* Normalizes keys before caching to avoid duplicate entries from formatting differences.
|
|
282
|
-
*/
|
|
283
|
-
var NormalizedCacheAdapter = class {
|
|
284
|
-
constructor(inner = new InMemoryCacheAdapter()) {
|
|
285
|
-
this.inner = inner;
|
|
286
|
-
}
|
|
287
|
-
normalize(key) {
|
|
288
|
-
return key.toLowerCase().trim().replace(/[^\p{L}\p{N}\s]/gu, " ").replace(/\s+/g, " ").trim();
|
|
289
|
-
}
|
|
290
|
-
async resolveKey(key) {
|
|
291
|
-
return this.normalize(key);
|
|
292
|
-
}
|
|
293
|
-
async get(key) {
|
|
294
|
-
return this.inner.get(this.normalize(key));
|
|
295
|
-
}
|
|
296
|
-
async set(key, value, ttl) {
|
|
297
|
-
await this.inner.set(this.normalize(key), value, ttl);
|
|
298
|
-
}
|
|
299
|
-
async delete(key) {
|
|
300
|
-
await this.inner.delete?.(this.normalize(key));
|
|
301
|
-
}
|
|
302
|
-
};
|
|
303
|
-
/**
|
|
304
|
-
* Two-tier cache with fast local L1 and shared L2.
|
|
305
|
-
* L2 hits are promoted back to L1.
|
|
306
|
-
*/
|
|
307
|
-
var TieredCacheAdapter = class {
|
|
308
|
-
constructor(l1, l2, l1Ttl) {
|
|
309
|
-
this.l1 = l1;
|
|
310
|
-
this.l2 = l2;
|
|
311
|
-
this.l1Ttl = l1Ttl;
|
|
312
|
-
}
|
|
313
|
-
/**
|
|
314
|
-
* Forwards to L1's `resolveKey` if it has one, otherwise L2's. L1 is
|
|
315
|
-
* preferred since `get()` checks L1 first, so its notion of "the same
|
|
316
|
-
* key" is the one that determines whether a lookup can skip L2 entirely.
|
|
317
|
-
*/
|
|
318
|
-
async resolveKey(key) {
|
|
319
|
-
if (this.l1.resolveKey) return this.l1.resolveKey(key);
|
|
320
|
-
if (this.l2.resolveKey) return this.l2.resolveKey(key);
|
|
321
|
-
return key;
|
|
322
|
-
}
|
|
323
|
-
async get(key) {
|
|
324
|
-
const l1Result = await this.l1.get(key);
|
|
325
|
-
if (l1Result.hit) return l1Result;
|
|
326
|
-
const l2Result = await this.l2.get(key);
|
|
327
|
-
if (l2Result.hit) await this.l1.set(key, l2Result.value, this.l1Ttl ?? 60);
|
|
328
|
-
return l2Result;
|
|
329
|
-
}
|
|
330
|
-
async set(key, value, ttl) {
|
|
331
|
-
await Promise.all([this.l1.set(key, value, this.l1Ttl ?? ttl), this.l2.set(key, value, ttl)]);
|
|
332
|
-
}
|
|
333
|
-
async delete(key) {
|
|
334
|
-
await Promise.all([this.l1.delete?.(key), this.l2.delete?.(key)]);
|
|
335
|
-
}
|
|
336
|
-
};
|
|
10
|
+
`);for(;r!==-1;){let e=n.slice(0,r);n=n.slice(r+2);let t=K(e);if(t===U)return;t!==W&&(yield t),r=n.indexOf(`
|
|
337
11
|
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
/**
|
|
341
|
-
* Runtime-safe check for whether a `call()` result is a `tool_calls`
|
|
342
|
-
* result. Prefer this over relying on TypeScript's static narrowing
|
|
343
|
-
* whenever `params` passed to `call()` wasn't a literal with `tools`
|
|
344
|
-
* inlined (see the "note on the overload" in `VernLLM.call`'s docs), in
|
|
345
|
-
* that case TS may have typed the result as plain `T` even though it's
|
|
346
|
-
* actually a `CallWithToolsResult<T>` at runtime, and this check works
|
|
347
|
-
* either way.
|
|
348
|
-
*/
|
|
349
|
-
function isToolCallResult(result) {
|
|
350
|
-
return typeof result === "object" && result !== null && "type" in result && result.type === "tool_calls" && Array.isArray(result.toolCalls);
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
//#endregion
|
|
354
|
-
//#region src/types/fallback.ts
|
|
355
|
-
/** Tool contract failures are the model ignoring the request, not a sick provider: repeating it elsewhere can't help. */
|
|
356
|
-
const TOOL_CONTRACT_CODES = new Set([
|
|
357
|
-
"unknown_tool",
|
|
358
|
-
"duplicate_tool_call_id",
|
|
359
|
-
"tool_choice_none_violated",
|
|
360
|
-
"unexpected_tool_calls"
|
|
361
|
-
]);
|
|
362
|
-
/**
|
|
363
|
-
* The default `fallbackOn` policy. Exported so a caller can wrap rather
|
|
364
|
-
* than replace it, e.g. `fallbackOn: (e, ctx) => myCheck(e) ? 'stop' : defaultFallbackOn(e, ctx)`.
|
|
365
|
-
*/
|
|
366
|
-
const defaultFallbackOn = (error) => {
|
|
367
|
-
if (error.type === "parse" || error.type === "validation" || error.type === "aborted") return "stop";
|
|
368
|
-
if (error.type === "quota_exceeded") return "stop";
|
|
369
|
-
if (error.code && TOOL_CONTRACT_CODES.has(error.code)) return "stop";
|
|
370
|
-
return "next";
|
|
371
|
-
};
|
|
372
|
-
/**
|
|
373
|
-
* Thrown when the chain gives up, whether because the last target failed
|
|
374
|
-
* or `fallbackOn` chose to stop early. Carries each attempt in order so
|
|
375
|
-
* an outage across providers stays debuggable without reproducing it.
|
|
376
|
-
* Extends `LLMError` so `isLLMError` and any `instanceof LLMError` check
|
|
377
|
-
* still passes, inheriting the last failure's `type`/`status`/`retryAfterMs`
|
|
378
|
-
* so existing type-based handling, including reading `retryAfterMs` on an
|
|
379
|
-
* `'api'`-typed error, keeps working on a fallback-exhausted error too.
|
|
380
|
-
*/
|
|
381
|
-
var FallbackExhaustedError = class extends LLMError {
|
|
382
|
-
constructor(attempts) {
|
|
383
|
-
const last = attempts[attempts.length - 1]?.error;
|
|
384
|
-
super(`${attempts.length} provider${attempts.length === 1 ? "" : "s"} attempted and failed: ${attempts.map((a) => `${a.provider}(${a.error.type})`).join(" then ")}`, "fallback_exhausted", {
|
|
385
|
-
status: last?.status,
|
|
386
|
-
cause: last,
|
|
387
|
-
retryAfterMs: last?.retryAfterMs,
|
|
388
|
-
code: "fallback_exhausted",
|
|
389
|
-
attempts
|
|
390
|
-
});
|
|
391
|
-
this.attempts = attempts;
|
|
392
|
-
}
|
|
393
|
-
/**
|
|
394
|
-
* `type: 'fallback_exhausted'` by itself says nothing about whether
|
|
395
|
-
* retrying could help; the reason the last target failed does. Defers to
|
|
396
|
-
* that attempt's own `retryable` instead of anything about this class's
|
|
397
|
-
* own type.
|
|
398
|
-
*/
|
|
399
|
-
get retryable() {
|
|
400
|
-
const last = this.attempts[this.attempts.length - 1]?.error;
|
|
401
|
-
return last ? last.retryable : super.retryable;
|
|
402
|
-
}
|
|
403
|
-
};
|
|
404
|
-
/** Narrows `err` to {@link FallbackExhaustedError}, for direct access to its `attempts` (`provider`/`model` per failed target) without a manual `instanceof` check. */
|
|
405
|
-
function isFallbackExhaustedError(err) {
|
|
406
|
-
return err instanceof FallbackExhaustedError;
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
//#endregion
|
|
410
|
-
//#region src/internal/execution/usage.utils.ts
|
|
411
|
-
/**
|
|
412
|
-
* Calls `params.reserveUsage`, if present, mapping any failure to a
|
|
413
|
-
* `quota_exceeded` LLMError (or an aborted error, if the signal fired
|
|
414
|
-
* during reservation). Returns whether a reservation was actually made,
|
|
415
|
-
* so callers know whether a later refund is needed. Shared by
|
|
416
|
-
* `withReservedUsage` and `withReservedUsageForStream`, which differ only
|
|
417
|
-
* in whether `coalesced` is caller-supplied or always `false`.
|
|
418
|
-
*/
|
|
419
|
-
async function reserve(params, coalesced, signal) {
|
|
420
|
-
if (!params.reserveUsage) return false;
|
|
421
|
-
try {
|
|
422
|
-
await params.reserveUsage({
|
|
423
|
-
coalesced,
|
|
424
|
-
signal
|
|
425
|
-
});
|
|
426
|
-
return true;
|
|
427
|
-
} catch (error) {
|
|
428
|
-
if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
429
|
-
throw new LLMError(error instanceof Error ? error.message : "Usage reservation failed", "quota_exceeded", { cause: error });
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
/**
|
|
433
|
-
* Builds a `(logMessage) => Promise<void>` refund function bound to the
|
|
434
|
-
* given hooks/coalesced/signal, reporting (instead of throwing) any error
|
|
435
|
-
* the refund hook itself raises, so a broken refund hook never masks the
|
|
436
|
-
* original error it was called to clean up after.
|
|
437
|
-
*/
|
|
438
|
-
function makeRefund(params, coalesced, signal, onRefundError) {
|
|
439
|
-
return async (logMessage) => {
|
|
440
|
-
try {
|
|
441
|
-
await params.refundUsage?.({
|
|
442
|
-
coalesced,
|
|
443
|
-
signal
|
|
444
|
-
});
|
|
445
|
-
} catch (refundError) {
|
|
446
|
-
onRefundError(logMessage, refundError);
|
|
447
|
-
}
|
|
448
|
-
};
|
|
449
|
-
}
|
|
450
|
-
/**
|
|
451
|
-
* Runs `getResult` after reserving usage, if a `reserveUsage` hook was
|
|
452
|
-
* provided. `refundUsage` fires only if a reservation was actually made.
|
|
453
|
-
* `onRefundError` is called (instead of throwing) whenever a refund attempt
|
|
454
|
-
* itself fails, so a broken refund hook never masks the original error.
|
|
455
|
-
*/
|
|
456
|
-
async function withReservedUsage(params, coalesced, getResult, signal, onRefundError) {
|
|
457
|
-
if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
458
|
-
const reserved = await reserve(params, coalesced, signal);
|
|
459
|
-
const refund = makeRefund(params, coalesced, signal, onRefundError);
|
|
460
|
-
if (signal?.aborted) {
|
|
461
|
-
if (reserved) await refund("[VernLLM] refundUsage failed after abort");
|
|
462
|
-
throw new LLMError("LLM request aborted", "aborted");
|
|
463
|
-
}
|
|
464
|
-
let result;
|
|
465
|
-
try {
|
|
466
|
-
result = await getResult();
|
|
467
|
-
} catch (error) {
|
|
468
|
-
if (reserved) await refund("[VernLLM] refundUsage failed");
|
|
469
|
-
throw error;
|
|
470
|
-
}
|
|
471
|
-
if (signal?.aborted) {
|
|
472
|
-
if (reserved) await refund("[VernLLM] refundUsage failed after abort");
|
|
473
|
-
throw new LLMError("LLM request aborted", "aborted");
|
|
474
|
-
}
|
|
475
|
-
return result;
|
|
476
|
-
}
|
|
477
|
-
/**
|
|
478
|
-
* Streaming counterpart to `withReservedUsage`. `withReservedUsage` assumes
|
|
479
|
-
* `getResult()` settling *is* the operation's final outcome, awaiting it
|
|
480
|
-
* synchronously before reserve/refund resolve. Streaming can't satisfy that:
|
|
481
|
-
* `call()` must return `{ chunks, finalResult }` as soon as the stream
|
|
482
|
-
* opens, well before the real outcome (validation, schema/tool-call checks)
|
|
483
|
-
* is known.
|
|
484
|
-
*
|
|
485
|
-
* Reserves usage before `openStream` runs, same failure mode as the
|
|
486
|
-
* non-streaming path if `reserveUsage` itself throws (mapped to
|
|
487
|
-
* `quota_exceeded`, nothing opened). If `openStream` itself throws (stream
|
|
488
|
-
* never opened), refunds synchronously and rethrows, exactly like
|
|
489
|
-
* `withReservedUsage` does today. If it succeeds, returns `{ chunks,
|
|
490
|
-
* finalResult }` immediately, refund/report is deferred onto
|
|
491
|
-
* `finalResult`'s continuation, since that's the only point the real
|
|
492
|
-
* outcome is known. This means `onUsageFailure` (and any refund) can fire
|
|
493
|
-
* well after this function itself has returned.
|
|
494
|
-
*/
|
|
495
|
-
async function withReservedUsageForStream(params, openStream, signal, onRefundError) {
|
|
496
|
-
if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
497
|
-
const reserved = await reserve(params, false, signal);
|
|
498
|
-
const refund = makeRefund(params, false, signal, onRefundError);
|
|
499
|
-
if (signal?.aborted) {
|
|
500
|
-
if (reserved) await refund("[VernLLM] refundUsage failed after abort");
|
|
501
|
-
throw new LLMError("LLM request aborted", "aborted");
|
|
502
|
-
}
|
|
503
|
-
let opened;
|
|
504
|
-
try {
|
|
505
|
-
opened = await openStream();
|
|
506
|
-
} catch (error) {
|
|
507
|
-
if (reserved) await refund("[VernLLM] refundUsage failed after stream-open failure");
|
|
508
|
-
throw error;
|
|
509
|
-
}
|
|
510
|
-
const finalResult = opened.finalResult.then((value) => value, async (error) => {
|
|
511
|
-
if (reserved) await refund("[VernLLM] refundUsage failed after stream error");
|
|
512
|
-
throw error;
|
|
513
|
-
});
|
|
514
|
-
finalResult.catch(() => {});
|
|
515
|
-
return {
|
|
516
|
-
chunks: opened.chunks,
|
|
517
|
-
finalResult
|
|
518
|
-
};
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
//#endregion
|
|
522
|
-
//#region src/internal/cache/replay.utils.ts
|
|
523
|
-
/**
|
|
524
|
-
* Converts an already-known cache value back into a plausible "text" form
|
|
525
|
-
* for a one-shot replay chunk: passed through unchanged if it's already a
|
|
526
|
-
* string (the `jsonMode: false` case), otherwise `JSON.stringify`'d (the
|
|
527
|
-
* `jsonMode: true` case, where the cached value is the *parsed* result, not
|
|
528
|
-
* the original raw text). This is a reasonable reconstruction, not a
|
|
529
|
-
* byte-identical replay of whatever text the model originally streamed,
|
|
530
|
-
* good enough for `for await (const c of chunks)` call sites that don't
|
|
531
|
-
* branch on hit vs. miss, which is the only thing a cache-hit replay needs
|
|
532
|
-
* to support.
|
|
533
|
-
*/
|
|
534
|
-
function toReplayText(value) {
|
|
535
|
-
return typeof value === "string" ? value : JSON.stringify(value) ?? "";
|
|
536
|
-
}
|
|
537
|
-
/**
|
|
538
|
-
* Builds a trivially-exhausted one-shot `chunks` iterable from an
|
|
539
|
-
* already-known value, used for a `cachedCall` cache hit, where there's no
|
|
540
|
-
* live generation to relay (see `VernLLM.cachedCall`'s docs). No `usage`
|
|
541
|
-
* chunk is emitted: a cache hit spent no real tokens, so there's nothing to
|
|
542
|
-
* report, matching how non-streaming `cachedCall` never calls `onUsage` on
|
|
543
|
-
* a hit either.
|
|
544
|
-
*
|
|
545
|
-
* `hasTools` must reflect whether the *original* call that produced this
|
|
546
|
-
* cached value had `tools` set, that's what determines whether `value` is
|
|
547
|
-
* `T` directly or a `CallWithToolsResult<T>` wrapper, and it isn't
|
|
548
|
-
* something that can be reliably guessed from the value's shape alone
|
|
549
|
-
* (a `schema`-validated `T` could coincidentally look like a
|
|
550
|
-
* `CallWithToolsResult`).
|
|
551
|
-
*/
|
|
552
|
-
function buildReplayChunks(value, hasTools) {
|
|
553
|
-
const items = [];
|
|
554
|
-
if (hasTools) {
|
|
555
|
-
const result = value;
|
|
556
|
-
if (result.type === "tool_calls") {
|
|
557
|
-
result.toolCalls.forEach((toolCall, index) => {
|
|
558
|
-
items.push({
|
|
559
|
-
type: "tool_call_delta",
|
|
560
|
-
index,
|
|
561
|
-
id: toolCall.id,
|
|
562
|
-
name: toolCall.name,
|
|
563
|
-
argsDelta: JSON.stringify(toolCall.arguments ?? {}),
|
|
564
|
-
complete: true
|
|
565
|
-
});
|
|
566
|
-
});
|
|
567
|
-
if (result.content) items.push({
|
|
568
|
-
type: "text-delta",
|
|
569
|
-
delta: result.content
|
|
570
|
-
});
|
|
571
|
-
} else items.push({
|
|
572
|
-
type: "text-delta",
|
|
573
|
-
delta: toReplayText(result.content)
|
|
574
|
-
});
|
|
575
|
-
} else items.push({
|
|
576
|
-
type: "text-delta",
|
|
577
|
-
delta: toReplayText(value)
|
|
578
|
-
});
|
|
579
|
-
return { async *[Symbol.asyncIterator]() {
|
|
580
|
-
for (const item of items) yield item;
|
|
581
|
-
} };
|
|
582
|
-
}
|
|
583
|
-
/**
|
|
584
|
-
* Streaming counterpart to `buildReplayChunks` for a `cachedCall` that
|
|
585
|
-
* *joined* an already-in-flight call for the same key rather than
|
|
586
|
-
* triggering one itself (see `runCachedStream`'s in-flight-coalescing
|
|
587
|
-
* path): there's no live stream to relay (it isn't this call's stream to
|
|
588
|
-
* relay, see the joiner-path comment in `runCachedStream`), but there's
|
|
589
|
-
* also no value yet, only a pending promise for one. Waits for `promise`,
|
|
590
|
-
* then delegates to `buildReplayChunks`. If `promise` rejects, iterating
|
|
591
|
-
* `chunks` throws that same error, consistent with how a live stream's
|
|
592
|
-
* `chunks` throws on a mid-stream failure.
|
|
593
|
-
*/
|
|
594
|
-
function buildReplayChunksFromPromise(promise, hasTools) {
|
|
595
|
-
return { async *[Symbol.asyncIterator]() {
|
|
596
|
-
const value = await promise;
|
|
597
|
-
yield* buildReplayChunks(value, hasTools);
|
|
598
|
-
} };
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
//#endregion
|
|
602
|
-
//#region src/internal/cache/cacheOrchestrator.ts
|
|
603
|
-
/**
|
|
604
|
-
* Owns cache key resolution, cache reads/writes, and in-flight coalescing
|
|
605
|
-
* for concurrent misses on the same key. Doesn't know about `CallExecutor`,
|
|
606
|
-
* retries, or providers at all: `fn`/`openStream` are opaque callbacks
|
|
607
|
-
* (`VernLLM.cachedCall` passes `() => this.call(...)`), so this class only
|
|
608
|
-
* needs the cache adapter and a logger. Extracted from `VernLLM` since
|
|
609
|
-
* caching and per-target call mechanics are independent concerns that
|
|
610
|
-
* happened to live on the same class.
|
|
611
|
-
*/
|
|
612
|
-
var CacheOrchestrator = class {
|
|
613
|
-
inFlight = new Map();
|
|
614
|
-
constructor(cache, logger) {
|
|
615
|
-
this.cache = cache;
|
|
616
|
-
this.logger = logger;
|
|
617
|
-
}
|
|
618
|
-
/** Resolves a cache key through the adapter when it supports normalization. */
|
|
619
|
-
async resolveCacheKey(key) {
|
|
620
|
-
return this.cache.resolveKey ? await this.cache.resolveKey(key) : key;
|
|
621
|
-
}
|
|
622
|
-
/**
|
|
623
|
-
* Removes a cached response by key when the configured cache adapter
|
|
624
|
-
* supports deletion. Cache invalidation is the caller's responsibility;
|
|
625
|
-
* only the application knows when cached data is stale.
|
|
626
|
-
*/
|
|
627
|
-
async deleteCache(key) {
|
|
628
|
-
if (!this.cache.delete) return;
|
|
629
|
-
try {
|
|
630
|
-
await this.cache.delete(await this.resolveCacheKey(key));
|
|
631
|
-
} catch (error) {
|
|
632
|
-
this.logger.warn(`[VernLLM] cache delete failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
633
|
-
}
|
|
634
|
-
}
|
|
635
|
-
/** Logs a failed refundUsage attempt via the configured logger. */
|
|
636
|
-
logRefundError(logMessage, error) {
|
|
637
|
-
this.logger.error(logMessage, { message: error instanceof Error ? error.message : "unknown" });
|
|
638
|
-
}
|
|
639
|
-
/**
|
|
640
|
-
* Reads from the cache, treating a failed adapter read as a miss rather
|
|
641
|
-
* than letting it fail the call. The request still falls through to a
|
|
642
|
-
* real provider call, but that fallback is now logged instead of silent.
|
|
643
|
-
*/
|
|
644
|
-
async getCached(key) {
|
|
645
|
-
try {
|
|
646
|
-
return await this.cache.get(key);
|
|
647
|
-
} catch (error) {
|
|
648
|
-
this.logger.warn(`[VernLLM] cache read failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
649
|
-
return { hit: false };
|
|
650
|
-
}
|
|
651
|
-
}
|
|
652
|
-
/**
|
|
653
|
-
* Internal cache primitive around caller-supplied logic. Concurrent misses
|
|
654
|
-
* for the same `cacheKey` share a single in-flight call, avoiding cache
|
|
655
|
-
* stampedes.
|
|
656
|
-
*
|
|
657
|
-
* Backs the public `VernLLM.cachedCall()`, which always composes this
|
|
658
|
-
* with `call()` so cached results get the same retry/timeout/
|
|
659
|
-
* circuit-breaker guarantees as any other LLM call.
|
|
660
|
-
*
|
|
661
|
-
* @param params `cacheKey`, `ttl`, `fn` (the work to run on a cache
|
|
662
|
-
* miss, typically `() => this.call(...)`), and optional
|
|
663
|
-
* `reserveUsage`/`refundUsage`/`signal`. See `InternalCacheParams`.
|
|
664
|
-
* @returns The cached value on a hit, or the result of `fn()` on a miss.
|
|
665
|
-
*/
|
|
666
|
-
async runCached(params) {
|
|
667
|
-
const resolvedKey = await this.resolveCacheKey(params.cacheKey);
|
|
668
|
-
const resolvedParams = resolvedKey === params.cacheKey ? params : {
|
|
669
|
-
...params,
|
|
670
|
-
cacheKey: resolvedKey
|
|
671
|
-
};
|
|
672
|
-
const cached = await this.getCached(resolvedKey);
|
|
673
|
-
if (cached.hit) return cached.value;
|
|
674
|
-
const existing = this.inFlight.get(resolvedKey);
|
|
675
|
-
if (existing) return withReservedUsage(resolvedParams, true, () => existing, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
|
|
676
|
-
return this.registerTrigger(resolvedParams);
|
|
677
|
-
}
|
|
678
|
-
/** Starts the shared fn() call for a cache miss and tracks it in the in-flight map until it settles. */
|
|
679
|
-
registerTrigger(params) {
|
|
680
|
-
const resultPromise = withReservedUsage(params, false, () => this.runAndCache(params), params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
|
|
681
|
-
this.inFlight.set(params.cacheKey, resultPromise);
|
|
682
|
-
resultPromise.catch(() => {}).finally(() => {
|
|
683
|
-
this.inFlight.delete(params.cacheKey);
|
|
684
|
-
});
|
|
685
|
-
return resultPromise;
|
|
686
|
-
}
|
|
687
|
-
/** Runs `fn` and writes its result to the cache. */
|
|
688
|
-
async runAndCache(params) {
|
|
689
|
-
const result = await params.fn();
|
|
690
|
-
try {
|
|
691
|
-
await this.cache.set(params.cacheKey, result, params.ttl);
|
|
692
|
-
} catch (error) {
|
|
693
|
-
this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
694
|
-
}
|
|
695
|
-
return result;
|
|
696
|
-
}
|
|
697
|
-
/**
|
|
698
|
-
* Streaming counterpart to `runCached`. Three cases:
|
|
699
|
-
*
|
|
700
|
-
* - Hit: no live generation to relay. Returns immediately with
|
|
701
|
-
* `finalResult` resolved to the cached value and a one-shot `chunks`
|
|
702
|
-
* replay built from it, so `for await (const c of chunks)` call sites
|
|
703
|
-
* work identically on a hit or a miss. No usage hooks fire, since
|
|
704
|
-
* nothing was actually spent.
|
|
705
|
-
* - Miss, nothing else in flight for this key: delegates to
|
|
706
|
-
* `registerStreamTrigger`, which opens the stream and relays its
|
|
707
|
-
* `chunks` live.
|
|
708
|
-
* - Miss, but another call for the same key is already in flight: this
|
|
709
|
-
* call has no live chunks of its own to relay, so it's treated like a
|
|
710
|
-
* delayed hit. `finalResult` shares the trigger's in-flight promise
|
|
711
|
-
* (the same in-flight map non-streaming `runCached` uses, so
|
|
712
|
-
* streaming and non-streaming calls for the same key coalesce
|
|
713
|
-
* against each other too), and `chunks` is a one-shot replay built
|
|
714
|
-
* once that promise resolves.
|
|
715
|
-
*/
|
|
716
|
-
async runCachedStream(params, hasTools) {
|
|
717
|
-
const resolvedKey = await this.resolveCacheKey(params.cacheKey);
|
|
718
|
-
const resolvedParams = resolvedKey === params.cacheKey ? params : {
|
|
719
|
-
...params,
|
|
720
|
-
cacheKey: resolvedKey
|
|
721
|
-
};
|
|
722
|
-
const cached = await this.getCached(resolvedKey);
|
|
723
|
-
if (cached.hit) {
|
|
724
|
-
const value = cached.value;
|
|
725
|
-
return {
|
|
726
|
-
chunks: buildReplayChunks(value, hasTools),
|
|
727
|
-
finalResult: Promise.resolve(value)
|
|
728
|
-
};
|
|
729
|
-
}
|
|
730
|
-
const existing = this.inFlight.get(resolvedKey);
|
|
731
|
-
if (existing) {
|
|
732
|
-
const finalResult = withReservedUsage(resolvedParams, true, () => existing, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
|
|
733
|
-
finalResult.catch(() => {});
|
|
734
|
-
return {
|
|
735
|
-
chunks: buildReplayChunksFromPromise(finalResult, hasTools),
|
|
736
|
-
finalResult
|
|
737
|
-
};
|
|
738
|
-
}
|
|
739
|
-
return this.registerStreamTrigger(resolvedParams);
|
|
740
|
-
}
|
|
741
|
-
/**
|
|
742
|
-
* Opens the shared stream for a cache miss and tracks its settled value
|
|
743
|
-
* in the in-flight map until it resolves or rejects. Writes to the cache
|
|
744
|
-
* on success only, matching `runAndCache`.
|
|
745
|
-
*
|
|
746
|
-
* Registers the in-flight promise synchronously, before anything async
|
|
747
|
-
* runs, so a concurrent `cachedCall` for the same key always sees it in
|
|
748
|
-
* time to join instead of triggering its own stream. Settlement is
|
|
749
|
-
* wired onto the whole `withReservedUsageForStream` call rather than a
|
|
750
|
-
* line inside its callback, so any failure point (reserving usage,
|
|
751
|
-
* opening the stream, or the stream itself) reliably settles the
|
|
752
|
-
* in-flight entry instead of leaving it stuck.
|
|
753
|
-
*/
|
|
754
|
-
registerStreamTrigger(params) {
|
|
755
|
-
let resolveInFlight;
|
|
756
|
-
let rejectInFlight;
|
|
757
|
-
const inFlightResult = new Promise((resolve, reject) => {
|
|
758
|
-
resolveInFlight = resolve;
|
|
759
|
-
rejectInFlight = reject;
|
|
760
|
-
});
|
|
761
|
-
this.inFlight.set(params.cacheKey, inFlightResult);
|
|
762
|
-
inFlightResult.catch(() => {}).finally(() => {
|
|
763
|
-
this.inFlight.delete(params.cacheKey);
|
|
764
|
-
});
|
|
765
|
-
const streamPromise = withReservedUsageForStream(params, async () => {
|
|
766
|
-
const opened = await params.openStream();
|
|
767
|
-
const trackedResult = opened.finalResult.then(async (value) => {
|
|
768
|
-
try {
|
|
769
|
-
await this.cache.set(params.cacheKey, value, params.ttl);
|
|
770
|
-
} catch (error) {
|
|
771
|
-
this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
772
|
-
}
|
|
773
|
-
return value;
|
|
774
|
-
}, (error) => {
|
|
775
|
-
throw error;
|
|
776
|
-
});
|
|
777
|
-
return {
|
|
778
|
-
chunks: opened.chunks,
|
|
779
|
-
finalResult: trackedResult
|
|
780
|
-
};
|
|
781
|
-
}, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
|
|
782
|
-
streamPromise.then((opened) => {
|
|
783
|
-
opened.finalResult.then(resolveInFlight, rejectInFlight);
|
|
784
|
-
}, (error) => {
|
|
785
|
-
rejectInFlight(error);
|
|
786
|
-
});
|
|
787
|
-
return streamPromise;
|
|
788
|
-
}
|
|
789
|
-
};
|
|
790
|
-
|
|
791
|
-
//#endregion
|
|
792
|
-
//#region src/circuitBreaker.ts
|
|
793
|
-
function newBucket() {
|
|
794
|
-
return {
|
|
795
|
-
state: "closed",
|
|
796
|
-
consecutiveFailures: 0,
|
|
797
|
-
openedAt: 0,
|
|
798
|
-
trialInFlight: false
|
|
799
|
-
};
|
|
800
|
-
}
|
|
801
|
-
/** Key a bucket lookup falls into when the call omitted `model` under `isolateByModel`. */
|
|
802
|
-
const UNLABELED_MODEL = "";
|
|
803
|
-
/**
|
|
804
|
-
* Per retry VernLLM-instance circuit breaker. Tracks consecutive failures across
|
|
805
|
-
* calls. Once the threshold is hit, short-circuits new calls with an
|
|
806
|
-
* LLMError('circuit_open') instead of hitting the provider, until the
|
|
807
|
-
* cooldown elapses and a single trial call is allowed through
|
|
808
|
-
*/
|
|
809
|
-
var CircuitBreaker = class {
|
|
810
|
-
threshold;
|
|
811
|
-
cooldownMs;
|
|
812
|
-
onStateChange;
|
|
813
|
-
/** Whether this breaker tracks failures per model instead of one shared circuit. Read by `CallExecutor`/`VernLLM` to report per-target in `getCircuitStates`. */
|
|
814
|
-
isolateByModel;
|
|
815
|
-
sharedBucket = newBucket();
|
|
816
|
-
bucketsByModel = new Map();
|
|
817
|
-
constructor(options = {}) {
|
|
818
|
-
this.threshold = options.threshold ?? 5;
|
|
819
|
-
this.cooldownMs = options.cooldownMs ?? 3e4;
|
|
820
|
-
this.onStateChange = options.onStateChange;
|
|
821
|
-
this.isolateByModel = options.isolateByModel ?? false;
|
|
822
|
-
}
|
|
823
|
-
/** Returns the bucket for a model if one already exists, without allocating. */
|
|
824
|
-
lookupBucket(model) {
|
|
825
|
-
if (!this.isolateByModel) return this.sharedBucket;
|
|
826
|
-
const key = model ?? UNLABELED_MODEL;
|
|
827
|
-
return this.bucketsByModel.get(key);
|
|
828
|
-
}
|
|
829
|
-
/** Creates and stores a bucket for a model when the first mutation needs one. */
|
|
830
|
-
ensureBucketFor(model) {
|
|
831
|
-
if (!this.isolateByModel) return this.sharedBucket;
|
|
832
|
-
const key = model ?? UNLABELED_MODEL;
|
|
833
|
-
let bucket = this.bucketsByModel.get(key);
|
|
834
|
-
if (!bucket) {
|
|
835
|
-
bucket = newBucket();
|
|
836
|
-
this.bucketsByModel.set(key, bucket);
|
|
837
|
-
}
|
|
838
|
-
return bucket;
|
|
839
|
-
}
|
|
840
|
-
/** Every state mutation routes through here, so `onStateChange` fires exactly once per real change. */
|
|
841
|
-
transition(bucket, to, model) {
|
|
842
|
-
if (to === bucket.state) return;
|
|
843
|
-
const from = bucket.state;
|
|
844
|
-
bucket.state = to;
|
|
845
|
-
this.onStateChange?.(from, to, bucket.consecutiveFailures, model);
|
|
846
|
-
}
|
|
847
|
-
/**
|
|
848
|
-
* Throws if the circuit is open and the cooldown hasn't elapsed, or if
|
|
849
|
-
* the circuit is half-open and a trial call is already in flight.
|
|
850
|
-
* Otherwise, if the circuit just became eligible for a trial (cooldown
|
|
851
|
-
* elapsed, or half-open with no trial currently running), this call
|
|
852
|
-
* becomes that trial
|
|
853
|
-
*/
|
|
854
|
-
assertClosed(model) {
|
|
855
|
-
const bucket = this.ensureBucketFor(model);
|
|
856
|
-
if (bucket.state === "closed") return;
|
|
857
|
-
if (bucket.state === "open") {
|
|
858
|
-
const elapsed = Date.now() - bucket.openedAt;
|
|
859
|
-
if (elapsed < this.cooldownMs) throw new LLMError(`Circuit open, provider has failed ${bucket.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open", { code: "circuit_cooling_down" });
|
|
860
|
-
bucket.trialInFlight = true;
|
|
861
|
-
this.transition(bucket, "half-open", model);
|
|
862
|
-
return;
|
|
863
|
-
}
|
|
864
|
-
if (bucket.trialInFlight) throw new LLMError("Circuit half-open. A trial request is already in flight. Try again shortly.", "circuit_open", { code: "circuit_trial_in_flight" });
|
|
865
|
-
bucket.trialInFlight = true;
|
|
866
|
-
}
|
|
867
|
-
recordSuccess(model) {
|
|
868
|
-
const bucket = this.lookupBucket(model);
|
|
869
|
-
if (!bucket) return;
|
|
870
|
-
bucket.consecutiveFailures = 0;
|
|
871
|
-
bucket.trialInFlight = false;
|
|
872
|
-
this.transition(bucket, "closed", model);
|
|
873
|
-
if (this.isolateByModel && bucket.state === "closed" && bucket.consecutiveFailures === 0) this.bucketsByModel.delete(model ?? UNLABELED_MODEL);
|
|
874
|
-
}
|
|
875
|
-
recordFailure(model) {
|
|
876
|
-
const bucket = this.ensureBucketFor(model);
|
|
877
|
-
bucket.consecutiveFailures += 1;
|
|
878
|
-
bucket.trialInFlight = false;
|
|
879
|
-
if (bucket.state === "half-open") {
|
|
880
|
-
bucket.openedAt = Date.now();
|
|
881
|
-
this.transition(bucket, "open", model);
|
|
882
|
-
return;
|
|
883
|
-
}
|
|
884
|
-
if (bucket.consecutiveFailures >= this.threshold) {
|
|
885
|
-
bucket.openedAt = Date.now();
|
|
886
|
-
this.transition(bucket, "open", model);
|
|
887
|
-
}
|
|
888
|
-
}
|
|
889
|
-
/**
|
|
890
|
-
* With `isolateByModel` off (the default), `model` is ignored and the
|
|
891
|
-
* one shared circuit's state is returned, unchanged from every version
|
|
892
|
-
* before this option existed. With `isolateByModel` on, returns that
|
|
893
|
-
* model's own state, `'closed'` for a model never seen yet, same as a
|
|
894
|
-
* fresh breaker.
|
|
895
|
-
*/
|
|
896
|
-
getState(model) {
|
|
897
|
-
return this.lookupBucket(model)?.state ?? "closed";
|
|
898
|
-
}
|
|
899
|
-
/**
|
|
900
|
-
* Manually opens the circuit, as if `threshold` consecutive failures had
|
|
901
|
-
* just happened, e.g. to pull a provider out of rotation ahead of known
|
|
902
|
-
* maintenance. Resets the cooldown window from now, same as a real
|
|
903
|
-
* threshold-crossing failure would, and clears any in-flight half-open
|
|
904
|
-
* trial since it no longer applies once the circuit is (re)opened.
|
|
905
|
-
*/
|
|
906
|
-
open(model) {
|
|
907
|
-
const bucket = this.ensureBucketFor(model);
|
|
908
|
-
bucket.openedAt = Date.now();
|
|
909
|
-
bucket.trialInFlight = false;
|
|
910
|
-
this.transition(bucket, "open", model);
|
|
911
|
-
}
|
|
912
|
-
/**
|
|
913
|
-
* Manually closes the circuit and resets its failure count, e.g. once a
|
|
914
|
-
* provider is confirmed healthy again without waiting out the cooldown.
|
|
915
|
-
* Mirrors `recordSuccess`'s bookkeeping (including dropping the
|
|
916
|
-
* per-model bucket under `isolateByModel`, once idle) but without
|
|
917
|
-
* requiring an actual successful call first.
|
|
918
|
-
*/
|
|
919
|
-
close(model) {
|
|
920
|
-
const bucket = this.ensureBucketFor(model);
|
|
921
|
-
bucket.consecutiveFailures = 0;
|
|
922
|
-
bucket.trialInFlight = false;
|
|
923
|
-
this.transition(bucket, "closed", model);
|
|
924
|
-
if (this.isolateByModel && bucket.state === "closed" && bucket.consecutiveFailures === 0) this.bucketsByModel.delete(model ?? UNLABELED_MODEL);
|
|
925
|
-
}
|
|
926
|
-
};
|
|
927
|
-
|
|
928
|
-
//#endregion
|
|
929
|
-
//#region src/internal/circuitBreaker.utils.ts
|
|
930
|
-
/**
|
|
931
|
-
* Builds a `(event) => void` reporter that no-ops when `onEvent` is unset,
|
|
932
|
-
* and otherwise calls it, swallowing and logging any error the handler
|
|
933
|
-
* throws so a broken `onEvent` can't break the call that triggered it.
|
|
934
|
-
* Shared by `buildCircuitBreaker` (which needs to report before any
|
|
935
|
-
* executor exists) and `CallExecutor.reportEvent`, kept independent of the
|
|
936
|
-
* executor for that reason.
|
|
937
|
-
*/
|
|
938
|
-
function makeEventReporter(onEvent, logger) {
|
|
939
|
-
return (event) => {
|
|
940
|
-
if (!onEvent) return;
|
|
941
|
-
try {
|
|
942
|
-
onEvent(event);
|
|
943
|
-
} catch (error) {
|
|
944
|
-
logger.error("[VernLLM] onEvent failed", { message: error instanceof Error ? error.message : "unknown" });
|
|
945
|
-
}
|
|
946
|
-
};
|
|
947
|
-
}
|
|
948
|
-
/**
|
|
949
|
-
* Builds the optional circuit breaker for one provider target, wiring its
|
|
950
|
-
* `onStateChange` to emit a `circuit_state` event and chain any
|
|
951
|
-
* caller-supplied `onStateChange`. Returns `undefined` when
|
|
952
|
-
* `circuitBreakerOption` is falsy, matching the option's own semantics.
|
|
953
|
-
*
|
|
954
|
-
* Lives outside `CallExecutor` (and outside `VernLLM`, once this were
|
|
955
|
-
* inlined) because the breaker has to exist *before* the executor it's
|
|
956
|
-
* passed into, so its construction can't be an executor concern.
|
|
957
|
-
* `onEvent` is called directly rather than through the executor for the
|
|
958
|
-
* same reason: nothing executor-shaped exists yet at this point.
|
|
959
|
-
*
|
|
960
|
-
* Takes the specific fields it needs (rather than a full `VernLLMOptions`)
|
|
961
|
-
* so it works identically for the primary target and for each fallback
|
|
962
|
-
* target, which carry their own `circuitBreaker` override alongside the
|
|
963
|
-
* shared `onEvent`.
|
|
964
|
-
*/
|
|
965
|
-
function buildCircuitBreaker(circuitBreakerOption, providerName, defaultModel, onEvent, logger) {
|
|
966
|
-
if (!circuitBreakerOption) return void 0;
|
|
967
|
-
const breakerOptions = typeof circuitBreakerOption === "object" ? circuitBreakerOption : void 0;
|
|
968
|
-
const userOnStateChange = breakerOptions?.onStateChange;
|
|
969
|
-
const reportEvent = makeEventReporter(onEvent, logger);
|
|
970
|
-
return new CircuitBreaker({
|
|
971
|
-
...breakerOptions,
|
|
972
|
-
onStateChange: (from, to, consecutiveFailures, model) => {
|
|
973
|
-
reportEvent({
|
|
974
|
-
kind: "circuit_state",
|
|
975
|
-
provider: providerName,
|
|
976
|
-
model: model ?? defaultModel,
|
|
977
|
-
from,
|
|
978
|
-
to,
|
|
979
|
-
consecutiveFailures
|
|
980
|
-
});
|
|
981
|
-
if (!userOnStateChange) return;
|
|
982
|
-
try {
|
|
983
|
-
userOnStateChange(from, to, consecutiveFailures, model);
|
|
984
|
-
} catch (error) {
|
|
985
|
-
logger.error("[VernLLM] circuitBreaker.onStateChange failed", { message: error instanceof Error ? error.message : "unknown" });
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
});
|
|
989
|
-
}
|
|
990
|
-
|
|
991
|
-
//#endregion
|
|
992
|
-
//#region src/internal/execution/retry.utils.ts
|
|
993
|
-
/**
|
|
994
|
-
* Default cap (ms) for both exponential backoff and honored Retry-After
|
|
995
|
-
* values, so a misbehaving/adversarial Retry-After can't stall a caller
|
|
996
|
-
* indefinitely
|
|
997
|
-
*/
|
|
998
|
-
const DEFAULT_MAX_DELAY_MS = 1e4;
|
|
999
|
-
/**
|
|
1000
|
-
* `setTimeout` silently clamps any delay above this (~24.8 days) or
|
|
1001
|
-
* `Infinity` down to ~1ms instead of erroring, so a caller passing
|
|
1002
|
-
* `Infinity` as "no timeout" gets the opposite of what they asked for.
|
|
1003
|
-
* Both timeout helpers below guard against this explicitly.
|
|
1004
|
-
*/
|
|
1005
|
-
const MAX_SETTIMEOUT_MS = 2147483647;
|
|
1006
|
-
/**
|
|
1007
|
-
* Resolves a timeout value to the number `setTimeout` should actually use,
|
|
1008
|
-
* or `undefined` when the timeout should be treated as disabled (0,
|
|
1009
|
-
* negative, or `Infinity`). Returning the resolved value directly, rather
|
|
1010
|
-
* than a boolean, lets callers narrow `number | undefined` to `number`
|
|
1011
|
-
* without an `as number` cast.
|
|
1012
|
-
*/
|
|
1013
|
-
function resolveActiveTimeoutMs(ms) {
|
|
1014
|
-
return !ms || ms <= 0 || ms === Infinity ? void 0 : ms;
|
|
1015
|
-
}
|
|
1016
|
-
/** Caps a timeout at the largest delay `setTimeout` actually honors. */
|
|
1017
|
-
function clampTimeoutMs(ms) {
|
|
1018
|
-
return Math.min(ms, MAX_SETTIMEOUT_MS);
|
|
1019
|
-
}
|
|
1020
|
-
/**
|
|
1021
|
-
* Runs an async function and cancels it if it takes longer than the given
|
|
1022
|
-
* timeout. Creates an internal abort controller that fires after the
|
|
1023
|
-
* timeout elapses, and combines it with any external signal the caller
|
|
1024
|
-
* passed in so either one can cancel the underlying call. If the internal
|
|
1025
|
-
* timeout triggers and the underlying operation aborts, the error is
|
|
1026
|
-
* converted into an LLMError with type "timeout". External cancellations
|
|
1027
|
-
* continue to propagate as aborted errors. The internal timer is always
|
|
1028
|
-
* cleared afterward, whether the function succeeds, fails, or is aborted,
|
|
1029
|
-
* so nothing is left running in the background.
|
|
1030
|
-
*
|
|
1031
|
-
* `timeoutMs` of `Infinity` (or any value beyond what `setTimeout` can
|
|
1032
|
-
* represent) disables the timeout rather than firing almost immediately.
|
|
1033
|
-
*/
|
|
1034
|
-
async function withTimeout(fn, timeoutMs, externalSignal) {
|
|
1035
|
-
const controller = new AbortController();
|
|
1036
|
-
const activeTimeoutMs = resolveActiveTimeoutMs(timeoutMs);
|
|
1037
|
-
const timer = activeTimeoutMs === void 0 ? void 0 : setTimeout(() => {
|
|
1038
|
-
controller.abort();
|
|
1039
|
-
}, clampTimeoutMs(activeTimeoutMs));
|
|
1040
|
-
const signal = externalSignal ? AbortSignal.any([externalSignal, controller.signal]) : controller.signal;
|
|
1041
|
-
try {
|
|
1042
|
-
return await fn(signal);
|
|
1043
|
-
} catch (err) {
|
|
1044
|
-
if (controller.signal.aborted && !externalSignal?.aborted && err instanceof DOMException && err.name === "AbortError") throw new LLMError("Request timed out", "timeout", { code: "request_timeout" });
|
|
1045
|
-
throw err;
|
|
1046
|
-
} finally {
|
|
1047
|
-
clearTimeout(timer);
|
|
1048
|
-
}
|
|
1049
|
-
}
|
|
1050
|
-
/**
|
|
1051
|
-
* Races one `iterator.next()` call against a per-call idle timer, to
|
|
1052
|
-
* bound the gap *between* chunks (unlike `withTimeout`, which only bounds
|
|
1053
|
-
* opening the stream and its first chunk). Without this, a connection
|
|
1054
|
-
* that streams one chunk then hangs would never fail.
|
|
1055
|
-
*
|
|
1056
|
-
* `timeoutMs` of 0/undefined/`Infinity` disables the check. Otherwise
|
|
1057
|
-
* rejects with `LLMError('timeout')` if `next()` doesn't settle in time.
|
|
1058
|
-
* The clock resets on every call, so the window is measured from the most
|
|
1059
|
-
* recent chunk, not from stream start.
|
|
1060
|
-
*
|
|
1061
|
-
* `onIdle`, if given, is called the moment the timer fires (before the
|
|
1062
|
-
* rejection), so callers can abort the underlying transport instead of
|
|
1063
|
-
* just walking away from an unread promise. `logger`, if given, records a
|
|
1064
|
-
* debug line if `next()` still settles *after* the idle timeout already
|
|
1065
|
-
* rejected. `resolve`/`reject` on an already-settled promise is otherwise
|
|
1066
|
-
* a silent no-op, so without this the late chunk (possibly the final
|
|
1067
|
-
* usage chunk) would vanish with no trace.
|
|
1068
|
-
*/
|
|
1069
|
-
function withChunkIdleTimeout(next, timeoutMs, onIdle, logger) {
|
|
1070
|
-
const activeTimeoutMs = resolveActiveTimeoutMs(timeoutMs);
|
|
1071
|
-
if (activeTimeoutMs === void 0) return next();
|
|
1072
|
-
let settled = false;
|
|
1073
|
-
return new Promise((resolve, reject) => {
|
|
1074
|
-
const timer = setTimeout(() => {
|
|
1075
|
-
settled = true;
|
|
1076
|
-
onIdle?.();
|
|
1077
|
-
reject(new LLMError(`No stream chunk received for ${activeTimeoutMs}ms (idle timeout)`, "timeout", { code: "idle_timeout" }));
|
|
1078
|
-
}, clampTimeoutMs(activeTimeoutMs));
|
|
1079
|
-
next().then((result) => {
|
|
1080
|
-
clearTimeout(timer);
|
|
1081
|
-
if (settled) {
|
|
1082
|
-
logger?.debug("[VernLLM] chunk resolved after idle timeout already fired; discarding");
|
|
1083
|
-
return;
|
|
1084
|
-
}
|
|
1085
|
-
settled = true;
|
|
1086
|
-
resolve(result);
|
|
1087
|
-
}, (error) => {
|
|
1088
|
-
clearTimeout(timer);
|
|
1089
|
-
if (settled) {
|
|
1090
|
-
logger?.debug("[VernLLM] chunk rejection arrived after idle timeout already fired; discarding");
|
|
1091
|
-
return;
|
|
1092
|
-
}
|
|
1093
|
-
settled = true;
|
|
1094
|
-
reject(error);
|
|
1095
|
-
});
|
|
1096
|
-
});
|
|
1097
|
-
}
|
|
1098
|
-
/**
|
|
1099
|
-
* Looks inside an unknown error value for a Retry-After header and
|
|
1100
|
-
* converts it to milliseconds. Checks `.headers` first (fetch-style,
|
|
1101
|
-
* Headers-like with `.get()`), then `.response.headers` (axios-style,
|
|
1102
|
-
* plain object) since different client libraries surface headers
|
|
1103
|
-
* differently. Supports both the delta-seconds form ("30") and the
|
|
1104
|
-
* HTTP-date form ("Wed, 21 Oct 2015 07:28:00 GMT"). The result is capped
|
|
1105
|
-
* at maxDelayMs. Returns undefined when no usable Retry-After is present
|
|
1106
|
-
*/
|
|
1107
|
-
function extractRetryAfterMs(err, maxDelayMs = DEFAULT_MAX_DELAY_MS) {
|
|
1108
|
-
if (!err || typeof err !== "object") return void 0;
|
|
1109
|
-
const error = err;
|
|
1110
|
-
const headers = error.headers ?? error.response?.headers;
|
|
1111
|
-
if (!headers || typeof headers !== "object") return void 0;
|
|
1112
|
-
const getter = headers;
|
|
1113
|
-
const raw = typeof getter.get === "function" ? getter.get("Retry-After") : Object.entries(headers).find(([name]) => name.toLowerCase() === "retry-after")?.at(1);
|
|
1114
|
-
if (typeof raw !== "string" || raw.trim() === "") return void 0;
|
|
1115
|
-
const trimmed = raw.trim();
|
|
1116
|
-
if (/^\d+$/.test(trimmed)) return Math.max(0, Math.min(Number(trimmed) * 1e3, maxDelayMs));
|
|
1117
|
-
const dateMs = Date.parse(trimmed);
|
|
1118
|
-
if (!Number.isNaN(dateMs)) return Math.max(0, Math.min(dateMs - Date.now(), maxDelayMs));
|
|
1119
|
-
return void 0;
|
|
1120
|
-
}
|
|
1121
|
-
/**
|
|
1122
|
-
* Exponential backoff with jitter, capped at maxDelayMs.
|
|
1123
|
-
* Jitter avoids thundering-herd retries when many callers back off in lockstep,
|
|
1124
|
-
* the cap prevents unbounded delays when maxRetries is high
|
|
1125
|
-
*/
|
|
1126
|
-
function getBackoffDelay(baseDelayMs, attempt, maxDelayMs = DEFAULT_MAX_DELAY_MS) {
|
|
1127
|
-
const exp = Math.min(baseDelayMs * 2 ** attempt, maxDelayMs);
|
|
1128
|
-
return exp / 2 + Math.random() * (exp / 2);
|
|
1129
|
-
}
|
|
1130
|
-
/**
|
|
1131
|
-
* Pauses execution for the given delay before a retry attempt. If an
|
|
1132
|
-
* abort signal is provided and it fires while waiting, the pending
|
|
1133
|
-
* timer is cancelled immediately and the wait rejects right away with
|
|
1134
|
-
* an aborted error instead of continuing to sit idle until the delay
|
|
1135
|
-
* would have finished on its own
|
|
1136
|
-
*/
|
|
1137
|
-
async function waitForRetry(delay, signal) {
|
|
1138
|
-
if (signal?.aborted) throw new LLMError("Operation aborted", "aborted");
|
|
1139
|
-
await new Promise((resolve, reject) => {
|
|
1140
|
-
const onAbort = () => {
|
|
1141
|
-
clearTimeout(timer);
|
|
1142
|
-
reject(new LLMError("Operation aborted", "aborted"));
|
|
1143
|
-
};
|
|
1144
|
-
const timer = setTimeout(() => {
|
|
1145
|
-
signal?.removeEventListener("abort", onAbort);
|
|
1146
|
-
resolve();
|
|
1147
|
-
}, delay);
|
|
1148
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1149
|
-
});
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
//#endregion
|
|
1153
|
-
//#region src/internal/execution/errors.utils.ts
|
|
1154
|
-
/**
|
|
1155
|
-
* Looks inside an unknown error value and pulls out an http status code
|
|
1156
|
-
* if one is present. Checks the status field first then the status code
|
|
1157
|
-
* field since different client libraries use different names for this,
|
|
1158
|
-
* falling back to AWS SDK v3's `$metadata.httpStatusCode` (e.g. Bedrock's
|
|
1159
|
-
* `ThrottlingException`), which doesn't set either of the other two.
|
|
1160
|
-
* Returns undefined when the error is not an object or carries no status
|
|
1161
|
-
*/
|
|
1162
|
-
function extractStatus(err) {
|
|
1163
|
-
if (!err || typeof err !== "object") return void 0;
|
|
1164
|
-
const error = err;
|
|
1165
|
-
if (typeof error.status === "number") return error.status;
|
|
1166
|
-
if (typeof error.statusCode === "number") return error.statusCode;
|
|
1167
|
-
if (typeof error.$metadata?.httpStatusCode === "number") return error.$metadata.httpStatusCode;
|
|
1168
|
-
return void 0;
|
|
1169
|
-
}
|
|
1170
|
-
/**
|
|
1171
|
-
* POSIX/libuv error codes libuv (and so Node's `fetch`/undici) attaches to
|
|
1172
|
-
* genuine transport-level failures: connection refused, DNS lookup
|
|
1173
|
-
* failure, connection reset mid-request, a connect that never completed,
|
|
1174
|
-
* DNS server unreachable, broken pipe, or host/network unreachable.
|
|
1175
|
-
* Deliberately narrow: only codes that can only mean "the connection
|
|
1176
|
-
* itself failed," not anything that could also indicate an application
|
|
1177
|
-
* error.
|
|
1178
|
-
*/
|
|
1179
|
-
const NETWORK_ERROR_CODES = new Set([
|
|
1180
|
-
"ECONNREFUSED",
|
|
1181
|
-
"ENOTFOUND",
|
|
1182
|
-
"ECONNRESET",
|
|
1183
|
-
"ETIMEDOUT",
|
|
1184
|
-
"EAI_AGAIN",
|
|
1185
|
-
"EPIPE",
|
|
1186
|
-
"ECONNABORTED",
|
|
1187
|
-
"EHOSTUNREACH",
|
|
1188
|
-
"ENETUNREACH"
|
|
1189
|
-
]);
|
|
1190
|
-
/** `fetch`'s own wording for a transport-level failure, across runtimes/browsers. */
|
|
1191
|
-
const NETWORK_ERROR_MESSAGES = new Set([
|
|
1192
|
-
"fetch failed",
|
|
1193
|
-
"failed to fetch",
|
|
1194
|
-
"load failed",
|
|
1195
|
-
"networkerror when attempting to fetch resource."
|
|
1196
|
-
]);
|
|
1197
|
-
/**
|
|
1198
|
-
* Whether `error` is, with reasonable confidence, a transport-level
|
|
1199
|
-
* failure (never reached the provider, as opposed to the provider itself
|
|
1200
|
-
* responding with an error) rather than some other unexpected exception.
|
|
1201
|
-
* Checked via explicit, well-known signals only, so a genuinely unknown
|
|
1202
|
-
* error never gets misclassified as a connection failure just because it
|
|
1203
|
-
* also lacked an HTTP status.
|
|
1204
|
-
*/
|
|
1205
|
-
function isNetworkError(error) {
|
|
1206
|
-
if (!error || typeof error !== "object") return false;
|
|
1207
|
-
const err = error;
|
|
1208
|
-
if (typeof err.code === "string" && NETWORK_ERROR_CODES.has(err.code)) return true;
|
|
1209
|
-
if (typeof err.message === "string" && NETWORK_ERROR_MESSAGES.has(err.message.toLowerCase())) return true;
|
|
1210
|
-
if (err.cause && typeof err.cause === "object") {
|
|
1211
|
-
const cause = err.cause;
|
|
1212
|
-
if (typeof cause.code === "string" && NETWORK_ERROR_CODES.has(cause.code)) return true;
|
|
1213
|
-
}
|
|
1214
|
-
return false;
|
|
1215
|
-
}
|
|
1216
|
-
function formatSafely(value) {
|
|
1217
|
-
try {
|
|
1218
|
-
return JSON.stringify(value, null, 2) ?? String(value);
|
|
1219
|
-
} catch {
|
|
1220
|
-
try {
|
|
1221
|
-
return String(value);
|
|
1222
|
-
} catch {
|
|
1223
|
-
return "[unprintable error]";
|
|
1224
|
-
}
|
|
1225
|
-
}
|
|
1226
|
-
}
|
|
1227
|
-
/**
|
|
1228
|
-
* Looks inside an unknown thrown value and pulls out a human-readable
|
|
1229
|
-
* description of it. Checks the `error` field first (the provider's raw
|
|
1230
|
-
* rejection body, JSON-stringified if possible) then falls back to the
|
|
1231
|
-
* message` field. Always returns a safe string, even when the thrown value
|
|
1232
|
-
* has hostile properties or cannot be serialized normally.
|
|
1233
|
-
*/
|
|
1234
|
-
function describeError(err) {
|
|
1235
|
-
if (err && typeof err === "object") try {
|
|
1236
|
-
const error = err;
|
|
1237
|
-
if (error.error !== void 0) return formatSafely(error.error);
|
|
1238
|
-
if (typeof error.message === "string") return error.message;
|
|
1239
|
-
} catch {}
|
|
1240
|
-
return formatSafely(err);
|
|
1241
|
-
}
|
|
1242
|
-
/**
|
|
1243
|
-
* Maps an HTTP status to its corresponding `LLMErrorCode`, derived purely
|
|
1244
|
-
* from the status itself so it applies the same way regardless of which
|
|
1245
|
-
* adapter or client raised the error. Used both when building a fresh
|
|
1246
|
-
* `LLMError` and when filling in a `code` on an already-normalized one
|
|
1247
|
-
* that doesn't have one yet, so the two paths can't drift apart.
|
|
1248
|
-
*/
|
|
1249
|
-
function codeForStatus(status) {
|
|
1250
|
-
switch (status) {
|
|
1251
|
-
case 429: return "provider_rate_limited";
|
|
1252
|
-
case 401: return "authentication";
|
|
1253
|
-
case 403: return "authorization";
|
|
1254
|
-
case 404: return "not_found";
|
|
1255
|
-
case 413: return "payload_too_large";
|
|
1256
|
-
default: return status >= 500 ? "server_error" : void 0;
|
|
1257
|
-
}
|
|
1258
|
-
}
|
|
1259
|
-
/**
|
|
1260
|
-
* Converts any thrown value into a well-typed LLMError. `attempts`, when
|
|
1261
|
-
* given, is the accumulated record of every attempt made before `error`
|
|
1262
|
-
* was thrown; it's passed straight into the constructed error's options
|
|
1263
|
-
* rather than assigned onto the error afterward, so `attempts` is always
|
|
1264
|
-
* settled once, through the constructor, like every other field on
|
|
1265
|
-
* `LLMError`.
|
|
1266
|
-
*/
|
|
1267
|
-
function normalizeError(error, signal, attempts) {
|
|
1268
|
-
if (signal?.aborted) return new LLMError("LLM request aborted", "aborted", { attempts });
|
|
1269
|
-
if (error instanceof LLMError) {
|
|
1270
|
-
if (error.code === void 0 && error.status !== void 0) error.code = codeForStatus(error.status);
|
|
1271
|
-
if (error.attempts === void 0 && attempts !== void 0) error.attempts = attempts;
|
|
1272
|
-
return error;
|
|
1273
|
-
}
|
|
1274
|
-
const status = extractStatus(error);
|
|
1275
|
-
const retryAfterMs = extractRetryAfterMs(error);
|
|
1276
|
-
if (status !== void 0) return new LLMError("LLM request failed", "api", {
|
|
1277
|
-
status,
|
|
1278
|
-
cause: error,
|
|
1279
|
-
retryAfterMs,
|
|
1280
|
-
code: codeForStatus(status),
|
|
1281
|
-
attempts
|
|
1282
|
-
});
|
|
1283
|
-
if (isNetworkError(error)) return new LLMError("LLM request failed", "network", {
|
|
1284
|
-
cause: error,
|
|
1285
|
-
retryAfterMs,
|
|
1286
|
-
code: "connection_failed",
|
|
1287
|
-
attempts
|
|
1288
|
-
});
|
|
1289
|
-
return new LLMError("LLM request failed", "unknown", {
|
|
1290
|
-
cause: error,
|
|
1291
|
-
retryAfterMs,
|
|
1292
|
-
attempts
|
|
1293
|
-
});
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
|
-
//#endregion
|
|
1297
|
-
//#region src/internal/execution/parse.utils.ts
|
|
1298
|
-
/** Default `parseJson`: `JSON.parse` wrapped in try/catch, returning `undefined` on failure. */
|
|
1299
|
-
function defaultParseJson(content) {
|
|
1300
|
-
try {
|
|
1301
|
-
return JSON.parse(content);
|
|
1302
|
-
} catch {
|
|
1303
|
-
return void 0;
|
|
1304
|
-
}
|
|
1305
|
-
}
|
|
1306
|
-
|
|
1307
|
-
//#endregion
|
|
1308
|
-
//#region src/internal/execution/wire.utils.ts
|
|
1309
|
-
/** Translates app-facing `ToolDefinition[]` into the OpenAI-shaped wire tools array. */
|
|
1310
|
-
function toWireTools(tools) {
|
|
1311
|
-
return tools.map((tool) => ({
|
|
1312
|
-
type: "function",
|
|
1313
|
-
function: {
|
|
1314
|
-
name: tool.name,
|
|
1315
|
-
description: tool.description,
|
|
1316
|
-
parameters: tool.parameters
|
|
1317
|
-
}
|
|
1318
|
-
}));
|
|
1319
|
-
}
|
|
1320
|
-
/** Translates app-facing `ToolCall[]` (e.g. from a replayed assistant turn) into wire tool_calls. */
|
|
1321
|
-
function toWireToolCalls(toolCalls) {
|
|
1322
|
-
return toolCalls.map((tc) => ({
|
|
1323
|
-
id: tc.id,
|
|
1324
|
-
type: "function",
|
|
1325
|
-
function: {
|
|
1326
|
-
name: tc.name,
|
|
1327
|
-
arguments: JSON.stringify(tc.arguments ?? {})
|
|
1328
|
-
}
|
|
1329
|
-
}));
|
|
1330
|
-
}
|
|
1331
|
-
/**
|
|
1332
|
-
* Parses the provider's wire-shaped `tool_calls` back into VernLLM's
|
|
1333
|
-
* `ToolCall[]`. Malformed argument JSON is a `'parse'` error, same
|
|
1334
|
-
* convention as malformed JSON response bodies elsewhere in VernLLM.
|
|
1335
|
-
*/
|
|
1336
|
-
function parseWireToolCalls(wireToolCalls) {
|
|
1337
|
-
return wireToolCalls.map((wc) => {
|
|
1338
|
-
let parsedArgs;
|
|
1339
|
-
try {
|
|
1340
|
-
parsedArgs = wc.function.arguments.trim() ? JSON.parse(wc.function.arguments) : {};
|
|
1341
|
-
} catch {
|
|
1342
|
-
throw new LLMError(`Invalid JSON arguments for tool call "${wc.function.name}"`, "parse", { code: "tool_arguments_parse_failed" });
|
|
1343
|
-
}
|
|
1344
|
-
return {
|
|
1345
|
-
id: wc.id,
|
|
1346
|
-
name: wc.function.name,
|
|
1347
|
-
arguments: parsedArgs
|
|
1348
|
-
};
|
|
1349
|
-
});
|
|
1350
|
-
}
|
|
1351
|
-
|
|
1352
|
-
//#endregion
|
|
1353
|
-
//#region src/internal/execution/requestBuilder.ts
|
|
1354
|
-
/**
|
|
1355
|
-
* Builds the wire request object for one call, applying per-instance
|
|
1356
|
-
* defaults (model, max tokens, temperature) and per-call overrides.
|
|
1357
|
-
* Owns every check that depends only on the caller's own input shape, not
|
|
1358
|
-
* on execution: history alternation, duplicate/empty tool lists,
|
|
1359
|
-
* `toolChoice` naming a real tool. All deterministic on the call site's
|
|
1360
|
-
* own input and never touch the network, so every throw here is
|
|
1361
|
-
* `type: 'invalid_params'`, not `'validation'` (which is reserved for the
|
|
1362
|
-
* model/provider's own response failing a contract check). Has no
|
|
1363
|
-
* knowledge of retry, timeouts, or the breaker, only
|
|
1364
|
-
* the three defaults a `FallbackTarget` can override per-target (see the
|
|
1365
|
-
* `defaultMaxTokens`/`defaultTemperature` overrides in the fallback
|
|
1366
|
-
* design), which is what keeps it separable from `CallExecutor`.
|
|
1367
|
-
*/
|
|
1368
|
-
var RequestBuilder = class {
|
|
1369
|
-
model;
|
|
1370
|
-
defaultMaxTokens;
|
|
1371
|
-
defaultTemperature;
|
|
1372
|
-
constructor(options) {
|
|
1373
|
-
this.model = options.model;
|
|
1374
|
-
this.defaultMaxTokens = options.defaultMaxTokens;
|
|
1375
|
-
this.defaultTemperature = options.defaultTemperature;
|
|
1376
|
-
}
|
|
1377
|
-
/** Applies per-call defaults and shapes params into the client's request object. */
|
|
1378
|
-
build(params) {
|
|
1379
|
-
const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model, reasoningEffort, jsonSchema, tools, toolChoice } = params;
|
|
1380
|
-
const temperature = params.temperature === void 0 ? this.defaultTemperature : params.temperature;
|
|
1381
|
-
if (tools && tools.length === 0) throw new LLMError("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.", "invalid_params");
|
|
1382
|
-
if (tools) {
|
|
1383
|
-
const seen = new Set();
|
|
1384
|
-
const duplicates = new Set();
|
|
1385
|
-
for (const tool of tools) {
|
|
1386
|
-
if (seen.has(tool.name)) duplicates.add(tool.name);
|
|
1387
|
-
seen.add(tool.name);
|
|
1388
|
-
}
|
|
1389
|
-
if (duplicates.size) throw new LLMError(`\`tools\` has duplicate name(s): [${[...duplicates].join(", ")}]. Tool names must be unique.`, "invalid_params", {
|
|
1390
|
-
code: "duplicate_tool_names",
|
|
1391
|
-
issues: { names: [...duplicates] }
|
|
1392
|
-
});
|
|
1393
|
-
}
|
|
1394
|
-
if (toolChoice && !tools) throw new LLMError("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.", "invalid_params");
|
|
1395
|
-
if (tools && typeof toolChoice === "object" && !tools.some((t) => t.name === toolChoice.name)) throw new LLMError(`toolChoice names "${toolChoice.name}", which is not in \`tools\` ([${tools.map((t) => t.name).join(", ")}]).`, "invalid_params", {
|
|
1396
|
-
code: "unknown_tool_choice",
|
|
1397
|
-
issues: {
|
|
1398
|
-
requested: toolChoice.name,
|
|
1399
|
-
available: tools.map((t) => t.name)
|
|
1400
|
-
}
|
|
1401
|
-
});
|
|
1402
|
-
const jsonMode = params.jsonMode ?? (tools ? false : true);
|
|
1403
|
-
const useJson = jsonMode || Boolean(jsonSchema);
|
|
1404
|
-
if (params.schema && !useJson) throw new LLMError("schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.", "invalid_params");
|
|
1405
|
-
const responseFormat = this.buildResponseFormat(jsonSchema, useJson);
|
|
1406
|
-
this.validateHistory(history);
|
|
1407
|
-
const request = {
|
|
1408
|
-
model,
|
|
1409
|
-
...temperature !== null ? { temperature } : {},
|
|
1410
|
-
max_tokens: maxTokens,
|
|
1411
|
-
...responseFormat ? { response_format: responseFormat } : {},
|
|
1412
|
-
...reasoningEffort ? { reasoning_effort: reasoningEffort } : {},
|
|
1413
|
-
...tools ? { tools: toWireTools(tools) } : {},
|
|
1414
|
-
...tools ? { tool_choice: this.buildWireToolChoice(toolChoice) } : {},
|
|
1415
|
-
messages: [
|
|
1416
|
-
...systemPrompt ? [{
|
|
1417
|
-
role: "system",
|
|
1418
|
-
content: systemPrompt
|
|
1419
|
-
}] : [],
|
|
1420
|
-
...history.flatMap((turn) => this.turnToWireMessages(turn)),
|
|
1421
|
-
{
|
|
1422
|
-
role: "user",
|
|
1423
|
-
content: userContent
|
|
1424
|
-
}
|
|
1425
|
-
]
|
|
1426
|
-
};
|
|
1427
|
-
return {
|
|
1428
|
-
useJson,
|
|
1429
|
-
model,
|
|
1430
|
-
request
|
|
1431
|
-
};
|
|
1432
|
-
}
|
|
1433
|
-
/**
|
|
1434
|
-
* Validates `history` alternates user/assistant turns, since providers
|
|
1435
|
-
* like Anthropic/Gemini reject or mishandle consecutive same-role turns.
|
|
1436
|
-
*/
|
|
1437
|
-
validateHistory(history) {
|
|
1438
|
-
let previousTurn;
|
|
1439
|
-
for (const [index, turn] of history.entries()) {
|
|
1440
|
-
if (turn.role === "tool") {
|
|
1441
|
-
if (previousTurn?.role !== "assistant" || !previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`, "invalid_params");
|
|
1442
|
-
if (!turn.toolResults?.length) throw new LLMError(`history[${index}] is a "tool" turn but has no toolResults`, "invalid_params");
|
|
1443
|
-
const requestedIds = new Set(previousTurn.toolCalls.map((tc) => tc.id));
|
|
1444
|
-
const resultIds = turn.toolResults.map((tr) => tr.toolCallId);
|
|
1445
|
-
const unknownIds = resultIds.filter((id) => !requestedIds.has(id));
|
|
1446
|
-
if (unknownIds.length) throw new LLMError(`history[${index}].toolResults references unknown toolCallId(s) [${unknownIds.join(", ")}]`, "invalid_params", {
|
|
1447
|
-
code: "unknown_tool_result_ids",
|
|
1448
|
-
issues: {
|
|
1449
|
-
historyIndex: index,
|
|
1450
|
-
ids: unknownIds
|
|
1451
|
-
}
|
|
1452
|
-
});
|
|
1453
|
-
const seenIds = new Set();
|
|
1454
|
-
const duplicateIds = new Set();
|
|
1455
|
-
for (const id of resultIds) {
|
|
1456
|
-
if (seenIds.has(id)) duplicateIds.add(id);
|
|
1457
|
-
seenIds.add(id);
|
|
1458
|
-
}
|
|
1459
|
-
if (duplicateIds.size) throw new LLMError(`history[${index}].toolResults has duplicate toolCallId(s) [${[...duplicateIds].join(", ")}]`, "invalid_params", {
|
|
1460
|
-
code: "duplicate_tool_result_ids",
|
|
1461
|
-
issues: {
|
|
1462
|
-
historyIndex: index,
|
|
1463
|
-
ids: [...duplicateIds]
|
|
1464
|
-
}
|
|
1465
|
-
});
|
|
1466
|
-
const missingIds = [...requestedIds].filter((id) => !resultIds.includes(id));
|
|
1467
|
-
if (missingIds.length) throw new LLMError(`history[${index}] is missing toolResults for toolCallId(s) [${missingIds.join(", ")}]`, "invalid_params", {
|
|
1468
|
-
code: "missing_tool_results",
|
|
1469
|
-
issues: {
|
|
1470
|
-
historyIndex: index,
|
|
1471
|
-
ids: missingIds
|
|
1472
|
-
}
|
|
1473
|
-
});
|
|
1474
|
-
} else {
|
|
1475
|
-
if (turn.role === previousTurn?.role) throw new LLMError(`history must alternate user/assistant turns: consecutive "${turn.role}" turns at history[${index - 1}] and history[${index}]`, "invalid_params");
|
|
1476
|
-
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] follows an assistant tool request without tool results`, "invalid_params");
|
|
1477
|
-
}
|
|
1478
|
-
previousTurn = turn;
|
|
1479
|
-
}
|
|
1480
|
-
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError("The last entry in history is an assistant tool request without tool results", "invalid_params");
|
|
1481
|
-
if (previousTurn?.role === "user") throw new LLMError("The last entry in history is a \"user\" turn, which would collide with the current userContent turn.", "invalid_params");
|
|
1482
|
-
}
|
|
1483
|
-
/** Maps VernLLM's app-facing `ToolChoice` onto the OpenAI-shaped wire `tool_choice`. */
|
|
1484
|
-
buildWireToolChoice(toolChoice) {
|
|
1485
|
-
if (!toolChoice || toolChoice === "auto") return "auto";
|
|
1486
|
-
if (toolChoice === "none" || toolChoice === "required") return toolChoice;
|
|
1487
|
-
return {
|
|
1488
|
-
type: "function",
|
|
1489
|
-
function: { name: toolChoice.name }
|
|
1490
|
-
};
|
|
1491
|
-
}
|
|
1492
|
-
/**
|
|
1493
|
-
* Expands one `ConversationTurn` into one or more wire messages. Plain
|
|
1494
|
-
* user/assistant turns map 1:1. An assistant turn with `toolCalls` maps
|
|
1495
|
-
* to an assistant message carrying wire-shaped `tool_calls`. A `'tool'`
|
|
1496
|
-
* turn expands into one wire `tool` message per `toolResult`, since
|
|
1497
|
-
* OpenAI-shaped wire format wants one message per tool_call_id.
|
|
1498
|
-
*/
|
|
1499
|
-
turnToWireMessages(turn) {
|
|
1500
|
-
if (turn.role === "tool") return (turn.toolResults ?? []).map((tr) => ({
|
|
1501
|
-
role: "tool",
|
|
1502
|
-
tool_call_id: tr.toolCallId,
|
|
1503
|
-
content: typeof tr.content === "string" ? tr.content : JSON.stringify(tr.content ?? null),
|
|
1504
|
-
...tr.isError ? { is_error: true } : {}
|
|
1505
|
-
}));
|
|
1506
|
-
if (turn.role === "assistant" && turn.toolCalls?.length) return [{
|
|
1507
|
-
role: "assistant",
|
|
1508
|
-
...turn.content ? { content: turn.content } : {},
|
|
1509
|
-
tool_calls: toWireToolCalls(turn.toolCalls)
|
|
1510
|
-
}];
|
|
1511
|
-
return [{
|
|
1512
|
-
role: turn.role,
|
|
1513
|
-
content: turn.content ?? ""
|
|
1514
|
-
}];
|
|
1515
|
-
}
|
|
1516
|
-
/**
|
|
1517
|
-
* Chooses the response format: a provider-native `jsonSchema` takes
|
|
1518
|
-
* priority when supplied (constrains generation directly), otherwise
|
|
1519
|
-
* falls back to the looser `json_object` mode when JSON output is
|
|
1520
|
-
* requested, or no format at all for plain text responses.
|
|
1521
|
-
*/
|
|
1522
|
-
buildResponseFormat(jsonSchema, useJson) {
|
|
1523
|
-
if (jsonSchema) return {
|
|
1524
|
-
type: "json_schema",
|
|
1525
|
-
json_schema: {
|
|
1526
|
-
name: jsonSchema.name,
|
|
1527
|
-
schema: jsonSchema.schema,
|
|
1528
|
-
strict: jsonSchema.strict ?? true,
|
|
1529
|
-
description: jsonSchema.description
|
|
1530
|
-
}
|
|
1531
|
-
};
|
|
1532
|
-
return useJson ? { type: "json_object" } : void 0;
|
|
1533
|
-
}
|
|
1534
|
-
};
|
|
1535
|
-
|
|
1536
|
-
//#endregion
|
|
1537
|
-
//#region src/internal/execution/streamAccumulator.ts
|
|
1538
|
-
/**
|
|
1539
|
-
* The streaming accumulator: wraps the raw `WireStreamChunk` iterator in
|
|
1540
|
-
* an async generator that yields translated `StreamChunk`s to the caller
|
|
1541
|
-
* live, as they arrive, with no per-chunk timeout and no bound on total
|
|
1542
|
-
* duration, and accumulates text/tool-call deltas internally so that
|
|
1543
|
-
* `finalize` can produce `finalResult` once the stream completes.
|
|
1544
|
-
*
|
|
1545
|
-
* Two separate try/catches: the iteration loop's catch handles errors
|
|
1546
|
-
* the transport itself throws, which aren't normalized yet, so that
|
|
1547
|
-
* happens here, alongside the one `onStreamFailure` call for them. The
|
|
1548
|
-
* second catch, around `finalize`, does not re-normalize or re-report,
|
|
1549
|
-
* since `finalize`'s caller (`finalizeResponse`) already does both
|
|
1550
|
-
* internally.
|
|
1551
|
-
*/
|
|
1552
|
-
function buildStreamResult(iterator, first, options) {
|
|
1553
|
-
const { requestId, model, providerName, isFallback, chunkIdleTimeoutMs, streamController, logger, signal } = options;
|
|
1554
|
-
let resolveFinal;
|
|
1555
|
-
let rejectFinal;
|
|
1556
|
-
const finalResult = new Promise((resolve, reject) => {
|
|
1557
|
-
resolveFinal = resolve;
|
|
1558
|
-
rejectFinal = reject;
|
|
1559
|
-
});
|
|
1560
|
-
finalResult.catch(() => {});
|
|
1561
|
-
const MAX_BUFFERED_CHUNKS = 1e4;
|
|
1562
|
-
const buffered = [];
|
|
1563
|
-
const pending = [];
|
|
1564
|
-
let streamDone = false;
|
|
1565
|
-
let streamError;
|
|
1566
|
-
let hasLoggedEviction = false;
|
|
1567
|
-
const push = (chunk) => {
|
|
1568
|
-
const waiter = pending.shift();
|
|
1569
|
-
if (waiter) {
|
|
1570
|
-
waiter.resolve({
|
|
1571
|
-
done: false,
|
|
1572
|
-
value: chunk
|
|
1573
|
-
});
|
|
1574
|
-
return;
|
|
1575
|
-
}
|
|
1576
|
-
buffered.push(chunk);
|
|
1577
|
-
if (buffered.length > MAX_BUFFERED_CHUNKS * 2) {
|
|
1578
|
-
if (!hasLoggedEviction) {
|
|
1579
|
-
hasLoggedEviction = true;
|
|
1580
|
-
logger.warn(`[VernLLM] stream chunk buffer exceeded cap (${MAX_BUFFERED_CHUNKS}), evicting ${buffered.length - MAX_BUFFERED_CHUNKS} oldest chunk(s); buffered=${buffered.length}. The chunks iterable was never read (or fell far behind) for this stream.`);
|
|
1581
|
-
}
|
|
1582
|
-
buffered.splice(0, buffered.length - MAX_BUFFERED_CHUNKS);
|
|
1583
|
-
}
|
|
1584
|
-
};
|
|
1585
|
-
const finish = () => {
|
|
1586
|
-
streamDone = true;
|
|
1587
|
-
for (const waiter of pending.splice(0)) waiter.resolve({
|
|
1588
|
-
done: true,
|
|
1589
|
-
value: void 0
|
|
1590
|
-
});
|
|
1591
|
-
};
|
|
1592
|
-
const fail = (error) => {
|
|
1593
|
-
streamDone = true;
|
|
1594
|
-
streamError = error;
|
|
1595
|
-
for (const waiter of pending.splice(0)) waiter.reject(error);
|
|
1596
|
-
};
|
|
1597
|
-
const chunks = { [Symbol.asyncIterator]() {
|
|
1598
|
-
return { next() {
|
|
1599
|
-
if (buffered.length) return Promise.resolve({
|
|
1600
|
-
done: false,
|
|
1601
|
-
value: buffered.shift()
|
|
1602
|
-
});
|
|
1603
|
-
if (streamDone) return streamError ? Promise.reject(streamError) : Promise.resolve({
|
|
1604
|
-
done: true,
|
|
1605
|
-
value: void 0
|
|
1606
|
-
});
|
|
1607
|
-
return new Promise((resolve, reject) => {
|
|
1608
|
-
pending.push({
|
|
1609
|
-
resolve,
|
|
1610
|
-
reject
|
|
1611
|
-
});
|
|
1612
|
-
});
|
|
1613
|
-
} };
|
|
1614
|
-
} };
|
|
1615
|
-
const toolCallAcc = new Map();
|
|
1616
|
-
let textAcc = "";
|
|
1617
|
-
let usage;
|
|
1618
|
-
(async () => {
|
|
1619
|
-
try {
|
|
1620
|
-
let result = first;
|
|
1621
|
-
while (!result.done) {
|
|
1622
|
-
const wireChunk = result.value;
|
|
1623
|
-
if (wireChunk.type === "ping") {} else if (wireChunk.type === "text-delta") {
|
|
1624
|
-
textAcc += wireChunk.delta;
|
|
1625
|
-
push({
|
|
1626
|
-
type: "text-delta",
|
|
1627
|
-
delta: wireChunk.delta
|
|
1628
|
-
});
|
|
1629
|
-
} else if (wireChunk.type === "tool_call_delta") {
|
|
1630
|
-
const entry = toolCallAcc.get(wireChunk.index) ?? { args: "" };
|
|
1631
|
-
entry.id ??= wireChunk.id;
|
|
1632
|
-
entry.name ??= wireChunk.name;
|
|
1633
|
-
entry.args += wireChunk.argumentsDelta ?? "";
|
|
1634
|
-
toolCallAcc.set(wireChunk.index, entry);
|
|
1635
|
-
push({
|
|
1636
|
-
type: "tool_call_delta",
|
|
1637
|
-
index: wireChunk.index,
|
|
1638
|
-
id: wireChunk.id,
|
|
1639
|
-
name: wireChunk.name,
|
|
1640
|
-
argsDelta: wireChunk.argumentsDelta,
|
|
1641
|
-
complete: wireChunk.complete
|
|
1642
|
-
});
|
|
1643
|
-
} else if (wireChunk.type === "usage") {
|
|
1644
|
-
usage = {
|
|
1645
|
-
promptTokens: wireChunk.usage.prompt_tokens ?? 0,
|
|
1646
|
-
completionTokens: wireChunk.usage.completion_tokens ?? 0,
|
|
1647
|
-
totalTokens: wireChunk.usage.total_tokens ?? 0,
|
|
1648
|
-
requestId,
|
|
1649
|
-
model,
|
|
1650
|
-
provider: providerName,
|
|
1651
|
-
usedFallback: isFallback
|
|
1652
|
-
};
|
|
1653
|
-
push({
|
|
1654
|
-
type: "usage",
|
|
1655
|
-
usage
|
|
1656
|
-
});
|
|
1657
|
-
}
|
|
1658
|
-
result = await withChunkIdleTimeout(() => iterator.next(), chunkIdleTimeoutMs, () => streamController.abort(), logger);
|
|
1659
|
-
}
|
|
1660
|
-
} catch (error) {
|
|
1661
|
-
try {
|
|
1662
|
-
await iterator.return?.();
|
|
1663
|
-
} catch {}
|
|
1664
|
-
streamController.abort();
|
|
1665
|
-
const normalized = normalizeError(error, signal);
|
|
1666
|
-
try {
|
|
1667
|
-
options.onStreamFailure(normalized, usage);
|
|
1668
|
-
} catch {}
|
|
1669
|
-
fail(normalized);
|
|
1670
|
-
rejectFinal(normalized);
|
|
1671
|
-
return;
|
|
1672
|
-
}
|
|
1673
|
-
finish();
|
|
1674
|
-
try {
|
|
1675
|
-
options.onStreamSuccess(usage);
|
|
1676
|
-
} catch {}
|
|
1677
|
-
try {
|
|
1678
|
-
const wireToolCalls = toolCallAcc.size ? [...toolCallAcc.entries()].sort(([indexA], [indexB]) => indexA - indexB).map(([, entry]) => ({
|
|
1679
|
-
id: entry.id ?? "",
|
|
1680
|
-
type: "function",
|
|
1681
|
-
function: {
|
|
1682
|
-
name: entry.name ?? "",
|
|
1683
|
-
arguments: entry.args
|
|
1684
|
-
}
|
|
1685
|
-
})) : void 0;
|
|
1686
|
-
const finalized = options.finalize(textAcc, wireToolCalls, usage);
|
|
1687
|
-
resolveFinal(finalized);
|
|
1688
|
-
} catch (error) {
|
|
1689
|
-
rejectFinal(error);
|
|
1690
|
-
}
|
|
1691
|
-
})();
|
|
1692
|
-
return {
|
|
1693
|
-
chunks,
|
|
1694
|
-
finalResult
|
|
1695
|
-
};
|
|
1696
|
-
}
|
|
1697
|
-
|
|
1698
|
-
//#endregion
|
|
1699
|
-
//#region src/internal/execution/callExecutor.ts
|
|
1700
|
-
/**
|
|
1701
|
-
* Everything one provider target needs to attempt a call: request
|
|
1702
|
-
* building, retry with backoff, the per-target breaker, the per-target
|
|
1703
|
-
* limiter. Never exported publicly. `VernLLM` holds one per target and
|
|
1704
|
-
* owns the fallback loop and caching on top.
|
|
1705
|
-
*/
|
|
1706
|
-
var CallExecutor = class {
|
|
1707
|
-
maxRetries;
|
|
1708
|
-
timeoutMs;
|
|
1709
|
-
chunkIdleTimeoutMs;
|
|
1710
|
-
baseDelayMs;
|
|
1711
|
-
nonRetryableStatus;
|
|
1712
|
-
parseJson;
|
|
1713
|
-
logger;
|
|
1714
|
-
redact;
|
|
1715
|
-
onUsage;
|
|
1716
|
-
onUsageFailure;
|
|
1717
|
-
reportEvent;
|
|
1718
|
-
breaker;
|
|
1719
|
-
limiter;
|
|
1720
|
-
isFallback;
|
|
1721
|
-
requestBuilder;
|
|
1722
|
-
constructor(providerName, client, model, options) {
|
|
1723
|
-
this.providerName = providerName;
|
|
1724
|
-
this.client = client;
|
|
1725
|
-
this.model = model;
|
|
1726
|
-
this.maxRetries = options.maxRetries;
|
|
1727
|
-
this.timeoutMs = options.timeoutMs;
|
|
1728
|
-
this.chunkIdleTimeoutMs = options.chunkIdleTimeoutMs;
|
|
1729
|
-
this.baseDelayMs = options.baseDelayMs;
|
|
1730
|
-
this.nonRetryableStatus = options.nonRetryableStatus;
|
|
1731
|
-
this.parseJson = options.parseJson ?? defaultParseJson;
|
|
1732
|
-
this.logger = options.logger;
|
|
1733
|
-
this.redact = options.redact;
|
|
1734
|
-
this.onUsage = options.onUsage;
|
|
1735
|
-
this.onUsageFailure = options.onUsageFailure;
|
|
1736
|
-
this.reportEvent = makeEventReporter(options.onEvent, this.logger);
|
|
1737
|
-
this.breaker = options.breaker;
|
|
1738
|
-
this.limiter = options.limiter;
|
|
1739
|
-
this.isFallback = options.isFallback ?? false;
|
|
1740
|
-
this.requestBuilder = new RequestBuilder({
|
|
1741
|
-
model,
|
|
1742
|
-
defaultMaxTokens: options.defaultMaxTokens,
|
|
1743
|
-
defaultTemperature: options.defaultTemperature
|
|
1744
|
-
});
|
|
1745
|
-
}
|
|
1746
|
-
getCircuitState(model) {
|
|
1747
|
-
return this.breaker?.getState(model);
|
|
1748
|
-
}
|
|
1749
|
-
/** Whether this target's breaker tracks failures per model. `false` if no breaker is configured. */
|
|
1750
|
-
get isolateByModel() {
|
|
1751
|
-
return this.breaker?.isolateByModel ?? false;
|
|
1752
|
-
}
|
|
1753
|
-
/** Manually opens this target's circuit breaker, if one is configured. No-op otherwise. */
|
|
1754
|
-
openCircuit(model) {
|
|
1755
|
-
this.breaker?.open(model);
|
|
1756
|
-
}
|
|
1757
|
-
/** Manually closes this target's circuit breaker, if one is configured. No-op otherwise. */
|
|
1758
|
-
closeCircuit(model) {
|
|
1759
|
-
this.breaker?.close(model);
|
|
1760
|
-
}
|
|
1761
|
-
/**
|
|
1762
|
-
* Throws if the breaker is open for this target/model, exactly like the
|
|
1763
|
-
* check `run`/`runStream` used to make internally. Exposed so `VernLLM`
|
|
1764
|
-
* can gate on it before reserving usage, avoiding a reserve-then-refund
|
|
1765
|
-
* round trip on a call that was never going to be attempted. `assertClosed`
|
|
1766
|
-
* has a stateful side effect (claiming a half-open trial slot), so it must
|
|
1767
|
-
* run exactly once per logical call: `run`/`runStream` no longer call it
|
|
1768
|
-
* themselves, this is now the only call site.
|
|
1769
|
-
*/
|
|
1770
|
-
assertBreakerClosed(model) {
|
|
1771
|
-
this.breaker?.assertClosed(model ?? this.model);
|
|
1772
|
-
}
|
|
1773
|
-
/**
|
|
1774
|
-
* Runs a single logical call against this target: retry with backoff,
|
|
1775
|
-
* normalized error on exhaustion. Mirrors the old `VernLLM.call`'s
|
|
1776
|
-
* non-streaming branch, minus cache/usage-reservation and the breaker
|
|
1777
|
-
* check, which stay one layer up since they aren't per-target concerns
|
|
1778
|
-
* (see `assertBreakerClosed`).
|
|
1779
|
-
*/
|
|
1780
|
-
async run(params, requestId, onAttempt) {
|
|
1781
|
-
const model = params.model ?? this.model;
|
|
1782
|
-
const attempts = [];
|
|
1783
|
-
try {
|
|
1784
|
-
return await this.retryWithBackoff((attempt) => this.executeCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
|
|
1785
|
-
} catch (error) {
|
|
1786
|
-
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1787
|
-
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
1788
|
-
this.logger.debug(`[VernLLM:${requestId}] error:\n${this.redactText(describeError(error))}`);
|
|
1789
|
-
throw normalized;
|
|
1790
|
-
}
|
|
1791
|
-
}
|
|
1792
|
-
/** Streaming counterpart to `run`. Mirrors the old streaming branch of `VernLLM.call`. */
|
|
1793
|
-
async runStream(params, requestId, onAttempt) {
|
|
1794
|
-
const model = params.model ?? this.model;
|
|
1795
|
-
const attempts = [];
|
|
1796
|
-
try {
|
|
1797
|
-
return await this.retryWithBackoff((attempt) => this.executeStreamCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
|
|
1798
|
-
} catch (error) {
|
|
1799
|
-
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1800
|
-
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
1801
|
-
this.logger.debug(`[VernLLM:${requestId}] stream-open error:\n${this.redactText(describeError(error))}`);
|
|
1802
|
-
throw normalized;
|
|
1803
|
-
}
|
|
1804
|
-
}
|
|
1805
|
-
/**
|
|
1806
|
-
* Performs a single attempt: builds the request (translating `tools` to
|
|
1807
|
-
* wire shape when present), dispatches it with a timeout, and shapes the
|
|
1808
|
-
* response into `T` or a `CallWithToolsResult<T>` when `params.tools` was
|
|
1809
|
-
* set. Throws on an empty response (no text and no tool_calls) so the
|
|
1810
|
-
* retry loop treats it like any other transient failure.
|
|
1811
|
-
*/
|
|
1812
|
-
async executeCall(params, requestId, attempt) {
|
|
1813
|
-
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
1814
|
-
let release;
|
|
1815
|
-
if (this.limiter) {
|
|
1816
|
-
const acquired = await this.limiter.acquire(this.limiter.estimate(request), params.signal);
|
|
1817
|
-
release = acquired.release;
|
|
1818
|
-
if (acquired.waitedMs > 0) this.reportEvent({
|
|
1819
|
-
kind: "rate_limited",
|
|
1820
|
-
requestId,
|
|
1821
|
-
provider: this.providerName,
|
|
1822
|
-
model,
|
|
1823
|
-
waitedMs: acquired.waitedMs,
|
|
1824
|
-
reason: acquired.reason ?? "rpm"
|
|
1825
|
-
});
|
|
1826
|
-
}
|
|
1827
|
-
try {
|
|
1828
|
-
const response = await withTimeout((attemptSignal) => this.client.chat.completions.create(request, { signal: attemptSignal }), this.timeoutMs, params.signal);
|
|
1829
|
-
const usage = this.extractUsage(response, requestId, model);
|
|
1830
|
-
release?.(this.actualTokensFor(usage));
|
|
1831
|
-
release = void 0;
|
|
1832
|
-
const rawContent = response.choices?.[0]?.message?.content;
|
|
1833
|
-
const wireToolCalls = response.choices?.[0]?.message?.tool_calls;
|
|
1834
|
-
return this.finalizeResponse(rawContent, wireToolCalls, params, useJson, model, usage, requestId, attempt);
|
|
1835
|
-
} finally {
|
|
1836
|
-
release?.();
|
|
1837
|
-
}
|
|
1838
|
-
}
|
|
1839
|
-
/** Applies `redact` (if configured); otherwise returns `text` unchanged. */
|
|
1840
|
-
redactText(text) {
|
|
1841
|
-
return this.redact ? this.redact(text) : text;
|
|
1842
|
-
}
|
|
1843
|
-
/**
|
|
1844
|
-
* Applies `redact` (if configured) to whatever the debug log is about
|
|
1845
|
-
* to show: real content when there is any, otherwise the tool-call
|
|
1846
|
-
* placeholder, which carries no user data and passes through
|
|
1847
|
-
* `redact` unchanged in practice but is included for a caller whose
|
|
1848
|
-
* `redact` does something structural (e.g. adding a marker) rather
|
|
1849
|
-
* than just scrubbing PII.
|
|
1850
|
-
*/
|
|
1851
|
-
redactedOutput(content, wireToolCalls) {
|
|
1852
|
-
return this.redactText(content ?? `[${wireToolCalls?.length ?? 0} tool call(s)]`);
|
|
1853
|
-
}
|
|
1854
|
-
/**
|
|
1855
|
-
* Shapes a fully-arrived response (content and/or tool_calls, already
|
|
1856
|
-
* extracted from the provider's payload) into `T` or a
|
|
1857
|
-
* `CallWithToolsResult<T>`. Reused by the streaming path once it has
|
|
1858
|
-
* buffered the full text/tool-call deltas, so there's no separate
|
|
1859
|
-
* parsing/validation logic for streaming.
|
|
1860
|
-
*
|
|
1861
|
-
* Normalizes and reports usage failure on error itself, so every caller
|
|
1862
|
-
* gets identical error handling without duplicating it.
|
|
1863
|
-
*/
|
|
1864
|
-
finalizeResponse(rawContent, wireToolCalls, params, useJson, model, usage, requestId, attempt) {
|
|
1865
|
-
try {
|
|
1866
|
-
const content = rawContent?.trim();
|
|
1867
|
-
if (!content && !wireToolCalls?.length) throw new LLMError("Empty LLM response", "api", { code: "empty_response" });
|
|
1868
|
-
this.logger.debug(`[VernLLM:${requestId}] output:\n${this.redactedOutput(content, wireToolCalls).slice(0, 800)}`);
|
|
1869
|
-
if (wireToolCalls?.length) {
|
|
1870
|
-
if (!params.tools) throw new LLMError("Provider returned tool_calls but no `tools` were sent with this call.", "validation", { code: "unexpected_tool_calls" });
|
|
1871
|
-
if (params.toolChoice === "none") throw new LLMError("Provider returned tool_calls despite toolChoice: 'none'.", "validation", { code: "tool_choice_none_violated" });
|
|
1872
|
-
const toolCalls = parseWireToolCalls(wireToolCalls);
|
|
1873
|
-
this.validateToolCallArguments(toolCalls, params.tools);
|
|
1874
|
-
this.breaker?.recordSuccess(model);
|
|
1875
|
-
this.reportUsage(usage);
|
|
1876
|
-
return {
|
|
1877
|
-
type: "tool_calls",
|
|
1878
|
-
toolCalls,
|
|
1879
|
-
...content ? { content } : {}
|
|
1880
|
-
};
|
|
1881
|
-
}
|
|
1882
|
-
const textContent = content ?? "";
|
|
1883
|
-
if (!useJson) {
|
|
1884
|
-
this.breaker?.recordSuccess(model);
|
|
1885
|
-
this.reportUsage(usage);
|
|
1886
|
-
return params.tools ? {
|
|
1887
|
-
type: "content",
|
|
1888
|
-
content: textContent
|
|
1889
|
-
} : textContent;
|
|
1890
|
-
}
|
|
1891
|
-
const result = this.parseAndValidate(textContent, params.schema);
|
|
1892
|
-
this.breaker?.recordSuccess(model);
|
|
1893
|
-
this.reportUsage(usage);
|
|
1894
|
-
return params.tools ? {
|
|
1895
|
-
type: "content",
|
|
1896
|
-
content: result
|
|
1897
|
-
} : result;
|
|
1898
|
-
} catch (error) {
|
|
1899
|
-
const normalized = normalizeError(error, params.signal);
|
|
1900
|
-
if (usage && normalized.type !== "aborted") this.reportUsageFailure(usage, normalized, attempt);
|
|
1901
|
-
throw normalized;
|
|
1902
|
-
}
|
|
1903
|
-
}
|
|
1904
|
-
/**
|
|
1905
|
-
* Opens a stream for a single attempt: builds the request exactly like
|
|
1906
|
-
* `executeCall`, then requires `createStream` on the client (a clear
|
|
1907
|
-
* `validation` error if the adapter doesn't support it). The timeout
|
|
1908
|
-
* wraps stream construction and the first `.next()` together, not just
|
|
1909
|
-
* construction: calling an `async function*` returns an iterator
|
|
1910
|
-
* synchronously without running its body until `.next()` is first
|
|
1911
|
-
* invoked, so timing only construction would time an operation that's
|
|
1912
|
-
* always instant, not the actual connection. Both are folded into a
|
|
1913
|
-
* single `withTimeout` so the same abort signal reaches whatever the
|
|
1914
|
-
* adapter's `createStream` uses internally for its first network
|
|
1915
|
-
* round-trip.
|
|
1916
|
-
*
|
|
1917
|
-
* Circuit-breaker success is recorded once the stream fully completes,
|
|
1918
|
-
* not on the first chunk arriving, so a connection that opens but then
|
|
1919
|
-
* dies mid-stream isn't masked as a success (see `buildStreamResult`).
|
|
1920
|
-
*/
|
|
1921
|
-
async executeStreamCall(params, requestId, attempt) {
|
|
1922
|
-
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
1923
|
-
const completions = this.client.chat.completions;
|
|
1924
|
-
if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "invalid_params", {
|
|
1925
|
-
code: "unsupported_capability",
|
|
1926
|
-
issues: { capability: "createStream" }
|
|
1927
|
-
});
|
|
1928
|
-
const createStream = completions.createStream.bind(completions);
|
|
1929
|
-
let release;
|
|
1930
|
-
if (this.limiter) {
|
|
1931
|
-
const acquired = await this.limiter.acquire(this.limiter.estimate(request), params.signal);
|
|
1932
|
-
release = acquired.release;
|
|
1933
|
-
if (acquired.waitedMs > 0) this.reportEvent({
|
|
1934
|
-
kind: "rate_limited",
|
|
1935
|
-
requestId,
|
|
1936
|
-
provider: this.providerName,
|
|
1937
|
-
model,
|
|
1938
|
-
waitedMs: acquired.waitedMs,
|
|
1939
|
-
reason: acquired.reason ?? "rpm"
|
|
1940
|
-
});
|
|
1941
|
-
}
|
|
1942
|
-
const streamController = new AbortController();
|
|
1943
|
-
const combinedExternal = params.signal ? AbortSignal.any([params.signal, streamController.signal]) : streamController.signal;
|
|
1944
|
-
try {
|
|
1945
|
-
const { iterator, first } = await withTimeout(async (attemptSignal) => {
|
|
1946
|
-
const streamIterator = createStream(request, { signal: attemptSignal })[Symbol.asyncIterator]();
|
|
1947
|
-
const firstResult = await streamIterator.next();
|
|
1948
|
-
return {
|
|
1949
|
-
iterator: streamIterator,
|
|
1950
|
-
first: firstResult
|
|
1951
|
-
};
|
|
1952
|
-
}, this.timeoutMs, combinedExternal);
|
|
1953
|
-
if (first.done) throw new LLMError("Empty LLM response", "api");
|
|
1954
|
-
const releaseAtOpen = release;
|
|
1955
|
-
const result = buildStreamResult(iterator, first, {
|
|
1956
|
-
requestId,
|
|
1957
|
-
model,
|
|
1958
|
-
providerName: this.providerName,
|
|
1959
|
-
isFallback: this.isFallback,
|
|
1960
|
-
chunkIdleTimeoutMs: params.chunkIdleTimeoutMs ?? this.chunkIdleTimeoutMs,
|
|
1961
|
-
streamController,
|
|
1962
|
-
logger: this.logger,
|
|
1963
|
-
signal: params.signal,
|
|
1964
|
-
onStreamSuccess: (usage) => {
|
|
1965
|
-
this.breaker?.recordSuccess(model);
|
|
1966
|
-
releaseAtOpen?.(this.actualTokensFor(usage));
|
|
1967
|
-
},
|
|
1968
|
-
onStreamFailure: (normalized, usage) => {
|
|
1969
|
-
if (normalized.type === "timeout") this.breaker?.recordFailure(model);
|
|
1970
|
-
if (usage && normalized.type !== "aborted") this.reportUsageFailure(usage, normalized, attempt, true);
|
|
1971
|
-
releaseAtOpen?.(this.actualTokensFor(usage));
|
|
1972
|
-
},
|
|
1973
|
-
finalize: (textAcc, wireToolCalls, usage) => this.finalizeResponse(textAcc, wireToolCalls, params, useJson, model, usage, requestId, attempt)
|
|
1974
|
-
});
|
|
1975
|
-
release = void 0;
|
|
1976
|
-
return result;
|
|
1977
|
-
} finally {
|
|
1978
|
-
release?.();
|
|
1979
|
-
}
|
|
1980
|
-
}
|
|
1981
|
-
/**
|
|
1982
|
-
* Checks every `ToolCall` against the `tools` that were offered, catching
|
|
1983
|
-
* a hallucinated tool name and a duplicate call id before either reaches
|
|
1984
|
-
* the application's dispatch table, then runs each tool's
|
|
1985
|
-
* `argumentsSchema`, if present.
|
|
1986
|
-
*
|
|
1987
|
-
* Contract failures (unknown name, duplicate id) are collected across
|
|
1988
|
-
* every call and thrown together as one `type: 'validation'` error with
|
|
1989
|
-
* `issues: ToolIssue[]`, since retrying a request that already has these
|
|
1990
|
-
* errors cannot help (excluded from retry by `type`) and a caller fixing
|
|
1991
|
-
* them wants to see every one, not just the first. Schema failures keep
|
|
1992
|
-
* the original single-error, `type: 'validation'` shape rather than being
|
|
1993
|
-
* folded into the aggregate, since they're a distinct failure kind from
|
|
1994
|
-
* the contract failures above.
|
|
1995
|
-
*/
|
|
1996
|
-
validateToolCallArguments(toolCalls, tools) {
|
|
1997
|
-
const known = new Map(tools.map((t) => [t.name, t]));
|
|
1998
|
-
const seenIds = new Set();
|
|
1999
|
-
const toolIssues = [];
|
|
2000
|
-
for (const call of toolCalls) {
|
|
2001
|
-
if (seenIds.has(call.id)) toolIssues.push({
|
|
2002
|
-
name: call.name,
|
|
2003
|
-
toolCallId: call.id,
|
|
2004
|
-
code: "duplicate_tool_call_id"
|
|
2005
|
-
});
|
|
2006
|
-
seenIds.add(call.id);
|
|
2007
|
-
if (!known.has(call.name)) toolIssues.push({
|
|
2008
|
-
name: call.name,
|
|
2009
|
-
toolCallId: call.id,
|
|
2010
|
-
code: "unknown_tool"
|
|
2011
|
-
});
|
|
2012
|
-
}
|
|
2013
|
-
if (toolIssues.length > 0) {
|
|
2014
|
-
const unknownTool = toolIssues.find((i) => i.code === "unknown_tool");
|
|
2015
|
-
const primary = unknownTool ? `Model requested tool "${unknownTool.name}", which was not in the tools offered ([${[...known.keys()].join(", ")}]).` : `Duplicate tool call id "${toolIssues[0].toolCallId}" in the model's response.`;
|
|
2016
|
-
const message = toolIssues.length > 1 ? `${primary} (${toolIssues.length} tool call issues total, see error.issues.)` : primary;
|
|
2017
|
-
throw new LLMError(message, "validation", {
|
|
2018
|
-
code: unknownTool ? "unknown_tool" : "duplicate_tool_call_id",
|
|
2019
|
-
issues: toolIssues
|
|
2020
|
-
});
|
|
2021
|
-
}
|
|
2022
|
-
for (const call of toolCalls) {
|
|
2023
|
-
const definition = known.get(call.name);
|
|
2024
|
-
if (!definition?.argumentsSchema) continue;
|
|
2025
|
-
const result = definition.argumentsSchema.safeParse(call.arguments);
|
|
2026
|
-
if (!result.success) throw new LLMError(`Arguments for tool call "${call.name}" failed validation`, "validation", { issues: result.error });
|
|
2027
|
-
}
|
|
2028
|
-
}
|
|
2029
|
-
/**
|
|
2030
|
-
* Runs `fn`, retrying with backoff according to `shouldRetry`. When
|
|
2031
|
-
* `attempts` is given, every failed attempt that is actually followed by
|
|
2032
|
-
* a retry is recorded, in order. This mirrors `LLMError.attempts`'s
|
|
2033
|
-
* contract: every attempt made before this error was thrown. The
|
|
2034
|
-
* terminal failure is never pushed since it isn't a prior attempt, it
|
|
2035
|
-
* is the error being thrown. `attempts` stays empty when nothing was
|
|
2036
|
-
* retried, so no separate bookkeeping is needed at the call sites.
|
|
2037
|
-
* Each failure is recorded as a snapshot (`LLMError.toSnapshot()`),
|
|
2038
|
-
* not the live `LLMError`, per `RetryAttempt`'s contract.
|
|
2039
|
-
*/
|
|
2040
|
-
async retryWithBackoff(fn, requestId, model, signal, onAttempt, attempts) {
|
|
2041
|
-
let lastError;
|
|
2042
|
-
for (let attempt = 0; attempt <= this.maxRetries; attempt++) try {
|
|
2043
|
-
if (attempt > 0) await this.recoverDelay(requestId, model, attempt, lastError, signal);
|
|
2044
|
-
onAttempt?.();
|
|
2045
|
-
return await fn(attempt);
|
|
2046
|
-
} catch (error) {
|
|
2047
|
-
lastError = error;
|
|
2048
|
-
const willRetry = attempt < this.maxRetries && this.shouldRetry(error, signal);
|
|
2049
|
-
if (!willRetry) break;
|
|
2050
|
-
attempts?.push({
|
|
2051
|
-
index: attempt,
|
|
2052
|
-
error: normalizeError(error, signal).toSnapshot()
|
|
2053
|
-
});
|
|
2054
|
-
}
|
|
2055
|
-
throw lastError;
|
|
2056
|
-
}
|
|
2057
|
-
/**
|
|
2058
|
-
* Pulls `TokenUsage` out of a raw response, if the provider reported it.
|
|
2059
|
-
* Extraction doesn't depend on what happens to the response afterward, so
|
|
2060
|
-
* a malformed body can still yield usage if the provider's usage block
|
|
2061
|
-
* itself came through intact.
|
|
2062
|
-
*/
|
|
2063
|
-
extractUsage(response, requestId, model) {
|
|
2064
|
-
if (!response.usage) return void 0;
|
|
2065
|
-
return {
|
|
2066
|
-
promptTokens: response.usage.prompt_tokens ?? 0,
|
|
2067
|
-
completionTokens: response.usage.completion_tokens ?? 0,
|
|
2068
|
-
totalTokens: response.usage.total_tokens ?? 0,
|
|
2069
|
-
requestId,
|
|
2070
|
-
model,
|
|
2071
|
-
provider: this.providerName,
|
|
2072
|
-
usedFallback: this.isFallback
|
|
2073
|
-
};
|
|
2074
|
-
}
|
|
2075
|
-
/**
|
|
2076
|
-
* The token count to reconcile the rate limiter against for a finished
|
|
2077
|
-
* attempt: `totalTokens` when reported, otherwise the sum of prompt and
|
|
2078
|
-
* completion tokens, matching `reportUsageFailure`'s own fallback below
|
|
2079
|
-
* for a hand-rolled client that reports the parts but omits the total.
|
|
2080
|
-
*/
|
|
2081
|
-
actualTokensFor(usage) {
|
|
2082
|
-
if (!usage) return void 0;
|
|
2083
|
-
return usage.totalTokens || usage.promptTokens + usage.completionTokens;
|
|
2084
|
-
}
|
|
2085
|
-
/** Reports token usage for a successful call, swallowing and logging any error `onUsage` throws. */
|
|
2086
|
-
reportUsage(usage) {
|
|
2087
|
-
if (!usage || !this.onUsage) return;
|
|
2088
|
-
try {
|
|
2089
|
-
this.onUsage(usage);
|
|
2090
|
-
} catch (error) {
|
|
2091
|
-
this.logger.error("[VernLLM] onUsage failed", { message: error instanceof Error ? error.message : "unknown" });
|
|
2092
|
-
}
|
|
2093
|
-
}
|
|
2094
|
-
/**
|
|
2095
|
-
* Reports token usage spent on an attempt that then failed, so it isn't
|
|
2096
|
-
* dropped alongside the error. Covers any error thrown after usage
|
|
2097
|
-
* extraction, since all of them happen only after a response (real
|
|
2098
|
-
* spend) already arrived. Swallows and logs any error `onUsageFailure`
|
|
2099
|
-
* itself throws.
|
|
2100
|
-
*/
|
|
2101
|
-
reportUsageFailure(usage, error, attempt, terminal = false) {
|
|
2102
|
-
const displayTokens = usage.totalTokens || usage.promptTokens + usage.completionTokens;
|
|
2103
|
-
const attemptText = terminal ? "mid-stream failure (terminal, no further attempts)" : `attempt ${attempt + 1}/${this.maxRetries + 1}`;
|
|
2104
|
-
this.logger.warn(`[VernLLM:${usage.requestId}] usage failure, ${attemptText}: type=${error.type} tokens=${displayTokens}`);
|
|
2105
|
-
if (!this.onUsageFailure) return;
|
|
2106
|
-
try {
|
|
2107
|
-
this.onUsageFailure(usage, error);
|
|
2108
|
-
} catch (hookError) {
|
|
2109
|
-
this.logger.error("[VernLLM] onUsageFailure failed", { message: hookError instanceof Error ? hookError.message : "unknown" });
|
|
2110
|
-
}
|
|
2111
|
-
}
|
|
2112
|
-
/** Parses response content as JSON and validates it against `schema` when supplied. */
|
|
2113
|
-
parseAndValidate(content, schema) {
|
|
2114
|
-
let parsed;
|
|
2115
|
-
try {
|
|
2116
|
-
parsed = this.parseJson(content);
|
|
2117
|
-
} catch {
|
|
2118
|
-
throw new LLMError("Invalid JSON response", "parse");
|
|
2119
|
-
}
|
|
2120
|
-
if (parsed === null || parsed === void 0) throw new LLMError("Invalid JSON response", "parse");
|
|
2121
|
-
if (!schema) return parsed;
|
|
2122
|
-
const result = schema.safeParse(parsed);
|
|
2123
|
-
if (!result.success) throw new LLMError("Schema validation failed", "validation", { issues: result.error });
|
|
2124
|
-
return result.data;
|
|
2125
|
-
}
|
|
2126
|
-
/**
|
|
2127
|
-
* Waits out the backoff delay for a retry attempt, honoring a
|
|
2128
|
-
* Retry-After header on the failed attempt's error when present.
|
|
2129
|
-
* Both Retry-After and plain exponential backoff are capped at the same
|
|
2130
|
-
* max delay (see `DEFAULT_MAX_DELAY_MS` in `retry.utils.ts`).
|
|
2131
|
-
*/
|
|
2132
|
-
async recoverDelay(requestId, model, attempt, error, signal) {
|
|
2133
|
-
const retryAfterMs = extractRetryAfterMs(error);
|
|
2134
|
-
const delay = retryAfterMs ?? getBackoffDelay(this.baseDelayMs, attempt);
|
|
2135
|
-
const retryAfterHonored = retryAfterMs !== void 0;
|
|
2136
|
-
this.logger.warn(`[VernLLM:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${Math.ceil(delay)}ms` + (retryAfterHonored ? " (honoring Retry-After)" : ""));
|
|
2137
|
-
this.reportEvent({
|
|
2138
|
-
kind: "retry",
|
|
2139
|
-
requestId,
|
|
2140
|
-
provider: this.providerName,
|
|
2141
|
-
model,
|
|
2142
|
-
attempt,
|
|
2143
|
-
maxRetries: this.maxRetries,
|
|
2144
|
-
delayMs: delay,
|
|
2145
|
-
retryAfterHonored,
|
|
2146
|
-
error: normalizeError(error, signal)
|
|
2147
|
-
});
|
|
2148
|
-
await waitForRetry(delay, signal);
|
|
2149
|
-
}
|
|
2150
|
-
/** Decides whether a failed attempt is worth retrying. */
|
|
2151
|
-
shouldRetry(error, signal) {
|
|
2152
|
-
if (signal?.aborted) return false;
|
|
2153
|
-
if (error instanceof LLMError && !error.retryable) return false;
|
|
2154
|
-
const status = extractStatus(error);
|
|
2155
|
-
return !(status !== void 0 && this.nonRetryableStatus.includes(status));
|
|
2156
|
-
}
|
|
2157
|
-
/**
|
|
2158
|
-
* Decides whether a failed attempt should count toward the circuit
|
|
2159
|
-
* breaker's failure threshold. A model hallucinating a tool name,
|
|
2160
|
-
* reusing a call id, or a provider ignoring `toolChoice: 'none'` isn't
|
|
2161
|
-
* the provider being unhealthy, it's a model/provider response defect
|
|
2162
|
-
* that will very likely recur regardless of provider health, so it
|
|
2163
|
-
* shouldn't push a healthy provider's circuit toward opening. Same for
|
|
2164
|
-
* a caller-input bug or a local rate-limit rejection: neither ever
|
|
2165
|
-
* reached the provider at all. This is exactly what `LLMError.retryable`
|
|
2166
|
-
* already excludes, so this defers to it directly.
|
|
2167
|
-
*/
|
|
2168
|
-
countsTowardBreaker(error) {
|
|
2169
|
-
return error.retryable;
|
|
2170
|
-
}
|
|
2171
|
-
};
|
|
2172
|
-
|
|
2173
|
-
//#endregion
|
|
2174
|
-
//#region src/internal/logger.utils.ts
|
|
2175
|
-
/**
|
|
2176
|
-
* Wraps a `Logger` so a throwing implementation can never break the call
|
|
2177
|
-
* it's trying to describe. `logger` is user-supplied (`VernLLMOptions.logger`),
|
|
2178
|
-
* so a custom logger that ships to a file, Datadog, etc. can throw for
|
|
2179
|
-
* reasons unrelated to VernLLM. Wrap once at construction so every
|
|
2180
|
-
* downstream `this.logger.warn(...)` call stays as-is and is safe by
|
|
2181
|
-
* construction, instead of guarding each call site individually.
|
|
2182
|
-
*/
|
|
2183
|
-
function createSafeLogger(logger) {
|
|
2184
|
-
return {
|
|
2185
|
-
debug: safe(logger, "debug"),
|
|
2186
|
-
warn: safe(logger, "warn"),
|
|
2187
|
-
error: safe(logger, "error")
|
|
2188
|
-
};
|
|
2189
|
-
}
|
|
2190
|
-
function safe(logger, method) {
|
|
2191
|
-
const fn = logger[method].bind(logger);
|
|
2192
|
-
return (...args) => {
|
|
2193
|
-
try {
|
|
2194
|
-
swallowRejection(fn(...args));
|
|
2195
|
-
} catch {}
|
|
2196
|
-
};
|
|
2197
|
-
}
|
|
2198
|
-
function isPromiseLike(value) {
|
|
2199
|
-
return typeof value?.then === "function";
|
|
2200
|
-
}
|
|
2201
|
-
function swallowRejection(result) {
|
|
2202
|
-
if (isPromiseLike(result)) Promise.resolve(result).catch(() => {});
|
|
2203
|
-
}
|
|
2204
|
-
|
|
2205
|
-
//#endregion
|
|
2206
|
-
//#region src/logger.ts
|
|
2207
|
-
/**
|
|
2208
|
-
* Default logger. `debug` is gated by the `debug` option on VernLLM
|
|
2209
|
-
* warn/error always fire since they indicate real problems (retries, cache failures)
|
|
2210
|
-
*/
|
|
2211
|
-
var ConsoleLogger = class {
|
|
2212
|
-
constructor(debugEnabled) {
|
|
2213
|
-
this.debugEnabled = debugEnabled;
|
|
2214
|
-
}
|
|
2215
|
-
debug(message) {
|
|
2216
|
-
if (this.debugEnabled) console.debug(message);
|
|
2217
|
-
}
|
|
2218
|
-
warn(message) {
|
|
2219
|
-
console.warn(message);
|
|
2220
|
-
}
|
|
2221
|
-
error(message, meta) {
|
|
2222
|
-
console.error(message, meta ?? "");
|
|
2223
|
-
}
|
|
2224
|
-
};
|
|
2225
|
-
|
|
2226
|
-
//#endregion
|
|
2227
|
-
//#region src/rateLimit.ts
|
|
2228
|
-
/** Default `estimateTokens`: chars/4 over every message's content, plus the requested `max_tokens`. */
|
|
2229
|
-
function defaultEstimateTokens(request) {
|
|
2230
|
-
const messagesChars = request.messages.reduce((sum, message) => {
|
|
2231
|
-
const content = message.content;
|
|
2232
|
-
if (typeof content === "string") return sum + content.length;
|
|
2233
|
-
if (content === void 0 || content === null) return sum;
|
|
2234
|
-
try {
|
|
2235
|
-
return sum + JSON.stringify(content).length;
|
|
2236
|
-
} catch {
|
|
2237
|
-
return sum;
|
|
2238
|
-
}
|
|
2239
|
-
}, 0);
|
|
2240
|
-
return Math.ceil(messagesChars / 4) + (request.max_tokens ?? 0);
|
|
2241
|
-
}
|
|
2242
|
-
/**
|
|
2243
|
-
* A capacity that refills continuously. Used for requests per minute and
|
|
2244
|
-
* tokens per minute, where `refillPerMs` is `capacity / 60000`, and for
|
|
2245
|
-
* concurrency, where `refillPerMs` is 0 and every release calls
|
|
2246
|
-
* `give(1)` instead of relying on the clock.
|
|
2247
|
-
*/
|
|
2248
|
-
var TokenBucket = class {
|
|
2249
|
-
available;
|
|
2250
|
-
lastRefill = Date.now();
|
|
2251
|
-
constructor(capacity, refillPerMs) {
|
|
2252
|
-
this.capacity = capacity;
|
|
2253
|
-
this.refillPerMs = refillPerMs;
|
|
2254
|
-
this.available = capacity;
|
|
2255
|
-
}
|
|
2256
|
-
refill() {
|
|
2257
|
-
if (this.refillPerMs === 0) return;
|
|
2258
|
-
const now = Date.now();
|
|
2259
|
-
const elapsedMs = now - this.lastRefill;
|
|
2260
|
-
this.available = Math.min(this.capacity, this.available + Math.max(0, elapsedMs) * this.refillPerMs);
|
|
2261
|
-
this.lastRefill = now;
|
|
2262
|
-
}
|
|
2263
|
-
/** Refills, then takes `amount` if available. Leaves the bucket untouched if it can't. */
|
|
2264
|
-
tryTake(amount) {
|
|
2265
|
-
this.refill();
|
|
2266
|
-
if (this.available < amount) return false;
|
|
2267
|
-
this.available -= amount;
|
|
2268
|
-
return true;
|
|
2269
|
-
}
|
|
2270
|
-
/**
|
|
2271
|
-
* Refills, then reports how many ms until this bucket could supply
|
|
2272
|
-
* `amount`, assuming nothing else takes from it meanwhile. Returns 0 if
|
|
2273
|
-
* it already can, `Infinity` if it never will on its own (a
|
|
2274
|
-
* concurrency bucket, `refillPerMs === 0`, only frees via `give`).
|
|
2275
|
-
*/
|
|
2276
|
-
msUntilAvailable(amount) {
|
|
2277
|
-
this.refill();
|
|
2278
|
-
if (this.available >= amount) return 0;
|
|
2279
|
-
if (this.refillPerMs === 0) return Infinity;
|
|
2280
|
-
return (amount - this.available) / this.refillPerMs;
|
|
2281
|
-
}
|
|
2282
|
-
/**
|
|
2283
|
-
* Gives capacity back. Not floored at 0: a bad token-usage estimate can
|
|
2284
|
-
* push `available` negative, and it self-corrects on the next refill
|
|
2285
|
-
* rather than being clamped away immediately. Only ceilinged at
|
|
2286
|
-
* `capacity`, so a give can never overfill the bucket.
|
|
2287
|
-
*/
|
|
2288
|
-
give(amount) {
|
|
2289
|
-
this.available = Math.min(this.capacity, this.available + amount);
|
|
2290
|
-
}
|
|
2291
|
-
/** The bucket's ceiling, e.g. so a request that could never fit can fail fast instead of queueing forever. */
|
|
2292
|
-
getCapacity() {
|
|
2293
|
-
return this.capacity;
|
|
2294
|
-
}
|
|
2295
|
-
};
|
|
2296
|
-
/**
|
|
2297
|
-
* `setTimeout` silently clamps any delay above this (~24.8 days) instead
|
|
2298
|
-
* of erroring, so an uncapped delay derived from a very small
|
|
2299
|
-
* `requestsPerMinute`/`tokensPerMinute` could wrap around to firing
|
|
2300
|
-
* almost immediately instead of waiting. Mirrors the same guard in
|
|
2301
|
-
* `withTimeout`/`withChunkIdleTimeout`.
|
|
2302
|
-
*/
|
|
2303
|
-
const MAX_WAKE_DELAY_MS = 2147483647;
|
|
2304
|
-
/**
|
|
2305
|
-
* Per-target rate limiter. Up to three buckets (requests/min, tokens/min,
|
|
2306
|
-
* concurrency) behind one FIFO queue, so a large call isn't starved by a
|
|
2307
|
-
* stream of small ones. Any bucket omitted from `options` has infinite
|
|
2308
|
-
* capacity and never blocks.
|
|
2309
|
-
*/
|
|
2310
|
-
var RateLimiter = class {
|
|
2311
|
-
requests;
|
|
2312
|
-
tokens;
|
|
2313
|
-
concurrency;
|
|
2314
|
-
maxQueueMs;
|
|
2315
|
-
maxQueueSize;
|
|
2316
|
-
estimateTokensFn;
|
|
2317
|
-
queue = [];
|
|
2318
|
-
/**
|
|
2319
|
-
* A single scheduled re-check for the head of the queue when it's
|
|
2320
|
-
* blocked on a bucket that refills on its own clock (rpm/tpm), so a
|
|
2321
|
-
* queue that nobody calls `acquire`/`release` on again isn't stuck
|
|
2322
|
-
* forever waiting for an external trigger to re-drain it. Not needed
|
|
2323
|
-
* for a concurrency block, which only clears via `release`.
|
|
2324
|
-
*/
|
|
2325
|
-
wakeTimer;
|
|
2326
|
-
constructor(options) {
|
|
2327
|
-
if (options.requestsPerMinute) this.requests = new TokenBucket(options.requestsPerMinute, options.requestsPerMinute / 6e4);
|
|
2328
|
-
if (options.tokensPerMinute) this.tokens = new TokenBucket(options.tokensPerMinute, options.tokensPerMinute / 6e4);
|
|
2329
|
-
if (options.maxConcurrent) this.concurrency = new TokenBucket(options.maxConcurrent, 0);
|
|
2330
|
-
this.maxQueueMs = options.maxQueueMs ?? 3e4;
|
|
2331
|
-
this.maxQueueSize = options.maxQueueSize ?? 0;
|
|
2332
|
-
this.estimateTokensFn = options.estimateTokens ?? defaultEstimateTokens;
|
|
2333
|
-
}
|
|
2334
|
-
/** Pre-flight token estimate for a request, per the configured (or default) heuristic. */
|
|
2335
|
-
estimate(request) {
|
|
2336
|
-
return this.estimateTokensFn(request);
|
|
2337
|
-
}
|
|
2338
|
-
/**
|
|
2339
|
-
* Waits for capacity in every configured bucket, then takes from each.
|
|
2340
|
-
* The returned `release` gives the concurrency slot back and reconciles
|
|
2341
|
-
* the token bucket against real usage; it must run in a `finally` block.
|
|
2342
|
-
*/
|
|
2343
|
-
async acquire(estimatedTokens, signal) {
|
|
2344
|
-
if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
2345
|
-
if (!Number.isFinite(estimatedTokens) || estimatedTokens < 0) throw new LLMError(`estimatedTokens must be a finite, non-negative number, got ${String(estimatedTokens)}`, "invalid_params");
|
|
2346
|
-
if (this.tokens && estimatedTokens > this.tokens.getCapacity()) throw new LLMError(`estimatedTokens (${estimatedTokens}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`, "rate_limited", { code: "rate_limit_capacity_exceeded" });
|
|
2347
|
-
if (this.queue.length === 0) {
|
|
2348
|
-
const attempt = this.tryAcquireBuckets(estimatedTokens);
|
|
2349
|
-
if (attempt.ok) return {
|
|
2350
|
-
release: this.makeRelease(estimatedTokens),
|
|
2351
|
-
waitedMs: 0
|
|
2352
|
-
};
|
|
2353
|
-
return this.enqueue(estimatedTokens, attempt.reason, signal);
|
|
2354
|
-
}
|
|
2355
|
-
if (this.maxQueueSize > 0 && this.queue.length >= this.maxQueueSize) throw this.queueFullError();
|
|
2356
|
-
return this.enqueue(estimatedTokens, void 0, signal);
|
|
2357
|
-
}
|
|
2358
|
-
queueFullError() {
|
|
2359
|
-
return new LLMError("Rate limit queue is full", "rate_limited", { code: "rate_limit_queue_full" });
|
|
2360
|
-
}
|
|
2361
|
-
enqueue(estimatedTokens, initialReason, signal) {
|
|
2362
|
-
return new Promise((resolvePromise, rejectPromise) => {
|
|
2363
|
-
const waiter = {
|
|
2364
|
-
estimatedTokens,
|
|
2365
|
-
enqueuedAt: Date.now(),
|
|
2366
|
-
lastReason: initialReason,
|
|
2367
|
-
resolve: (result) => {
|
|
2368
|
-
cleanup();
|
|
2369
|
-
resolvePromise(result);
|
|
2370
|
-
},
|
|
2371
|
-
reject: (error) => {
|
|
2372
|
-
cleanup();
|
|
2373
|
-
if (this.wakeTimer) {
|
|
2374
|
-
clearTimeout(this.wakeTimer);
|
|
2375
|
-
this.wakeTimer = void 0;
|
|
2376
|
-
}
|
|
2377
|
-
this.drain();
|
|
2378
|
-
rejectPromise(error);
|
|
2379
|
-
}
|
|
2380
|
-
};
|
|
2381
|
-
let queueTimer;
|
|
2382
|
-
const onAbort = () => {
|
|
2383
|
-
waiter.reject(new LLMError("LLM request aborted", "aborted"));
|
|
2384
|
-
};
|
|
2385
|
-
const cleanup = () => {
|
|
2386
|
-
if (queueTimer) clearTimeout(queueTimer);
|
|
2387
|
-
signal?.removeEventListener("abort", onAbort);
|
|
2388
|
-
const index = this.queue.indexOf(waiter);
|
|
2389
|
-
if (index !== -1) this.queue.splice(index, 1);
|
|
2390
|
-
};
|
|
2391
|
-
if (this.maxQueueMs > 0) queueTimer = setTimeout(() => {
|
|
2392
|
-
waiter.reject(new LLMError("Rate limit queue timed out before capacity was available", "rate_limited", { code: "rate_limit_queue_timeout" }));
|
|
2393
|
-
}, this.maxQueueMs);
|
|
2394
|
-
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2395
|
-
this.queue.push(waiter);
|
|
2396
|
-
this.drain();
|
|
2397
|
-
});
|
|
2398
|
-
}
|
|
2399
|
-
/**
|
|
2400
|
-
* Checks and takes from every configured bucket as one atomic unit: if
|
|
2401
|
-
* any bucket lacks capacity, whatever was already taken from the
|
|
2402
|
-
* earlier ones in this attempt is rolled back before reporting which
|
|
2403
|
-
* bucket blocked.
|
|
2404
|
-
*/
|
|
2405
|
-
tryAcquireBuckets(estimatedTokens) {
|
|
2406
|
-
const taken = [];
|
|
2407
|
-
const take = (bucket, amount) => {
|
|
2408
|
-
if (!bucket) return true;
|
|
2409
|
-
if (!bucket.tryTake(amount)) return false;
|
|
2410
|
-
taken.push({
|
|
2411
|
-
bucket,
|
|
2412
|
-
amount
|
|
2413
|
-
});
|
|
2414
|
-
return true;
|
|
2415
|
-
};
|
|
2416
|
-
if (!take(this.concurrency, 1)) return {
|
|
2417
|
-
ok: false,
|
|
2418
|
-
reason: "concurrency"
|
|
2419
|
-
};
|
|
2420
|
-
if (!take(this.requests, 1)) {
|
|
2421
|
-
for (const entry of taken) entry.bucket.give(entry.amount);
|
|
2422
|
-
return {
|
|
2423
|
-
ok: false,
|
|
2424
|
-
reason: "rpm"
|
|
2425
|
-
};
|
|
2426
|
-
}
|
|
2427
|
-
if (!take(this.tokens, estimatedTokens)) {
|
|
2428
|
-
for (const entry of taken) entry.bucket.give(entry.amount);
|
|
2429
|
-
return {
|
|
2430
|
-
ok: false,
|
|
2431
|
-
reason: "tpm"
|
|
2432
|
-
};
|
|
2433
|
-
}
|
|
2434
|
-
return { ok: true };
|
|
2435
|
-
}
|
|
2436
|
-
/** Drains the queue head first. Stops at the first waiter that still can't proceed, so no one is starved out of turn. */
|
|
2437
|
-
drain() {
|
|
2438
|
-
while (this.queue.length > 0) {
|
|
2439
|
-
const waiter = this.queue[0];
|
|
2440
|
-
const attempt = this.tryAcquireBuckets(waiter.estimatedTokens);
|
|
2441
|
-
if (!attempt.ok) {
|
|
2442
|
-
waiter.lastReason = attempt.reason;
|
|
2443
|
-
this.scheduleWake(attempt.reason, waiter.estimatedTokens);
|
|
2444
|
-
return;
|
|
2445
|
-
}
|
|
2446
|
-
const waitedMs = Date.now() - waiter.enqueuedAt;
|
|
2447
|
-
waiter.resolve({
|
|
2448
|
-
release: this.makeRelease(waiter.estimatedTokens),
|
|
2449
|
-
waitedMs,
|
|
2450
|
-
reason: waiter.lastReason
|
|
2451
|
-
});
|
|
2452
|
-
}
|
|
2453
|
-
}
|
|
2454
|
-
/**
|
|
2455
|
-
* Schedules a one-shot re-check of the queue for whenever the bucket
|
|
2456
|
-
* that's currently blocking the head waiter should next have enough
|
|
2457
|
-
* capacity. A no-op for a concurrency block (only `release` can clear
|
|
2458
|
-
* that) or while a wake is already pending.
|
|
2459
|
-
*/
|
|
2460
|
-
scheduleWake(reason, estimatedTokens) {
|
|
2461
|
-
if (this.wakeTimer) return;
|
|
2462
|
-
const ms = reason === "rpm" ? this.requests?.msUntilAvailable(1) : reason === "tpm" ? this.tokens?.msUntilAvailable(estimatedTokens) : void 0;
|
|
2463
|
-
if (ms === void 0 || !Number.isFinite(ms)) return;
|
|
2464
|
-
const delay = Math.min(Math.max(1, Math.ceil(ms)), MAX_WAKE_DELAY_MS);
|
|
2465
|
-
this.wakeTimer = setTimeout(() => {
|
|
2466
|
-
this.wakeTimer = void 0;
|
|
2467
|
-
this.drain();
|
|
2468
|
-
}, delay);
|
|
2469
|
-
}
|
|
2470
|
-
/**
|
|
2471
|
-
* Builds the one-shot release closure for an acquired slot. Only the
|
|
2472
|
-
* concurrency bucket is given back on release; the requests-per-minute
|
|
2473
|
-
* bucket is a real spend that only recovers via its own refill, and the
|
|
2474
|
-
* tokens bucket is reconciled against `actualTokens` rather than fully
|
|
2475
|
-
* refunded, since real tokens really were spent.
|
|
2476
|
-
*/
|
|
2477
|
-
makeRelease(estimatedTokens) {
|
|
2478
|
-
let released = false;
|
|
2479
|
-
return (actualTokens) => {
|
|
2480
|
-
if (released) return;
|
|
2481
|
-
released = true;
|
|
2482
|
-
this.concurrency?.give(1);
|
|
2483
|
-
if (this.tokens && actualTokens !== void 0 && Number.isFinite(actualTokens)) this.tokens.give(estimatedTokens - actualTokens);
|
|
2484
|
-
this.drain();
|
|
2485
|
-
};
|
|
2486
|
-
}
|
|
2487
|
-
};
|
|
2488
|
-
|
|
2489
|
-
//#endregion
|
|
2490
|
-
//#region src/vernLLM.ts
|
|
2491
|
-
/**
|
|
2492
|
-
* A resilient layer around an LLM chat completions client. This is VernLLM!
|
|
2493
|
-
*
|
|
2494
|
-
* Adds retry with backoff and jitter, per-attempt timeouts, an optional
|
|
2495
|
-
* circuit breaker, JSON parsing with optional schema validation, usage
|
|
2496
|
-
* tracking, and an optional response cache. All configurable, all opt-in
|
|
2497
|
-
* beyond sensible defaults.
|
|
2498
|
-
*/
|
|
2499
|
-
var VernLLM = class {
|
|
2500
|
-
logger;
|
|
2501
|
-
/**
|
|
2502
|
-
* One `CallExecutor` per provider target: index 0 is the primary,
|
|
2503
|
-
* everything after it is a `fallback` target, in the order declared.
|
|
2504
|
-
* Each owns its own request building, retry/timeout, circuit breaker,
|
|
2505
|
-
* and rate limiter. `call()` walks this array in `runFallbackChain`,
|
|
2506
|
-
* moving to the next entry only when `fallbackOn` says to.
|
|
2507
|
-
*/
|
|
2508
|
-
executors;
|
|
2509
|
-
/** Decides whether a failed target is followed by the next one or the chain stops. See `VernLLMOptions['fallbackOn']`. */
|
|
2510
|
-
fallbackOn;
|
|
2511
|
-
/** Reports a `'fallback'` event when the chain moves to the next target. Shared `onEvent` plumbing, same as every executor's. */
|
|
2512
|
-
reportEvent;
|
|
2513
|
-
/**
|
|
2514
|
-
* Owns cache key resolution, cache reads/writes, and in-flight
|
|
2515
|
-
* coalescing for `cachedCall()`. Independent of `executor`: it only
|
|
2516
|
-
* ever calls back into `this.call()` as an opaque function.
|
|
2517
|
-
*/
|
|
2518
|
-
cacheOrchestrator;
|
|
2519
|
-
/**
|
|
2520
|
-
* @param options Client, model, and tunables. Defaults: `maxRetries` 1,
|
|
2521
|
-
* `timeoutMs` 25000, `baseDelayMs` 500, `defaultMaxTokens` 1000,
|
|
2522
|
-
* `defaultTemperature` 0.2, `cache` an in-memory adapter,
|
|
2523
|
-
* `nonRetryableStatus` `[400, 401, 403, 404, 422]`, `debug` false.
|
|
2524
|
-
*/
|
|
2525
|
-
constructor(options) {
|
|
2526
|
-
this.logger = createSafeLogger(options.logger ?? new ConsoleLogger(options.debug ?? false));
|
|
2527
|
-
const providerName = options.name ?? "primary";
|
|
2528
|
-
this.cacheOrchestrator = new CacheOrchestrator(options.cache ?? new InMemoryCacheAdapter(), this.logger);
|
|
2529
|
-
this.fallbackOn = options.fallbackOn ?? defaultFallbackOn;
|
|
2530
|
-
this.reportEvent = makeEventReporter(options.onEvent, this.logger);
|
|
2531
|
-
const primaryDefaultTemperature = options.defaultTemperature === void 0 ? .2 : options.defaultTemperature;
|
|
2532
|
-
const primaryTarget = {
|
|
2533
|
-
client: options.client,
|
|
2534
|
-
model: options.model,
|
|
2535
|
-
name: providerName,
|
|
2536
|
-
maxRetries: options.maxRetries,
|
|
2537
|
-
timeoutMs: options.timeoutMs,
|
|
2538
|
-
chunkIdleTimeoutMs: options.chunkIdleTimeoutMs,
|
|
2539
|
-
baseDelayMs: options.baseDelayMs,
|
|
2540
|
-
defaultMaxTokens: options.defaultMaxTokens,
|
|
2541
|
-
defaultTemperature: primaryDefaultTemperature,
|
|
2542
|
-
nonRetryableStatus: options.nonRetryableStatus,
|
|
2543
|
-
circuitBreaker: options.circuitBreaker,
|
|
2544
|
-
rateLimit: options.rateLimit
|
|
2545
|
-
};
|
|
2546
|
-
const declaredFallbacks = Array.isArray(options.fallback) ? options.fallback : options.fallback ? [options.fallback] : [];
|
|
2547
|
-
const targets = [primaryTarget, ...declaredFallbacks];
|
|
2548
|
-
this.executors = targets.map((target, i) => {
|
|
2549
|
-
const isFallback = i > 0;
|
|
2550
|
-
const name = target.name ?? (isFallback ? `fallback[${i - 1}]` : providerName);
|
|
2551
|
-
const breaker = buildCircuitBreaker(target.circuitBreaker, name, target.model, options.onEvent, this.logger);
|
|
2552
|
-
return new CallExecutor(name, target.client, target.model, {
|
|
2553
|
-
maxRetries: target.maxRetries ?? options.maxRetries ?? 1,
|
|
2554
|
-
timeoutMs: target.timeoutMs ?? options.timeoutMs ?? 25e3,
|
|
2555
|
-
chunkIdleTimeoutMs: target.chunkIdleTimeoutMs ?? options.chunkIdleTimeoutMs ?? 3e4,
|
|
2556
|
-
baseDelayMs: target.baseDelayMs ?? options.baseDelayMs ?? 500,
|
|
2557
|
-
defaultMaxTokens: target.defaultMaxTokens ?? options.defaultMaxTokens ?? 1e3,
|
|
2558
|
-
defaultTemperature: target.defaultTemperature === void 0 ? primaryDefaultTemperature : target.defaultTemperature,
|
|
2559
|
-
nonRetryableStatus: target.nonRetryableStatus ?? options.nonRetryableStatus ?? [
|
|
2560
|
-
400,
|
|
2561
|
-
401,
|
|
2562
|
-
403,
|
|
2563
|
-
404,
|
|
2564
|
-
422
|
|
2565
|
-
],
|
|
2566
|
-
parseJson: options.parseJson,
|
|
2567
|
-
logger: this.logger,
|
|
2568
|
-
redact: options.redact,
|
|
2569
|
-
onUsage: options.onUsage,
|
|
2570
|
-
onUsageFailure: options.onUsageFailure,
|
|
2571
|
-
onEvent: options.onEvent,
|
|
2572
|
-
breaker,
|
|
2573
|
-
limiter: target.rateLimit ? new RateLimiter(target.rateLimit) : void 0,
|
|
2574
|
-
isFallback
|
|
2575
|
-
});
|
|
2576
|
-
});
|
|
2577
|
-
}
|
|
2578
|
-
/** Logs a failed refundUsage attempt via the configured logger. */
|
|
2579
|
-
logRefundError(logMessage, error) {
|
|
2580
|
-
this.logger.error(logMessage, { message: error instanceof Error ? error.message : "unknown" });
|
|
2581
|
-
}
|
|
2582
|
-
/**
|
|
2583
|
-
* Walks `this.executors` in order, running `attempt` against each until
|
|
2584
|
-
* one succeeds or every target has failed. `run` on a lone target
|
|
2585
|
-
* (no `fallback` configured) throws exactly what it throws today: the
|
|
2586
|
-
* loop's single iteration path is unchanged from pre-fallback behavior.
|
|
2587
|
-
*
|
|
2588
|
-
* For streaming, `attempt` is `executor.runStream`, whose own retries
|
|
2589
|
-
* only cover *opening* the stream (see `CallExecutor.runStream`). A
|
|
2590
|
-
* mid-stream failure surfaces through `finalResult` after this function
|
|
2591
|
-
* has already returned, so it's never seen here and never falls over,
|
|
2592
|
-
* per the streaming limitation: splicing a second model's output into a
|
|
2593
|
-
* response the consumer has already partially rendered would corrupt
|
|
2594
|
-
* it.
|
|
2595
|
-
*/
|
|
2596
|
-
async runFallbackChain(params, requestId, attempt, skipBreakerCheckForFirst = false) {
|
|
2597
|
-
const attempts = [];
|
|
2598
|
-
for (let i = 0; i < this.executors.length; i++) {
|
|
2599
|
-
const executor = this.executors[i];
|
|
2600
|
-
const startedAt = Date.now();
|
|
2601
|
-
let attemptCount = 0;
|
|
2602
|
-
try {
|
|
2603
|
-
if (!(i === 0 && skipBreakerCheckForFirst)) executor.assertBreakerClosed(params.model);
|
|
2604
|
-
const result = await attempt(executor, () => {
|
|
2605
|
-
attemptCount += 1;
|
|
2606
|
-
});
|
|
2607
|
-
return {
|
|
2608
|
-
result,
|
|
2609
|
-
executor,
|
|
2610
|
-
index: i,
|
|
2611
|
-
attemptCount
|
|
2612
|
-
};
|
|
2613
|
-
} catch (error) {
|
|
2614
|
-
const normalized = normalizeError(error, params.signal);
|
|
2615
|
-
attempts.push({
|
|
2616
|
-
index: i - 1,
|
|
2617
|
-
provider: executor.providerName,
|
|
2618
|
-
model: params.model ?? executor.model,
|
|
2619
|
-
error: normalized.toSnapshot()
|
|
2620
|
-
});
|
|
2621
|
-
const isLast = i === this.executors.length - 1;
|
|
2622
|
-
const policyDecision = this.fallbackOn(normalized, { isLastTarget: isLast });
|
|
2623
|
-
const decision = isLast ? "stop" : policyDecision;
|
|
2624
|
-
if (decision === "stop") throw attempts.length > 1 ? new FallbackExhaustedError(attempts) : normalized;
|
|
2625
|
-
const next = this.executors[i + 1];
|
|
2626
|
-
this.reportEvent({
|
|
2627
|
-
kind: "fallback",
|
|
2628
|
-
requestId,
|
|
2629
|
-
from: executor.providerName,
|
|
2630
|
-
to: next.providerName,
|
|
2631
|
-
fromIndex: i - 1,
|
|
2632
|
-
toIndex: i,
|
|
2633
|
-
error: normalized,
|
|
2634
|
-
elapsedMs: Date.now() - startedAt
|
|
2635
|
-
});
|
|
2636
|
-
}
|
|
2637
|
-
}
|
|
2638
|
-
throw new LLMError("No provider targets configured", "invalid_params");
|
|
2639
|
-
}
|
|
2640
|
-
async call(params) {
|
|
2641
|
-
if (params.signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
2642
|
-
const requestId = params.requestId ?? (0, crypto.randomUUID)();
|
|
2643
|
-
const soleTarget = this.executors.length === 1;
|
|
2644
|
-
if (soleTarget) this.executors[0].assertBreakerClosed(params.model);
|
|
2645
|
-
if (params.stream) return withReservedUsageForStream(params, async () => {
|
|
2646
|
-
const { result } = await this.runFallbackChain(params, requestId, (executor, onAttempt) => executor.runStream(params, requestId, onAttempt), soleTarget);
|
|
2647
|
-
return result;
|
|
2648
|
-
}, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
|
|
2649
|
-
return withReservedUsage(params, false, async () => {
|
|
2650
|
-
const { result, executor, index, attemptCount } = await this.runFallbackChain(params, requestId, (target, onAttempt) => target.run(params, requestId, onAttempt), soleTarget);
|
|
2651
|
-
if (params.meta) params.meta.current = {
|
|
2652
|
-
provider: executor.providerName,
|
|
2653
|
-
model: params.model ?? executor.model,
|
|
2654
|
-
fallbackIndex: index - 1,
|
|
2655
|
-
usedFallback: index > 0,
|
|
2656
|
-
attempts: attemptCount
|
|
2657
|
-
};
|
|
2658
|
-
return result;
|
|
2659
|
-
}, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
|
|
2660
|
-
}
|
|
2661
|
-
/**
|
|
2662
|
-
* Thin delegator kept private on `VernLLM` (rather than only existing on
|
|
2663
|
-
* `CacheOrchestrator`) since it's the one caching primitive exercised
|
|
2664
|
-
* directly by white-box tests, independent of the public `cachedCall()`
|
|
2665
|
-
* surface.
|
|
2666
|
-
*/
|
|
2667
|
-
runCached(params) {
|
|
2668
|
-
return this.cacheOrchestrator.runCached(params);
|
|
2669
|
-
}
|
|
2670
|
-
/**
|
|
2671
|
-
* Removes a cached response by key when the configured cache adapter
|
|
2672
|
-
* supports deletion. Cache invalidation is the caller's responsibility;
|
|
2673
|
-
* only the application knows when cached data is stale.
|
|
2674
|
-
*
|
|
2675
|
-
* @param key The raw cache key (resolved through the adapter's
|
|
2676
|
-
* `resolveKey`, if any, before deletion).
|
|
2677
|
-
*/
|
|
2678
|
-
async deleteCache(key) {
|
|
2679
|
-
await this.cacheOrchestrator.deleteCache(key);
|
|
2680
|
-
}
|
|
2681
|
-
async cachedCall(params) {
|
|
2682
|
-
const { call: callParams,...cacheParams } = params;
|
|
2683
|
-
const restCallParams = callParams;
|
|
2684
|
-
if (restCallParams.reserveUsage || restCallParams.refundUsage) throw new LLMError("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.", "invalid_params");
|
|
2685
|
-
if (restCallParams.stream) {
|
|
2686
|
-
const streamParams = restCallParams;
|
|
2687
|
-
return this.cacheOrchestrator.runCachedStream({
|
|
2688
|
-
...cacheParams,
|
|
2689
|
-
openStream: () => this.call(streamParams)
|
|
2690
|
-
}, Boolean(restCallParams.tools));
|
|
2691
|
-
}
|
|
2692
|
-
return this.runCached({
|
|
2693
|
-
...cacheParams,
|
|
2694
|
-
fn: () => this.call(restCallParams)
|
|
2695
|
-
});
|
|
2696
|
-
}
|
|
2697
|
-
/**
|
|
2698
|
-
* @param target.index Which target to read. Defaults to the primary.
|
|
2699
|
-
* @param target.model Which model bucket to read, if the target isolates by model.
|
|
2700
|
-
* @returns The breaker state, or `undefined` if that target has no breaker.
|
|
2701
|
-
* @throws {RangeError} If `target.index` names no target. Lets a real
|
|
2702
|
-
* target with no breaker (`undefined`) stay distinguishable from a
|
|
2703
|
-
* target that doesn't exist.
|
|
2704
|
-
*/
|
|
2705
|
-
getCircuitState(target) {
|
|
2706
|
-
const executor = this.resolveExecutor(target?.index ?? 0, "getCircuitState");
|
|
2707
|
-
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "getCircuitState");
|
|
2708
|
-
return executor.getCircuitState(target?.model ?? executor.model);
|
|
2709
|
-
}
|
|
2710
|
-
/**
|
|
2711
|
-
* @param model Which model bucket to read, for targets that isolate by model.
|
|
2712
|
-
* @returns Every target's state, in chain order.
|
|
2713
|
-
*/
|
|
2714
|
-
getCircuitStates(model) {
|
|
2715
|
-
return this.executors.map((executor, index) => ({
|
|
2716
|
-
provider: executor.providerName,
|
|
2717
|
-
index,
|
|
2718
|
-
isFallback: index > 0,
|
|
2719
|
-
isolateByModel: executor.isolateByModel,
|
|
2720
|
-
state: executor.getCircuitState(model ?? executor.model)
|
|
2721
|
-
}));
|
|
2722
|
-
}
|
|
2723
|
-
/**
|
|
2724
|
-
* Manually opens a target's breaker, e.g. to pull a provider out of
|
|
2725
|
-
* rotation ahead of known maintenance instead of waiting for it to fail.
|
|
2726
|
-
*
|
|
2727
|
-
* @param target.index Which target to open. Defaults to the primary.
|
|
2728
|
-
* @param target.model Which model bucket to open, if the target isolates by model.
|
|
2729
|
-
* @throws {RangeError} If `target.index` names no target.
|
|
2730
|
-
*/
|
|
2731
|
-
openCircuit(target) {
|
|
2732
|
-
const executor = this.resolveExecutor(target?.index ?? 0, "openCircuit");
|
|
2733
|
-
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "openCircuit");
|
|
2734
|
-
executor.openCircuit(target?.model ?? executor.model);
|
|
2735
|
-
}
|
|
2736
|
-
/**
|
|
2737
|
-
* Manually closes a target's breaker, e.g. once a provider is confirmed
|
|
2738
|
-
* healthy again without waiting out the cooldown.
|
|
2739
|
-
*
|
|
2740
|
-
* @param target.index Which target to close. Defaults to the primary.
|
|
2741
|
-
* @param target.model Which model bucket to close, if the target isolates by model.
|
|
2742
|
-
* @throws {RangeError} If `target.index` names no target.
|
|
2743
|
-
*/
|
|
2744
|
-
closeCircuit(target) {
|
|
2745
|
-
const executor = this.resolveExecutor(target?.index ?? 0, "closeCircuit");
|
|
2746
|
-
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "closeCircuit");
|
|
2747
|
-
executor.closeCircuit(target?.model ?? executor.model);
|
|
2748
|
-
}
|
|
2749
|
-
/** Resolves a target index so every circuit-breaker method agrees on what counts as valid. */
|
|
2750
|
-
resolveExecutor(index, caller) {
|
|
2751
|
-
const executor = this.executors[index];
|
|
2752
|
-
if (!executor) throw new RangeError(`${caller}: no target at index ${index} (chain has ${this.executors.length} target${this.executors.length === 1 ? "" : "s"})`);
|
|
2753
|
-
return executor;
|
|
2754
|
-
}
|
|
2755
|
-
/** Warns when `model` can't do anything on this target, so it's never silently ignored. */
|
|
2756
|
-
warnIfModelUnsupported(isolateByModel, model, caller) {
|
|
2757
|
-
if (model !== void 0 && !isolateByModel) this.logger.warn(`[VernLLM] ${caller}: \`model: '${model}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`);
|
|
2758
|
-
}
|
|
2759
|
-
};
|
|
2760
|
-
|
|
2761
|
-
//#endregion
|
|
2762
|
-
//#region src/adapters/internal/sse.ts
|
|
2763
|
-
/**
|
|
2764
|
-
* Parses a Server-Sent-Events byte/text stream into the JSON payload of
|
|
2765
|
-
* each `data:` frame, in arrival order. Generic over transport: works with
|
|
2766
|
-
* anything that hands back progressively-arriving `Uint8Array` or `string`
|
|
2767
|
-
* chunks via async iteration: native `fetch`'s `response.body` (wrapped
|
|
2768
|
-
* to be iterable, see `webStreamToAsyncIterable` in `fetch.ts`), axios's
|
|
2769
|
-
* Node `Readable` (already async-iterable, no wrapping needed), etc, so
|
|
2770
|
-
* this framing layer doesn't care which transport produced the bytes.
|
|
2771
|
-
*
|
|
2772
|
-
* Follows the SSE spec's frame-delimiting rules closely enough for LLM
|
|
2773
|
-
* streaming responses: frames are separated by a blank line, each frame
|
|
2774
|
-
* may carry one or more `data:` lines (joined with `\n` per spec when
|
|
2775
|
-
* there's more than one), `:`-prefixed lines are comments and ignored, and
|
|
2776
|
-
* other SSE fields (`event:`, `id:`, `retry:`) are ignored since VernLLM
|
|
2777
|
-
* only needs the payload. A frame whose data is exactly `[DONE]` (the
|
|
2778
|
-
* sentinel several providers, notably OpenAI, send to mark stream end)
|
|
2779
|
-
* ends iteration without yielding it.
|
|
2780
|
-
*
|
|
2781
|
-
* Line endings: `\r\n` and bare `\r` (both legal per the SSE spec, alongside `\n`) are normalized
|
|
2782
|
-
* to `\n` before frame splitting. A `\r` at the very end of the currently-buffered text is left
|
|
2783
|
-
* alone until either more text arrives (in case it's the first half of a split `\r\n` pair) or the
|
|
2784
|
-
* stream ends, so a `\r\n` pair split across two transport chunks is never misread as two blank
|
|
2785
|
-
* lines.
|
|
2786
|
-
*
|
|
2787
|
-
* Malformed JSON in a frame throws `LLMError('parse')`, consistent with
|
|
2788
|
-
* how malformed JSON is handled elsewhere in VernLLM.
|
|
2789
|
-
*/
|
|
2790
|
-
async function* parseSseStream(source) {
|
|
2791
|
-
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
2792
|
-
let buffer = "";
|
|
2793
|
-
for await (const chunk of source) {
|
|
2794
|
-
let text;
|
|
2795
|
-
try {
|
|
2796
|
-
text = typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true });
|
|
2797
|
-
} catch (cause) {
|
|
2798
|
-
throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
|
|
2799
|
-
}
|
|
2800
|
-
buffer = (buffer + text).replace(/\r\n/g, "\n").replace(/\r(?!$)/g, "\n");
|
|
2801
|
-
let boundary$1 = buffer.indexOf("\n\n");
|
|
2802
|
-
while (boundary$1 !== -1) {
|
|
2803
|
-
const frame = buffer.slice(0, boundary$1);
|
|
2804
|
-
buffer = buffer.slice(boundary$1 + 2);
|
|
2805
|
-
const event = parseSseFrame(frame);
|
|
2806
|
-
if (event === DONE) return;
|
|
2807
|
-
if (event !== NO_DATA) yield event;
|
|
2808
|
-
boundary$1 = buffer.indexOf("\n\n");
|
|
2809
|
-
}
|
|
2810
|
-
}
|
|
2811
|
-
try {
|
|
2812
|
-
buffer += decoder.decode();
|
|
2813
|
-
} catch (cause) {
|
|
2814
|
-
throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
|
|
2815
|
-
}
|
|
2816
|
-
buffer = buffer.replace(/\r$/, "\n");
|
|
2817
|
-
let boundary = buffer.indexOf("\n\n");
|
|
2818
|
-
while (boundary !== -1) {
|
|
2819
|
-
const frame = buffer.slice(0, boundary);
|
|
2820
|
-
buffer = buffer.slice(boundary + 2);
|
|
2821
|
-
const event = parseSseFrame(frame);
|
|
2822
|
-
if (event === DONE) return;
|
|
2823
|
-
if (event !== NO_DATA) yield event;
|
|
2824
|
-
boundary = buffer.indexOf("\n\n");
|
|
2825
|
-
}
|
|
2826
|
-
const trailing = buffer.trim();
|
|
2827
|
-
if (trailing) {
|
|
2828
|
-
const event = parseSseFrame(trailing);
|
|
2829
|
-
if (event !== DONE && event !== NO_DATA) yield event;
|
|
2830
|
-
}
|
|
2831
|
-
}
|
|
2832
|
-
const DONE = Symbol("sse-stream-done");
|
|
2833
|
-
const NO_DATA = Symbol("sse-frame-no-data");
|
|
2834
|
-
/**
|
|
2835
|
-
* Sentinel yielded by `parseSseStream` for a comment-only frame (no
|
|
2836
|
-
* `data:` payload), the mechanism providers use for SSE keep-alive
|
|
2837
|
-
* pings. Exported so a consumer (e.g. `fromFetch`) can react to "still
|
|
2838
|
-
* alive" separately from a genuinely empty frame (`NO_DATA`, kept internal).
|
|
2839
|
-
*/
|
|
2840
|
-
const SSE_PING = Symbol("sse-frame-ping");
|
|
2841
|
-
/** Extracts and JSON-parses the `data:` payload of one SSE frame (the text between two blank lines). */
|
|
2842
|
-
function parseSseFrame(frame) {
|
|
2843
|
-
const dataLines = [];
|
|
2844
|
-
let sawComment = false;
|
|
2845
|
-
for (const line of frame.split("\n")) {
|
|
2846
|
-
if (line.startsWith(":")) {
|
|
2847
|
-
sawComment = true;
|
|
2848
|
-
continue;
|
|
2849
|
-
}
|
|
2850
|
-
if (!line.startsWith("data:")) continue;
|
|
2851
|
-
dataLines.push(line.startsWith("data: ") ? line.slice(6) : line.slice(5));
|
|
2852
|
-
}
|
|
2853
|
-
if (!dataLines.length) return sawComment ? SSE_PING : NO_DATA;
|
|
2854
|
-
const data = dataLines.join("\n");
|
|
2855
|
-
if (data === "[DONE]") return DONE;
|
|
2856
|
-
try {
|
|
2857
|
-
return JSON.parse(data);
|
|
2858
|
-
} catch (cause) {
|
|
2859
|
-
throw new LLMError(`Invalid JSON in SSE frame: ${data.slice(0, 200)}`, "parse", {
|
|
2860
|
-
cause,
|
|
2861
|
-
code: "stream_frame_invalid"
|
|
2862
|
-
});
|
|
2863
|
-
}
|
|
2864
|
-
}
|
|
2865
|
-
|
|
2866
|
-
//#endregion
|
|
2867
|
-
//#region src/adapters/internal/imageFormat.ts
|
|
2868
|
-
/**
|
|
2869
|
-
* MIME types accepted for `ImageBlock.mimeType` across all adapters. This is
|
|
2870
|
-
* the intersection of what Anthropic, Gemini, OpenAI-compatible, and Bedrock
|
|
2871
|
-
* Converse all natively support, so a `ContentBlock[]` that validates for
|
|
2872
|
-
* one provider validates for all of them.
|
|
2873
|
-
*/
|
|
2874
|
-
const SUPPORTED_IMAGE_MIME_TYPES = [
|
|
2875
|
-
"image/png",
|
|
2876
|
-
"image/jpeg",
|
|
2877
|
-
"image/gif",
|
|
2878
|
-
"image/webp"
|
|
2879
|
-
];
|
|
2880
|
-
/**
|
|
2881
|
-
* Validates an `ImageBlock.mimeType` against the shared supported set.
|
|
2882
|
-
* Throws a non-retryable `LLMError('invalid_params')`, since an unsupported
|
|
2883
|
-
* mimeType is a bug in the caller's own input, deterministic before any
|
|
2884
|
-
* request is built, the same class of failure as every other check in
|
|
2885
|
-
* `RequestBuilder`.
|
|
2886
|
-
*/
|
|
2887
|
-
function assertSupportedImageMimeType(mimeType) {
|
|
2888
|
-
if (SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType)) return mimeType;
|
|
2889
|
-
throw new LLMError(`Unsupported image mimeType "${mimeType}": expected one of ${SUPPORTED_IMAGE_MIME_TYPES.join(", ")}`, "invalid_params");
|
|
2890
|
-
}
|
|
2891
|
-
|
|
2892
|
-
//#endregion
|
|
2893
|
-
//#region src/adapters/internal/nativeStructuredOutput.ts
|
|
2894
|
-
/** Resolves whether `model` is covered by a caller-supplied allow-list/predicate. */
|
|
2895
|
-
function supportsNativeStructuredOutput(model, override) {
|
|
2896
|
-
if (!override) return false;
|
|
2897
|
-
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
2898
|
-
}
|
|
2899
|
-
|
|
2900
|
-
//#endregion
|
|
2901
|
-
//#region src/adapters/anthropic.ts
|
|
2902
|
-
/**
|
|
2903
|
-
* Translates a VernLLM `ContentBlock[]` (our provider-agnostic multimodal
|
|
2904
|
-
* shape) into Anthropic's native content-block array: text blocks pass
|
|
2905
|
-
* through as-is, image blocks become `{ type: 'image', source: { type:
|
|
2906
|
-
* 'base64', media_type, data } }`.
|
|
2907
|
-
*/
|
|
2908
|
-
function toAnthropicContent(blocks) {
|
|
2909
|
-
return blocks.map((block) => block.type === "image" ? {
|
|
2910
|
-
type: "image",
|
|
2911
|
-
source: {
|
|
2912
|
-
type: "base64",
|
|
2913
|
-
media_type: assertSupportedImageMimeType(block.mimeType),
|
|
2914
|
-
data: block.data
|
|
2915
|
-
}
|
|
2916
|
-
} : {
|
|
2917
|
-
type: "text",
|
|
2918
|
-
text: block.text
|
|
2919
|
-
});
|
|
2920
|
-
}
|
|
2921
|
-
/**
|
|
2922
|
-
* Asserts a caller-supplied JSON Schema is an object schema before it's
|
|
2923
|
-
* used as Anthropic's `Tool.input_schema`, which (like every other
|
|
2924
|
-
* provider's function-calling API) requires `type: 'object'`. VernLLM's own
|
|
2925
|
-
* public `tools`/`jsonSchema` APIs accept freeform `Record<string,
|
|
2926
|
-
* unknown>` JSON Schema, so nothing upstream guarantees this at compile
|
|
2927
|
-
* time; this is the runtime check that stands in for that, so a schema
|
|
2928
|
-
* missing (or mistyping) `type: 'object'` fails loudly and immediately
|
|
2929
|
-
* instead of being silently forwarded to Anthropic malformed.
|
|
2930
|
-
*/
|
|
2931
|
-
function assertObjectSchema(schema, toolName) {
|
|
2932
|
-
if (schema.type !== "object") throw new LLMError(`Tool "${toolName}"'s schema must have "type": "object" (Anthropic requires object-shaped tool parameters).`, "validation");
|
|
2933
|
-
return schema;
|
|
2934
|
-
}
|
|
2935
|
-
/**
|
|
2936
|
-
* Translates VernLLM's OpenAI-shaped wire `tool_choice` into Anthropic's
|
|
2937
|
-
* `{ type: 'auto' | 'any' | 'none' | 'tool', name? }` shape. `'required'`
|
|
2938
|
-
* maps to `'any'` (Anthropic's "must call some tool" equivalent).
|
|
2939
|
-
*/
|
|
2940
|
-
function toAnthropicToolChoice(toolChoice) {
|
|
2941
|
-
if (!toolChoice || toolChoice === "auto") return { type: "auto" };
|
|
2942
|
-
if (toolChoice === "none") return { type: "none" };
|
|
2943
|
-
if (toolChoice === "required") return { type: "any" };
|
|
2944
|
-
return {
|
|
2945
|
-
type: "tool",
|
|
2946
|
-
name: toolChoice.function.name
|
|
2947
|
-
};
|
|
2948
|
-
}
|
|
2949
|
-
/**
|
|
2950
|
-
* Maps VernLLM's OpenAI-shaped wire `tools`/`tool_choice` into Anthropic's
|
|
2951
|
-
* `tools`/`tool_choice` shape. Shared by the two call sites that build real
|
|
2952
|
-
* (non-schema-forced) tool definitions: the plain tools-only branch, and
|
|
2953
|
-
* the native-structured-output branch, which sends real tools alongside
|
|
2954
|
-
* `output_config` rather than instead of it.
|
|
2955
|
-
*/
|
|
2956
|
-
function buildAnthropicTools(tools, toolChoiceParam) {
|
|
2957
|
-
return {
|
|
2958
|
-
tools: tools.map((t) => ({
|
|
2959
|
-
name: t.function.name,
|
|
2960
|
-
description: t.function.description,
|
|
2961
|
-
input_schema: assertObjectSchema(t.function.parameters, t.function.name)
|
|
2962
|
-
})),
|
|
2963
|
-
toolChoice: toAnthropicToolChoice(toolChoiceParam)
|
|
2964
|
-
};
|
|
2965
|
-
}
|
|
2966
|
-
/**
|
|
2967
|
-
* Builds the Anthropic-shaped request body from VernLLM's wire params,
|
|
2968
|
-
* shared between `create` and `createStream` so both go through identical
|
|
2969
|
-
* translation (system prompt, message shaping, and the jsonSchema →
|
|
2970
|
-
* forced-single-tool mapping all happen exactly once, not once per entry
|
|
2971
|
-
* point).
|
|
2972
|
-
*
|
|
2973
|
-
* Returns `toolName` alongside the body: when set, the model was forced to
|
|
2974
|
-
* call a single synthetic tool standing in for `jsonSchema` output (the
|
|
2975
|
-
* legacy path, for models without native structured-output support), and
|
|
2976
|
-
* both `create` and `createStream` need to know this so they can unwrap
|
|
2977
|
-
* that tool call back into plain text content instead of treating it like
|
|
2978
|
-
* a real tool call. On the native path (model supports `output_config`),
|
|
2979
|
-
* `toolName` is `undefined`: the schema-conforming JSON already arrives as
|
|
2980
|
-
* ordinary text content, nothing to unwrap, and any real tool calls in
|
|
2981
|
-
* `params.tools` are left for the normal, non-forced tool-call handling
|
|
2982
|
-
* both `create` and `createStream` already do when `toolName` is unset.
|
|
2983
|
-
*/
|
|
2984
|
-
function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
2985
|
-
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
2986
|
-
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
2987
|
-
const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
|
|
2988
|
-
const schemaName = jsonSchema?.name.trim();
|
|
2989
|
-
if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
|
|
2990
|
-
const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
|
|
2991
|
-
if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Anthropic model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there, which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromAnthropic's \`nativeStructuredOutputModels\` option once you've confirmed it supports Anthropic's \`output_config.format\`.`, "validation");
|
|
2992
|
-
let toolName;
|
|
2993
|
-
let jsonInstruction;
|
|
2994
|
-
let outputFormat;
|
|
2995
|
-
let tools;
|
|
2996
|
-
let toolChoice;
|
|
2997
|
-
if (jsonSchema && isNative) {
|
|
2998
|
-
outputFormat = {
|
|
2999
|
-
type: "json_schema",
|
|
3000
|
-
schema: jsonSchema.schema
|
|
3001
|
-
};
|
|
3002
|
-
if (params.tools?.length) ({tools, toolChoice} = buildAnthropicTools(params.tools, params.tool_choice));
|
|
3003
|
-
} else if (jsonSchema && schemaName) {
|
|
3004
|
-
const { schema, description, strict } = jsonSchema;
|
|
3005
|
-
toolName = schemaName;
|
|
3006
|
-
tools = [{
|
|
3007
|
-
name: toolName,
|
|
3008
|
-
description,
|
|
3009
|
-
input_schema: assertObjectSchema(schema, toolName),
|
|
3010
|
-
strict
|
|
3011
|
-
}];
|
|
3012
|
-
toolChoice = {
|
|
3013
|
-
type: "tool",
|
|
3014
|
-
name: toolName
|
|
3015
|
-
};
|
|
3016
|
-
} else if (params.response_format?.type === "json_object") jsonInstruction = "Respond with valid JSON only, no prose or markdown fences.";
|
|
3017
|
-
if (!jsonSchema && params.tools?.length) ({tools, toolChoice} = buildAnthropicTools(params.tools, params.tool_choice));
|
|
3018
|
-
const system = [systemMessage?.content, jsonInstruction].filter(Boolean).join("\n\n");
|
|
3019
|
-
const body = {
|
|
3020
|
-
model: params.model,
|
|
3021
|
-
max_tokens: params.max_tokens,
|
|
3022
|
-
...params.temperature !== void 0 ? { temperature: params.temperature } : {},
|
|
3023
|
-
system: system || void 0,
|
|
3024
|
-
messages: mergeConsecutiveToolResults$1(conversationMessages.map((m) => toAnthropicMessage(m))),
|
|
3025
|
-
...tools ? {
|
|
3026
|
-
tools,
|
|
3027
|
-
tool_choice: toolChoice
|
|
3028
|
-
} : {},
|
|
3029
|
-
...outputFormat ? { output_config: { format: outputFormat } } : {}
|
|
3030
|
-
};
|
|
3031
|
-
return {
|
|
3032
|
-
body,
|
|
3033
|
-
toolName
|
|
3034
|
-
};
|
|
3035
|
-
}
|
|
3036
|
-
/**
|
|
3037
|
-
* Wraps an Anthropic SDK client so it satisfies the same `LLMClient`
|
|
3038
|
-
* interface VernLLM uses for OpenAI/Groq.
|
|
3039
|
-
*
|
|
3040
|
-
* `response_format: json_schema`, on a model covered by
|
|
3041
|
-
* `options.nativeStructuredOutputModels`, is sent as `output_config.format`,
|
|
3042
|
-
* its own request field, independent of `tools`/`tool_choice`, so it can be
|
|
3043
|
-
* combined with real, caller-supplied `tools` in the same request. Only
|
|
3044
|
-
* `type` and `schema` are sent on this path, the real Anthropic API's
|
|
3045
|
-
* `output_config.format` has no `name`/`description`/`strict` fields.
|
|
3046
|
-
*
|
|
3047
|
-
* On any other model (the default, since `nativeStructuredOutputModels` is
|
|
3048
|
-
* opt-in), `response_format: json_schema` is mapped to Anthropic's forced
|
|
3049
|
-
* tool-use instead: a single tool is defined with `input_schema` set to
|
|
3050
|
-
* the caller's schema, `description` forwarded when provided, and `strict`
|
|
3051
|
-
* forwarded when set, and `tool_choice` forces the model to call it. This
|
|
3052
|
-
* legacy path cannot be combined with real `tools` (both would need the
|
|
3053
|
-
* same `tools`/`tool_choice` field), and a call that tries throws
|
|
3054
|
-
* `LLMError('validation')` before reaching the API. Provider-constrained
|
|
3055
|
-
* schema matching applies only when `strict: true` is forwarded and
|
|
3056
|
-
* supported.
|
|
3057
|
-
*
|
|
3058
|
-
* `response_format: json_object` (no schema to build a tool from) falls
|
|
3059
|
-
* back to a system-prompt instruction, since there's nothing to constrain
|
|
3060
|
-
* generation against. Unlike `jsonSchema`, this combines with real `tools`
|
|
3061
|
-
* freely on every model: it's a prompt nudge, not a request field, so
|
|
3062
|
-
* there's nothing for it to collide with.
|
|
3063
|
-
*/
|
|
3064
|
-
function fromAnthropic(anthropicClient, options) {
|
|
3065
|
-
const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
|
|
3066
|
-
const rawMessagesCreate = anthropicClient.messages.create.bind(anthropicClient.messages);
|
|
3067
|
-
return { chat: { completions: {
|
|
3068
|
-
async create(params, options$1) {
|
|
3069
|
-
const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels);
|
|
3070
|
-
const response = await anthropicClient.messages.create(body, options$1);
|
|
3071
|
-
let text;
|
|
3072
|
-
let wireToolCalls;
|
|
3073
|
-
if (toolName) {
|
|
3074
|
-
const toolUse = response.content.find((block) => block.type === "tool_use" && block.name === toolName);
|
|
3075
|
-
if (!toolUse) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
|
|
3076
|
-
if (!toolUse.input || typeof toolUse.input !== "object" || Array.isArray(toolUse.input)) throw new LLMError(`Anthropic returned invalid structured output for tool "${toolName}". Expected an object.`, "validation");
|
|
3077
|
-
text = JSON.stringify(toolUse.input);
|
|
3078
|
-
} else {
|
|
3079
|
-
text = response.content.filter((block) => block.type === "text").map((block) => block.text ?? "").join("");
|
|
3080
|
-
const toolUses = response.content.filter((block) => block.type === "tool_use");
|
|
3081
|
-
if (toolUses.length) wireToolCalls = toolUses.map((block) => ({
|
|
3082
|
-
id: block.id,
|
|
3083
|
-
type: "function",
|
|
3084
|
-
function: {
|
|
3085
|
-
name: block.name,
|
|
3086
|
-
arguments: JSON.stringify(block.input ?? {})
|
|
3087
|
-
}
|
|
3088
|
-
}));
|
|
3089
|
-
}
|
|
3090
|
-
return {
|
|
3091
|
-
choices: [{ message: {
|
|
3092
|
-
content: text,
|
|
3093
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3094
|
-
} }],
|
|
3095
|
-
usage: {
|
|
3096
|
-
prompt_tokens: response.usage?.input_tokens,
|
|
3097
|
-
completion_tokens: response.usage?.output_tokens,
|
|
3098
|
-
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0)
|
|
3099
|
-
}
|
|
3100
|
-
};
|
|
3101
|
-
},
|
|
3102
|
-
async *createStream(params, options$1) {
|
|
3103
|
-
const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels);
|
|
3104
|
-
const stream = await rawMessagesCreate({
|
|
3105
|
-
...body,
|
|
3106
|
-
stream: true
|
|
3107
|
-
}, options$1);
|
|
3108
|
-
const blockKinds = new Map();
|
|
3109
|
-
let inputTokens = 0;
|
|
3110
|
-
let sawJsonTool = false;
|
|
3111
|
-
for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
|
|
3112
|
-
else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
|
|
3113
|
-
const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
|
|
3114
|
-
blockKinds.set(event.index, kind);
|
|
3115
|
-
if (kind === "json-tool") sawJsonTool = true;
|
|
3116
|
-
else if (!toolName) yield {
|
|
3117
|
-
type: "tool_call_delta",
|
|
3118
|
-
index: event.index,
|
|
3119
|
-
id: event.content_block.id,
|
|
3120
|
-
name: event.content_block.name
|
|
3121
|
-
};
|
|
3122
|
-
} else blockKinds.set(event.index, "text");
|
|
3123
|
-
else if (event.type === "content_block_delta") {
|
|
3124
|
-
if (event.delta.type === "text_delta") {
|
|
3125
|
-
if (!toolName) yield {
|
|
3126
|
-
type: "text-delta",
|
|
3127
|
-
delta: event.delta.text
|
|
3128
|
-
};
|
|
3129
|
-
} else if (event.delta.type === "input_json_delta") {
|
|
3130
|
-
const kind = blockKinds.get(event.index);
|
|
3131
|
-
if (kind === "json-tool") yield {
|
|
3132
|
-
type: "text-delta",
|
|
3133
|
-
delta: event.delta.partial_json
|
|
3134
|
-
};
|
|
3135
|
-
else if (!toolName) yield {
|
|
3136
|
-
type: "tool_call_delta",
|
|
3137
|
-
index: event.index,
|
|
3138
|
-
argumentsDelta: event.delta.partial_json
|
|
3139
|
-
};
|
|
3140
|
-
}
|
|
3141
|
-
} else if (event.type === "message_delta") {
|
|
3142
|
-
const outputTokens = event.usage?.output_tokens ?? 0;
|
|
3143
|
-
yield {
|
|
3144
|
-
type: "usage",
|
|
3145
|
-
usage: {
|
|
3146
|
-
prompt_tokens: inputTokens,
|
|
3147
|
-
completion_tokens: outputTokens,
|
|
3148
|
-
total_tokens: inputTokens + outputTokens
|
|
3149
|
-
}
|
|
3150
|
-
};
|
|
3151
|
-
} else if (event.type === "ping") yield { type: "ping" };
|
|
3152
|
-
if (toolName && !sawJsonTool) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
|
|
3153
|
-
}
|
|
3154
|
-
} } };
|
|
3155
|
-
}
|
|
3156
|
-
/**
|
|
3157
|
-
* Anthropic requires strict role alternation, so the per-wire-message
|
|
3158
|
-
* mapping above (one `{role:'user', content:[tool_result]}` per VernLLM
|
|
3159
|
-
* wire tool message) needs merging back together when an assistant turn
|
|
3160
|
-
* requested more than one tool: multiple consecutive user turns would
|
|
3161
|
-
* violate that alternation, and Anthropic's API rejects it outright. This
|
|
3162
|
-
* merges any run of tool-result-only user messages into one, with all
|
|
3163
|
-
* their tool_result blocks combined, the shape Anthropic expects for "here
|
|
3164
|
-
* are the results of everything you just asked for."
|
|
3165
|
-
*/
|
|
3166
|
-
function mergeConsecutiveToolResults$1(messages) {
|
|
3167
|
-
const isToolResultOnly = (m) => m.role === "user" && Array.isArray(m.content) && m.content.length > 0 && m.content.every((b) => b.type === "tool_result");
|
|
3168
|
-
const merged = [];
|
|
3169
|
-
for (const m of messages) {
|
|
3170
|
-
const prev = merged.at(-1);
|
|
3171
|
-
if (isToolResultOnly(m) && prev && isToolResultOnly(prev)) prev.content.push(...m.content);
|
|
3172
|
-
else merged.push(m);
|
|
3173
|
-
}
|
|
3174
|
-
return merged;
|
|
3175
|
-
}
|
|
3176
|
-
/**
|
|
3177
|
-
* Translates one VernLLM wire message (OpenAI-shaped: plain user/assistant
|
|
3178
|
-
* turns, an assistant turn with `tool_calls`, or a `tool` turn) into
|
|
3179
|
-
* Anthropic's `{ role: 'user' | 'assistant', content }` shape.
|
|
3180
|
-
*/
|
|
3181
|
-
function toAnthropicMessage(m) {
|
|
3182
|
-
if (m.role === "tool") return {
|
|
3183
|
-
role: "user",
|
|
3184
|
-
content: [{
|
|
3185
|
-
type: "tool_result",
|
|
3186
|
-
tool_use_id: m.tool_call_id,
|
|
3187
|
-
content: m.content,
|
|
3188
|
-
...m.is_error ? { is_error: true } : {}
|
|
3189
|
-
}]
|
|
3190
|
-
};
|
|
3191
|
-
if (m.role === "assistant" && m.tool_calls?.length) {
|
|
3192
|
-
const blocks = [];
|
|
3193
|
-
if (m.content) blocks.push({
|
|
3194
|
-
type: "text",
|
|
3195
|
-
text: m.content
|
|
3196
|
-
});
|
|
3197
|
-
for (const tc of m.tool_calls) {
|
|
3198
|
-
let input;
|
|
3199
|
-
try {
|
|
3200
|
-
input = tc.function.arguments.trim() ? JSON.parse(tc.function.arguments) : {};
|
|
3201
|
-
} catch (cause) {
|
|
3202
|
-
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
|
|
3203
|
-
}
|
|
3204
|
-
if (input === null || Array.isArray(input) || typeof input !== "object") throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) arguments must be a JSON object.`, "validation");
|
|
3205
|
-
blocks.push({
|
|
3206
|
-
type: "tool_use",
|
|
3207
|
-
id: tc.id,
|
|
3208
|
-
name: tc.function.name,
|
|
3209
|
-
input
|
|
3210
|
-
});
|
|
3211
|
-
}
|
|
3212
|
-
return {
|
|
3213
|
-
role: "assistant",
|
|
3214
|
-
content: blocks
|
|
3215
|
-
};
|
|
3216
|
-
}
|
|
3217
|
-
return {
|
|
3218
|
-
role: m.role,
|
|
3219
|
-
content: Array.isArray(m.content) ? toAnthropicContent(m.content) : m.content ?? ""
|
|
3220
|
-
};
|
|
3221
|
-
}
|
|
3222
|
-
|
|
3223
|
-
//#endregion
|
|
3224
|
-
//#region src/adapters/gemini.ts
|
|
3225
|
-
/**
|
|
3226
|
-
* Translates a VernLLM `ContentBlock[]` into Gemini's native `parts` array:
|
|
3227
|
-
* text blocks become `{ text }`, image blocks become inline data parts
|
|
3228
|
-
* (`{ inlineData: { mimeType, data } }`), Gemini's shape for embedding raw
|
|
3229
|
-
* base64 image bytes directly in the request.
|
|
3230
|
-
*/
|
|
3231
|
-
function toGeminiParts(blocks) {
|
|
3232
|
-
return blocks.map((block) => block.type === "image" ? { inlineData: {
|
|
3233
|
-
mimeType: assertSupportedImageMimeType(block.mimeType),
|
|
3234
|
-
data: block.data
|
|
3235
|
-
} } : { text: block.text });
|
|
3236
|
-
}
|
|
3237
|
-
/** Maps VernLLM's OpenAI-shaped wire `tool_choice` onto Gemini's `functionCallingConfig`. */
|
|
3238
|
-
function toGeminiToolConfig(toolChoice) {
|
|
3239
|
-
if (!toolChoice || toolChoice === "auto") return { functionCallingConfig: { mode: "AUTO" } };
|
|
3240
|
-
if (toolChoice === "none") return { functionCallingConfig: { mode: "NONE" } };
|
|
3241
|
-
if (toolChoice === "required") return { functionCallingConfig: { mode: "ANY" } };
|
|
3242
|
-
return { functionCallingConfig: {
|
|
3243
|
-
mode: "ANY",
|
|
3244
|
-
allowedFunctionNames: [toolChoice.function.name]
|
|
3245
|
-
} };
|
|
3246
|
-
}
|
|
3247
|
-
/**
|
|
3248
|
-
* Translates one VernLLM wire message into a Gemini `contents` entry.
|
|
3249
|
-
* Gemini has no separate 'tool' role: a prior assistant tool request
|
|
3250
|
-
* becomes a `'model'` turn with `functionCall` parts, and its result
|
|
3251
|
-
* becomes a `'user'` turn with `functionResponse` parts.
|
|
3252
|
-
*/
|
|
3253
|
-
function toGeminiContent(m) {
|
|
3254
|
-
if (m.role === "tool") return {
|
|
3255
|
-
role: "user",
|
|
3256
|
-
parts: [{ functionResponse: {
|
|
3257
|
-
name: m.tool_call_id,
|
|
3258
|
-
response: parseToolResult(m.content)
|
|
3259
|
-
} }]
|
|
3260
|
-
};
|
|
3261
|
-
if (m.role === "assistant" && m.tool_calls?.length) {
|
|
3262
|
-
const parts = [];
|
|
3263
|
-
if (typeof m.content === "string" && m.content) parts.push({ text: m.content });
|
|
3264
|
-
parts.push(...m.tool_calls.map((tc) => ({ functionCall: {
|
|
3265
|
-
name: tc.function.name,
|
|
3266
|
-
args: parseToolArguments(tc.function.arguments, tc.function.name)
|
|
3267
|
-
} })));
|
|
3268
|
-
return {
|
|
3269
|
-
role: "model",
|
|
3270
|
-
parts
|
|
3271
|
-
};
|
|
3272
|
-
}
|
|
3273
|
-
return {
|
|
3274
|
-
role: m.role === "assistant" ? "model" : "user",
|
|
3275
|
-
parts: Array.isArray(m.content) ? toGeminiParts(m.content) : [{ text: m.content ?? "" }]
|
|
3276
|
-
};
|
|
3277
|
-
}
|
|
3278
|
-
function parseToolArguments(text, toolName) {
|
|
3279
|
-
let parsed;
|
|
3280
|
-
try {
|
|
3281
|
-
parsed = text.trim() ? JSON.parse(text) : {};
|
|
3282
|
-
} catch (cause) {
|
|
3283
|
-
throw new LLMError(`Tool call "${toolName}" arguments are not valid JSON.`, "parse", {
|
|
3284
|
-
cause,
|
|
3285
|
-
code: "tool_arguments_parse_failed"
|
|
3286
|
-
});
|
|
3287
|
-
}
|
|
3288
|
-
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") throw new LLMError(`Tool call "${toolName}" arguments must be a JSON object.`, "validation");
|
|
3289
|
-
return parsed;
|
|
3290
|
-
}
|
|
3291
|
-
function parseToolResult(text) {
|
|
3292
|
-
try {
|
|
3293
|
-
return text.trim() ? JSON.parse(text) : "";
|
|
3294
|
-
} catch {
|
|
3295
|
-
return text;
|
|
3296
|
-
}
|
|
3297
|
-
}
|
|
3298
|
-
/**
|
|
3299
|
-
* Gemini expects the results of everything the model asked for in one turn
|
|
3300
|
-
* to arrive together as multiple `functionResponse` parts on a single
|
|
3301
|
-
* `'user'` entry, not as separate consecutive `'user'` entries. The
|
|
3302
|
-
* per-wire-message mapping above produces one `'user'` entry per VernLLM
|
|
3303
|
-
* wire tool message, so when an assistant turn requested more than one
|
|
3304
|
-
* tool, this merges the resulting run of functionResponse-only `'user'`
|
|
3305
|
-
* entries back into one.
|
|
3306
|
-
*/
|
|
3307
|
-
function mergeConsecutiveFunctionResponses(contents) {
|
|
3308
|
-
const isFunctionResponseOnly = (c) => c.role === "user" && c.parts.length > 0 && c.parts.every((p) => "functionResponse" in p);
|
|
3309
|
-
const merged = [];
|
|
3310
|
-
for (const c of contents) {
|
|
3311
|
-
const prev = merged.at(-1);
|
|
3312
|
-
if (isFunctionResponseOnly(c) && prev && isFunctionResponseOnly(prev)) prev.parts.push(...c.parts);
|
|
3313
|
-
else merged.push(c);
|
|
3314
|
-
}
|
|
3315
|
-
return merged;
|
|
3316
|
-
}
|
|
3317
|
-
/**
|
|
3318
|
-
* Builds the Gemini-shaped request from VernLLM's wire params, shared
|
|
3319
|
-
* between `create` and `createStream` so both go through identical
|
|
3320
|
-
* translation (contents shaping, `responseSchema`/`responseMimeType`
|
|
3321
|
-
* mapping, and tool/toolConfig translation all happen exactly once).
|
|
3322
|
-
* `abortSignal` is folded into `config` by the caller (`create`/
|
|
3323
|
-
* `createStream`), once the request options are available.
|
|
3324
|
-
*/
|
|
3325
|
-
function buildGeminiRequest(params) {
|
|
3326
|
-
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
3327
|
-
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
3328
|
-
const wantsJson = Boolean(params.response_format);
|
|
3329
|
-
const config = {
|
|
3330
|
-
...params.temperature !== void 0 ? { temperature: params.temperature } : {},
|
|
3331
|
-
maxOutputTokens: params.max_tokens,
|
|
3332
|
-
...systemMessage ? { systemInstruction: { parts: [{ text: systemMessage.content }] } } : {}
|
|
3333
|
-
};
|
|
3334
|
-
if (wantsJson) config.responseMimeType = "application/json";
|
|
3335
|
-
if (params.response_format?.type === "json_schema") {
|
|
3336
|
-
const { schema, description } = params.response_format.json_schema;
|
|
3337
|
-
config.responseSchema = {
|
|
3338
|
-
...schema,
|
|
3339
|
-
...description ? { description } : {}
|
|
3340
|
-
};
|
|
3341
|
-
}
|
|
3342
|
-
if (params.tools?.length) {
|
|
3343
|
-
config.tools = [{ functionDeclarations: params.tools.map((t) => ({
|
|
3344
|
-
name: t.function.name,
|
|
3345
|
-
description: t.function.description,
|
|
3346
|
-
parameters: t.function.parameters
|
|
3347
|
-
})) }];
|
|
3348
|
-
config.toolConfig = toGeminiToolConfig(params.tool_choice);
|
|
3349
|
-
}
|
|
3350
|
-
return {
|
|
3351
|
-
model: params.model,
|
|
3352
|
-
contents: mergeConsecutiveFunctionResponses(conversationMessages.map((m) => toGeminiContent(m))),
|
|
3353
|
-
config
|
|
3354
|
-
};
|
|
3355
|
-
}
|
|
3356
|
-
/**
|
|
3357
|
-
* Wraps a Gemini client so it satisfies the `LLMClient` interface VernLLM
|
|
3358
|
-
* uses for OpenAI-compatible APIs. Gemini's shape differs on nearly every
|
|
3359
|
-
* axis: a `contents` array instead of `messages`, a separate
|
|
3360
|
-
* `systemInstruction` field instead of a `system` role message,
|
|
3361
|
-
* `generationConfig` instead of top-level `temperature`/`max_tokens`, and
|
|
3362
|
-
* native JSON Schema support via `responseMimeType: 'application/json'` +
|
|
3363
|
-
* `responseSchema`. `reasoning_effort` has no equivalent. Gemini's thinking
|
|
3364
|
-
* models use a token budget, not an effort tier, so it's dropped, same as
|
|
3365
|
-
* Anthropic.
|
|
3366
|
-
*
|
|
3367
|
-
* `tools` maps to Gemini's native `functionDeclarations`/`functionCall`;
|
|
3368
|
-
* `tool_choice` maps to `toolConfig.functionCallingConfig`. Gemini accepts
|
|
3369
|
-
* `responseSchema` and `tools` in the same request natively, so both are
|
|
3370
|
-
* set independently here and no special-casing is needed for the
|
|
3371
|
-
* combination, unlike `fromAnthropic`/`fromBedrock`.
|
|
3372
|
-
*
|
|
3373
|
-
* `createStream` calls `generateContentStream` (optional on `GeminiClient`
|
|
3374
|
-
*, required only if the caller sets `stream: true`) and translates each
|
|
3375
|
-
* partial response into `WireStreamChunk`s. Unlike OpenAI/Anthropic,
|
|
3376
|
-
* Gemini's own function-calling API doesn't stream tool-call arguments
|
|
3377
|
-
* incrementally: a `functionCall` part always arrives whole in one chunk,
|
|
3378
|
-
* so each one is emitted as a single, complete `tool_call_delta` (a
|
|
3379
|
-
* one-shot "delta" containing the full arguments) rather than accumulated
|
|
3380
|
-
* fragments, that's a real difference in the underlying API, not
|
|
3381
|
-
* something this adapter can smooth over. `usageMetadata` is (per Gemini's
|
|
3382
|
-
* own behavior) only reliably present on the last chunk, so the `usage`
|
|
3383
|
-
* `WireStreamChunk` is emitted once, after the stream completes, from
|
|
3384
|
-
* whichever chunk's `usageMetadata` was seen last.
|
|
3385
|
-
*/
|
|
3386
|
-
function fromGemini(geminiClient) {
|
|
3387
|
-
return { chat: { completions: {
|
|
3388
|
-
async create(params, options) {
|
|
3389
|
-
const request = buildGeminiRequest(params);
|
|
3390
|
-
request.config = {
|
|
3391
|
-
...request.config,
|
|
3392
|
-
abortSignal: options.signal
|
|
3393
|
-
};
|
|
3394
|
-
const response = await geminiClient.generateContent(request);
|
|
3395
|
-
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
3396
|
-
const text = parts.map((p) => p.text ?? "").join("");
|
|
3397
|
-
const functionCalls = parts.filter((p) => p.functionCall);
|
|
3398
|
-
let wireToolCalls;
|
|
3399
|
-
if (functionCalls.length) wireToolCalls = functionCalls.map((p) => ({
|
|
3400
|
-
id: p.functionCall.name,
|
|
3401
|
-
type: "function",
|
|
3402
|
-
function: {
|
|
3403
|
-
name: p.functionCall.name,
|
|
3404
|
-
arguments: JSON.stringify(p.functionCall.args ?? {})
|
|
3405
|
-
}
|
|
3406
|
-
}));
|
|
3407
|
-
return {
|
|
3408
|
-
choices: [{ message: {
|
|
3409
|
-
content: text,
|
|
3410
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3411
|
-
} }],
|
|
3412
|
-
usage: {
|
|
3413
|
-
prompt_tokens: response.usageMetadata?.promptTokenCount,
|
|
3414
|
-
completion_tokens: response.usageMetadata?.candidatesTokenCount,
|
|
3415
|
-
total_tokens: response.usageMetadata?.totalTokenCount
|
|
3416
|
-
}
|
|
3417
|
-
};
|
|
3418
|
-
},
|
|
3419
|
-
async *createStream(params, options) {
|
|
3420
|
-
if (!geminiClient.generateContentStream) throw new LLMError("stream: true requires a Gemini client with generateContentStream", "invalid_params", {
|
|
3421
|
-
code: "unsupported_capability",
|
|
3422
|
-
issues: { capability: "generateContentStream" }
|
|
3423
|
-
});
|
|
3424
|
-
const request = buildGeminiRequest(params);
|
|
3425
|
-
request.config = {
|
|
3426
|
-
...request.config,
|
|
3427
|
-
abortSignal: options.signal
|
|
3428
|
-
};
|
|
3429
|
-
const stream = await geminiClient.generateContentStream(request);
|
|
3430
|
-
let toolCallIndex = 0;
|
|
3431
|
-
let lastUsage;
|
|
3432
|
-
for await (const chunk of stream) {
|
|
3433
|
-
const parts = chunk.candidates?.[0]?.content?.parts ?? [];
|
|
3434
|
-
for (const part of parts) {
|
|
3435
|
-
if (part.text) yield {
|
|
3436
|
-
type: "text-delta",
|
|
3437
|
-
delta: part.text
|
|
3438
|
-
};
|
|
3439
|
-
if (part.functionCall) {
|
|
3440
|
-
yield {
|
|
3441
|
-
type: "tool_call_delta",
|
|
3442
|
-
index: toolCallIndex,
|
|
3443
|
-
id: part.functionCall.name,
|
|
3444
|
-
name: part.functionCall.name,
|
|
3445
|
-
argumentsDelta: JSON.stringify(part.functionCall.args ?? {}),
|
|
3446
|
-
complete: true
|
|
3447
|
-
};
|
|
3448
|
-
toolCallIndex++;
|
|
3449
|
-
}
|
|
3450
|
-
}
|
|
3451
|
-
if (chunk.usageMetadata) lastUsage = chunk.usageMetadata;
|
|
3452
|
-
}
|
|
3453
|
-
if (lastUsage) yield {
|
|
3454
|
-
type: "usage",
|
|
3455
|
-
usage: {
|
|
3456
|
-
prompt_tokens: lastUsage.promptTokenCount,
|
|
3457
|
-
completion_tokens: lastUsage.candidatesTokenCount,
|
|
3458
|
-
total_tokens: lastUsage.totalTokenCount
|
|
3459
|
-
}
|
|
3460
|
-
};
|
|
3461
|
-
}
|
|
3462
|
-
} } };
|
|
3463
|
-
}
|
|
3464
|
-
|
|
3465
|
-
//#endregion
|
|
3466
|
-
//#region src/adapters/bedrock.ts
|
|
3467
|
-
/** Maps a `ContentBlock` image MIME type, already validated, to Converse's `format` enum. */
|
|
3468
|
-
function toBedrockImageFormat(mimeType) {
|
|
3469
|
-
switch (assertSupportedImageMimeType(mimeType)) {
|
|
3470
|
-
case "image/png": return "png";
|
|
3471
|
-
case "image/jpeg": return "jpeg";
|
|
3472
|
-
case "image/gif": return "gif";
|
|
3473
|
-
case "image/webp": return "webp";
|
|
3474
|
-
}
|
|
3475
|
-
}
|
|
3476
|
-
/**
|
|
3477
|
-
* Decodes base64 image data into the raw `Uint8Array` bytes Converse's
|
|
3478
|
-
* `image.source.bytes` expects (unlike Anthropic/Gemini/OpenAI, which all
|
|
3479
|
-
* take base64 strings directly). Uses `Buffer`, since this adapter, like
|
|
3480
|
-
* the rest of the package, targets Node.
|
|
3481
|
-
*/
|
|
3482
|
-
function decodeBase64(data) {
|
|
3483
|
-
return new Uint8Array(Buffer.from(data, "base64"));
|
|
3484
|
-
}
|
|
3485
|
-
/**
|
|
3486
|
-
* Translates a VernLLM `ContentBlock[]` into Converse's native content-block
|
|
3487
|
-
* array: text blocks pass through as `{ text }`, image blocks become
|
|
3488
|
-
* `{ image: { format, source: { bytes } } }` with the base64 payload decoded
|
|
3489
|
-
* to raw bytes, since Converse doesn't accept base64 strings directly.
|
|
3490
|
-
*/
|
|
3491
|
-
function toBedrockContent(blocks) {
|
|
3492
|
-
return blocks.map((block) => block.type === "image" ? { image: {
|
|
3493
|
-
format: toBedrockImageFormat(block.mimeType),
|
|
3494
|
-
source: { bytes: decodeBase64(block.data) }
|
|
3495
|
-
} } : { text: block.text });
|
|
3496
|
-
}
|
|
3497
|
-
/**
|
|
3498
|
-
* Maps VernLLM's OpenAI-shaped wire `tools`/`tool_choice` into Converse's
|
|
3499
|
-
* `toolConfig` shape. Shared by the two call sites that build real
|
|
3500
|
-
* (non-schema-forced) tool definitions: the plain tools-only branch, and
|
|
3501
|
-
* the native-structured-output branch, which sends real tools alongside
|
|
3502
|
-
* `outputConfig` rather than instead of it.
|
|
3503
|
-
*/
|
|
3504
|
-
function buildBedrockToolConfig(tools, toolChoiceParam) {
|
|
3505
|
-
return {
|
|
3506
|
-
tools: tools.map((t) => ({ toolSpec: {
|
|
3507
|
-
name: t.function.name,
|
|
3508
|
-
description: t.function.description,
|
|
3509
|
-
inputSchema: { json: t.function.parameters }
|
|
3510
|
-
} })),
|
|
3511
|
-
toolChoice: toBedrockToolChoice(toolChoiceParam)
|
|
3512
|
-
};
|
|
3513
|
-
}
|
|
3514
|
-
/**
|
|
3515
|
-
* Builds the Converse-shaped request from VernLLM's wire params, shared
|
|
3516
|
-
* between `create` and `createStream` so both go through identical
|
|
3517
|
-
* translation (system prompt, message shaping, the jsonSchema →
|
|
3518
|
-
* forced-single-tool mapping, and the `toolUseSupportedModels` preflight
|
|
3519
|
-
* check all happen exactly once).
|
|
3520
|
-
*
|
|
3521
|
-
* Returns `toolName` alongside the request: when set, the model was forced
|
|
3522
|
-
* to call a single synthetic tool standing in for `jsonSchema` output (the
|
|
3523
|
-
* legacy path, for models not covered by `nativeStructuredOutputModels`),
|
|
3524
|
-
* and both `create` and `createStream` need to know this so they can
|
|
3525
|
-
* unwrap that tool call back into plain text content instead of treating
|
|
3526
|
-
* it like a real tool call. On the native path (model covered by
|
|
3527
|
-
* `nativeStructuredOutputModels`), `toolName` is `undefined`: the
|
|
3528
|
-
* schema-conforming JSON already arrives as ordinary text content, nothing
|
|
3529
|
-
* to unwrap, and any real tool calls in `params.tools` are left for the
|
|
3530
|
-
* normal, non-forced tool-call handling both `create` and `createStream`
|
|
3531
|
-
* already do when `toolName` is unset.
|
|
3532
|
-
*/
|
|
3533
|
-
function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels) {
|
|
3534
|
-
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
3535
|
-
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
3536
|
-
const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
|
|
3537
|
-
const schemaName = jsonSchema?.name.trim();
|
|
3538
|
-
if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
|
|
3539
|
-
const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
|
|
3540
|
-
if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Bedrock model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there (via \`toolConfig\`), which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromBedrock's \`nativeStructuredOutputModels\` option once you've confirmed it supports Converse's \`outputConfig.textFormat\`.`, "validation");
|
|
3541
|
-
let toolName;
|
|
3542
|
-
let jsonInstruction;
|
|
3543
|
-
let toolConfig;
|
|
3544
|
-
let outputConfig;
|
|
3545
|
-
if (jsonSchema && isNative) {
|
|
3546
|
-
const { schema, description } = jsonSchema;
|
|
3547
|
-
outputConfig = { textFormat: {
|
|
3548
|
-
type: "json_schema",
|
|
3549
|
-
structure: { jsonSchema: {
|
|
3550
|
-
schema: JSON.stringify(schema),
|
|
3551
|
-
name: schemaName,
|
|
3552
|
-
description
|
|
3553
|
-
} }
|
|
3554
|
-
} };
|
|
3555
|
-
} else if (jsonSchema && schemaName) {
|
|
3556
|
-
const { schema, description, strict } = jsonSchema;
|
|
3557
|
-
toolName = schemaName;
|
|
3558
|
-
toolConfig = {
|
|
3559
|
-
tools: [{ toolSpec: {
|
|
3560
|
-
name: toolName,
|
|
3561
|
-
description,
|
|
3562
|
-
inputSchema: { json: schema },
|
|
3563
|
-
strict
|
|
3564
|
-
} }],
|
|
3565
|
-
toolChoice: { tool: { name: toolName } }
|
|
3566
|
-
};
|
|
3567
|
-
} else if (params.response_format?.type === "json_object") jsonInstruction = "Respond with valid JSON only, no prose or markdown fences.";
|
|
3568
|
-
if (params.tools?.length && !toolName) toolConfig = buildBedrockToolConfig(params.tools, params.tool_choice);
|
|
3569
|
-
if (jsonSchema && toolConfig && toolUseSupportedModels) {
|
|
3570
|
-
const isSupported = Array.isArray(toolUseSupportedModels) ? toolUseSupportedModels.includes(params.model) : toolUseSupportedModels(params.model);
|
|
3571
|
-
if (!isSupported) throw new LLMError(`Bedrock model "${params.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`, "invalid_params", {
|
|
3572
|
-
code: "unsupported_capability",
|
|
3573
|
-
issues: { capability: "toolUseSupportedModels" }
|
|
3574
|
-
});
|
|
3575
|
-
}
|
|
3576
|
-
const systemParts = [systemMessage?.content, jsonInstruction].filter((s) => Boolean(s));
|
|
3577
|
-
const request = {
|
|
3578
|
-
modelId: params.model,
|
|
3579
|
-
messages: mergeConsecutiveToolResults(conversationMessages.map((m) => toBedrockMessage(m))),
|
|
3580
|
-
system: systemParts.length ? systemParts.map((text) => ({ text })) : void 0,
|
|
3581
|
-
inferenceConfig: {
|
|
3582
|
-
...params.temperature !== void 0 ? { temperature: params.temperature } : {},
|
|
3583
|
-
maxTokens: params.max_tokens
|
|
3584
|
-
},
|
|
3585
|
-
...toolConfig ? { toolConfig } : {},
|
|
3586
|
-
...outputConfig ? { outputConfig } : {}
|
|
3587
|
-
};
|
|
3588
|
-
return {
|
|
3589
|
-
request,
|
|
3590
|
-
toolName
|
|
3591
|
-
};
|
|
3592
|
-
}
|
|
3593
|
-
/**
|
|
3594
|
-
* Wraps a Bedrock Converse-API client so it satisfies the `LLMClient`
|
|
3595
|
-
* interface VernLLM uses for OpenAI/Groq. The Converse API is unified
|
|
3596
|
-
* across Bedrock's model families (Anthropic, Titan, Llama, Mistral, etc.),
|
|
3597
|
-
* so unlike raw per-model Bedrock invocation, this one adapter works
|
|
3598
|
-
* regardless of which underlying model `modelId` points at, as long as
|
|
3599
|
-
* that model supports Converse (most current-generation ones do)
|
|
3600
|
-
*
|
|
3601
|
-
* `response_format: json_schema`, on a model covered by
|
|
3602
|
-
* `options.nativeStructuredOutputModels` (opt-in, unset by default), is
|
|
3603
|
-
* sent as `outputConfig.textFormat`, its own request field, independent of
|
|
3604
|
-
* `toolConfig`, so it can be combined with real, caller-supplied `tools`
|
|
3605
|
-
* in the same request. Matches the real Converse API's shape exactly: the
|
|
3606
|
-
* schema is nested under `structure.jsonSchema` and JSON-encoded as a
|
|
3607
|
-
* string, not the parsed object `toolConfig`'s tool schemas use, and there
|
|
3608
|
-
* is no `strict` field on this path.
|
|
3609
|
-
*
|
|
3610
|
-
* On any other model (the default), `response_format: json_schema` is
|
|
3611
|
-
* mapped to Converse's `toolConfig` instead: a single tool is defined from
|
|
3612
|
-
* the schema, description, and strictness settings, and `toolChoice`
|
|
3613
|
-
* forces the model to call it. This legacy path cannot be combined with
|
|
3614
|
-
* real `tools` (both would need the same `toolConfig`), and a call that
|
|
3615
|
-
* tries throws `LLMError('validation')` before reaching the API.
|
|
3616
|
-
* Provider-constrained schema matching applies only when `strict: true` is
|
|
3617
|
-
* forwarded and supported. Native tool support varies by model family;
|
|
3618
|
-
* pass `toolUseSupportedModels` to preflight-check it (see
|
|
3619
|
-
* `BedrockAdapterOptions`), otherwise a `jsonSchema` call to an
|
|
3620
|
-
* unsupported model surfaces Bedrock's raw error unchanged.
|
|
3621
|
-
*
|
|
3622
|
-
* `response_format: json_object` (no schema to build a tool from) and
|
|
3623
|
-
* `reasoning_effort` (no Converse equivalent) fall back to a system-prompt
|
|
3624
|
-
* instruction and are dropped respectively. Unlike `jsonSchema`,
|
|
3625
|
-
* `json_object` combines with real `tools` freely on every model: it's a
|
|
3626
|
-
* prompt nudge, not a request field, so there's nothing for it to collide
|
|
3627
|
-
* with.
|
|
3628
|
-
*
|
|
3629
|
-
* `tools` alone maps to Converse's native `toolConfig`/`toolUse`/
|
|
3630
|
-
* `toolResult`; `tool_choice` maps to `toolConfig.toolChoice`.
|
|
3631
|
-
*
|
|
3632
|
-
* `createStream` calls `converseStream` (optional on `BedrockConverseClient`
|
|
3633
|
-
*, required only if the caller sets `stream: true`) and translates its
|
|
3634
|
-
* `contentBlockStart`/`contentBlockDelta`/`metadata` events into
|
|
3635
|
-
* `WireStreamChunk`s. Content blocks are tracked by `contentBlockIndex`,
|
|
3636
|
-
* same as `fromAnthropic`'s block-index tracking (Converse's streaming
|
|
3637
|
-
* shape is structurally close to Anthropic's own, both being tool-use-aware
|
|
3638
|
-
* content-block streams), including the same `json-tool` unwrapping: a
|
|
3639
|
-
* `jsonSchema`-forced tool's `toolUse.input` deltas are re-emitted as
|
|
3640
|
-
* `text-delta`, not `tool_call_delta`, so the accumulated result lands in
|
|
3641
|
-
* `finalizeResponse`'s `content` path exactly like the non-streaming
|
|
3642
|
-
* `create` branch above unwraps it.
|
|
3643
|
-
*/
|
|
3644
|
-
function fromBedrock(bedrockClient, options) {
|
|
3645
|
-
const toolUseSupportedModels = options?.toolUseSupportedModels;
|
|
3646
|
-
const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
|
|
3647
|
-
return { chat: { completions: {
|
|
3648
|
-
async create(params, requestOptions) {
|
|
3649
|
-
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels);
|
|
3650
|
-
const response = await bedrockClient.converse(request, requestOptions);
|
|
3651
|
-
let text;
|
|
3652
|
-
let wireToolCalls;
|
|
3653
|
-
if (toolName) {
|
|
3654
|
-
const toolUseBlock = response.output?.message?.content?.find((block) => block.toolUse?.name === toolName);
|
|
3655
|
-
text = toolUseBlock?.toolUse ? JSON.stringify(toolUseBlock.toolUse.input) : "";
|
|
3656
|
-
} else {
|
|
3657
|
-
const blocks = response.output?.message?.content ?? [];
|
|
3658
|
-
text = blocks.map((c) => c.text ?? "").join("");
|
|
3659
|
-
const toolUses = blocks.filter((block) => Boolean(block.toolUse));
|
|
3660
|
-
if (toolUses.length) wireToolCalls = toolUses.map((block, i) => {
|
|
3661
|
-
const toolUse = block.toolUse;
|
|
3662
|
-
if (!toolUse.name) throw new LLMError(`Bedrock returned a toolUse block without a name at index ${i}.`, "validation");
|
|
3663
|
-
return {
|
|
3664
|
-
id: toolUse.toolUseId ?? `${toolUse.name}_${i}`,
|
|
3665
|
-
type: "function",
|
|
3666
|
-
function: {
|
|
3667
|
-
name: toolUse.name,
|
|
3668
|
-
arguments: JSON.stringify(toolUse.input ?? {})
|
|
3669
|
-
}
|
|
3670
|
-
};
|
|
3671
|
-
});
|
|
3672
|
-
}
|
|
3673
|
-
return {
|
|
3674
|
-
choices: [{ message: {
|
|
3675
|
-
content: text,
|
|
3676
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3677
|
-
} }],
|
|
3678
|
-
usage: {
|
|
3679
|
-
prompt_tokens: response.usage?.inputTokens,
|
|
3680
|
-
completion_tokens: response.usage?.outputTokens,
|
|
3681
|
-
total_tokens: response.usage?.totalTokens
|
|
3682
|
-
}
|
|
3683
|
-
};
|
|
3684
|
-
},
|
|
3685
|
-
async *createStream(params, requestOptions) {
|
|
3686
|
-
if (!bedrockClient.converseStream) throw new LLMError("stream: true requires a Bedrock client with converseStream", "invalid_params", {
|
|
3687
|
-
code: "unsupported_capability",
|
|
3688
|
-
issues: { capability: "converseStream" }
|
|
3689
|
-
});
|
|
3690
|
-
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels);
|
|
3691
|
-
const { stream } = await bedrockClient.converseStream(request, requestOptions);
|
|
3692
|
-
const blockKinds = new Map();
|
|
3693
|
-
for await (const event of stream) if ("contentBlockStart" in event) {
|
|
3694
|
-
const { contentBlockIndex, start } = event.contentBlockStart;
|
|
3695
|
-
if (start?.toolUse) {
|
|
3696
|
-
const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
|
|
3697
|
-
blockKinds.set(contentBlockIndex, kind);
|
|
3698
|
-
if (kind === "tool_use" && !toolName) yield {
|
|
3699
|
-
type: "tool_call_delta",
|
|
3700
|
-
index: contentBlockIndex,
|
|
3701
|
-
id: start.toolUse.toolUseId,
|
|
3702
|
-
name: start.toolUse.name
|
|
3703
|
-
};
|
|
3704
|
-
} else blockKinds.set(contentBlockIndex, "text");
|
|
3705
|
-
} else if ("contentBlockDelta" in event) {
|
|
3706
|
-
const { contentBlockIndex, delta } = event.contentBlockDelta;
|
|
3707
|
-
if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
|
|
3708
|
-
type: "text-delta",
|
|
3709
|
-
delta: delta.text
|
|
3710
|
-
};
|
|
3711
|
-
else if (delta && "toolUse" in delta && delta.toolUse?.input !== void 0) {
|
|
3712
|
-
const kind = blockKinds.get(contentBlockIndex);
|
|
3713
|
-
if (kind === "json-tool") yield {
|
|
3714
|
-
type: "text-delta",
|
|
3715
|
-
delta: delta.toolUse.input
|
|
3716
|
-
};
|
|
3717
|
-
else if (!toolName) yield {
|
|
3718
|
-
type: "tool_call_delta",
|
|
3719
|
-
index: contentBlockIndex,
|
|
3720
|
-
argumentsDelta: delta.toolUse.input
|
|
3721
|
-
};
|
|
3722
|
-
}
|
|
3723
|
-
} else if ("metadata" in event && event.metadata.usage) yield {
|
|
3724
|
-
type: "usage",
|
|
3725
|
-
usage: {
|
|
3726
|
-
prompt_tokens: event.metadata.usage.inputTokens,
|
|
3727
|
-
completion_tokens: event.metadata.usage.outputTokens,
|
|
3728
|
-
total_tokens: event.metadata.usage.totalTokens
|
|
3729
|
-
}
|
|
3730
|
-
};
|
|
3731
|
-
else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api", {
|
|
3732
|
-
status: 429,
|
|
3733
|
-
code: "provider_rate_limited"
|
|
3734
|
-
});
|
|
3735
|
-
else if ("validationException" in event) throw new LLMError(event.validationException.message ?? "Bedrock rejected the request mid-stream", "validation");
|
|
3736
|
-
else if ("internalServerException" in event || "serviceUnavailableException" in event || "modelStreamErrorException" in event) {
|
|
3737
|
-
const detail = "internalServerException" in event && event.internalServerException.message || "serviceUnavailableException" in event && event.serviceUnavailableException.message || "modelStreamErrorException" in event && event.modelStreamErrorException.message || "Bedrock reported a mid-stream error";
|
|
3738
|
-
const status = "modelStreamErrorException" in event && event.modelStreamErrorException.originalStatusCode || "serviceUnavailableException" in event && 503 || 500;
|
|
3739
|
-
throw new LLMError(detail, "api", {
|
|
3740
|
-
status,
|
|
3741
|
-
code: status >= 500 ? "server_error" : void 0
|
|
3742
|
-
});
|
|
3743
|
-
}
|
|
3744
|
-
}
|
|
3745
|
-
} } };
|
|
3746
|
-
}
|
|
3747
|
-
/** Maps VernLLM's OpenAI-shaped wire `tool_choice` onto Converse's `toolChoice`. */
|
|
3748
|
-
function toBedrockToolChoice(toolChoice) {
|
|
3749
|
-
if (!toolChoice || toolChoice === "auto") return { auto: {} };
|
|
3750
|
-
if (toolChoice === "required") return { any: {} };
|
|
3751
|
-
if (toolChoice === "none") throw new LLMError("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.", "invalid_params", {
|
|
3752
|
-
code: "unsupported_capability",
|
|
3753
|
-
issues: { capability: "toolChoice: 'none'" }
|
|
3754
|
-
});
|
|
3755
|
-
return { tool: { name: toolChoice.function.name } };
|
|
3756
|
-
}
|
|
3757
|
-
/**
|
|
3758
|
-
* Translates one VernLLM wire message into Converse's
|
|
3759
|
-
* `{ role: 'user' | 'assistant', content }` shape.
|
|
3760
|
-
*/
|
|
3761
|
-
function toBedrockMessage(m) {
|
|
3762
|
-
if (m.role === "tool") return {
|
|
3763
|
-
role: "user",
|
|
3764
|
-
content: [{ toolResult: {
|
|
3765
|
-
toolUseId: m.tool_call_id,
|
|
3766
|
-
content: [{ text: m.content }],
|
|
3767
|
-
status: m.is_error ? "error" : "success"
|
|
3768
|
-
} }]
|
|
3769
|
-
};
|
|
3770
|
-
if (m.role === "assistant" && m.tool_calls?.length) {
|
|
3771
|
-
const blocks = [];
|
|
3772
|
-
if (m.content) blocks.push({ text: m.content });
|
|
3773
|
-
for (const tc of m.tool_calls) {
|
|
3774
|
-
let input;
|
|
3775
|
-
if (!tc.function.arguments.trim()) input = {};
|
|
3776
|
-
else try {
|
|
3777
|
-
input = JSON.parse(tc.function.arguments);
|
|
3778
|
-
} catch (cause) {
|
|
3779
|
-
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
|
|
3780
|
-
}
|
|
3781
|
-
blocks.push({ toolUse: {
|
|
3782
|
-
toolUseId: tc.id,
|
|
3783
|
-
name: tc.function.name,
|
|
3784
|
-
input
|
|
3785
|
-
} });
|
|
3786
|
-
}
|
|
3787
|
-
return {
|
|
3788
|
-
role: "assistant",
|
|
3789
|
-
content: blocks
|
|
3790
|
-
};
|
|
3791
|
-
}
|
|
3792
|
-
return {
|
|
3793
|
-
role: m.role,
|
|
3794
|
-
content: Array.isArray(m.content) ? toBedrockContent(m.content) : [{ text: m.content ?? "" }]
|
|
3795
|
-
};
|
|
3796
|
-
}
|
|
3797
|
-
/**
|
|
3798
|
-
* Converse expects the results of everything the model asked for in one
|
|
3799
|
-
* turn to arrive together as multiple `toolResult` content blocks on a
|
|
3800
|
-
* single `'user'` message, not as separate consecutive `'user'` messages.
|
|
3801
|
-
* The per-wire-message mapping above produces one `'user'` message per
|
|
3802
|
-
* VernLLM wire tool message, so when an assistant turn requested more than
|
|
3803
|
-
* one tool, this merges the resulting run of toolResult-only `'user'`
|
|
3804
|
-
* messages back into one.
|
|
3805
|
-
*/
|
|
3806
|
-
function mergeConsecutiveToolResults(messages) {
|
|
3807
|
-
const isToolResultOnly = (m) => m.role === "user" && m.content.length > 0 && m.content.every((b) => "toolResult" in b);
|
|
3808
|
-
const merged = [];
|
|
3809
|
-
for (const m of messages) {
|
|
3810
|
-
const prev = merged.at(-1);
|
|
3811
|
-
if (isToolResultOnly(m) && prev && isToolResultOnly(prev)) prev.content.push(...m.content);
|
|
3812
|
-
else merged.push(m);
|
|
3813
|
-
}
|
|
3814
|
-
return merged;
|
|
3815
|
-
}
|
|
3816
|
-
|
|
3817
|
-
//#endregion
|
|
3818
|
-
//#region src/adapters/fetch.ts
|
|
3819
|
-
/**
|
|
3820
|
-
* Wraps a WHATWG `ReadableStream` (what `response.body` is) so it can be
|
|
3821
|
-
* consumed with `for await`. Implemented via `getReader()` rather than
|
|
3822
|
-
* relying on `ReadableStream` having a native `Symbol.asyncIterator`,
|
|
3823
|
-
* that support varies across runtimes/versions, and this works everywhere
|
|
3824
|
-
* a `ReadableStream` does.
|
|
3825
|
-
*/
|
|
3826
|
-
async function* webStreamToAsyncIterable(stream) {
|
|
3827
|
-
const reader = stream.getReader();
|
|
3828
|
-
try {
|
|
3829
|
-
for (;;) {
|
|
3830
|
-
const { done, value } = await reader.read();
|
|
3831
|
-
if (done) return;
|
|
3832
|
-
if (value) yield value;
|
|
3833
|
-
}
|
|
3834
|
-
} finally {
|
|
3835
|
-
try {
|
|
3836
|
-
await reader.cancel();
|
|
3837
|
-
} catch {}
|
|
3838
|
-
reader.releaseLock();
|
|
3839
|
-
}
|
|
3840
|
-
}
|
|
3841
|
-
/** Default `requestStream`: native `fetch`, with the same error/`.status` contract non-streaming errors get. */
|
|
3842
|
-
async function defaultRequestStream(url, init) {
|
|
3843
|
-
const res = await fetch(url, init);
|
|
3844
|
-
if (!res.ok) {
|
|
3845
|
-
const body = await res.text().catch(() => "");
|
|
3846
|
-
const err = new Error(`Fetch adapter stream request failed (${res.status}): ${body.slice(0, 500)}`);
|
|
3847
|
-
err.status = res.status;
|
|
3848
|
-
err.headers = res.headers;
|
|
3849
|
-
throw err;
|
|
3850
|
-
}
|
|
3851
|
-
if (!res.body) throw new Error("Fetch adapter stream request received a response with no body.");
|
|
3852
|
-
return webStreamToAsyncIterable(res.body);
|
|
3853
|
-
}
|
|
3854
|
-
/** Builds the shared `{ method, headers, body? }` request-init for both `create` and `createStream`. */
|
|
3855
|
-
async function buildRequestInit(config, params, requestBody) {
|
|
3856
|
-
const url = typeof config.url === "function" ? config.url(params) : config.url;
|
|
3857
|
-
const headers = typeof config.headers === "function" ? await config.headers() : config.headers;
|
|
3858
|
-
const method = config.method ?? "POST";
|
|
3859
|
-
const supportsBody = !["GET", "HEAD"].includes(method.toUpperCase());
|
|
3860
|
-
return {
|
|
3861
|
-
url,
|
|
3862
|
-
method,
|
|
3863
|
-
headers: supportsBody ? {
|
|
3864
|
-
"Content-Type": "application/json",
|
|
3865
|
-
...headers
|
|
3866
|
-
} : { ...headers },
|
|
3867
|
-
...supportsBody ? { body: JSON.stringify(requestBody) } : {}
|
|
3868
|
-
};
|
|
3869
|
-
}
|
|
3870
|
-
/**
|
|
3871
|
-
* A fetch-based escape hatch for providers with no SDK, or where pulling one
|
|
3872
|
-
* in isnt worth it. You supply the URL, headers, and two small mapping
|
|
3873
|
-
* functions; this handles the HTTP call and slots the result into the same
|
|
3874
|
-
* `LLMClient` shape every other adapter produces, so retries, timeouts,
|
|
3875
|
-
* the circuit breaker, and JSON/schema handling all still work unmodified
|
|
3876
|
-
*
|
|
3877
|
-
* Non-2xx responses throw an error with `.status` set to the HTTP status
|
|
3878
|
-
* code, so VernLLMs `nonRetryableStatus` handling (e.g. failing fast on
|
|
3879
|
-
* 401/403) applies here too
|
|
3880
|
-
*
|
|
3881
|
-
* Tool calling works the same way as every other adapter: `mapRequest`
|
|
3882
|
-
* receives the full `ChatRequest`, including `tools`/`toolChoice`, so it can
|
|
3883
|
-
* translate them into whatever shape the provider's wire format expects
|
|
3884
|
-
* (typically an OpenAI-`function`-wrapped `tools` array plus a `tool_choice`
|
|
3885
|
-
* field). On the way back, `mapResponse` may return a `toolCalls` array
|
|
3886
|
-
* (id/name/JSON-encoded-arguments-string per call) alongside or instead of
|
|
3887
|
-
* `content`; VernLLM parses and (if `argumentsSchema` was set) validates
|
|
3888
|
-
* those arguments the same way it does for every other adapter. For
|
|
3889
|
-
* `stream: true`, tool-call deltas go through the existing
|
|
3890
|
-
* `mapStreamEvent` seam via `WireStreamChunk`'s `tool_call_delta` variant,
|
|
3891
|
-
* no separate config is needed for streaming vs non-streaming tool calls.
|
|
3892
|
-
*
|
|
3893
|
-
|
|
3894
|
-
* `createStream` requires `mapStreamEvent` (there's no non-streaming
|
|
3895
|
-
* response to fall back on, unlike the other three optional streaming
|
|
3896
|
-
* seams). It opens the request via `requestStream` (defaults to native
|
|
3897
|
-
* `fetch`), splits the raw bytes into individual events via
|
|
3898
|
-
* `parseStreamFrames` (defaults to SSE framing, see `parseSseStream`),
|
|
3899
|
-
* and translates each event into `WireStreamChunk`(s) via
|
|
3900
|
-
* `mapStreamEvent`. Both seams are overridable per-config for providers
|
|
3901
|
-
* that don't fit the SSE-over-fetch default. If a custom `request`
|
|
3902
|
-
* transport is configured, `requestStream` must be configured too,
|
|
3903
|
-
* `requestStream` never silently falls back to `request` (see
|
|
3904
|
-
* `createStream`'s own comment for why), so a `stream: true` call with
|
|
3905
|
-
* `request` set but no `requestStream` throws a clear
|
|
3906
|
-
* `LLMError('validation')` instead of quietly using unrelated native
|
|
3907
|
-
* `fetch`.
|
|
3908
|
-
*/
|
|
3909
|
-
function fromFetch(config) {
|
|
3910
|
-
return { chat: { completions: {
|
|
3911
|
-
async create(params, options) {
|
|
3912
|
-
const { url, method, headers, body } = await buildRequestInit(config, params, config.mapRequest(params));
|
|
3913
|
-
const request = config.request ?? fetch;
|
|
3914
|
-
const res = await request(url, {
|
|
3915
|
-
method,
|
|
3916
|
-
headers,
|
|
3917
|
-
body,
|
|
3918
|
-
signal: options.signal
|
|
3919
|
-
});
|
|
3920
|
-
if (!res.ok) {
|
|
3921
|
-
const responseBody = await res.text().catch(() => "");
|
|
3922
|
-
const err = new Error(`Fetch adapter request failed (${res.status}): ${responseBody.slice(0, 500)}`);
|
|
3923
|
-
err.status = res.status;
|
|
3924
|
-
err.headers = res.headers;
|
|
3925
|
-
throw err;
|
|
3926
|
-
}
|
|
3927
|
-
const json = await res.json();
|
|
3928
|
-
const { content, usage, toolCalls } = config.mapResponse(json);
|
|
3929
|
-
const wireToolCalls = toolCalls?.length ? toolCalls.map((tc) => ({
|
|
3930
|
-
id: tc.id,
|
|
3931
|
-
type: "function",
|
|
3932
|
-
function: {
|
|
3933
|
-
name: tc.name,
|
|
3934
|
-
arguments: tc.arguments
|
|
3935
|
-
}
|
|
3936
|
-
})) : void 0;
|
|
3937
|
-
return {
|
|
3938
|
-
choices: [{ message: {
|
|
3939
|
-
content,
|
|
3940
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3941
|
-
} }],
|
|
3942
|
-
usage: usage ? {
|
|
3943
|
-
prompt_tokens: usage.promptTokens,
|
|
3944
|
-
completion_tokens: usage.completionTokens,
|
|
3945
|
-
total_tokens: usage.totalTokens
|
|
3946
|
-
} : void 0
|
|
3947
|
-
};
|
|
3948
|
-
},
|
|
3949
|
-
async *createStream(params, options) {
|
|
3950
|
-
if (!config.mapStreamEvent) throw new LLMError("stream: true requires mapStreamEvent to be configured on fromFetch", "invalid_params", {
|
|
3951
|
-
code: "unsupported_capability",
|
|
3952
|
-
issues: { capability: "mapStreamEvent" }
|
|
3953
|
-
});
|
|
3954
|
-
if (config.request && !config.requestStream) throw new LLMError("`stream: true` requires `requestStream` to be configured on fromFetch when a custom `request` transport is set. `requestStream` does not fall back to `request` (it needs an async-iterable byte stream, which `RequestLike`'s buffered `ResponseLike` has no way to provide), without it, `stream: true` would silently use plain native `fetch` instead of your configured transport. Add a `requestStream` that opens the same connection your `request` does, or omit `request` if native `fetch` is fine for both.", "invalid_params", {
|
|
3955
|
-
code: "unsupported_capability",
|
|
3956
|
-
issues: { capability: "requestStream" }
|
|
3957
|
-
});
|
|
3958
|
-
const { url, method, headers, body } = await buildRequestInit(config, params, config.mapRequest(params));
|
|
3959
|
-
const requestStream = config.requestStream ?? defaultRequestStream;
|
|
3960
|
-
const parseFrames = config.parseStreamFrames ?? parseSseStream;
|
|
3961
|
-
const byteStream = await requestStream(url, {
|
|
3962
|
-
method,
|
|
3963
|
-
headers,
|
|
3964
|
-
body,
|
|
3965
|
-
signal: options.signal
|
|
3966
|
-
});
|
|
3967
|
-
for await (const event of parseFrames(byteStream)) {
|
|
3968
|
-
if (event === SSE_PING) {
|
|
3969
|
-
yield { type: "ping" };
|
|
3970
|
-
continue;
|
|
3971
|
-
}
|
|
3972
|
-
const wireChunks = config.mapStreamEvent(event);
|
|
3973
|
-
if (!wireChunks) continue;
|
|
3974
|
-
if (Array.isArray(wireChunks)) yield* wireChunks;
|
|
3975
|
-
else yield wireChunks;
|
|
3976
|
-
}
|
|
3977
|
-
}
|
|
3978
|
-
} } };
|
|
3979
|
-
}
|
|
3980
|
-
|
|
3981
|
-
//#endregion
|
|
3982
|
-
//#region src/adapters/openaiCompatible.ts
|
|
3983
|
-
/**
|
|
3984
|
-
* Translates a VernLLM `ContentBlock[]` into OpenAI's wire-level content
|
|
3985
|
-
* array. Text blocks become `{ type: 'text', text }`; image blocks become
|
|
3986
|
-
* `{ type: 'image_url', image_url: { url } }` with the base64 payload
|
|
3987
|
-
* inlined as a `data:` URL, since our `ContentBlock` shape (`{ type:
|
|
3988
|
-
* 'image', data, mimeType }`) is provider-agnostic and doesn't itself match
|
|
3989
|
-
* OpenAI's wire format.
|
|
3990
|
-
*/
|
|
3991
|
-
function toOpenAIContent(blocks) {
|
|
3992
|
-
return blocks.map((block) => block.type === "image" ? {
|
|
3993
|
-
type: "image_url",
|
|
3994
|
-
image_url: { url: `data:${assertSupportedImageMimeType(block.mimeType)};base64,${block.data}` }
|
|
3995
|
-
} : {
|
|
3996
|
-
type: "text",
|
|
3997
|
-
text: block.text
|
|
3998
|
-
});
|
|
3999
|
-
}
|
|
4000
|
-
/**
|
|
4001
|
-
* Translates VernLLM's provider-agnostic `messages` (the one part of a
|
|
4002
|
-
* request that isn't a pure passthrough for OpenAI-compatible clients) into
|
|
4003
|
-
* OpenAI's native wire shape. Shared between `create` and `createStream` so
|
|
4004
|
-
* both go through identical message translation.
|
|
4005
|
-
*/
|
|
4006
|
-
function toOpenAIMessages(params) {
|
|
4007
|
-
return params.messages.map((m) => {
|
|
4008
|
-
if (m.role === "user" && Array.isArray(m.content)) return {
|
|
4009
|
-
...m,
|
|
4010
|
-
content: toOpenAIContent(m.content)
|
|
4011
|
-
};
|
|
4012
|
-
if (m.role === "tool") {
|
|
4013
|
-
const { is_error: _isError,...openAIToolMessage } = m;
|
|
4014
|
-
return openAIToolMessage;
|
|
4015
|
-
}
|
|
4016
|
-
return m;
|
|
4017
|
-
});
|
|
4018
|
-
}
|
|
4019
|
-
/**
|
|
4020
|
-
* Translates one OpenAI-shaped SSE chunk into zero or more `WireStreamChunk`s.
|
|
4021
|
-
* A single chunk can carry a text delta, one or more tool-call argument
|
|
4022
|
-
* deltas (each keyed by `index`, OpenAI's own convention for streaming
|
|
4023
|
-
* parallel tool calls, mirrored directly by VernLLM's `tool_call_delta`
|
|
4024
|
-
* shape so accumulation composes without translation), and/or a final
|
|
4025
|
-
* usage block (present only when `stream_options.include_usage` is set,
|
|
4026
|
-
* which this adapter always sets).
|
|
4027
|
-
*/
|
|
4028
|
-
function* toWireStreamChunks(chunk) {
|
|
4029
|
-
const delta = chunk.choices?.[0]?.delta;
|
|
4030
|
-
if (delta?.content) yield {
|
|
4031
|
-
type: "text-delta",
|
|
4032
|
-
delta: delta.content
|
|
4033
|
-
};
|
|
4034
|
-
if (delta?.tool_calls?.length) for (const toolCall of delta.tool_calls) yield {
|
|
4035
|
-
type: "tool_call_delta",
|
|
4036
|
-
index: toolCall.index,
|
|
4037
|
-
id: toolCall.id,
|
|
4038
|
-
name: toolCall.function?.name,
|
|
4039
|
-
argumentsDelta: toolCall.function?.arguments
|
|
4040
|
-
};
|
|
4041
|
-
if (chunk.usage) yield {
|
|
4042
|
-
type: "usage",
|
|
4043
|
-
usage: chunk.usage
|
|
4044
|
-
};
|
|
4045
|
-
}
|
|
4046
|
-
function fromOpenAICompatible(client, options = {}) {
|
|
4047
|
-
const raw = client;
|
|
4048
|
-
const { supportsStreamUsage = true } = options;
|
|
4049
|
-
const rawCreate = raw.chat.completions.create.bind(raw.chat.completions);
|
|
4050
|
-
return { chat: { completions: {
|
|
4051
|
-
async create(params, options$1) {
|
|
4052
|
-
const messages = toOpenAIMessages(params);
|
|
4053
|
-
return raw.chat.completions.create({
|
|
4054
|
-
...params,
|
|
4055
|
-
messages
|
|
4056
|
-
}, options$1);
|
|
4057
|
-
},
|
|
4058
|
-
async *createStream(params, options$1) {
|
|
4059
|
-
const messages = toOpenAIMessages(params);
|
|
4060
|
-
const stream = await rawCreate({
|
|
4061
|
-
...params,
|
|
4062
|
-
messages,
|
|
4063
|
-
stream: true,
|
|
4064
|
-
...supportsStreamUsage ? { stream_options: { include_usage: true } } : {}
|
|
4065
|
-
}, options$1);
|
|
4066
|
-
for await (const chunk of stream) yield* toWireStreamChunks(chunk);
|
|
4067
|
-
}
|
|
4068
|
-
} } };
|
|
4069
|
-
}
|
|
4070
|
-
/**
|
|
4071
|
-
* Named alias for the OpenAI SDK itself. A raw `new OpenAI(...)` instance
|
|
4072
|
-
* structurally matches most of `LLMClient`, but newer `openai` SDK major
|
|
4073
|
-
* versions have widened `ChatCompletionContentPart` (e.g. adding a `file`
|
|
4074
|
-
* variant) in ways that no longer structurally satisfy VernLLM's
|
|
4075
|
-
* provider-agnostic `ContentBlock[]` on `userContent`, so passing the SDK
|
|
4076
|
-
* instance directly can fail to typecheck depending on the installed
|
|
4077
|
-
* `openai` version. Wrapping with `fromOpenAI()` (a plain alias of
|
|
4078
|
-
* `fromOpenAICompatible()`) sidesteps that by translating through
|
|
4079
|
-
* `unknown` at the boundary, and also picks up multimodal image
|
|
4080
|
-
* translation and `createStream` wiring that a raw client doesn't have.
|
|
4081
|
-
* See Migration Notes for details.
|
|
4082
|
-
*/
|
|
4083
|
-
const fromOpenAI = fromOpenAICompatible;
|
|
4084
|
-
/** Groqs SDK matches the OpenAI wire format */
|
|
4085
|
-
const fromGroq = fromOpenAICompatible;
|
|
4086
|
-
/**
|
|
4087
|
-
* Mistrals `chat.completions`-shaped client (or their OpenAI-compat
|
|
4088
|
-
* endpoint). Mistral supports `stream_options.include_usage` (added after
|
|
4089
|
-
* an earlier period where it returned a 422 for unrecognized fields, per
|
|
4090
|
-
* Mistral's changelog and streaming docs), so this is a plain alias like
|
|
4091
|
-
* the others, `supportsStreamUsage` defaults to `true`.
|
|
4092
|
-
*/
|
|
4093
|
-
const fromMistral = fromOpenAICompatible;
|
|
4094
|
-
/** DeepSeeks API is OpenAI-compatible */
|
|
4095
|
-
const fromDeepSeek = fromOpenAICompatible;
|
|
4096
|
-
/** Cerebras inference API is OpenAI-compatible */
|
|
4097
|
-
const fromCerebras = fromOpenAICompatible;
|
|
4098
|
-
/** Together AIs API is OpenAI-compatible */
|
|
4099
|
-
const fromTogether = fromOpenAICompatible;
|
|
4100
|
-
/** Fireworks AIs API is OpenAI-compatible */
|
|
4101
|
-
const fromFireworks = fromOpenAICompatible;
|
|
4102
|
-
/**
|
|
4103
|
-
* Ollama exposes an OpenAI-compatible endpoint at `/v1/chat/completions`
|
|
4104
|
-
* (as opposed to its native `/api/chat` format, which differs). Point an
|
|
4105
|
-
* OpenAI SDK instances `baseURL` at your Ollama server and pass it here:
|
|
4106
|
-
* this does not talk to Ollamas native API directly.
|
|
4107
|
-
*/
|
|
4108
|
-
const fromOllama = fromOpenAICompatible;
|
|
4109
|
-
/** OpenRouter's API is OpenAI-compatible */
|
|
4110
|
-
const fromOpenRouter = fromOpenAICompatible;
|
|
4111
|
-
/** Perplexity's API is OpenAI-compatible */
|
|
4112
|
-
const fromPerplexity = fromOpenAICompatible;
|
|
4113
|
-
/** DeepInfra's API is OpenAI-compatible */
|
|
4114
|
-
const fromDeepInfra = fromOpenAICompatible;
|
|
4115
|
-
/** Novita's API is OpenAI-compatible */
|
|
4116
|
-
const fromNovita = fromOpenAICompatible;
|
|
4117
|
-
/** Hyperbolic's API is OpenAI-compatible */
|
|
4118
|
-
const fromHyperbolic = fromOpenAICompatible;
|
|
4119
|
-
/** Moonshot's (Kimi) API is OpenAI-compatible */
|
|
4120
|
-
const fromMoonshot = fromOpenAICompatible;
|
|
4121
|
-
/** Zhipu's (GLM) API is OpenAI-compatible */
|
|
4122
|
-
const fromZhipu = fromOpenAICompatible;
|
|
4123
|
-
/**
|
|
4124
|
-
* LM Studio exposes an OpenAI-compatible endpoint at `/v1/chat/completions`.
|
|
4125
|
-
* Point an OpenAI SDK instance's `baseURL` at your local LM Studio server.
|
|
4126
|
-
*/
|
|
4127
|
-
const fromLMStudio = fromOpenAICompatible;
|
|
4128
|
-
/**
|
|
4129
|
-
* vLLM's OpenAI-compatible server mode exposes `/v1/chat/completions`.
|
|
4130
|
-
* Point an OpenAI SDK instance's `baseURL` at your vLLM server.
|
|
4131
|
-
*/
|
|
4132
|
-
const fromVLLM = fromOpenAICompatible;
|
|
4133
|
-
/** xAI's Grok API is OpenAI-compatible */
|
|
4134
|
-
const fromXAI = fromOpenAICompatible;
|
|
4135
|
-
/** NVIDIA NIM's hosted and self-hosted endpoints are OpenAI-compatible */
|
|
4136
|
-
const fromNvidiaNIM = fromOpenAICompatible;
|
|
4137
|
-
/** Vercel AI Gateway is OpenAI-compatible */
|
|
4138
|
-
const fromVercelAIGateway = fromOpenAICompatible;
|
|
4139
|
-
/** Cloudflare Workers AI exposes an OpenAI-compatible endpoint */
|
|
4140
|
-
const fromCloudflareWorkersAI = fromOpenAICompatible;
|
|
4141
|
-
/** Nebius AI Studio is OpenAI-compatible */
|
|
4142
|
-
const fromNebius = fromOpenAICompatible;
|
|
4143
|
-
/** SambaNova Cloud's API is OpenAI-compatible */
|
|
4144
|
-
const fromSambaNova = fromOpenAICompatible;
|
|
4145
|
-
/** Baseten's model hosting exposes an OpenAI-compatible endpoint */
|
|
4146
|
-
const fromBaseten = fromOpenAICompatible;
|
|
4147
|
-
/** Featherless AI's API is OpenAI-compatible */
|
|
4148
|
-
const fromFeatherless = fromOpenAICompatible;
|
|
4149
|
-
/** Friendli AI's serving endpoint is OpenAI-compatible */
|
|
4150
|
-
const fromFriendli = fromOpenAICompatible;
|
|
4151
|
-
/** SiliconFlow's API is OpenAI-compatible */
|
|
4152
|
-
const fromSiliconFlow = fromOpenAICompatible;
|
|
4153
|
-
/** Parasail's inference API is OpenAI-compatible */
|
|
4154
|
-
const fromParasail = fromOpenAICompatible;
|
|
4155
|
-
/** StepFun's API is OpenAI-compatible */
|
|
4156
|
-
const fromStepFun = fromOpenAICompatible;
|
|
4157
|
-
/** MiniMax's API is OpenAI-compatible */
|
|
4158
|
-
const fromMiniMax = fromOpenAICompatible;
|
|
4159
|
-
/** Lambda Labs' Inference API is OpenAI-compatible */
|
|
4160
|
-
const fromLambdaLabs = fromOpenAICompatible;
|
|
4161
|
-
/** Snowflake Cortex's LLM endpoint is OpenAI-compatible */
|
|
4162
|
-
const fromSnowflakeCortex = fromOpenAICompatible;
|
|
4163
|
-
/** Anyscale Endpoints' API is OpenAI-compatible */
|
|
4164
|
-
const fromAnyscale = fromOpenAICompatible;
|
|
4165
|
-
/** Lepton AI's inference API is OpenAI-compatible */
|
|
4166
|
-
const fromLepton = fromOpenAICompatible;
|
|
4167
|
-
/** Inference.net's API is OpenAI-compatible */
|
|
4168
|
-
const fromInferenceNet = fromOpenAICompatible;
|
|
4169
|
-
/** Infermatic's API is OpenAI-compatible */
|
|
4170
|
-
const fromInfermatic = fromOpenAICompatible;
|
|
4171
|
-
/** AtlasCloud's inference API is OpenAI-compatible */
|
|
4172
|
-
const fromAtlasCloud = fromOpenAICompatible;
|
|
4173
|
-
/** 01.AI's (Yi models) API is OpenAI-compatible */
|
|
4174
|
-
const from01AI = fromOpenAICompatible;
|
|
4175
|
-
|
|
4176
|
-
//#endregion
|
|
4177
|
-
exports.CircuitBreaker = CircuitBreaker
|
|
4178
|
-
exports.ConsoleLogger = ConsoleLogger
|
|
4179
|
-
exports.FallbackExhaustedError = FallbackExhaustedError
|
|
4180
|
-
exports.InMemoryCacheAdapter = InMemoryCacheAdapter
|
|
4181
|
-
exports.LLMError = LLMError
|
|
4182
|
-
exports.NormalizedCacheAdapter = NormalizedCacheAdapter
|
|
4183
|
-
exports.RateLimiter = RateLimiter
|
|
4184
|
-
exports.SSE_PING = SSE_PING
|
|
4185
|
-
exports.TieredCacheAdapter = TieredCacheAdapter
|
|
4186
|
-
exports.VernLLM = VernLLM
|
|
4187
|
-
exports.defaultEstimateTokens = defaultEstimateTokens
|
|
4188
|
-
exports.defaultFallbackOn = defaultFallbackOn
|
|
4189
|
-
exports.from01AI = from01AI
|
|
4190
|
-
exports.fromAnthropic = fromAnthropic
|
|
4191
|
-
exports.fromAnyscale = fromAnyscale
|
|
4192
|
-
exports.fromAtlasCloud = fromAtlasCloud
|
|
4193
|
-
exports.fromBaseten = fromBaseten
|
|
4194
|
-
exports.fromBedrock = fromBedrock
|
|
4195
|
-
exports.fromCerebras = fromCerebras
|
|
4196
|
-
exports.fromCloudflareWorkersAI = fromCloudflareWorkersAI
|
|
4197
|
-
exports.fromDeepInfra = fromDeepInfra
|
|
4198
|
-
exports.fromDeepSeek = fromDeepSeek
|
|
4199
|
-
exports.fromFeatherless = fromFeatherless
|
|
4200
|
-
exports.fromFetch = fromFetch
|
|
4201
|
-
exports.fromFireworks = fromFireworks
|
|
4202
|
-
exports.fromFriendli = fromFriendli
|
|
4203
|
-
exports.fromGemini = fromGemini
|
|
4204
|
-
exports.fromGroq = fromGroq
|
|
4205
|
-
exports.fromHyperbolic = fromHyperbolic
|
|
4206
|
-
exports.fromInferenceNet = fromInferenceNet
|
|
4207
|
-
exports.fromInfermatic = fromInfermatic
|
|
4208
|
-
exports.fromLMStudio = fromLMStudio
|
|
4209
|
-
exports.fromLambdaLabs = fromLambdaLabs
|
|
4210
|
-
exports.fromLepton = fromLepton
|
|
4211
|
-
exports.fromMiniMax = fromMiniMax
|
|
4212
|
-
exports.fromMistral = fromMistral
|
|
4213
|
-
exports.fromMoonshot = fromMoonshot
|
|
4214
|
-
exports.fromNebius = fromNebius
|
|
4215
|
-
exports.fromNovita = fromNovita
|
|
4216
|
-
exports.fromNvidiaNIM = fromNvidiaNIM
|
|
4217
|
-
exports.fromOllama = fromOllama
|
|
4218
|
-
exports.fromOpenAI = fromOpenAI
|
|
4219
|
-
exports.fromOpenAICompatible = fromOpenAICompatible
|
|
4220
|
-
exports.fromOpenRouter = fromOpenRouter
|
|
4221
|
-
exports.fromParasail = fromParasail
|
|
4222
|
-
exports.fromPerplexity = fromPerplexity
|
|
4223
|
-
exports.fromSambaNova = fromSambaNova
|
|
4224
|
-
exports.fromSiliconFlow = fromSiliconFlow
|
|
4225
|
-
exports.fromSnowflakeCortex = fromSnowflakeCortex
|
|
4226
|
-
exports.fromStepFun = fromStepFun
|
|
4227
|
-
exports.fromTogether = fromTogether
|
|
4228
|
-
exports.fromVLLM = fromVLLM
|
|
4229
|
-
exports.fromVercelAIGateway = fromVercelAIGateway
|
|
4230
|
-
exports.fromXAI = fromXAI
|
|
4231
|
-
exports.fromZhipu = fromZhipu
|
|
4232
|
-
exports.hasIssues = hasIssues
|
|
4233
|
-
exports.isFallbackExhaustedError = isFallbackExhaustedError
|
|
4234
|
-
exports.isLLMError = isLLMError
|
|
4235
|
-
exports.isToolCallResult = isToolCallResult
|
|
4236
|
-
exports.parseSseStream = parseSseStream
|
|
12
|
+
`)}let i=n.trim();if(i){let e=K(i);e!==U&&e!==W&&(yield e)}}const U=Symbol(`sse-stream-done`),W=Symbol(`sse-frame-no-data`),G=Symbol(`sse-frame-ping`);function K(e){let t=[],n=!1;for(let r of e.split(`
|
|
13
|
+
`)){if(r.startsWith(`:`)){n=!0;continue}if(!r.startsWith(`data:`))continue;t.push(r.startsWith(`data: `)?r.slice(6):r.slice(5))}if(!t.length)return n?G:W;let r=t.join(`
|
|
14
|
+
`);if(r===`[DONE]`)return U;try{return JSON.parse(r)}catch(e){throw new b(`Invalid JSON in SSE frame: ${r.slice(0,200)}`,`parse`,{cause:e,code:`stream_frame_invalid`})}}const Ue=[`image/png`,`image/jpeg`,`image/gif`,`image/webp`];function q(e){if(Ue.includes(e))return e;throw new b(`Unsupported image mimeType "${e}": expected one of ${Ue.join(`, `)}`,`invalid_params`)}function J(e,t){return t?Array.isArray(t)?t.includes(e):t(e):!1}const Y={minimal:1024,low:4096,medium:16e3,high:32e3};function X(e){if(!e)return Y;let t={...Y,...e};if(!(t.minimal<t.low&&t.low<t.medium&&t.medium<t.high))throw new b(`reasoningEffortTokens must keep tiers in strictly ascending order (minimal < low < medium < high), got ${JSON.stringify(t)}. An out-of-order override doesn't just misrank tiers, it can make some of them unreachable.`,`invalid_params`);return t}function Z(e,t=Y){return t[e]}function Q(e,t=Y){return e<=t.minimal?`minimal`:e<=t.low?`low`:e<=t.medium?`medium`:`high`}function We(e){let t=/opus-(\d+)(?:-(\d+))?/.exec(e);if(!t)return null;let n=t[2],r=n===void 0||n.length>=8?0:Number(n);return[Number(t[1]),r]}function Ge(e){let t=We(e);if(t){let[e,n]=t;return e>4||e===4&&n>=7}return[`sonnet-5`,`fable-5`,`mythos`].some(t=>e.includes(t))}function Ke(e,t){return Ge(e)?!0:t?Array.isArray(t)?t.includes(e):t(e):!1}function qe(e,t){return!Ke(e,t)}function Je(e,t){if(e<1024)throw new b(`budgetTokens (${e}) is below Anthropic's minimum of 1024. Raise budgetTokens, or use a reasoningEffort tier of 'low' or above with the default conversion table.`,`invalid_params`);if(e>=t)throw new b(`budgetTokens (${e}) must be less than maxTokens (${t}); the thinking budget and the reply share the same max_tokens ceiling on Anthropic. Raise maxTokens, or lower budgetTokens/reasoningEffort.`,`invalid_params`)}function Ye(e){if(e)throw new b(`budgetTokens/reasoningEffort was set alongside ${e}. Anthropic rejects thinking combined with a tool_choice that forces tool use, the model has to be able to reply with plain text for thinking to run. Use toolChoice: 'auto' (or omit toolChoice) for this call, or drop budgetTokens/reasoningEffort for it.`,`invalid_params`)}function Xe(e){return e===`minimal`?`low`:e}function Ze(e,t){return $e(t,e.toUpperCase())}function Qe(e){let t=/gemini-\d+\.(\d+)/.exec(e);return t?Number(t[1]):0}function $e(e,t){if(!e.includes(`pro`))return t;let n=et(e);if(n===null||n<3)return t;let r=Qe(e);return r===0?t===`HIGH`?`HIGH`:`LOW`:t===`MINIMAL`?`LOW`:t}function et(e){let t=/gemini-(\d+)/.exec(e);return t?Number(t[1]):null}function tt(e){let t=et(e);return t!==null&&t>=3}function nt(e,t){return tt(e)?!0:t?Array.isArray(t)?t.includes(e):t(e):!1}function rt(e){return e.map(e=>e.type===`image`?{type:`image`,source:{type:`base64`,media_type:q(e.mimeType),data:e.data}}:{type:`text`,text:e.text})}function it(e,t){if(e.type!==`object`)throw new b(`Tool "${t}"'s schema must have "type": "object" (Anthropic requires object-shaped tool parameters).`,`validation`);return e}function at(e){return!e||e===`auto`?{type:`auto`}:e===`none`?{type:`none`}:e===`required`?{type:`any`}:{type:`tool`,name:e.function.name}}function ot(e,t){return{tools:e.map(e=>({name:e.function.name,description:e.function.description,input_schema:it(e.function.parameters,e.function.name)})),toolChoice:at(t)}}function st(e,t,n,r){let i=e.messages.find(e=>e.role===`system`),a=e.messages.filter(e=>e.role===`user`||e.role===`assistant`||e.role===`tool`),o=e.response_format?.type===`json_schema`?e.response_format.json_schema:void 0,s=o?.name.trim();if(o&&!s)throw new b(`json_schema.name must not be empty.`,`validation`);let c=!!o&&J(e.model,t);if(o&&e.tools?.length&&!c)throw new b(`Anthropic model "${e.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there, which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromAnthropic's \`nativeStructuredOutputModels\` option once you've confirmed it supports Anthropic's \`output_config.format\`.`,`validation`);if(e.response_format?.type===`json_object`)throw new b('response_format: "json_object" is not supported on Anthropic. Unlike OpenAI, Anthropic has no API-level field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a "respond with JSON only" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real API-level constraint (Anthropic\'s native output_config.format on covered models, or a forced single tool call otherwise).',`validation`);let l,u,d,f;if(o&&c)u={type:`json_schema`,schema:o.schema},e.tools?.length&&({tools:d,toolChoice:f}=ot(e.tools,e.tool_choice));else if(o&&s){let{schema:e,description:t,strict:n}=o;l=s,d=[{name:l,description:t,input_schema:it(e,l),strict:n}],f={type:`tool`,name:l}}!o&&e.tools?.length&&({tools:d,toolChoice:f}=ot(e.tools,e.tool_choice));let p,m;if(e.budget_tokens!==void 0||e.reasoning_effort!==void 0)if(Ye(f?.type===`tool`?`toolChoice forcing the "${f.name}" tool`:f?.type===`any`?`toolChoice: 'required' (Anthropic's "any" tool_choice)`:void 0),qe(e.model,r)){let t=e.budget_tokens??Z(e.reasoning_effort,n);Je(t,e.max_tokens),p={type:`enabled`,budget_tokens:t}}else{let t=e.reasoning_effort??Q(e.budget_tokens,n);p={type:`adaptive`},m=Xe(t)}let h=i?.content,g=p?void 0:e.temperature,_={model:e.model,max_tokens:e.max_tokens,...g===void 0?{}:{temperature:g},system:h||void 0,messages:lt(a.map(e=>ut(e))),...d?{tools:d,tool_choice:f}:{},...u||m?{output_config:{...u?{format:u}:{},...m?{effort:m}:{}}}:{},...p?{thinking:p}:{}};return{body:_,toolName:l}}function ct(e,t){let n=t?.nativeStructuredOutputModels,r=X(t?.reasoningEffortTokens),i=t?.adaptiveOnlyModels,a=e.messages.create.bind(e.messages);return{supportsJsonObjectMode:!1,chat:{completions:{async create(t,a){let{body:o,toolName:s}=st(t,n,r,i),c=await e.messages.create(o,a),l,u;if(s){let e=c.content.find(e=>e.type===`tool_use`&&e.name===s);if(!e)throw new b(`Anthropic did not return the required structured output tool "${s}".`,`validation`);if(!e.input||typeof e.input!=`object`||Array.isArray(e.input))throw new b(`Anthropic returned invalid structured output for tool "${s}". Expected an object.`,`validation`);l=JSON.stringify(e.input)}else{l=c.content.filter(e=>e.type===`text`).map(e=>e.text??``).join(``);let e=c.content.filter(e=>e.type===`tool_use`);e.length&&(u=e.map(e=>({id:e.id,type:`function`,function:{name:e.name,arguments:JSON.stringify(e.input??{})}})))}return{choices:[{message:{content:l,...u?{tool_calls:u}:{}}}],usage:{prompt_tokens:c.usage?.input_tokens,completion_tokens:c.usage?.output_tokens,total_tokens:(c.usage?.input_tokens??0)+(c.usage?.output_tokens??0),...c.usage?.output_tokens_details?.thinking_tokens===void 0?{}:{completion_tokens_details:{reasoning_tokens:c.usage.output_tokens_details.thinking_tokens}}}}},async*createStream(e,t){let{body:o,toolName:s}=st(e,n,r,i),c=await a({...o,stream:!0},t),l=new Map,u=0,d=!1;for await(let e of c)if(e.type===`message_start`)u=e.message.usage?.input_tokens??0;else if(e.type===`content_block_start`)if(e.content_block.type===`tool_use`){let t=e.content_block.name===s?`json-tool`:`tool_use`;l.set(e.index,t),t===`json-tool`?d=!0:s||(yield{type:`tool_call_delta`,index:e.index,id:e.content_block.id,name:e.content_block.name})}else l.set(e.index,`text`);else if(e.type===`content_block_delta`){if(e.delta.type===`text_delta`)s||(yield{type:`text-delta`,delta:e.delta.text});else if(e.delta.type===`input_json_delta`){let t=l.get(e.index);t===`json-tool`?yield{type:`text-delta`,delta:e.delta.partial_json}:s||(yield{type:`tool_call_delta`,index:e.index,argumentsDelta:e.delta.partial_json})}}else if(e.type===`message_delta`){let t=e.usage?.output_tokens??0,n=e.usage?.output_tokens_details?.thinking_tokens;yield{type:`usage`,usage:{prompt_tokens:u,completion_tokens:t,total_tokens:u+t,...n===void 0?{}:{completion_tokens_details:{reasoning_tokens:n}}}}}else e.type===`ping`&&(yield{type:`ping`});if(s&&!d)throw new b(`Anthropic did not return the required structured output tool "${s}".`,`validation`)}}}}}function lt(e){let t=e=>e.role===`user`&&Array.isArray(e.content)&&e.content.length>0&&e.content.every(e=>e.type===`tool_result`),n=[];for(let r of e){let e=n.at(-1);t(r)&&e&&t(e)?e.content.push(...r.content):n.push(r)}return n}function ut(e){if(e.role===`tool`)return{role:`user`,content:[{type:`tool_result`,tool_use_id:e.tool_call_id,content:e.content,...e.is_error?{is_error:!0}:{}}]};if(e.role===`assistant`&&e.tool_calls?.length){let t=[];e.content&&t.push({type:`text`,text:e.content});for(let n of e.tool_calls){let e;try{e=n.function.arguments.trim()?JSON.parse(n.function.arguments):{}}catch(e){throw new b(`Assistant tool call "${n.function.name}" (${n.id}) has arguments that are not valid JSON.`,`validation`,{cause:e})}if(e===null||Array.isArray(e)||typeof e!=`object`)throw new b(`Assistant tool call "${n.function.name}" (${n.id}) arguments must be a JSON object.`,`validation`);t.push({type:`tool_use`,id:n.id,name:n.function.name,input:e})}return{role:`assistant`,content:t}}return{role:e.role,content:Array.isArray(e.content)?rt(e.content):e.content??``}}function dt(e){return e.map(e=>e.type===`image`?{inlineData:{mimeType:q(e.mimeType),data:e.data}}:{text:e.text})}function ft(e){return!e||e===`auto`?{functionCallingConfig:{mode:`AUTO`}}:e===`none`?{functionCallingConfig:{mode:`NONE`}}:e===`required`?{functionCallingConfig:{mode:`ANY`}}:{functionCallingConfig:{mode:`ANY`,allowedFunctionNames:[e.function.name]}}}function pt(e){if(e.role===`tool`)return{role:`user`,parts:[{functionResponse:{name:e.tool_call_id,response:ht(e.content)}}]};if(e.role===`assistant`&&e.tool_calls?.length){let t=[];return typeof e.content==`string`&&e.content&&t.push({text:e.content}),t.push(...e.tool_calls.map(e=>({functionCall:{name:e.function.name,args:mt(e.function.arguments,e.function.name)}}))),{role:`model`,parts:t}}return{role:e.role===`assistant`?`model`:`user`,parts:Array.isArray(e.content)?dt(e.content):[{text:e.content??``}]}}function mt(e,t){let n;try{n=e.trim()?JSON.parse(e):{}}catch(e){throw new b(`Tool call "${t}" arguments are not valid JSON.`,`parse`,{cause:e,code:`tool_arguments_parse_failed`})}if(!n||Array.isArray(n)||typeof n!=`object`)throw new b(`Tool call "${t}" arguments must be a JSON object.`,`validation`);return n}function ht(e){let t;try{t=e.trim()?JSON.parse(e):``}catch{t=e}return t&&!Array.isArray(t)&&typeof t==`object`?t:{output:t}}function gt(e){let t=e=>e.role===`user`&&e.parts.length>0&&e.parts.every(e=>`functionResponse`in e),n=[];for(let r of e){let e=n.at(-1);t(r)&&e&&t(e)?e.parts.push(...r.parts):n.push(r)}return n}function _t(e,t,n){let r=e.messages.find(e=>e.role===`system`),i=e.messages.filter(e=>e.role===`user`||e.role===`assistant`||e.role===`tool`),a=!!e.response_format,o={...e.temperature===void 0?{}:{temperature:e.temperature},maxOutputTokens:e.max_tokens,...r?{systemInstruction:{parts:[{text:r.content}]}}:{}};if(a&&(o.responseMimeType=`application/json`),e.response_format?.type===`json_schema`){let{schema:t,description:n}=e.response_format.json_schema;o.responseSchema={...t,...n?{description:n}:{}}}if(e.tools?.length&&(o.tools=[{functionDeclarations:e.tools.map(e=>({name:e.function.name,description:e.function.description,parameters:e.function.parameters}))}],o.toolConfig=ft(e.tool_choice)),nt(e.model,n)){let n=e.reasoning_effort??(e.budget_tokens===void 0?void 0:Q(e.budget_tokens,t));n!==void 0&&(o.thinkingConfig={thinkingLevel:Ze(n,e.model)})}else{let n=e.budget_tokens??(e.reasoning_effort?Z(e.reasoning_effort,t):void 0);n!==void 0&&(o.thinkingConfig={thinkingBudget:n})}return{model:e.model,contents:gt(i.map(e=>pt(e))),config:o}}function vt(e,t){let n=X(t?.reasoningEffortTokens),r=t?.thinkingLevelModels,i=e.models??e;if(typeof i.generateContent!=`function`)throw new b(`fromGemini requires a client with generateContent: pass ai.models, or the whole ai client (fromGemini(ai)).`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`generateContent`}});let a=i.generateContent.bind(i),o=typeof i.generateContentStream==`function`?i.generateContentStream.bind(i):void 0;return{chat:{completions:{async create(e,t){let i=_t(e,n,r);i.config={...i.config,abortSignal:t.signal};let o=await a(i),s=o.candidates?.[0]?.content?.parts??[],c=s.map(e=>e.text??``).join(``),l=s.filter(e=>e.functionCall),u;return l.length&&(u=l.map(e=>({id:e.functionCall.name,type:`function`,function:{name:e.functionCall.name,arguments:JSON.stringify(e.functionCall.args??{})}}))),{choices:[{message:{content:c,...u?{tool_calls:u}:{}}}],usage:{prompt_tokens:o.usageMetadata?.promptTokenCount,completion_tokens:o.usageMetadata?.candidatesTokenCount,total_tokens:o.usageMetadata?.totalTokenCount,...o.usageMetadata?.thoughtsTokenCount===void 0?{}:{completion_tokens_details:{reasoning_tokens:o.usageMetadata.thoughtsTokenCount}}}}},async*createStream(e,t){if(!o)throw new b(`stream: true requires a Gemini client with generateContentStream`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`generateContentStream`}});let i=_t(e,n,r);i.config={...i.config,abortSignal:t.signal};let a=await o(i),s=0,c;for await(let e of a){let t=e.candidates?.[0]?.content?.parts??[];for(let e of t)e.text&&(yield{type:`text-delta`,delta:e.text}),e.functionCall&&(yield{type:`tool_call_delta`,index:s,id:e.functionCall.name,name:e.functionCall.name,argumentsDelta:JSON.stringify(e.functionCall.args??{}),complete:!0},s++);e.usageMetadata&&(c=e.usageMetadata)}c&&(yield{type:`usage`,usage:{prompt_tokens:c.promptTokenCount,completion_tokens:c.candidatesTokenCount,total_tokens:c.totalTokenCount,...c.thoughtsTokenCount===void 0?{}:{completion_tokens_details:{reasoning_tokens:c.thoughtsTokenCount}}}})}}}}}function yt(e){return e.includes(`claude`)}function bt(e){switch(q(e)){case`image/png`:return`png`;case`image/jpeg`:return`jpeg`;case`image/gif`:return`gif`;case`image/webp`:return`webp`}}function xt(e){return new Uint8Array(Buffer.from(e,`base64`))}function St(e){return e.map(e=>e.type===`image`?{image:{format:bt(e.mimeType),source:{bytes:xt(e.data)}}}:{text:e.text})}function Ct(e,t){return{tools:e.map(e=>({toolSpec:{name:e.function.name,description:e.function.description,inputSchema:{json:e.function.parameters}}})),toolChoice:At(t)}}function wt(e,t,n,r,i){let a=e.messages.find(e=>e.role===`system`),o=e.messages.filter(e=>e.role===`user`||e.role===`assistant`||e.role===`tool`),s=e.response_format?.type===`json_schema`?e.response_format.json_schema:void 0,c=s?.name.trim();if(s&&!c)throw new b(`json_schema.name must not be empty.`,`validation`);let l=!!s&&J(e.model,n);if(s&&e.tools?.length&&!l)throw new b(`Bedrock model "${e.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there (via \`toolConfig\`), which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromBedrock's \`nativeStructuredOutputModels\` option once you've confirmed it supports Converse's \`outputConfig.textFormat\`.`,`validation`);if(e.response_format?.type===`json_object`)throw new b('response_format: "json_object" is not supported on Bedrock. Converse has no field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a "respond with JSON only" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real constraint (Converse\'s native outputConfig.textFormat on covered models, or a forced tool call otherwise).',`validation`);let u,d,f;if(s&&l){let{schema:e,description:t}=s;f={textFormat:{type:`json_schema`,structure:{jsonSchema:{schema:JSON.stringify(e),name:c,description:t}}}}}else if(s&&c){let{schema:e,description:t,strict:n}=s;u=c,d={tools:[{toolSpec:{name:u,description:t,inputSchema:{json:e},strict:n}}],toolChoice:{tool:{name:u}}}}if(e.tools?.length&&!u&&(d=Ct(e.tools,e.tool_choice)),s&&d&&t){let n=Array.isArray(t)?t.includes(e.model):t(e.model);if(!n)throw new b(`Bedrock model "${e.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`toolUseSupportedModels`}})}let p,m;if(yt(e.model)&&(e.budget_tokens!==void 0||e.reasoning_effort!==void 0)){let t=d?.toolChoice;if(Ye(t&&`tool`in t?`toolChoice forcing the "${t.tool?.name}" tool`:t&&`any`in t?`toolChoice: 'required' (Converse's "any" tool_choice)`:void 0),qe(e.model,i)){let t=e.budget_tokens??Z(e.reasoning_effort,r);Je(t,e.max_tokens),p={thinking:{type:`enabled`,budget_tokens:t}}}else{let t=e.reasoning_effort??Q(e.budget_tokens,r);p={thinking:{type:`adaptive`}},m=Xe(t)}}let h=p?void 0:e.temperature,g={modelId:e.model,messages:Mt(o.map(e=>jt(e))),system:a?.content?[{text:a.content}]:void 0,inferenceConfig:{...h===void 0?{}:{temperature:h},maxTokens:e.max_tokens},...d?{toolConfig:d}:{},...f||m?{outputConfig:{...f??{},...m?{effort:m}:{}}}:{},...p?{additionalModelRequestFields:p}:{}};return{request:g,toolName:u}}function Tt(e){return typeof e.send==`function`}function Et(e){if(`messageStart`in e)return{messageStart:e.messageStart};if(`contentBlockStart`in e)return{contentBlockStart:e.contentBlockStart};if(`contentBlockDelta`in e)return{contentBlockDelta:e.contentBlockDelta};if(`contentBlockStop`in e)return{contentBlockStop:e.contentBlockStop};if(`messageStop`in e)return{messageStop:e.messageStop};if(`metadata`in e)return{metadata:e.metadata};if(`internalServerException`in e)return{internalServerException:e.internalServerException};if(`modelStreamErrorException`in e)return{modelStreamErrorException:e.modelStreamErrorException};if(`validationException`in e)return{validationException:e.validationException};if(`throttlingException`in e)return{throttlingException:e.throttlingException};if(`serviceUnavailableException`in e)return{serviceUnavailableException:e.serviceUnavailableException}}async function*Dt(e){for await(let t of e){let e=Et(t);e&&(yield e)}}function Ot(e){let t;function n(){return t??=import(`@aws-sdk/client-bedrock-runtime`).then(e=>e,e=>{throw t=void 0,new b(`fromBedrock requires "@aws-sdk/client-bedrock-runtime" to be installed to use a raw AWS SDK client (it is not a dependency of vern-llm itself). Install it, or pass your own object with .converse()/.converseStream() methods instead.`,`validation`,{cause:e})}),t}return{converse:async(t,r)=>{let{ConverseCommand:i}=await n();return e.send(new i(t),{abortSignal:r.signal})},converseStream:async(t,r)=>{let{ConverseStreamCommand:i}=await n(),a=await e.send(new i(t),{abortSignal:r.signal});if(!a.stream)throw new b(`Bedrock ConverseStreamCommand response did not include a stream. This can happen if the request or the model doesn't actually support Converse streaming.`,`api`,{code:`server_error`});return{stream:Dt(a.stream)}}}}function kt(e,t){let n=Tt(e)?Ot(e):e,r=t?.toolUseSupportedModels,i=t?.nativeStructuredOutputModels,a=X(t?.reasoningEffortTokens),o=t?.adaptiveOnlyModels;return{supportsJsonObjectMode:!1,chat:{completions:{async create(e,t){let{request:s,toolName:c}=wt(e,r,i,a,o),l=await n.converse(s,t),u,d;if(c){let e=l.output?.message?.content?.find(e=>e.toolUse?.name===c);u=e?.toolUse?JSON.stringify(e.toolUse.input):``}else{let e=l.output?.message?.content??[];u=e.map(e=>e.text??``).join(``);let t=e.filter(e=>!!e.toolUse);t.length&&(d=t.map((e,t)=>{let n=e.toolUse;if(!n.name)throw new b(`Bedrock returned a toolUse block without a name at index ${t}.`,`validation`);return{id:n.toolUseId??`${n.name}_${t}`,type:`function`,function:{name:n.name,arguments:JSON.stringify(n.input??{})}}}))}return{choices:[{message:{content:u,...d?{tool_calls:d}:{}}}],usage:{prompt_tokens:l.usage?.inputTokens,completion_tokens:l.usage?.outputTokens,total_tokens:l.usage?.totalTokens}}},async*createStream(e,t){if(!n.converseStream)throw new b(`stream: true requires a Bedrock client with converseStream`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`converseStream`}});let{request:s,toolName:c}=wt(e,r,i,a,o),{stream:l}=await n.converseStream(s,t),u=new Map;for await(let e of l)if(`contentBlockStart`in e){let{contentBlockIndex:t,start:n}=e.contentBlockStart;if(n?.toolUse){let e=n.toolUse.name===c?`json-tool`:`tool_use`;u.set(t,e),e===`tool_use`&&!c&&(yield{type:`tool_call_delta`,index:t,id:n.toolUse.toolUseId,name:n.toolUse.name})}else u.set(t,`text`)}else if(`contentBlockDelta`in e){let{contentBlockIndex:t,delta:n}=e.contentBlockDelta;if(n&&`text`in n&&n.text!==void 0&&!c)yield{type:`text-delta`,delta:n.text};else if(n&&`toolUse`in n&&n.toolUse?.input!==void 0){let e=u.get(t);e===`json-tool`?yield{type:`text-delta`,delta:n.toolUse.input}:c||(yield{type:`tool_call_delta`,index:t,argumentsDelta:n.toolUse.input})}}else if(`metadata`in e&&e.metadata.usage)yield{type:`usage`,usage:{prompt_tokens:e.metadata.usage.inputTokens,completion_tokens:e.metadata.usage.outputTokens,total_tokens:e.metadata.usage.totalTokens}};else if(`throttlingException`in e)throw new b(e.throttlingException.message??`Bedrock throttled the request mid-stream`,`api`,{status:429,code:`provider_rate_limited`});else if(`validationException`in e)throw new b(e.validationException.message??`Bedrock rejected the request mid-stream`,`validation`);else if(`internalServerException`in e||`serviceUnavailableException`in e||`modelStreamErrorException`in e){let t=`internalServerException`in e&&e.internalServerException.message||`serviceUnavailableException`in e&&e.serviceUnavailableException.message||`modelStreamErrorException`in e&&e.modelStreamErrorException.message||`Bedrock reported a mid-stream error`,n=`modelStreamErrorException`in e&&e.modelStreamErrorException.originalStatusCode||`serviceUnavailableException`in e&&503||500;throw new b(t,`api`,{status:n,code:n>=500?`server_error`:void 0})}}}}}}function At(e){if(!e||e===`auto`)return{auto:{}};if(e===`required`)return{any:{}};if(e===`none`)throw new b("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.",`invalid_params`,{code:`unsupported_capability`,issues:{capability:`toolChoice: 'none'`}});return{tool:{name:e.function.name}}}function jt(e){if(e.role===`tool`)return{role:`user`,content:[{toolResult:{toolUseId:e.tool_call_id,content:[{text:e.content}],status:e.is_error?`error`:`success`}}]};if(e.role===`assistant`&&e.tool_calls?.length){let t=[];e.content&&t.push({text:e.content});for(let n of e.tool_calls){let e;if(!n.function.arguments.trim())e={};else try{e=JSON.parse(n.function.arguments)}catch(e){throw new b(`Assistant tool call "${n.function.name}" (${n.id}) has arguments that are not valid JSON.`,`validation`,{cause:e})}t.push({toolUse:{toolUseId:n.id,name:n.function.name,input:e}})}return{role:`assistant`,content:t}}return{role:e.role,content:Array.isArray(e.content)?St(e.content):[{text:e.content??``}]}}function Mt(e){let t=e=>e.role===`user`&&e.content.length>0&&e.content.every(e=>`toolResult`in e),n=[];for(let r of e){let e=n.at(-1);t(r)&&e&&t(e)?e.content.push(...r.content):n.push(r)}return n}async function*Nt(e){let t=e.getReader();try{for(;;){let{done:e,value:n}=await t.read();if(e)return;n&&(yield n)}}finally{try{await t.cancel()}catch{}t.releaseLock()}}async function Pt(e,t){let n=await fetch(e,t);if(!n.ok){let e=await n.text().catch(()=>``),t=Error(`Fetch adapter stream request failed (${n.status}): ${e.slice(0,500)}`);throw t.status=n.status,t.headers=n.headers,t}if(!n.body)throw Error(`Fetch adapter stream request received a response with no body.`);return Nt(n.body)}async function Ft(e,t,n){let r=typeof e.url==`function`?e.url(t):e.url,i=typeof e.headers==`function`?await e.headers():e.headers,a=e.method??`POST`,o=![`GET`,`HEAD`].includes(a.toUpperCase());return{url:r,method:a,headers:o?{"Content-Type":`application/json`,...i}:{...i},...o?{body:JSON.stringify(n)}:{}}}function It(e){return{chat:{completions:{async create(t,n){let{url:r,method:i,headers:a,body:o}=await Ft(e,t,e.mapRequest(t)),s=e.request??fetch,c=await s(r,{method:i,headers:a,body:o,signal:n.signal});if(!c.ok){let e=await c.text().catch(()=>``),t=Error(`Fetch adapter request failed (${c.status}): ${e.slice(0,500)}`);throw t.status=c.status,t.headers=c.headers,t}let l=await c.json(),{content:u,usage:d,toolCalls:f}=e.mapResponse(l),p=f?.length?f.map(e=>({id:e.id,type:`function`,function:{name:e.name,arguments:e.arguments}})):void 0;return{choices:[{message:{content:u,...p?{tool_calls:p}:{}}}],usage:d?{prompt_tokens:d.promptTokens,completion_tokens:d.completionTokens,total_tokens:d.totalTokens}:void 0}},async*createStream(t,n){if(!e.mapStreamEvent)throw new b(`stream: true requires mapStreamEvent to be configured on fromFetch`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`mapStreamEvent`}});if(e.request&&!e.requestStream)throw new b("`stream: true` requires `requestStream` to be configured on fromFetch when a custom `request` transport is set. `requestStream` does not fall back to `request` (it needs an async-iterable byte stream, which `RequestLike`'s buffered `ResponseLike` has no way to provide), without it, `stream: true` would silently use plain native `fetch` instead of your configured transport. Add a `requestStream` that opens the same connection your `request` does, or omit `request` if native `fetch` is fine for both.",`invalid_params`,{code:`unsupported_capability`,issues:{capability:`requestStream`}});let{url:r,method:i,headers:a,body:o}=await Ft(e,t,e.mapRequest(t)),s=e.requestStream??Pt,c=e.parseStreamFrames??He,l=await s(r,{method:i,headers:a,body:o,signal:n.signal});for await(let t of c(l)){if(t===G){yield{type:`ping`};continue}let n=e.mapStreamEvent(t);if(!n)continue;Array.isArray(n)?yield*n:yield n}}}}}}function Lt(e,t){if(e.budget_tokens===void 0)return e;let{budget_tokens:n,...r}=e;return r.reasoning_effort===void 0?{...r,reasoning_effort:Q(n,t)}:r}function Rt(e){return e.map(e=>e.type===`image`?{type:`image_url`,image_url:{url:`data:${q(e.mimeType)};base64,${e.data}`}}:{type:`text`,text:e.text})}function zt(e){return e.messages.map(e=>{if(e.role===`user`&&Array.isArray(e.content))return{...e,content:Rt(e.content)};if(e.role===`tool`){let{is_error:t,...n}=e;return n}return e})}function*Bt(e){let t=e.choices?.[0]?.delta;if(t?.content&&(yield{type:`text-delta`,delta:t.content}),t?.tool_calls?.length)for(let e of t.tool_calls)yield{type:`tool_call_delta`,index:e.index,id:e.id,name:e.function?.name,argumentsDelta:e.function?.arguments};e.usage&&(yield{type:`usage`,usage:e.usage})}function $(e,t={}){let n=e,{supportsStreamUsage:r=!0}=t,i=X(t.reasoningEffortTokens),a=n.chat.completions.create.bind(n.chat.completions);return{chat:{completions:{async create(e,t){let r=zt(e);return n.chat.completions.create(Lt({...e,messages:r},i),t)},async*createStream(e,t){let n=zt(e),o=await a(Lt({...e,messages:n,stream:!0,...r?{stream_options:{include_usage:!0}}:{}},i),t);for await(let e of o)yield*Bt(e)}}}}}const Vt=$,Ht=$,Ut=$,Wt=$,Gt=$,Kt=$,qt=$,Jt=$,Yt=$,Xt=$,Zt=$,Qt=$,$t=$,en=$,tn=$,nn=$,rn=$,an=$,on=$,sn=$,cn=$,ln=$,un=$,dn=$,fn=$,pn=$,mn=$,hn=$,gn=$,_n=$,vn=$,yn=$,bn=$,xn=$,Sn=$,Cn=$,wn=$,Tn=$;exports.CircuitBreaker=fe,exports.ConsoleLogger=Fe,exports.FallbackExhaustedError=E,exports.InMemoryCacheAdapter=S,exports.LLMError=b,exports.NormalizedCacheAdapter=C,exports.RateLimiter=Re,exports.SSE_PING=G,exports.TieredCacheAdapter=w,exports.VernLLM=ze,exports.defaultEstimateTokens=Ie,exports.defaultFallbackOn=ne,exports.defineCachedCallParams=Ve,exports.defineCallParams=Be,exports.from01AI=Tn,exports.fromAnthropic=ct,exports.fromAnyscale=bn,exports.fromAtlasCloud=wn,exports.fromBaseten=dn,exports.fromBedrock=kt,exports.fromCerebras=Gt,exports.fromCloudflareWorkersAI=cn,exports.fromDeepInfra=Zt,exports.fromDeepSeek=Wt,exports.fromFeatherless=fn,exports.fromFetch=It,exports.fromFireworks=qt,exports.fromFriendli=pn,exports.fromGemini=vt,exports.fromGroq=Ht,exports.fromHyperbolic=$t,exports.fromInferenceNet=Sn,exports.fromInfermatic=Cn,exports.fromLMStudio=nn,exports.fromLambdaLabs=vn,exports.fromLepton=xn,exports.fromMiniMax=_n,exports.fromMistral=Ut,exports.fromMoonshot=en,exports.fromNebius=ln,exports.fromNovita=Qt,exports.fromNvidiaNIM=on,exports.fromOllama=Jt,exports.fromOpenAI=Vt,exports.fromOpenAICompatible=$,exports.fromOpenRouter=Yt,exports.fromParasail=hn,exports.fromPerplexity=Xt,exports.fromSambaNova=un,exports.fromSiliconFlow=mn,exports.fromSnowflakeCortex=yn,exports.fromStepFun=gn,exports.fromTogether=Kt,exports.fromVLLM=rn,exports.fromVercelAIGateway=sn,exports.fromXAI=an,exports.fromZhipu=tn,exports.hasIssues=ee,exports.isFallbackExhaustedError=re,exports.isLLMError=x,exports.isToolCallResult=T,exports.parseSseStream=He;
|
|
4237
15
|
//# sourceMappingURL=index.cjs.map
|