vern-llm 1.3.0 → 1.4.0

package/dist/index.js

@@ -28,26 +28,43 @@ var CircuitBreaker = class {
 	openedAt = 0;
 	threshold;
 	cooldownMs;
+	/**
+	* True while a single half-open trial call is in flight. Guards against
+	* multiple concurrent callers all treating themselves as "the" trial once
+	* the cooldown elapses
+	*/
+	trialInFlight = false;
 	constructor(options = {}) {
 		this.threshold = options.threshold ?? 5;
 		this.cooldownMs = options.cooldownMs ?? 3e4;
 	}
-	/** Throws if the circuit is open and the cooldown hasnt elapsed */
+	/**
+	* Throws if the circuit is open and the cooldown hasn't elapsed, or if
+	* the circuit is half-open and a trial call is already in flight.
+	* Otherwise, if the circuit just became eligible for a trial (cooldown
+	* elapsed, or half-open with no trial currently running), this call
+	* becomes that trial
+	*/
 	assertClosed() {
-		if (this.state !== "open") return;
-		const elapsed = Date.now() - this.openedAt;
-		if (elapsed >= this.cooldownMs) {
+		if (this.state === "closed") return;
+		if (this.state === "open") {
+			const elapsed = Date.now() - this.openedAt;
+			if (elapsed < this.cooldownMs) throw new LLMError(`Circuit open — provider has failed ${this.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open");
 			this.state = "half-open";
+			this.trialInFlight = true;
 			return;
 		}
-		throw new LLMError(`Circuit open — provider has failed ${this.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open");
+		if (this.trialInFlight) throw new LLMError("Circuit half-open. A trial request is already in flight. Try again shortly.", "circuit_open");
+		this.trialInFlight = true;
 	}
 	recordSuccess() {
 		this.consecutiveFailures = 0;
 		this.state = "closed";
+		this.trialInFlight = false;
 	}
 	recordFailure() {
 		this.consecutiveFailures += 1;
+		this.trialInFlight = false;
 		if (this.state === "half-open") {
 			this.state = "open";
 			this.openedAt = Date.now();
@@ -112,11 +129,40 @@ async function withTimeout(fn, timeoutMs, externalSignal) {
 	}
 }
 /**
+* Default cap (ms) for both exponential backoff and honored Retry-After
+* values, so a misbehaving/adversarial Retry-After can't stall a caller
+* indefinitely
+*/
+const DEFAULT_MAX_DELAY_MS = 1e4;
+/**
+* Looks inside an unknown error value for a Retry-After header and
+* converts it to milliseconds. Checks `.headers` first (fetch-style,
+* Headers-like with `.get()`), then `.response.headers` (axios-style,
+* plain object) since different client libraries surface headers
+* differently. Supports both the delta-seconds form ("30") and the
+* HTTP-date form ("Wed, 21 Oct 2015 07:28:00 GMT"). The result is capped
+* at maxDelayMs. Returns undefined when no usable Retry-After is present
+*/
+function extractRetryAfterMs(err, maxDelayMs = DEFAULT_MAX_DELAY_MS) {
+	if (!err || typeof err !== "object") return void 0;
+	const error = err;
+	const headers = error.headers ?? error.response?.headers;
+	if (!headers || typeof headers !== "object") return void 0;
+	const getter = headers;
+	const raw = typeof getter.get === "function" ? getter.get("Retry-After") : Object.entries(headers).find(([name]) => name.toLowerCase() === "retry-after")?.at(1);
+	if (typeof raw !== "string" || raw.trim() === "") return void 0;
+	const trimmed = raw.trim();
+	if (/^\d+$/.test(trimmed)) return Math.max(0, Math.min(Number(trimmed) * 1e3, maxDelayMs));
+	const dateMs = Date.parse(trimmed);
+	if (!Number.isNaN(dateMs)) return Math.max(0, Math.min(dateMs - Date.now(), maxDelayMs));
+	return void 0;
+}
+/**
 * Exponential backoff with jitter, capped at maxDelayMs.
 * Jitter avoids thundering-herd retries when many callers back off in lockstep,
 * the cap prevents unbounded delays when maxRetries is high
 */
-function getBackoffDelay(baseDelayMs, attempt, maxDelayMs = 1e4) {
+function getBackoffDelay(baseDelayMs, attempt, maxDelayMs = DEFAULT_MAX_DELAY_MS) {
 	const exp = Math.min(baseDelayMs * 2 ** attempt, maxDelayMs);
 	return exp / 2 + Math.random() * (exp / 2);
 }
@@ -234,6 +280,7 @@ var VernLLM = class {
 	defaultMaxTokens;
 	cache;
 	nonRetryableStatus;
+	inFlight = new Map();
 	parseJson;
 	onUsage;
 	logger;
@@ -327,7 +374,7 @@ var VernLLM = class {
 	async retryWithBackoff(fn, requestId, signal) {
 		let lastError;
 		for (let attempt = 0; attempt <= this.maxRetries; attempt++) try {
-			if (attempt > 0) await this.recoverDelay(requestId, attempt, signal);
+			if (attempt > 0) await this.recoverDelay(requestId, attempt, lastError, signal);
 			return await fn();
 		} catch (error) {
 			lastError = error;
@@ -475,12 +522,16 @@ var VernLLM = class {
 	}
 	/**
 	* Waits out the backoff delay for a given retry attempt, logging the
-	* attempt for observability before the wait begins. Rejects early if
-	* the signal aborts during the wait
+	* attempt for observability before the wait begins. Honors a
+	* Retry-After header on the failed attempt's error when present
+	* (capped at the same maxDelayMs as backoff), otherwise falls back to
+	* exponential backoff exactly as before. Rejects early if the signal
+	* aborts during the wait
 	*/
-	async recoverDelay(requestId, attempt, signal) {
-		const delay = getBackoffDelay(this.baseDelayMs, attempt);
-		this.logger.warn(`[vern:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${delay}ms`);
+	async recoverDelay(requestId, attempt, error, signal) {
+		const retryAfterMs = extractRetryAfterMs(error);
+		const delay = retryAfterMs ?? getBackoffDelay(this.baseDelayMs, attempt);
+		this.logger.warn(`[vern:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${delay}ms` + (retryAfterMs !== void 0 ? " (honoring Retry-After)" : ""));
 		await waitForRetry(delay, signal);
 	}
 	/**
@@ -509,26 +560,55 @@ var VernLLM = class {
 		await this.cache.delete(key);
 	}
 	/**
-	* Thin cache wrapper around caller supplied logic. `params.fn` is expected
-	* to be a call that itself invokes `this.call(...)` (see `cachedLLMCall`
-	* below for a convenience wrapper that wires this up automatically),
-	* `cachedCall` does not itself apply retry/timeout policy.
+	* Cache wrapper around caller-supplied logic. `params.fn` should invoke
+	* `this.call(...)` (see `cachedLLMCall`); retry/timeout handling is left
+	* to the caller.
+	*
+	* Concurrent misses for the same `cacheKey` share a single in-flight call,
+	* avoiding cache stampedes. Each caller still receives its own
+	* `reserveUsage`/`refundUsage` callbacks with coalescing metadata.
 	*/
 	async cachedCall(params) {
 		const cached = await this.cache.get(params.cacheKey);
 		if (cached.hit) return cached.value;
+		const existing = this.inFlight.get(params.cacheKey);
+		const coalesced = existing !== void 0;
+		const resultPromise = existing ?? this.registerTrigger(params, coalesced);
+		if (coalesced) return this.withRefundOnFailure(params, coalesced, async () => {
+			await params.reserveUsage?.({ coalesced });
+			return resultPromise;
+		});
+		return this.withRefundOnFailure(params, coalesced, () => resultPromise);
+	}
+	/** Starts the shared fn() call for a cache miss, reserving usage first, and registers it in the in-flight map until it settles */
+	registerTrigger(params, coalesced) {
+		const resultPromise = (async () => {
+			await params.reserveUsage?.({ coalesced });
+			return this.runAndCache(params);
+		})();
+		this.inFlight.set(params.cacheKey, resultPromise);
+		resultPromise.catch(() => {}).finally(() => {
+			this.inFlight.delete(params.cacheKey);
+		});
+		return resultPromise;
+	}
+	/** Runs `fn` and writes its result to the cache. Only ever called once per cacheKey per in-flight window, from registerTrigger */
+	async runAndCache(params) {
+		const result = await params.fn();
 		try {
-			await params.reserveUsage?.();
-			const result = await params.fn();
-			try {
-				await this.cache.set(params.cacheKey, result, params.ttl);
-			} catch (error) {
-				this.logger.error("[VernLLM] cache write failed", { message: error instanceof Error ? error.message : "unknown" });
-			}
-			return result;
+			await this.cache.set(params.cacheKey, result, params.ttl);
+		} catch (error) {
+			this.logger.error("[VernLLM] cache write failed", { message: error instanceof Error ? error.message : "unknown" });
+		}
+		return result;
+	}
+	/** Awaits `run`, calling this caller's own refundUsage (tagged with whether it was coalesced) if it rejects, then rethrows the original error */
+	async withRefundOnFailure(params, coalesced, run) {
+		try {
+			return await run();
 		} catch (error) {
 			try {
-				await params.refundUsage?.();
+				await params.refundUsage?.({ coalesced });
 			} catch (refundError) {
 				this.logger.error("[VernLLM] refundUsage failed", { message: refundError instanceof Error ? refundError.message : "unknown" });
 			}
@@ -556,9 +636,53 @@ var VernLLM = class {
 	}
 };
 
+//#endregion
+//#region src/internal/imageFormat.ts
+/**
+* MIME types accepted for `ImageBlock.mimeType` across all adapters. This is
+* the intersection of what Anthropic, Gemini, OpenAI-compatible, and Bedrock
+* Converse all natively support, so a `ContentBlock[]` that validates for
+* one provider validates for all of them.
+*/
+const SUPPORTED_IMAGE_MIME_TYPES = [
+	"image/png",
+	"image/jpeg",
+	"image/gif",
+	"image/webp"
+];
+/**
+* Validates an `ImageBlock.mimeType` against the shared supported set.
+* Throws a non-retryable `LLMError('validation')`, since an unsupported
+* mimeType is a permanent failure, retrying the same input can't fix it,
+* the same way a schema-validation or JSON-parse failure isn't retried.
+*/
+function assertSupportedImageMimeType(mimeType) {
+	if (SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType)) return mimeType;
+	throw new LLMError(`Unsupported image mimeType "${mimeType}": expected one of ${SUPPORTED_IMAGE_MIME_TYPES.join(", ")}`, "validation");
+}
+
 //#endregion
 //#region src/adapters/anthropic.ts
 /**
+* Translates a VernLLM `ContentBlock[]` (our provider-agnostic multimodal
+* shape) into Anthropic's native content-block array: text blocks pass
+* through as-is, image blocks become `{ type: 'image', source: { type:
+* 'base64', media_type, data } }`.
+*/
+function toAnthropicContent(blocks) {
+	return blocks.map((block) => block.type === "image" ? {
+		type: "image",
+		source: {
+			type: "base64",
+			media_type: assertSupportedImageMimeType(block.mimeType),
+			data: block.data
+		}
+	} : {
+		type: "text",
+		text: block.text
+	});
+}
+/**
 * Wraps an Anthropic SDK client so it satisfies the same `LLMClient`
 * interface VernLLM uses for OpenAI/Groq.
 *
@@ -596,7 +720,7 @@ function fromAnthropic(anthropicClient) {
 			system: system || void 0,
 			messages: conversationMessages.map((m) => ({
 				role: m.role,
-				content: m.content
+				content: Array.isArray(m.content) ? toAnthropicContent(m.content) : m.content
 			})),
 			...tools ? {
 				tools,
@@ -625,6 +749,18 @@ function fromAnthropic(anthropicClient) {
 //#endregion
 //#region src/adapters/gemini.ts
 /**
+* Translates a VernLLM `ContentBlock[]` into Gemini's native `parts` array:
+* text blocks become `{ text }`, image blocks become inline data parts
+* (`{ inlineData: { mimeType, data } }`), Geminis shape for embedding raw
+* base64 image bytes directly in the request.
+*/
+function toGeminiParts(blocks) {
+	return blocks.map((block) => block.type === "image" ? { inlineData: {
+		mimeType: assertSupportedImageMimeType(block.mimeType),
+		data: block.data
+	} } : { text: block.text });
+}
+/**
 * Wraps a Gemini client so it satisfies the `LLMClient` interface VernLLM
 * uses for OpenAI/Groq. Geminis shape differs on nearly every axis: a
 * `contents` array instead of `messages`, a separate `systemInstruction`
@@ -650,7 +786,7 @@ function fromGemini(geminiClient) {
 			model: params.model,
 			contents: conversationMessages.map((m) => ({
 				role: m.role === "assistant" ? "model" : "user",
-				parts: [{ text: m.content }]
+				parts: Array.isArray(m.content) ? toGeminiParts(m.content) : [{ text: m.content }]
 			})),
 			systemInstruction: systemMessage ? { parts: [{ text: systemMessage.content }] } : void 0,
 			generationConfig
@@ -669,6 +805,36 @@ function fromGemini(geminiClient) {
 
 //#endregion
 //#region src/adapters/bedrock.ts
+/** Maps a `ContentBlock` image MIME type, already validated, to Converse's `format` enum. */
+function toBedrockImageFormat(mimeType) {
+	switch (assertSupportedImageMimeType(mimeType)) {
+		case "image/png": return "png";
+		case "image/jpeg": return "jpeg";
+		case "image/gif": return "gif";
+		case "image/webp": return "webp";
+	}
+}
+/**
+* Decodes base64 image data into the raw `Uint8Array` bytes Converse's
+* `image.source.bytes` expects (unlike Anthropic/Gemini/OpenAI, which all
+* take base64 strings directly). Uses `Buffer`, since this adapter, like
+* the rest of the package, targets Node.
+*/
+function decodeBase64(data) {
+	return new Uint8Array(Buffer.from(data, "base64"));
+}
+/**
+* Translates a VernLLM `ContentBlock[]` into Converse's native content-block
+* array: text blocks pass through as `{ text }`, image blocks become
+* `{ image: { format, source: { bytes } } }` with the base64 payload decoded
+* to raw bytes, since Converse doesn't accept base64 strings directly.
+*/
+function toBedrockContent(blocks) {
+	return blocks.map((block) => block.type === "image" ? { image: {
+		format: toBedrockImageFormat(block.mimeType),
+		source: { bytes: decodeBase64(block.data) }
+	} } : { text: block.text });
+}
 /**
 * Wraps a Bedrock Converse-API client so it satisfies the `LLMClient`
 * interface VernLLM uses for OpenAI/Groq. The Converse API is unified
@@ -710,7 +876,7 @@ function fromBedrock(bedrockClient) {
 			modelId: params.model,
 			messages: conversationMessages.map((m) => ({
 				role: m.role,
-				content: [{ text: m.content }]
+				content: Array.isArray(m.content) ? toBedrockContent(m.content) : [{ text: m.content }]
 			})),
 			system: systemParts.length ? systemParts.map((text$1) => ({ text: text$1 })) : void 0,
 			inferenceConfig: {
@@ -752,19 +918,23 @@ function fromFetch(config) {
 	return { chat: { completions: { async create(params, options) {
 		const url = typeof config.url === "function" ? config.url(params) : config.url;
 		const headers = typeof config.headers === "function" ? await config.headers() : config.headers;
-		const res = await fetch(url, {
-			method: config.method ?? "POST",
-			headers: {
+		const method = config.method ?? "POST";
+		const request = config.request ?? fetch;
+		const supportsBody = !["GET", "HEAD"].includes(method.toUpperCase());
+		const res = await request(url, {
+			method,
+			headers: supportsBody ? {
 				"Content-Type": "application/json",
 				...headers
-			},
-			body: JSON.stringify(config.mapRequest(params)),
+			} : { ...headers },
+			...supportsBody ? { body: JSON.stringify(config.mapRequest(params)) } : {},
 			signal: options.signal
 		});
 		if (!res.ok) {
 			const body = await res.text().catch(() => "");
 			const err = new Error(`Fetch adapter request failed (${res.status}): ${body.slice(0, 500)}`);
 			err.status = res.status;
+			err.headers = res.headers;
 			throw err;
 		}
 		const json = await res.json();
@@ -783,13 +953,36 @@ function fromFetch(config) {
 //#endregion
 //#region src/adapters/openaiCompatible.ts
 /**
-* Passthrough adapter for any SDK/client whose `chat.completions.create`
-* already matches the OpenAI wire format 1:1 : this covers most hosted
-* inference providers, since "OpenAI-compatible" is a de facto standard for
-* chat completion APIs. No transformation happens here, this exists purely
-* so call sites read clearly (`fromMistral(client)` vs handing a Mistral
-* client to something typed for OpenAI) and so a real transformation could
-* be added later, per-provider, without a breaking change.
+* Translates a VernLLM `ContentBlock[]` into OpenAI's wire-level content
+* array. Text blocks become `{ type: 'text', text }`; image blocks become
+* `{ type: 'image_url', image_url: { url } }` with the base64 payload
+* inlined as a `data:` URL, since our `ContentBlock` shape (`{ type:
+* 'image', data, mimeType }`) is provider-agnostic and doesn't itself match
+* OpenAI's wire format.
+*/
+function toOpenAIContent(blocks) {
+	return blocks.map((block) => block.type === "image" ? {
+		type: "image_url",
+		image_url: { url: `data:${assertSupportedImageMimeType(block.mimeType)};base64,${block.data}` }
+	} : {
+		type: "text",
+		text: block.text
+	});
+}
+/**
+* Adapter for any SDK/client whose `chat.completions.create` already
+* matches the OpenAI wire format: this covers most hosted inference
+* providers, since "OpenAI-compatible" is a de facto standard for chat
+* completion APIs. Almost everything passes straight through untouched,
+* this exists purely so call sites read clearly (`fromMistral(client)` vs
+* handing a Mistral client to something typed for OpenAI) and so a real
+* transformation could be added later, per-provider, without a breaking
+* change.
+*
+* The one thing that isn't a pure passthrough: a `ContentBlock[]`
+* `userContent` is translated into OpenAI's native `image_url` content-part
+* shape, since VernLLM's `ContentBlock` is intentionally provider-agnostic
+* rather than a copy of any one provider's wire format.
 *
 * Not every SDKs own TypeScript types line up exactly with `LLMClient`
 * (extra fields, stricter unions, etc.), so this takes `unknown` and casts:
@@ -797,7 +990,17 @@ function fromFetch(config) {
 * receives over the wire, not the SDKs TS types.
 */
 function fromOpenAICompatible(client) {
-	return client;
+	const raw = client;
+	return { chat: { completions: { async create(params, options) {
+		const messages = params.messages.map((m) => m.role === "user" && Array.isArray(m.content) ? {
+			...m,
+			content: toOpenAIContent(m.content)
+		} : m);
+		return raw.chat.completions.create({
+			...params,
+			messages
+		}, options);
+	} } } };
 }
 /** Groqs SDK matches the OpenAI wire format */
 const fromGroq = fromOpenAICompatible;