smoltalk 0.8.4 → 0.10.0

package/dist/client.js

@@ -62,6 +62,7 @@ export function getClient(config) {
         ...config,
         apiKey: { ...resolvedKeys, ...config.apiKey },
         model: modelName,
+        provider,
     };
     switch (provider) {
         case "anthropic":

package/dist/clients/anthropic.js

@@ -204,7 +204,7 @@ export class SmolAnthropic extends BaseClient {
         }
         this.client = new Anthropic({ apiKey });
         this.logger = getLogger();
-        this.model = new Model(config.model, undefined, config.modelData);
+        this.model = new Model(config.model, config.provider, config.modelData);
     }
     getModel() {
         return this.model.getModel();

package/dist/clients/baseClient.d.ts

@@ -1,5 +1,11 @@
 import { StatelogClient } from "../statelogClient.js";
 import { PromptResult, Result, SmolClient, SmolConfig, StreamChunk } from "../types.js";
+export type ClientAttachmentCapabilities = {
+    /** Non-audio attachment modalities this client's serializers can render. */
+    inputModalities: readonly ("image" | "pdf")[];
+    /** Audio containers (by primary extension) accepted inline; empty = no audio. */
+    audioFormats: readonly string[];
+};
 export declare class BaseClient implements SmolClient {
     protected config: SmolConfig;
     protected statelogClient?: StatelogClient;
@@ -14,7 +20,13 @@ export declare class BaseClient implements SmolClient {
     }): Promise<Result<PromptResult>>;
     checkMessageLimit(promptConfig: SmolConfig): Result<PromptResult> | null;
     /**
-     * Gate on input modalities and resolve any image/PDF attachment refs
+     * What this client can accept as attachments. Subclasses override to declare
+     * more (or fewer). Checked against the messages, alongside the model's own
+     * declared modalities, before any serialization runs.
+     */
+    protected attachmentCapabilities(): ClientAttachmentCapabilities;
+    /**
+     * Gate on input modalities and resolve any image/PDF/audio attachment refs
      * (path/url/bytes → base64) before the synchronous serializers run. Returns
      * the (possibly rewritten) config on success, or a Failure to surface. Shared
      * by textSync and textStream so the two paths can't diverge.

package/dist/clients/baseClient.js

@@ -6,11 +6,21 @@ import { stripCodeFence } from "../util/util.js";
 import { success, failure, } from "../types.js";
 import { validateHostedTools } from "../util/hostedTools.js";
 import { resolveMessageAttachments, messagesHaveAttachments, DEFAULT_MAX_ATTACHMENT_BYTES } from "./resolveAttachments.js";
-import { validateModalities } from "../util/modalities.js";
+import { neededInputModalities, MODALITIES_REQUIRING_DECLARATION, } from "../util/modalities.js";
+import { modelSupportsInputModality } from "../models.js";
 import { resolveProvider } from "../util/provider.js";
 import { isUnconstrainedSchema } from "../util/jsonSchema.js";
 import { z } from "zod";
 const DEFAULT_NUM_RETRIES = 2;
+function clientSupportsAttachment(capabilities, modality) {
+    if (modality === "audio") {
+        return capabilities.audioFormats.length > 0;
+    }
+    if (modality === "image" || modality === "pdf") {
+        return capabilities.inputModalities.includes(modality);
+    }
+    return false;
+}
 export class BaseClient {
     config;
     statelogClient;
@@ -57,24 +67,43 @@ export class BaseClient {
         return null;
     }
     /**
-     * Gate on input modalities and resolve any image/PDF attachment refs
+     * What this client can accept as attachments. Subclasses override to declare
+     * more (or fewer). Checked against the messages, alongside the model's own
+     * declared modalities, before any serialization runs.
+     */
+    attachmentCapabilities() {
+        return { inputModalities: ["image", "pdf"], audioFormats: [] };
+    }
+    /**
+     * Gate on input modalities and resolve any image/PDF/audio attachment refs
      * (path/url/bytes → base64) before the synchronous serializers run. Returns
      * the (possibly rewritten) config on success, or a Failure to surface. Shared
      * by textSync and textStream so the two paths can't diverge.
      */
     async prepareAttachments(config) {
-        const modalityResult = validateModalities(config);
-        if (modalityResult && modalityResult.success === false) {
-            return modalityResult;
-        }
-        if (!messagesHaveAttachments(config.messages)) {
+        const needed = neededInputModalities(config.messages);
+        if (needed.length === 0 && !messagesHaveAttachments(config.messages)) {
             return success(config);
         }
         const provider = resolveProvider(config.model, config.provider, config.modelData);
+        const capabilities = this.attachmentCapabilities();
+        for (const modality of needed) {
+            if (!clientSupportsAttachment(capabilities, modality)) {
+                return failure(`${modality[0].toUpperCase()}${modality.slice(1)} input is not supported by the "${provider}" provider.`);
+            }
+            const supported = modelSupportsInputModality(config.model, modality, config.modelData, provider);
+            if (supported === false) {
+                return failure(`Model ${config.model} does not support ${modality} input.`);
+            }
+            if (supported === undefined && MODALITIES_REQUIRING_DECLARATION.has(modality)) {
+                return failure(`Model ${config.model} does not support ${modality} input.`);
+            }
+        }
         const maxBytes = config.attachments?.maxBytes ?? DEFAULT_MAX_ATTACHMENT_BYTES;
         const resolved = await resolveMessageAttachments(config.messages, {
             provider,
             maxBytes,
+            audioFormats: capabilities.audioFormats,
         });
         if (!resolved.success) {
             return resolved;

package/dist/clients/google.d.ts

@@ -3,6 +3,7 @@ import { PromptResult, Result, SmolClient, SmolConfig, StreamChunk } from "../ty
 import { BaseClient } from "./baseClient.js";
 import { ModelName } from "../models.js";
 import { HostedToolResult } from "../types.js";
+import type { Message } from "../classes/message/index.js";
 export type SmolGoogleConfig = SmolConfig;
 export declare function googleWebSearchEntries(hostedTools?: string[]): any[];
 /**
@@ -20,6 +21,7 @@ export declare function googleWebSearchEntries(hostedTools?: string[]): any[];
  * See egonSchiele/agency-lang#495.
  */
 export declare function geminiSupportsToolCirculation(model: string): boolean;
+export declare function reorderToolResultsForGemini(messages: Message[]): Message[];
 export declare function parseGoogleHostedTools(result: any, provider: string, model: string): HostedToolResult[];
 type GeneratedRequest = {
     contents: Content[];

package/dist/clients/google.js

@@ -39,6 +39,118 @@ export function geminiSupportsToolCirculation(model) {
         return true;
     return parseInt(m[1], 10) >= 3;
 }
+// Reorder each round's tool results to match the order of the calls that
+// produced them. Two documented Gemini behaviors make this necessary:
+//   - A function call carries an optional `id`, and a response is paired back to
+//     its call by echoing that id.
+//     https://ai.google.dev/gemini-api/docs/function-calling
+//   - Parts must be returned in the order received (responses in call order:
+//     FC1,FC2 -> FR1,FR2), and the thought signature rides ONLY the first
+//     function call of a parallel batch; omitting it 400s on Gemini 3. So part
+//     order is load-bearing independent of ids.
+//     https://ai.google.dev/gemini-api/docs/generate-content/thought-signatures
+// Current Gemini 3 preview models are observed to emit no function-call ids, so
+// on them pairing is purely positional. A caller whose results arrive in
+// completion order (rather than call order) would otherwise feed tool A's answer
+// to tool B and break thought-signature validation.
+//
+// For every assistant message that carries toolCalls, the run of ToolMessages
+// immediately following it is reordered. The run ends at the first non-tool
+// message: any results the caller interleaved AFTER a non-tool message are left
+// where they are — moving messages across an interloper is repair, not reorder.
+//   1. By id, when both the call and a response have non-empty ids (order-free,
+//      takes global priority so an id match is never stolen by a name match).
+//   2. By name + occurrence otherwise: the k-th response named X answers the
+//      k-th call named X.
+// NEVER drop a message: a response matching no call (or any surplus) is kept at
+// the end of the run in its original relative order. A reorder that lost a
+// message would turn a mispairing bug into a missing-result bug, which is worse.
+export function reorderToolResultsForGemini(messages) {
+    const out = [];
+    let i = 0;
+    while (i < messages.length) {
+        const msg = messages[i];
+        out.push(msg);
+        const toolCalls = msg.role === "assistant" ? msg.toolCalls : undefined;
+        if (!toolCalls || toolCalls.length === 0) {
+            i += 1;
+            continue;
+        }
+        // Collect the contiguous run of tool results that answers this round.
+        let j = i + 1;
+        const run = [];
+        while (j < messages.length && messages[j].role === "tool") {
+            run.push(messages[j]);
+            j += 1;
+        }
+        if (run.length === 0) {
+            i += 1;
+            continue;
+        }
+        out.push(...orderRunToMatchCalls(toolCalls, run));
+        i = j;
+    }
+    return out;
+}
+function orderRunToMatchCalls(toolCalls, run) {
+    const used = new Array(run.length).fill(false);
+    // assignment[callIndex] = index into `run`, or -1 if that call has no result.
+    const assignment = new Array(toolCalls.length).fill(-1);
+    function runId(k) {
+        const m = run[k];
+        if (m.role === "tool") {
+            return m.tool_call_id;
+        }
+        return "";
+    }
+    function runName(k) {
+        const m = run[k];
+        if (m.role === "tool") {
+            return m.name;
+        }
+        return "";
+    }
+    // Pass 1: id pairing (both sides non-empty), global priority.
+    toolCalls.forEach((call, ci) => {
+        if (call.id === "")
+            return;
+        const ri = run.findIndex((_r, k) => !used[k] && runId(k) !== "" && runId(k) === call.id);
+        if (ri !== -1) {
+            assignment[ci] = ri;
+            used[ri] = true;
+        }
+    });
+    // Pass 2: name + occurrence, for calls still unmatched. findIndex takes the
+    // first unused same-name response, so the k-th call named X pairs with the
+    // k-th response named X.
+    //
+    // Garbage-in caveat: if a call and its only same-name response carry DIFFERENT
+    // non-empty ids (one side lost or mangled its id), pass 1 misses and pass 2
+    // pairs them by name — emitting functionCall.id != functionResponse.id in that
+    // slot, which an id-pairing model would see as a contradiction. The input was
+    // already inconsistent; we pair positionally rather than drop the result. Not
+    // a reorder bug.
+    toolCalls.forEach((call, ci) => {
+        if (assignment[ci] !== -1)
+            return;
+        const ri = run.findIndex((_r, k) => !used[k] && runName(k) === call.name);
+        if (ri !== -1) {
+            assignment[ci] = ri;
+            used[ri] = true;
+        }
+    });
+    const ordered = [];
+    for (const ri of assignment) {
+        if (ri !== -1)
+            ordered.push(run[ri]);
+    }
+    // Never drop: append any unmatched/surplus responses in original order.
+    for (let k = 0; k < run.length; k++) {
+        if (!used[k])
+            ordered.push(run[k]);
+    }
+    return ordered;
+}
 export function parseGoogleHostedTools(result, provider, model) {
     const queries = [];
     const sources = [];
@@ -97,7 +209,7 @@ export class SmolGoogle extends BaseClient {
         }
         this.client = new GoogleGenAI({ apiKey });
         this.logger = getLogger();
-        this.model = new Model(config.model, undefined, config.modelData);
+        this.model = new Model(config.model, config.provider, config.modelData);
     }
     getClient() {
         return this.client;
@@ -136,7 +248,12 @@ export class SmolGoogle extends BaseClient {
             }
             return true;
         });
-        const messages = contentMessages.map((msg) => msg.toGoogleMessage());
+        // Normalize tool-result ordering before conversion: Gemini pairs each
+        // functionResponse to its functionCall (by id on 3.5+, strictly by position
+        // on the Gemini 3 family), so results must leave in call order regardless of
+        // the order the caller supplied them. See reorderToolResultsForGemini.
+        const orderedMessages = reorderToolResultsForGemini(contentMessages);
+        const messages = orderedMessages.map((msg) => msg.toGoogleMessage());
         const tools = (config.tools || []).map((tool) => {
             return zodToGoogleTool(tool.name, tool.schema, {
                 description: tool.description,
@@ -341,7 +458,12 @@ export class SmolGoogle extends BaseClient {
                         const functionCall = part.functionCall;
                         // Gemini 3 rides the thought signature on the same part as the
                         // function call; capture it so it can be echoed back during tool use.
-                        toolCalls.push(new ToolCall("", functionCall.name, functionCall.args, {
+                        toolCalls.push(
+                        // Keep functionCall.id so id-based pairing can round-trip. Do NOT
+                        // fall back to the name (as the streaming path does for its Map
+                        // key): two parallel calls to the same tool would share a fake id
+                        // and re-create the pairing bug at the id layer.
+                        new ToolCall(functionCall.id || "", functionCall.name, functionCall.args, {
                             thoughtSignature: part.thoughtSignature,
                         }));
                     }

package/dist/clients/ollama.js

@@ -20,7 +20,7 @@ export class SmolOllama extends BaseClient {
     constructor(config) {
         super(config);
         this.logger = getLogger();
-        this.model = new Model(config.model, undefined, config.modelData);
+        this.model = new Model(config.model, config.provider, config.modelData);
         const apiKey = config.apiKey?.ollama;
         if (apiKey) {
             this.client = new Ollama({

package/dist/clients/openai.d.ts

@@ -1,7 +1,7 @@
 import OpenAI from "openai";
 import { PromptResult, Result, SmolClient, SmolConfig, StreamChunk, HostedToolResult } from "../types.js";
 import { EgonLog } from "../util/logger.js";
-import { BaseClient } from "./baseClient.js";
+import { BaseClient, type ClientAttachmentCapabilities } from "./baseClient.js";
 import { ModelName } from "../models.js";
 import { Model } from "../model.js";
 import { CostEstimate, TokenUsage } from "../types.js";
@@ -11,6 +11,7 @@ export declare class SmolOpenAi extends BaseClient implements SmolClient {
     protected logger: EgonLog;
     protected model: Model;
     constructor(config: SmolOpenAiConfig);
+    protected attachmentCapabilities(): ClientAttachmentCapabilities;
     /**
      * Build the `new OpenAI({...})` options. Subclasses override to inject a
      * different baseURL or to pull the key from a different config field.

package/dist/clients/openai.js

@@ -20,7 +20,11 @@ export class SmolOpenAi extends BaseClient {
         const options = this.resolveClientOptions(config);
         this.client = new OpenAI(options);
         this.logger = getLogger();
-        this.model = new Model(config.model, undefined, config.modelData);
+        this.model = new Model(config.model, config.provider, config.modelData);
+    }
+    attachmentCapabilities() {
+        // Chat Completions input_audio accepts inline mp3/wav only.
+        return { inputModalities: ["image", "pdf"], audioFormats: ["mp3", "wav"] };
     }
     /**
      * Build the `new OpenAI({...})` options. Subclasses override to inject a
@@ -72,14 +76,22 @@ export class SmolOpenAi extends BaseClient {
         let cost;
         if (usageData) {
             const cached = usageData.prompt_tokens_details?.cached_tokens ?? 0;
+            const audioIn = usageData.prompt_tokens_details?.audio_tokens ?? 0;
+            const audioOut = usageData.completion_tokens_details?.audio_tokens ?? 0;
             usage = {
-                inputTokens: Math.max(0, (usageData.prompt_tokens || 0) - cached),
-                outputTokens: usageData.completion_tokens || 0,
+                inputTokens: Math.max(0, (usageData.prompt_tokens || 0) - cached - audioIn),
+                outputTokens: Math.max(0, (usageData.completion_tokens || 0) - audioOut),
                 totalTokens: usageData.total_tokens,
             };
             if (cached > 0) {
                 usage.cachedInputTokens = cached;
             }
+            if (audioIn > 0) {
+                usage.inputAudioTokens = audioIn;
+            }
+            if (audioOut > 0) {
+                usage.outputAudioTokens = audioOut;
+            }
             // Prefer provider-supplied cost when available (e.g. OpenRouter
             // usage.cost, DeepInfra usage.estimated_cost, LiteLLM header).
             // Fall back to the smoltalk model-registry calculation.

package/dist/clients/openaiCompat.d.ts

@@ -1,4 +1,5 @@
 import { SmolOpenAi } from "./openai.js";
+import type { ClientAttachmentCapabilities } from "./baseClient.js";
 import type { SmolConfig } from "../types.js";
 /**
  * Generic OpenAI-compatible client. Use when pointing smoltalk at any
@@ -14,6 +15,7 @@ import type { SmolConfig } from "../types.js";
  * cost stays undefined — that's expected for arbitrary backends).
  */
 export declare class SmolOpenAiCompat extends SmolOpenAi {
+    protected attachmentCapabilities(): ClientAttachmentCapabilities;
     protected resolveClientOptions(config: SmolConfig): {
         apiKey: string;
         baseURL: string;

package/dist/clients/openaiCompat.js

@@ -14,6 +14,11 @@ import { resolveApiKey, resolveBaseUrl } from "../util/provider.js";
  * cost stays undefined — that's expected for arbitrary backends).
  */
 export class SmolOpenAiCompat extends SmolOpenAi {
+    // Compat endpoints speak the Chat Completions wire format but do not get
+    // OpenAI's input_audio handling — declare no audio support.
+    attachmentCapabilities() {
+        return { inputModalities: ["image", "pdf"], audioFormats: [] };
+    }
     resolveClientOptions(config) {
         const apiKey = resolveApiKey("openai-compat", config);
         const baseURL = resolveBaseUrl("openai-compat", config);

package/dist/clients/openaiResponses.js

@@ -62,7 +62,7 @@ export class SmolOpenAiResponses extends BaseClient {
         }
         this.client = new OpenAI({ apiKey });
         this.logger = getLogger();
-        this.model = new Model(config.model, undefined, config.modelData);
+        this.model = new Model(config.model, config.provider, config.modelData);
     }
     getClient() {
         return this.client;

package/dist/clients/resolveAttachments.d.ts

@@ -1,11 +1,15 @@
 import { Message } from "../classes/message/index.js";
 import { Result } from "../types.js";
 export declare const DEFAULT_MAX_ATTACHMENT_BYTES: number;
+type ResolveOptions = {
+    provider: string;
+    maxBytes: number;
+    /** Audio containers (by primary extension) the target client accepts inline. */
+    audioFormats: readonly string[];
+};
 /** Whether any user message carries an image/file attachment part. */
 export declare function messagesHaveAttachments(messages: Message[]): boolean;
 /** Whether `provider` accepts a remote URL directly for this part type. */
 export declare function acceptsRemoteUrl(provider: string, partType: "image" | "file"): boolean;
-export declare function resolveMessageAttachments(messages: Message[], options: {
-    provider: string;
-    maxBytes: number;
-}): Promise<Result<Message[]>>;
+export declare function resolveMessageAttachments(messages: Message[], options: ResolveOptions): Promise<Result<Message[]>>;
+export {};

package/dist/clients/resolveAttachments.js

@@ -1,6 +1,7 @@
 import { UserMessage } from "../classes/message/index.js";
-import { normalizeImageRef } from "../util/imageRef.js";
+import { normalizeBlob } from "../util/blobRef.js";
 import { fileFamily } from "../util/attachments.js";
+import { audioFormatForMime } from "../util/mime.js";
 import { success, failure } from "../types.js";
 export const DEFAULT_MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024;
 const URL_IMAGE_PROVIDERS = new Set([
@@ -24,7 +25,7 @@ export function messagesHaveAttachments(messages) {
             continue;
         }
         for (const part of parts) {
-            if (part.type === "image" || part.type === "file") {
+            if (part.type === "image" || part.type === "file" || part.type === "audio") {
                 return true;
             }
         }
@@ -38,6 +39,100 @@ export function acceptsRemoteUrl(provider, partType) {
     }
     return URL_PDF_PROVIDERS.has(provider);
 }
+/** Load a ref to inline base64, gated to `allowed` MIME prefixes. Throws on failure. */
+async function toBase64Source(source, allowed, maxBytes) {
+    const { data, mimeType } = await normalizeBlob(source, {
+        allowedMimePrefixes: allowed,
+        maxBytes,
+    });
+    return { kind: "base64", base64: Buffer.from(data).toString("base64"), mimeType };
+}
+/** Error message when a providerFile ref targets the wrong provider family, else null. */
+function providerFileError(fileProvider, targetProvider) {
+    const family = fileFamily(targetProvider);
+    if (family === null || fileProvider !== family) {
+        return (`Attachment references a "${fileProvider}" file, but this call targets provider ` +
+            `"${targetProvider}" (file family ${family ?? "none"}).`);
+    }
+    return null;
+}
+// Audio has no providerFile/URL passthrough: Chat input_audio requires
+// inline base64, so every audio source is normalized here.
+async function resolveAudioPart(part, options) {
+    try {
+        const source = await toBase64Source(part.source, ["audio/"], options.maxBytes);
+        const audioFormat = audioFormatForMime(source.mimeType);
+        if (audioFormat === null || !options.audioFormats.includes(audioFormat.extension)) {
+            return failure(`Audio input for provider "${options.provider}" supports only ` +
+                `${options.audioFormats.join(", ")}; got "${source.mimeType}".`);
+        }
+        const resolved = { type: "audio", source };
+        if (part.filename !== undefined) {
+            resolved.filename = part.filename;
+        }
+        return success(resolved);
+    }
+    catch (err) {
+        return failure(`Failed to load audio attachment: ${err.message}`);
+    }
+}
+async function resolveImagePart(part, options) {
+    // Provider file references are validated and passed through (no download/cap).
+    if (part.source.kind === "providerFile") {
+        const mismatch = providerFileError(part.source.provider, options.provider);
+        if (mismatch !== null) {
+            return failure(mismatch);
+        }
+        if (options.provider === "openai") {
+            return failure("An image file reference requires the openai-responses provider (OpenAI Chat Completions has no image-by-file_id form).");
+        }
+        return success(part);
+    }
+    // Passthrough: keep a url ref when the target provider accepts a remote URL.
+    if (part.source.kind === "url" && acceptsRemoteUrl(options.provider, "image")) {
+        return success(part);
+    }
+    try {
+        const source = await toBase64Source(part.source, ["image/"], options.maxBytes);
+        return success({ type: "image", source });
+    }
+    catch (err) {
+        return failure(`Failed to load image attachment: ${err.message}`);
+    }
+}
+async function resolveFilePart(part, options) {
+    // Provider file references are validated and passed through (no download/cap).
+    if (part.source.kind === "providerFile") {
+        const mismatch = providerFileError(part.source.provider, options.provider);
+        if (mismatch !== null) {
+            return failure(mismatch);
+        }
+        return success(part);
+    }
+    // Passthrough: keep a url ref when the target provider accepts a remote URL.
+    if (part.source.kind === "url" && acceptsRemoteUrl(options.provider, "file")) {
+        return success(part);
+    }
+    try {
+        const source = await toBase64Source(part.source, ["application/pdf"], options.maxBytes);
+        return success({ type: "file", source, filename: part.filename });
+    }
+    catch (err) {
+        return failure(`Failed to load file attachment: ${err.message}`);
+    }
+}
+async function resolveUserPart(part, options) {
+    if (part.type === "text") {
+        return success(part);
+    }
+    if (part.type === "audio") {
+        return resolveAudioPart(part, options);
+    }
+    if (part.type === "image") {
+        return resolveImagePart(part, options);
+    }
+    return resolveFilePart(part, options);
+}
 export async function resolveMessageAttachments(messages, options) {
     const out = [];
     for (const msg of messages) {
@@ -52,55 +147,11 @@ export async function resolveMessageAttachments(messages, options) {
         }
         const resolvedParts = [];
         for (const part of parts) {
-            if (part.type === "text") {
-                resolvedParts.push(part);
-                continue;
-            }
-            // Provider file references are validated and passed through (no download/cap).
-            if (part.source.kind === "providerFile") {
-                const family = fileFamily(options.provider);
-                if (family === null || part.source.provider !== family) {
-                    return failure(`Attachment references a "${part.source.provider}" file, but this call targets provider ` +
-                        `"${options.provider}" (file family ${family ?? "none"}).`);
-                }
-                if (part.type === "image" && options.provider === "openai") {
-                    return failure("An image file reference requires the openai-responses provider (OpenAI Chat Completions has no image-by-file_id form).");
-                }
-                resolvedParts.push(part);
-                continue;
-            }
-            // Passthrough: keep a url ref when the target provider accepts a remote URL.
-            if (part.source.kind === "url" && acceptsRemoteUrl(options.provider, part.type)) {
-                resolvedParts.push(part);
-                continue;
-            }
-            let allowed;
-            if (part.type === "image") {
-                allowed = ["image/"];
-            }
-            else {
-                allowed = ["application/pdf"];
-            }
-            try {
-                const { data, mimeType } = await normalizeImageRef(part.source, {
-                    allowedMimePrefixes: allowed,
-                    maxBytes: options.maxBytes,
-                });
-                const source = {
-                    kind: "base64",
-                    base64: Buffer.from(data).toString("base64"),
-                    mimeType,
-                };
-                if (part.type === "image") {
-                    resolvedParts.push({ type: "image", source });
-                }
-                else {
-                    resolvedParts.push({ type: "file", source, filename: part.filename });
-                }
-            }
-            catch (err) {
-                return failure(`Failed to load ${part.type} attachment: ${err.message}`);
+            const resolved = await resolveUserPart(part, options);
+            if (!resolved.success) {
+                return resolved;
             }
+            resolvedParts.push(resolved.value);
         }
         out.push(new UserMessage(resolvedParts, { name: msg.name, rawData: msg.rawData }));
     }

package/dist/embed.d.ts

@@ -16,6 +16,8 @@ export type EmbedConfig = {
         deepInfra?: string;
         liteLlm?: string;
         openAiCompat?: string;
+        /** Arbitrary provider names, for keys targeting a custom-registered provider. */
+        [provider: string]: string | undefined;
     };
     /** Custom base URLs, nested by provider. */
     baseUrl?: {
@@ -23,6 +25,8 @@ export type EmbedConfig = {
         deepInfra?: string;
         liteLlm?: string;
         openAiCompat?: string;
+        /** Arbitrary provider names, for URLs targeting a custom-registered provider. */
+        [provider: string]: string | undefined;
     };
     metadata?: Record<string, unknown>;
     modelData?: ModelDataBlob;

package/dist/files.d.ts

@@ -1,7 +1,7 @@
 import { Result } from "./types/result.js";
 import type { SmolConfig } from "./types.js";
 import { ProviderFileRef } from "./classes/message/contentParts.js";
-import { BlobRef } from "./util/imageRef.js";
+import { BlobRef } from "./util/blobRef.js";
 /** Default cap on a resolved upload's size. Callers can raise it via opts.maxBytes. */
 export declare const DEFAULT_UPLOAD_BYTES: number;
 /**

package/dist/files.js

@@ -1,5 +1,5 @@
 import { failure } from "./types/result.js";
-import { loadBlob } from "./util/imageRef.js";
+import { loadBlob } from "./util/blobRef.js";
 import { fileFamily } from "./util/attachments.js";
 import { resolveApiKey } from "./util/provider.js";
 import { openaiFileProvider } from "./files/openai.js";

package/dist/image/google.js

@@ -1,7 +1,7 @@
 import { GoogleGenAI } from "@google/genai";
 import { success, failure } from "../types/result.js";
 import { getModel, isImageModel } from "../models.js";
-import { normalizeImageRef } from "../util/imageRef.js";
+import { normalizeBlob } from "../util/blobRef.js";
 import { COST_DECIMAL_PLACES, round } from "../util/util.js";
 export async function googleImage(input, config, apiKey) {
     try {
@@ -9,7 +9,7 @@ export async function googleImage(input, config, apiKey) {
         const client = new GoogleGenAI({ apiKey });
         const parts = [{ text: normalized.prompt }];
         if (normalized.images && normalized.images.length > 0) {
-            const normalizedImages = await Promise.all(normalized.images.map((ref) => normalizeImageRef(ref)));
+            const normalizedImages = await Promise.all(normalized.images.map((ref) => normalizeBlob(ref)));
             for (const img of normalizedImages) {
                 parts.push({
                     inlineData: {