smoltalk 0.5.1 → 0.7.0

package/dist/image.js

@@ -1,7 +1,8 @@
 import { failure } from "./types/result.js";
-import { resolveProvider, resolveApiKey } from "./util/provider.js";
+import { resolveProvider, resolveApiKey, resolveBaseUrl } from "./util/provider.js";
 import { openaiImage } from "./image/openai.js";
 import { googleImage } from "./image/google.js";
+import { openaiCompatImage } from "./image/openaiCompat.js";
 // Null-prototype so provider names like "toString"/"__proto__" can't collide
 // with Object.prototype or pollute the registry.
 const registeredImageProviders = Object.create(null);
@@ -27,16 +28,40 @@ export async function image(input, config) {
         case "openai":
         case "openai-responses": {
             if (!apiKey) {
-                return failure("No OpenAI API key provided. Set openAiApiKey in config or the OPENAI_API_KEY environment variable.");
+                return failure("No OpenAI API key provided. Set config.apiKey.openAi or the OPENAI_API_KEY environment variable.");
             }
             return openaiImage(input, config, apiKey);
         }
         case "google": {
             if (!apiKey) {
-                return failure("No Google API key provided. Set googleApiKey in config or the GEMINI_API_KEY environment variable.");
+                return failure("No Google API key provided. Set config.apiKey.google or the GEMINI_API_KEY environment variable.");
             }
             return googleImage(input, config, apiKey);
         }
+        case "openrouter":
+            return failure("openrouter does not expose an OpenAI-compatible images endpoint; use openai-compat or litellm instead.");
+        case "deepinfra":
+            return failure("deepinfra exposes per-model image endpoints rather than the OpenAI `/images/generations` shape; use openai-compat with a specific backend or litellm instead.");
+        case "litellm": {
+            if (!apiKey) {
+                return failure("No LiteLLM API key provided. Set config.apiKey.liteLlm or the LITELLM_API_KEY environment variable.");
+            }
+            const baseURL = resolveBaseUrl("litellm", config);
+            if (!baseURL) {
+                return failure("No LiteLLM base URL provided. Set config.baseUrl.liteLlm or the LITELLM_BASE_URL environment variable.");
+            }
+            return openaiCompatImage(input, config, apiKey, baseURL);
+        }
+        case "openai-compat": {
+            if (!apiKey) {
+                return failure("No openai-compat API key provided. Set config.apiKey.openAiCompat or the OPENAI_COMPAT_API_KEY environment variable.");
+            }
+            const baseURL = resolveBaseUrl("openai-compat", config);
+            if (!baseURL) {
+                return failure("No openai-compat base URL provided. Set config.baseUrl.openAiCompat or the OPENAI_COMPAT_BASE_URL environment variable.");
+            }
+            return openaiCompatImage(input, config, apiKey, baseURL);
+        }
         default: {
             const custom = registeredImageProviders[provider];
             if (custom) {

package/dist/index.d.ts

@@ -11,5 +11,9 @@ export * from "./functions.js";
 export * from "./classes/ToolCall.js";
 export * from "./embed.js";
 export * from "./image.js";
+export { uploadFile, deleteFile, registerFileProvider, DEFAULT_UPLOAD_BYTES } from "./files.js";
+export type { UploadFileOptions, FileProviderContext, FileProvider } from "./files.js";
+export { normalizeImageRef, loadBlob } from "./util/imageRef.js";
+export type { BlobRef } from "./util/imageRef.js";
 export { getLogger, EgonLog } from "./util/logger.js";
 export type { LogLevel } from "./util/logger.js";

package/dist/index.js

@@ -11,4 +11,7 @@ export * from "./functions.js";
 export * from "./classes/ToolCall.js";
 export * from "./embed.js";
 export * from "./image.js";
+// Explicit (not `export *`) so the test-only `_resetForTests` stays off the public surface.
+export { uploadFile, deleteFile, registerFileProvider, DEFAULT_UPLOAD_BYTES } from "./files.js";
+export { normalizeImageRef, loadBlob } from "./util/imageRef.js";
 export { getLogger, EgonLog } from "./util/logger.js";

package/dist/models.d.ts

@@ -1,14 +1,18 @@
 import { z } from "zod";
 import { type ModelDataBlob, type HostedTool, type HostedToolPrice } from "./modelData.js";
-export declare const providers: readonly ["ollama", "openai", "openai-responses", "anthropic", "google", "replicate", "modal"];
+export declare const providers: readonly ["ollama", "openai", "openai-responses", "anthropic", "google", "replicate", "modal", "openrouter", "deepinfra", "litellm", "openai-compat"];
 export declare const ProviderSchema: z.ZodEnum<{
-    ollama: "ollama";
     openai: "openai";
-    "openai-responses": "openai-responses";
     anthropic: "anthropic";
     google: "google";
+    "openai-responses": "openai-responses";
+    ollama: "ollama";
     replicate: "replicate";
     modal: "modal";
+    openrouter: "openrouter";
+    deepinfra: "deepinfra";
+    litellm: "litellm";
+    "openai-compat": "openai-compat";
 }>;
 export type Provider = z.infer<typeof ProviderSchema>;
 export type BaseModel = {
@@ -1450,7 +1454,22 @@ export declare function registerTextModel(model: Omit<TextModel, "type"> & {
 export declare function registerModelData(blob: ModelDataBlob): void;
 export declare function clearModelData(): void;
 export declare function getRegisteredModelData(): ModelDataBlob | null;
+/**
+ * All known models: the baked-in catalog merged with any registered
+ * (refreshed) model data and optional per-request overrides. This is the same
+ * data `getModel` searches, exposed as a list so callers can enumerate, filter,
+ * or display the catalog (e.g. a model picker). Fields worth filtering on live
+ * on `ModelType`: `provider`, `openWeights`, `inputTokenCost` / `outputTokenCost`,
+ * `maxInputTokens`, and `modalities`.
+ */
+export declare function getAllModels(requestData?: ModelDataBlob): ModelType[];
 export declare function getModel(modelName: ModelName, requestData?: ModelDataBlob): ModelType | undefined;
+/**
+ * Whether a model is known to accept the given input modality ("image", "pdf", …).
+ * Returns undefined when the model is unknown or carries no `modalities` data —
+ * callers should treat undefined as "don't gate".
+ */
+export declare function modelSupportsInputModality(modelName: ModelName, modality: string, requestData?: ModelDataBlob): boolean | undefined;
 export declare function getHostedTools(opts?: {
     provider?: string;
     model?: string;

package/dist/models.js

@@ -8,6 +8,10 @@ export const providers = [
     "google",
     "replicate",
     "modal",
+    "openrouter",
+    "deepinfra",
+    "litellm",
+    "openai-compat",
 ];
 export const ProviderSchema = z.enum(providers);
 export const speechToTextModels = [
@@ -1557,6 +1561,18 @@ export const hostedTools = [
         models: ["gemini-3-pro-preview", "gemini-3.1-pro-preview", "gemini-3-flash-preview", "gemini-3.5-flash", "gemini-3.1-flash-lite"],
         pricing: { unit: "per_call", note: "Gemini 3 family only; see Google pricing." },
     },
+    {
+        name: "web_search",
+        provider: "openrouter",
+        category: "web_search",
+        description: "Web search via OpenRouter's web plugin (Exa-backed). Adds citations as annotations.",
+        providerToolId: "web",
+        pricing: {
+            unit: "per_call",
+            amount: 0.02,
+            note: "$4 per 1,000 results returned by the web plugin × default 5 results/call = $0.02/call. Plus token cost for inserted context. If you override max_results, scale this accordingly.",
+        },
+    },
 ];
 export const registeredTextModels = [];
 export function registerTextModel(model) {
@@ -1581,7 +1597,15 @@ function baselineModels() {
         ...embeddingsModels,
     ];
 }
-export function getModel(modelName, requestData) {
+/**
+ * All known models: the baked-in catalog merged with any registered
+ * (refreshed) model data and optional per-request overrides. This is the same
+ * data `getModel` searches, exposed as a list so callers can enumerate, filter,
+ * or display the catalog (e.g. a model picker). Fields worth filtering on live
+ * on `ModelType`: `provider`, `openWeights`, `inputTokenCost` / `outputTokenCost`,
+ * `maxInputTokens`, and `modalities`.
+ */
+export function getAllModels(requestData) {
     let models = baselineModels();
     if (registeredModelData) {
         models = mergeModelData(models, registeredModelData.models);
@@ -1589,7 +1613,26 @@ export function getModel(modelName, requestData) {
     if (requestData) {
         models = mergeModelData(models, requestData.models);
     }
-    return models.find((model) => model.modelName === modelName);
+    return models;
+}
+export function getModel(modelName, requestData) {
+    return getAllModels(requestData).find((model) => model.modelName === modelName);
+}
+/**
+ * Whether a model is known to accept the given input modality ("image", "pdf", …).
+ * Returns undefined when the model is unknown or carries no `modalities` data —
+ * callers should treat undefined as "don't gate".
+ */
+export function modelSupportsInputModality(modelName, modality, requestData) {
+    const model = getModel(modelName, requestData);
+    if (!model || model.type !== "text") {
+        return undefined;
+    }
+    const inputs = model.modalities?.input;
+    if (!inputs) {
+        return undefined;
+    }
+    return inputs.includes(modality);
 }
 export function getHostedTools(opts = {}) {
     // Start from a copy so callers can never mutate the baseline registry.
@@ -1658,4 +1701,4 @@ export function isEmbeddingsModel(model) {
 }
 export const ModelNameSchema = z
     .string()
-    .regex(/^[a-zA-Z0-9._:-]+$/, "Model name must only contain letters, numbers, dots, underscores, hyphens, and colons");
+    .regex(/^[a-zA-Z0-9._:@/-]+$/, "Model name must only contain letters, numbers, dots, underscores, hyphens, colons, slashes, and @");

package/dist/statelogClient.js

@@ -1,5 +1,6 @@
 import { nanoid } from "nanoid";
 import { failure, mergeResults, success } from "./types/result.js";
+import { redactAttachments } from "./util/redact.js";
 export function mergeUploadResults(_results) {
     const results = mergeResults(_results);
     if (!results.success) {
@@ -277,7 +278,7 @@ export class StatelogClient {
         const postBody = JSON.stringify({
             trace_id: this.traceId,
             project_id: this.projectId,
-            data: { ...body, timestamp: new Date().toISOString() },
+            data: redactAttachments({ ...body, timestamp: new Date().toISOString() }),
         });
         if (this.host.toLowerCase() === "stdout") {
             console.log(postBody);

package/dist/types.d.ts

@@ -1,4 +1,5 @@
 export * from "./types/result.js";
+export * from "./classes/message/contentParts.js";
 import { LogLevel } from "./util/logger.js";
 import z, { ZodType } from "zod";
 import { Message } from "./classes/message/index.js";
@@ -16,16 +17,28 @@ export type SmolConfig = {
     model: ModelName;
     /** Override the provider for the given model (e.g., use a custom endpoint for an OpenAI-compatible model). */
     provider?: string;
-    /** API key for OpenAI. Required when using OpenAI models. */
-    openAiApiKey?: string;
-    /** API key for Google Gemini. Required when using Google models. */
-    googleApiKey?: string;
-    /** API key for Anthropic. Required when using Anthropic/Claude models. */
-    anthropicApiKey?: string;
-    /** API key for Ollama. Only needed when connecting to a cloud-hosted Ollama instance. */
-    ollamaApiKey?: string;
-    /** Base URL for the Ollama server. Defaults to localhost if not set. (Ollama only) */
-    ollamaHost?: string;
+    /** API keys, nested by provider. Each key falls back to its conventional env var
+     *  if unset (e.g. apiKey.openAi → OPENAI_API_KEY). */
+    apiKey?: {
+        openAi?: string;
+        google?: string;
+        anthropic?: string;
+        ollama?: string;
+        openRouter?: string;
+        deepInfra?: string;
+        liteLlm?: string;
+        openAiCompat?: string;
+    };
+    /** Custom base URLs, nested by provider. Defaults are baked in where applicable
+     *  (e.g. openrouter, deepinfra); litellm and openai-compat require an explicit URL. */
+    baseUrl?: {
+        /** Host URL for the Ollama server. Defaults to http://localhost:11434 (or $OLLAMA_HOST). */
+        ollama?: string;
+        openRouter?: string;
+        deepInfra?: string;
+        liteLlm?: string;
+        openAiCompat?: string;
+    };
     /** Log level for internal debug logging. */
     logLevel?: LogLevel;
     /** Configuration for Statelog observability/tracing integration. */
@@ -45,6 +58,11 @@ export type SmolConfig = {
     }>;
     /** Arbitrary metadata passed to custom model providers. */
     metadata?: Record<string, any>;
+    /** Options for image/PDF attachments on user messages. */
+    attachments?: {
+        /** Max resolved attachment size in bytes. Default 20 MB. Over → Failure. */
+        maxBytes?: number;
+    };
     /** Refreshed model/hosted-tool data (from refreshModels) to layer over the baked-in registry for this call. */
     modelData?: ModelDataBlob;
     /** The conversation messages to send to the model. */
@@ -161,10 +179,6 @@ export interface SmolClient {
     textStream(config: SmolConfig): AsyncGenerator<StreamChunk>;
     _textStream(config: SmolConfig): AsyncGenerator<StreamChunk>;
 }
-export type TextPart = {
-    type: "text";
-    text: string;
-};
 /** Loose variant of SmolConfig for `getClient()` — messages are not required at construction time. */
 export type SmolClientConfig = Omit<SmolConfig, "messages"> & {
     messages?: Message[];
@@ -174,10 +188,6 @@ export type ThinkingBlock = {
     text: string;
     signature: string;
 };
-export declare const TextPartSchema: z.ZodObject<{
-    type: z.ZodLiteral<"text">;
-    text: z.ZodString;
-}, z.z.core.$strip>;
 export declare const ThinkingBlockSchema: z.ZodObject<{
     text: z.ZodString;
     signature: z.ZodString;

package/dist/types.js

@@ -1,4 +1,5 @@
 export * from "./types/result.js";
+export * from "./classes/message/contentParts.js";
 import z from "zod";
 export * from "./types/costEstimate.js";
 export * from "./types/tokenUsage.js";
@@ -13,10 +14,6 @@ export function promptResult({ output, toolCalls, thinkingBlocks, usage, cost, m
         hostedToolResults,
     };
 }
-export const TextPartSchema = z.object({
-    type: z.literal("text"),
-    text: z.string(),
-});
 export const ThinkingBlockSchema = z.object({
     text: z.string(),
     signature: z.string(),

package/dist/util/attachments.d.ts

@@ -0,0 +1,34 @@
+import type { ImageRef } from "./imageRef.js";
+/** Filename sent to providers that require one when a file part omits its own. */
+export declare const DEFAULT_ATTACHMENT_FILENAME = "attachment.pdf";
+/** A file part's filename, falling back to the shared default when unset. */
+export declare function attachmentFilename(filename?: string): string;
+/**
+ * Extract base64 + MIME from an ImageRef that has already been resolved to
+ * in-memory bytes. `path`/`url` refs must be resolved (via the BaseClient
+ * normalize pass) before serialization — reaching here with one is a bug.
+ */
+export declare function refToBase64(ref: ImageRef): {
+    base64: string;
+    mimeType: string;
+};
+export declare function toDataUri(base64: string, mimeType: string): string;
+/** The Files-API namespace for a provider, or null when it has no Files API. */
+export declare function fileFamily(provider: string): "openai" | "anthropic" | "google" | null;
+/**
+ * OpenAI-style image URL: a passthrough `url` ref stays a remote URL; anything
+ * else becomes a base64 data URI.
+ */
+export declare function openAiImageUrl(ref: ImageRef): string;
+/**
+ * Anthropic image/document source: a passthrough `url` ref becomes a url source;
+ * anything else becomes a base64 source.
+ */
+export declare function anthropicSource(ref: ImageRef): {
+    type: "url";
+    url: string;
+} | {
+    type: "base64";
+    media_type: string;
+    data: string;
+};

package/dist/util/attachments.js

@@ -0,0 +1,62 @@
+/** Filename sent to providers that require one when a file part omits its own. */
+export const DEFAULT_ATTACHMENT_FILENAME = "attachment.pdf";
+/** A file part's filename, falling back to the shared default when unset. */
+export function attachmentFilename(filename) {
+    if (filename === undefined) {
+        return DEFAULT_ATTACHMENT_FILENAME;
+    }
+    return filename;
+}
+/**
+ * Extract base64 + MIME from an ImageRef that has already been resolved to
+ * in-memory bytes. `path`/`url` refs must be resolved (via the BaseClient
+ * normalize pass) before serialization — reaching here with one is a bug.
+ */
+export function refToBase64(ref) {
+    if (ref.kind === "base64") {
+        return { base64: ref.base64, mimeType: ref.mimeType };
+    }
+    if (ref.kind === "bytes") {
+        return { base64: Buffer.from(ref.data).toString("base64"), mimeType: ref.mimeType };
+    }
+    throw new Error(`Attachment ref must be resolved to base64 or bytes before serialization ` +
+        `(got kind="${ref.kind}"). This is a smoltalk bug.`);
+}
+export function toDataUri(base64, mimeType) {
+    return `data:${mimeType};base64,${base64}`;
+}
+/** The Files-API namespace for a provider, or null when it has no Files API. */
+export function fileFamily(provider) {
+    if (provider === "openai" || provider === "openai-responses") {
+        return "openai";
+    }
+    if (provider === "anthropic") {
+        return "anthropic";
+    }
+    if (provider === "google") {
+        return "google";
+    }
+    return null;
+}
+/**
+ * OpenAI-style image URL: a passthrough `url` ref stays a remote URL; anything
+ * else becomes a base64 data URI.
+ */
+export function openAiImageUrl(ref) {
+    if (ref.kind === "url") {
+        return ref.url;
+    }
+    const { base64, mimeType } = refToBase64(ref);
+    return toDataUri(base64, mimeType);
+}
+/**
+ * Anthropic image/document source: a passthrough `url` ref becomes a url source;
+ * anything else becomes a base64 source.
+ */
+export function anthropicSource(ref) {
+    if (ref.kind === "url") {
+        return { type: "url", url: ref.url };
+    }
+    const { base64, mimeType } = refToBase64(ref);
+    return { type: "base64", media_type: mimeType, data: base64 };
+}

package/dist/util/hostedTools.js

@@ -27,7 +27,11 @@ export function validateHostedTools(requested, model, provider, modelData) {
             if (!effectiveProvider) {
                 effectiveProvider = getModel(model, modelData)?.provider ?? "unknown";
             }
-            return `${name} is a hosted capability; ${model} (${effectiveProvider}) doesn't offer it — pass a search function as a tool instead.`;
+            let err = `${name} is a hosted capability; ${model} (${effectiveProvider}) doesn't offer it — pass a search function as a tool instead.`;
+            if (effectiveProvider === "openai") {
+                err += " You can also use the provider 'openai-responses' to access hosted tools for OpenAI models (you're currently using the base 'openai' provider).";
+            }
+            return err;
         }
     }
     return null;

package/dist/util/imageRef.d.ts

@@ -20,10 +20,25 @@ export type ImageRef = {
      */
     timeoutMs?: number;
 };
+/** Neutral alias of the source union for non-image uses (uploads via {@link loadBlob}). */
+export type BlobRef = ImageRef;
 export type NormalizedImage = {
     data: Uint8Array;
     mimeType: string;
 };
 /** Default timeout for fetching image URLs during normalization (60 seconds). */
 export declare const DEFAULT_FETCH_TIMEOUT_MS = 60000;
-export declare function normalizeImageRef(ref: ImageRef): Promise<NormalizedImage>;
+export declare function normalizeImageRef(ref: ImageRef, options?: {
+    allowedMimePrefixes?: string[];
+    maxBytes?: number;
+}): Promise<NormalizedImage>;
+/**
+ * Load a source to bytes WITHOUT the image MIME gate (used by file uploads,
+ * which accept any type). Enforces `maxBytes` across all kinds.
+ */
+export declare function loadBlob(ref: ImageRef, options?: {
+    maxBytes?: number;
+}): Promise<{
+    data: Uint8Array;
+    mimeType?: string;
+}>;

package/dist/util/imageRef.js

@@ -1,4 +1,4 @@
-import { readFile } from "node:fs/promises";
+import { readFile, stat } from "node:fs/promises";
 import { extname } from "node:path";
 /** Default timeout for fetching image URLs during normalization (60 seconds). */
 export const DEFAULT_FETCH_TIMEOUT_MS = 60_000;
@@ -8,8 +8,70 @@ const EXT_TO_MIME = {
     ".jpeg": "image/jpeg",
     ".webp": "image/webp",
     ".gif": "image/gif",
+    ".pdf": "application/pdf",
 };
-export async function normalizeImageRef(ref) {
+function isAllowedMime(mimeType, allowedPrefixes) {
+    for (const prefix of allowedPrefixes) {
+        if (mimeType.startsWith(prefix)) {
+            return true;
+        }
+    }
+    return false;
+}
+export async function normalizeImageRef(ref, options = {}) {
+    const allowed = options.allowedMimePrefixes ?? ["image/"];
+    const result = await loadRef(ref, allowed, options.maxBytes);
+    if (options.maxBytes !== undefined && result.data.length > options.maxBytes) {
+        throw new Error(`Attachment exceeds the maximum size of ${options.maxBytes} bytes ` +
+            `(got ${result.data.length} bytes).`);
+    }
+    if (!result.mimeType) {
+        throw new Error("internal: gated load returned no mimeType");
+    }
+    return { data: result.data, mimeType: result.mimeType };
+}
+/**
+ * Load a source to bytes WITHOUT the image MIME gate (used by file uploads,
+ * which accept any type). Enforces `maxBytes` across all kinds.
+ */
+export async function loadBlob(ref, options = {}) {
+    const result = await loadRef(ref, null, options.maxBytes);
+    if (options.maxBytes !== undefined && result.data.length > options.maxBytes) {
+        throw tooLargeError(options.maxBytes, result.data.length);
+    }
+    return result;
+}
+function tooLargeError(maxBytes, got) {
+    return new Error(`Attachment exceeds the maximum size of ${maxBytes} bytes (got ${got} bytes).`);
+}
+/**
+ * Read a response body into memory, aborting as soon as the running total
+ * exceeds `maxBytes` so a large or hostile URL can't exhaust memory. Falls back
+ * to buffering whole when the body isn't streamable.
+ */
+async function readBodyWithLimit(res, maxBytes) {
+    if (maxBytes === undefined || !res.body) {
+        return new Uint8Array(await res.arrayBuffer());
+    }
+    const reader = res.body.getReader();
+    const chunks = [];
+    let total = 0;
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done) {
+            break;
+        }
+        total += value.byteLength;
+        if (total > maxBytes) {
+            await reader.cancel();
+            throw tooLargeError(maxBytes, `>${total}`);
+        }
+        chunks.push(value);
+    }
+    // Pass the known total so concat doesn't re-scan the chunk list for its length.
+    return new Uint8Array(Buffer.concat(chunks, total));
+}
+async function loadRef(ref, allowed, maxBytes) {
     switch (ref.kind) {
         case "bytes":
             return { data: ref.data, mimeType: ref.mimeType };
@@ -19,31 +81,54 @@ export async function normalizeImageRef(ref) {
                 mimeType: ref.mimeType,
             };
         case "path": {
+            // S1: bound memory — reject on stat before reading the whole file.
+            if (maxBytes !== undefined) {
+                const info = await stat(ref.path);
+                if (info.size > maxBytes) {
+                    throw tooLargeError(maxBytes, info.size);
+                }
+            }
             const buf = await readFile(ref.path);
             const ext = extname(ref.path).toLowerCase();
             const inferred = EXT_TO_MIME[ext];
             const mimeType = ref.mimeType ?? inferred;
-            if (!mimeType || !mimeType.startsWith("image/")) {
-                throw new Error(`Could not determine image MIME type for path "${ref.path}". ` +
-                    `Pass an explicit mimeType (e.g. "image/png") on the ImageRef.`);
+            if (allowed !== null && (!mimeType || !isAllowedMime(mimeType, allowed))) {
+                throw new Error(`Could not determine an allowed MIME type for path "${ref.path}". ` +
+                    `Allowed: ${allowed.join(", ")}. Pass an explicit mimeType on the ImageRef.`);
             }
             return { data: new Uint8Array(buf), mimeType };
         }
         case "url": {
+            // S2: only http(s) — reject file:/ftp:/gopher:/data: etc.
+            let scheme;
+            try {
+                scheme = new URL(ref.url).protocol;
+            }
+            catch {
+                throw new Error(`Invalid attachment URL: "${ref.url}".`);
+            }
+            if (scheme !== "http:" && scheme !== "https:") {
+                throw new Error(`Unsupported URL scheme "${scheme}" for attachment; only http(s) is allowed.`);
+            }
             const timeoutMs = ref.timeoutMs ?? DEFAULT_FETCH_TIMEOUT_MS;
             const res = await fetch(ref.url, {
                 signal: AbortSignal.timeout(timeoutMs),
             });
             if (!res.ok) {
-                throw new Error(`Failed to fetch image from ${ref.url}: ${res.status}`);
+                throw new Error(`Failed to fetch attachment from ${ref.url}: ${res.status}`);
+            }
+            // Reject on an honest Content-Length before reading a single byte of body.
+            const declaredLength = Number(res.headers.get("content-length"));
+            if (maxBytes !== undefined && declaredLength > maxBytes) {
+                throw tooLargeError(maxBytes, declaredLength);
             }
-            const buf = new Uint8Array(await res.arrayBuffer());
+            const buf = await readBodyWithLimit(res, maxBytes);
             const contentType = res.headers.get("content-type") ?? undefined;
             const mimeType = ref.mimeType ?? contentType;
-            if (!mimeType || !mimeType.startsWith("image/")) {
-                throw new Error(`Could not determine image MIME type for URL "${ref.url}". ` +
+            if (allowed !== null && (!mimeType || !isAllowedMime(mimeType, allowed))) {
+                throw new Error(`Could not determine an allowed MIME type for URL "${ref.url}". ` +
                     `Response Content-Type was "${contentType ?? "missing"}". ` +
-                    `Pass an explicit mimeType (e.g. "image/png") on the ImageRef.`);
+                    `Allowed: ${allowed.join(", ")}. Pass an explicit mimeType on the ImageRef.`);
             }
             return { data: buf, mimeType };
         }

package/dist/util/modalities.d.ts

@@ -0,0 +1,2 @@
+import { PromptResult, Result, SmolConfig } from "../types.js";
+export declare function validateModalities(config: SmolConfig): Result<PromptResult> | null;

package/dist/util/modalities.js

@@ -0,0 +1,31 @@
+import { UserMessage } from "../classes/message/index.js";
+import { modelSupportsInputModality } from "../models.js";
+import { failure } from "../types.js";
+export function validateModalities(config) {
+    let needsImage = false;
+    let needsPdf = false;
+    for (const msg of config.messages) {
+        if (!(msg instanceof UserMessage)) {
+            continue;
+        }
+        const parts = msg.getContentParts();
+        if (parts === null) {
+            continue;
+        }
+        for (const part of parts) {
+            if (part.type === "image") {
+                needsImage = true;
+            }
+            if (part.type === "file") {
+                needsPdf = true;
+            }
+        }
+    }
+    if (needsImage && modelSupportsInputModality(config.model, "image", config.modelData) === false) {
+        return failure(`Model ${config.model} does not support image input.`);
+    }
+    if (needsPdf && modelSupportsInputModality(config.model, "pdf", config.modelData) === false) {
+        return failure(`Model ${config.model} does not support PDF/document input.`);
+    }
+    return null;
+}