smoltalk 0.9.0 → 0.10.1

package/dist/types/tokenUsage.js

@@ -4,6 +4,8 @@ export const TokenUsageSchema = z.object({
     outputTokens: z.number(),
     cachedInputTokens: z.number().optional(),
     cacheCreationInputTokens: z.number().optional(),
+    inputAudioTokens: z.number().optional(),
+    outputAudioTokens: z.number().optional(),
     totalTokens: z.number().optional(),
 });
 export function addTokenUsage(_a, _b) {
@@ -22,6 +24,8 @@ export function addTokenUsage(_a, _b) {
         outputTokens: a.outputTokens + b.outputTokens,
         cachedInputTokens: (a.cachedInputTokens || 0) + (b.cachedInputTokens || 0),
         cacheCreationInputTokens: (a.cacheCreationInputTokens || 0) + (b.cacheCreationInputTokens || 0),
+        inputAudioTokens: (a.inputAudioTokens || 0) + (b.inputAudioTokens || 0),
+        outputAudioTokens: (a.outputAudioTokens || 0) + (b.outputAudioTokens || 0),
         totalTokens: (a.totalTokens || 0) + (b.totalTokens || 0),
     };
 }

package/dist/types.d.ts

@@ -30,6 +30,10 @@ export type SmolConfig = {
         deepInfra?: string;
         liteLlm?: string;
         openAiCompat?: string;
+        groq?: string;
+        /** Arbitrary provider names, for keys targeting a custom-registered provider
+         *  (e.g. registerProvider("acme", ...)), keyed by the exact registered name. */
+        [provider: string]: string | undefined;
     };
     /** Custom base URLs, nested by provider. Defaults are baked in where applicable
      *  (e.g. openrouter, deepinfra); litellm and openai-compat require an explicit URL. */

package/dist/util/attachments.d.ts

@@ -1,4 +1,4 @@
-import type { ImageRef } from "./imageRef.js";
+import type { ImageRef } from "./blobRef.js";
 /** Filename sent to providers that require one when a file part omits its own. */
 export declare const DEFAULT_ATTACHMENT_FILENAME = "attachment.pdf";
 /** A file part's filename, falling back to the shared default when unset. */

package/dist/util/audioMime.d.ts

@@ -0,0 +1,26 @@
+export type SpeakFormat = "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm";
+export type TranscriptionAudioType = {
+    extension: string;
+    filename: string;
+};
+export declare function transcriptionAudioType(mime: string): TranscriptionAudioType | null;
+export declare function chatAudioFormat(mime: string): "mp3" | "wav" | null;
+export declare const SPEECH_FORMAT_TO_MIME: Record<SpeakFormat, string>;
+export declare function isSpeakFormat(value: string): value is SpeakFormat;
+/**
+ * Translate a canonical/alias audio MIME to the wire value Google expects.
+ * Google documents MP3 as `audio/mp3`, whereas this repo canonicalizes it to
+ * `audio/mpeg`; everything else passes through canonicalized.
+ */
+export declare function googleAudioWireMime(mimeType: string): string;
+export type PcmWavOptions = {
+    sampleRateHz: number;
+    channels: number;
+    bitsPerSample: number;
+};
+/**
+ * Wrap raw signed-integer little-endian PCM in a 44-byte RIFF/WAVE header so it
+ * becomes a directly-playable .wav. Pure function, no dependency. Used for
+ * Gemini TTS output, which is only ever raw PCM.
+ */
+export declare function pcmToWav(pcm: Uint8Array, opts: PcmWavOptions): Uint8Array;

package/dist/util/audioMime.js

@@ -0,0 +1,74 @@
+import { audioFormatForMime, canonicalizeMime } from "./mime.js";
+export function transcriptionAudioType(mime) {
+    const format = audioFormatForMime(mime);
+    if (format === null) {
+        return null;
+    }
+    return { extension: format.extension, filename: `audio.${format.extension}` };
+}
+export function chatAudioFormat(mime) {
+    const format = audioFormatForMime(mime);
+    if (format === null) {
+        return null;
+    }
+    if (format.extension === "mp3" || format.extension === "wav") {
+        return format.extension;
+    }
+    return null;
+}
+// PCM from OpenAI is headerless s16le / 24kHz / mono, which audio/L16 (big-endian
+// per RFC) would misdescribe — use octet-stream + structured metadata instead.
+export const SPEECH_FORMAT_TO_MIME = {
+    mp3: "audio/mpeg",
+    opus: "audio/ogg",
+    aac: "audio/aac",
+    flac: "audio/flac",
+    wav: "audio/wav",
+    pcm: "application/octet-stream",
+};
+// Object.hasOwn (not `in`) so prototype keys like "toString"/"__proto__"
+// never pass the guard.
+export function isSpeakFormat(value) {
+    return Object.hasOwn(SPEECH_FORMAT_TO_MIME, value);
+}
+/**
+ * Translate a canonical/alias audio MIME to the wire value Google expects.
+ * Google documents MP3 as `audio/mp3`, whereas this repo canonicalizes it to
+ * `audio/mpeg`; everything else passes through canonicalized.
+ */
+export function googleAudioWireMime(mimeType) {
+    const format = audioFormatForMime(mimeType);
+    const canonical = format?.mimeType ?? canonicalizeMime(mimeType);
+    return canonical === "audio/mpeg" ? "audio/mp3" : canonical;
+}
+/**
+ * Wrap raw signed-integer little-endian PCM in a 44-byte RIFF/WAVE header so it
+ * becomes a directly-playable .wav. Pure function, no dependency. Used for
+ * Gemini TTS output, which is only ever raw PCM.
+ */
+export function pcmToWav(pcm, opts) {
+    const { sampleRateHz, channels, bitsPerSample } = opts;
+    const blockAlign = (channels * bitsPerSample) / 8;
+    const byteRate = sampleRateHz * blockAlign;
+    const out = new Uint8Array(44 + pcm.length);
+    const view = new DataView(out.buffer);
+    const writeAscii = (offset, s) => {
+        for (let i = 0; i < s.length; i++)
+            view.setUint8(offset + i, s.charCodeAt(i));
+    };
+    writeAscii(0, "RIFF");
+    view.setUint32(4, 36 + pcm.length, true);
+    writeAscii(8, "WAVE");
+    writeAscii(12, "fmt ");
+    view.setUint32(16, 16, true); // PCM fmt chunk size
+    view.setUint16(20, 1, true); // audio format = PCM
+    view.setUint16(22, channels, true);
+    view.setUint32(24, sampleRateHz, true);
+    view.setUint32(28, byteRate, true);
+    view.setUint16(32, blockAlign, true);
+    view.setUint16(34, bitsPerSample, true);
+    writeAscii(36, "data");
+    view.setUint32(40, pcm.length, true);
+    out.set(pcm, 44);
+    return out;
+}

package/dist/util/{imageRef.d.ts → blobRef.d.ts}

@@ -1,4 +1,4 @@
-export type ImageRef = {
+export type BlobRef = {
     kind: "bytes";
     data: Uint8Array;
     mimeType: string;
@@ -20,23 +20,23 @@ export type ImageRef = {
      */
     timeoutMs?: number;
 };
-/** Neutral alias of the source union for non-image uses (uploads via {@link loadBlob}). */
-export type BlobRef = ImageRef;
-export type NormalizedImage = {
+/** Alias of {@link BlobRef} for image call sites, where it documents intent. */
+export type ImageRef = BlobRef;
+export type NormalizedBlob = {
     data: Uint8Array;
     mimeType: string;
 };
-/** Default timeout for fetching image URLs during normalization (60 seconds). */
+/** Default timeout for fetching attachment URLs during normalization (60 seconds). */
 export declare const DEFAULT_FETCH_TIMEOUT_MS = 60000;
-export declare function normalizeImageRef(ref: ImageRef, options?: {
+export declare function normalizeBlob(ref: BlobRef, options?: {
     allowedMimePrefixes?: string[];
     maxBytes?: number;
-}): Promise<NormalizedImage>;
+}): Promise<NormalizedBlob>;
 /**
- * Load a source to bytes WITHOUT the image MIME gate (used by file uploads,
+ * Load a source to bytes WITHOUT the MIME gate (used by file uploads,
  * which accept any type). Enforces `maxBytes` across all kinds.
  */
-export declare function loadBlob(ref: ImageRef, options?: {
+export declare function loadBlob(ref: BlobRef, options?: {
     maxBytes?: number;
 }): Promise<{
     data: Uint8Array;

package/dist/util/{imageRef.js → blobRef.js}

@@ -1,15 +1,8 @@
 import { readFile, stat } from "node:fs/promises";
 import { extname } from "node:path";
-/** Default timeout for fetching image URLs during normalization (60 seconds). */
+import { EXT_TO_MIME } from "./mime.js";
+/** Default timeout for fetching attachment URLs during normalization (60 seconds). */
 export const DEFAULT_FETCH_TIMEOUT_MS = 60_000;
-const EXT_TO_MIME = {
-    ".png": "image/png",
-    ".jpg": "image/jpeg",
-    ".jpeg": "image/jpeg",
-    ".webp": "image/webp",
-    ".gif": "image/gif",
-    ".pdf": "application/pdf",
-};
 function isAllowedMime(mimeType, allowedPrefixes) {
     for (const prefix of allowedPrefixes) {
         if (mimeType.startsWith(prefix)) {
@@ -18,7 +11,7 @@ function isAllowedMime(mimeType, allowedPrefixes) {
     }
     return false;
 }
-export async function normalizeImageRef(ref, options = {}) {
+export async function normalizeBlob(ref, options = {}) {
     const allowed = options.allowedMimePrefixes ?? ["image/"];
     const result = await loadRef(ref, allowed, options.maxBytes);
     if (options.maxBytes !== undefined && result.data.length > options.maxBytes) {
@@ -31,7 +24,7 @@ export async function normalizeImageRef(ref, options = {}) {
     return { data: result.data, mimeType: result.mimeType };
 }
 /**
- * Load a source to bytes WITHOUT the image MIME gate (used by file uploads,
+ * Load a source to bytes WITHOUT the MIME gate (used by file uploads,
  * which accept any type). Enforces `maxBytes` across all kinds.
  */
 export async function loadBlob(ref, options = {}) {
@@ -94,7 +87,7 @@ async function loadRef(ref, allowed, maxBytes) {
             const mimeType = ref.mimeType ?? inferred;
             if (allowed !== null && (!mimeType || !isAllowedMime(mimeType, allowed))) {
                 throw new Error(`Could not determine an allowed MIME type for path "${ref.path}". ` +
-                    `Allowed: ${allowed.join(", ")}. Pass an explicit mimeType on the ImageRef.`);
+                    `Allowed: ${allowed.join(", ")}. Pass an explicit mimeType on the BlobRef.`);
             }
             return { data: new Uint8Array(buf), mimeType };
         }
@@ -128,7 +121,7 @@ async function loadRef(ref, allowed, maxBytes) {
             if (allowed !== null && (!mimeType || !isAllowedMime(mimeType, allowed))) {
                 throw new Error(`Could not determine an allowed MIME type for URL "${ref.url}". ` +
                     `Response Content-Type was "${contentType ?? "missing"}". ` +
-                    `Allowed: ${allowed.join(", ")}. Pass an explicit mimeType on the ImageRef.`);
+                    `Allowed: ${allowed.join(", ")}. Pass an explicit mimeType on the BlobRef.`);
             }
             return { data: buf, mimeType };
         }

package/dist/util/googleAudioUsage.d.ts

@@ -0,0 +1,14 @@
+import type { GenerateContentResponseUsageMetadata } from "@google/genai";
+import type { TokenUsage } from "../types/tokenUsage.js";
+export type GoogleAudioDirection = "input" | "output";
+/**
+ * Normalize Gemini's usage metadata into smoltalk's TokenUsage, separating the
+ * audio bucket for the given direction so the token-priced cost engine can bill
+ * it at the audio rate. STT sets audioDirection "input" (audio prompt tokens);
+ * TTS sets "output" (audio candidate tokens).
+ *
+ * Thinking tokens (`thoughtsTokenCount`) are reported separately from
+ * `candidatesTokenCount` and are always text — they go into the text-output
+ * bucket, never the audio-output bucket.
+ */
+export declare function normalizeGoogleAudioUsage(metadata: GenerateContentResponseUsageMetadata | undefined, audioDirection: GoogleAudioDirection): TokenUsage | undefined;

package/dist/util/googleAudioUsage.js

@@ -0,0 +1,52 @@
+function modalityTokens(details, modality) {
+    return (details ?? [])
+        // `modality` is a MediaModality enum; compare by its string value.
+        .filter((detail) => String(detail.modality) === modality)
+        .reduce((sum, detail) => sum + (detail.tokenCount ?? 0), 0);
+}
+/**
+ * Normalize Gemini's usage metadata into smoltalk's TokenUsage, separating the
+ * audio bucket for the given direction so the token-priced cost engine can bill
+ * it at the audio rate. STT sets audioDirection "input" (audio prompt tokens);
+ * TTS sets "output" (audio candidate tokens).
+ *
+ * Thinking tokens (`thoughtsTokenCount`) are reported separately from
+ * `candidatesTokenCount` and are always text — they go into the text-output
+ * bucket, never the audio-output bucket.
+ */
+export function normalizeGoogleAudioUsage(metadata, audioDirection) {
+    if (metadata === undefined) {
+        return undefined;
+    }
+    const prompt = metadata.promptTokenCount ?? 0;
+    const candidates = metadata.candidatesTokenCount ?? 0;
+    const thoughts = metadata.thoughtsTokenCount ?? 0;
+    if (audioDirection === "input") {
+        const audioIn = modalityTokens(metadata.promptTokensDetails, "AUDIO");
+        const usage = {
+            inputTokens: Math.max(0, prompt - audioIn),
+            outputTokens: candidates + thoughts,
+        };
+        if (audioIn > 0)
+            usage.inputAudioTokens = audioIn;
+        if (metadata.totalTokenCount !== undefined)
+            usage.totalTokens = metadata.totalTokenCount;
+        return usage;
+    }
+    // Output (TTS): split the audio bucket out of candidate tokens. Only assume
+    // "all candidates are audio" when the details array is missing/empty; when it
+    // is present, trust its AUDIO total even if that total is 0 (otherwise
+    // purely-text candidate tokens would be mispriced at the audio rate).
+    const details = metadata.candidatesTokensDetails;
+    const detailsPresent = Array.isArray(details) && details.length > 0;
+    const audioOut = detailsPresent ? modalityTokens(details, "AUDIO") : candidates;
+    const usage = {
+        inputTokens: prompt,
+        outputTokens: Math.max(0, candidates - audioOut) + thoughts,
+    };
+    if (audioOut > 0)
+        usage.outputAudioTokens = audioOut;
+    if (metadata.totalTokenCount !== undefined)
+        usage.totalTokens = metadata.totalTokenCount;
+    return usage;
+}

package/dist/util/mime.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Shared extension/MIME knowledge for attachments and audio. One table per
+ * format; every other ext↔MIME mapping in the package derives from these so
+ * the forward and inverse maps can't drift apart.
+ */
+export type AudioFormat = {
+    /** Primary file extension, without the dot. */
+    extension: string;
+    /** Canonical MIME type. */
+    mimeType: string;
+    /** Other MIME strings that identify the same container. */
+    aliasMimeTypes: readonly string[];
+    /** Other extensions that map to this format. */
+    aliasExtensions: readonly string[];
+};
+export declare const AUDIO_FORMATS: readonly AudioFormat[];
+/** Extension (with leading dot, lowercase) → MIME, across images, PDF, and audio. */
+export declare const EXT_TO_MIME: Record<string, string>;
+export declare function canonicalizeMime(mime: string): string;
+/** The audio format a MIME string identifies, or null when unrecognized. */
+export declare function audioFormatForMime(mime: string): AudioFormat | null;

package/dist/util/mime.js

@@ -0,0 +1,54 @@
+/**
+ * Shared extension/MIME knowledge for attachments and audio. One table per
+ * format; every other ext↔MIME mapping in the package derives from these so
+ * the forward and inverse maps can't drift apart.
+ */
+export const AUDIO_FORMATS = [
+    { extension: "mp3", mimeType: "audio/mpeg", aliasMimeTypes: ["audio/mp3"], aliasExtensions: ["mpeg", "mpga"] },
+    { extension: "wav", mimeType: "audio/wav", aliasMimeTypes: ["audio/x-wav"], aliasExtensions: [] },
+    { extension: "m4a", mimeType: "audio/m4a", aliasMimeTypes: ["audio/x-m4a"], aliasExtensions: [] },
+    { extension: "mp4", mimeType: "audio/mp4", aliasMimeTypes: ["video/mp4"], aliasExtensions: [] },
+    { extension: "ogg", mimeType: "audio/ogg", aliasMimeTypes: [], aliasExtensions: [] },
+    { extension: "flac", mimeType: "audio/flac", aliasMimeTypes: [], aliasExtensions: [] },
+    { extension: "webm", mimeType: "audio/webm", aliasMimeTypes: [], aliasExtensions: [] },
+    { extension: "aac", mimeType: "audio/aac", aliasMimeTypes: [], aliasExtensions: [] },
+    { extension: "aiff", mimeType: "audio/aiff", aliasMimeTypes: ["audio/x-aiff"], aliasExtensions: ["aif"] },
+];
+const IMAGE_AND_DOCUMENT_EXT_TO_MIME = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".webp": "image/webp",
+    ".gif": "image/gif",
+    ".pdf": "application/pdf",
+};
+function buildExtToMime() {
+    const table = { ...IMAGE_AND_DOCUMENT_EXT_TO_MIME };
+    for (const format of AUDIO_FORMATS) {
+        table[`.${format.extension}`] = format.mimeType;
+        for (const alias of format.aliasExtensions) {
+            table[`.${alias}`] = format.mimeType;
+        }
+    }
+    return table;
+}
+/** Extension (with leading dot, lowercase) → MIME, across images, PDF, and audio. */
+export const EXT_TO_MIME = buildExtToMime();
+// Strips parameters (e.g. ";codecs=opus") and normalizes case, so MediaRecorder-
+// style MIME strings like "audio/webm;codecs=opus" or "AUDIO/MPEG" still match.
+export function canonicalizeMime(mime) {
+    return mime.split(";")[0].trim().toLowerCase();
+}
+/** The audio format a MIME string identifies, or null when unrecognized. */
+export function audioFormatForMime(mime) {
+    const canonical = canonicalizeMime(mime);
+    for (const format of AUDIO_FORMATS) {
+        if (format.mimeType === canonical) {
+            return format;
+        }
+        if (format.aliasMimeTypes.includes(canonical)) {
+            return format;
+        }
+    }
+    return null;
+}

package/dist/util/modalities.d.ts

@@ -1,2 +1,6 @@
-import { PromptResult, Result, SmolConfig } from "../types.js";
-export declare function validateModalities(config: SmolConfig): Result<PromptResult> | null;
+import { Message } from "../classes/message/index.js";
+/** Modalities a model must positively declare in its data block — for these,
+ *  "unknown" means "unsupported" (audio serialization is model-specific). */
+export declare const MODALITIES_REQUIRING_DECLARATION: ReadonlySet<string>;
+/** Which non-text input modalities the user messages actually use. */
+export declare function neededInputModalities(messages: Message[]): string[];

package/dist/util/modalities.js

@@ -1,10 +1,11 @@
 import { UserMessage } from "../classes/message/index.js";
-import { modelSupportsInputModality } from "../models.js";
-import { failure } from "../types.js";
-export function validateModalities(config) {
-    let needsImage = false;
-    let needsPdf = false;
-    for (const msg of config.messages) {
+/** Modalities a model must positively declare in its data block — for these,
+ *  "unknown" means "unsupported" (audio serialization is model-specific). */
+export const MODALITIES_REQUIRING_DECLARATION = new Set(["audio"]);
+/** Which non-text input modalities the user messages actually use. */
+export function neededInputModalities(messages) {
+    const needed = new Set();
+    for (const msg of messages) {
         if (!(msg instanceof UserMessage)) {
             continue;
         }
@@ -14,18 +15,15 @@ export function validateModalities(config) {
         }
         for (const part of parts) {
             if (part.type === "image") {
-                needsImage = true;
+                needed.add("image");
             }
             if (part.type === "file") {
-                needsPdf = true;
+                needed.add("pdf");
+            }
+            if (part.type === "audio") {
+                needed.add("audio");
             }
         }
     }
-    if (needsImage && modelSupportsInputModality(config.model, "image", config.modelData) === false) {
-        return failure(`Model ${config.model} does not support image input.`);
-    }
-    if (needsPdf && modelSupportsInputModality(config.model, "pdf", config.modelData) === false) {
-        return failure(`Model ${config.model} does not support PDF/document input.`);
-    }
-    return null;
+    return [...needed];
 }

package/dist/util/provider.d.ts

@@ -19,6 +19,9 @@ type NestedKeyConfig = {
         deepInfra?: string;
         liteLlm?: string;
         openAiCompat?: string;
+        groq?: string;
+        /** Arbitrary provider names, for keys targeting a custom-registered provider. */
+        [provider: string]: string | undefined;
     };
     baseUrl?: {
         ollama?: string;

package/dist/util/provider.js

@@ -37,8 +37,10 @@ export function resolveApiKey(provider, config) {
             return k?.liteLlm || process.env.LITELLM_API_KEY;
         case "openai-compat":
             return k?.openAiCompat || process.env.OPENAI_COMPAT_API_KEY;
+        case "groq":
+            return k?.groq || process.env.GROQ_API_KEY;
         default:
-            return undefined;
+            return config.apiKey?.[provider];
     }
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smoltalk",
-  "version": "0.9.0",
+  "version": "0.10.1",
   "description": "A common interface for LLM APIs",
   "homepage": "https://github.com/egonSchiele/smoltalk",
   "files": [