smoltalk 0.10.0 → 0.10.1

package/dist/speech.d.ts

@@ -2,6 +2,7 @@ import type { ModelDataBlob } from "./modelData.js";
 import type { SmolConfig } from "./types.js";
 import { Result } from "./types/result.js";
 import { CostEstimate } from "./types/costEstimate.js";
+import { TokenUsage } from "./types/tokenUsage.js";
 import { BaseSpeechClient, SpeechClientConfig } from "./speech/baseSpeechClient.js";
 export type SpeakOptions = {
     model: string;
@@ -9,9 +10,12 @@ export type SpeakOptions = {
     provider?: string;
     modelData?: ModelDataBlob;
     apiKey?: SmolConfig["apiKey"];
+    baseUrl?: SmolConfig["baseUrl"];
     format?: string;
     speed?: number;
     metadata?: Record<string, unknown>;
+    /** Abort the in-flight provider request when this signal fires. */
+    abortSignal?: AbortSignal;
 };
 export type PcmAudioMetadata = {
     sampleRateHz: number;
@@ -23,6 +27,7 @@ export type SpeechResult = {
     audio: Uint8Array;
     mimeType: string;
     pcm?: PcmAudioMetadata;
+    usage?: TokenUsage;
     cost?: CostEstimate;
     raw?: unknown;
 };

package/dist/speech.js

@@ -3,9 +3,15 @@ import { redactSecret } from "./util/redact.js";
 import { getLogger } from "./util/logger.js";
 import { resolveProvider, resolveApiKey } from "./util/provider.js";
 import { OpenAISpeechClient } from "./speech/openai.js";
+import { GroqSpeechClient } from "./speech/groq.js";
+import { GoogleSpeechClient } from "./speech/google.js";
+import { OpenAiCompatSpeechClient } from "./speech/openaiCompat.js";
 // Checked before the user registry so a registered "openai" can't hijack the built-in.
 const builtinClients = Object.create(null);
 builtinClients["openai"] = OpenAISpeechClient;
+builtinClients["groq"] = GroqSpeechClient;
+builtinClients["google"] = GoogleSpeechClient;
+builtinClients["openai-compat"] = OpenAiCompatSpeechClient;
 // Null-prototype so provider names like "toString"/"__proto__" can't collide
 // with Object.prototype or pollute the registry.
 const registered = Object.create(null);

package/dist/transcription/baseTranscriptionClient.d.ts

@@ -1,4 +1,5 @@
 import type { ModelDataBlob } from "../modelData.js";
+import type { SmolConfig } from "../types.js";
 import { Result } from "../types/result.js";
 import { BlobRef } from "../util/blobRef.js";
 import type { TranscriptionResult } from "../transcription.js";
@@ -9,12 +10,16 @@ export type TranscriptionClientConfig = {
     provider: string;
     /** Resolved API key; empty string when none was found. */
     apiKey: string;
+    /** Base-URL map (for OpenAI-compatible providers); read via resolveBaseUrl. */
+    baseUrl?: SmolConfig["baseUrl"];
     modelData?: ModelDataBlob;
     language?: string;
     prompt?: string;
     timestampGranularity?: "segment" | "word";
     maxBytes?: number;
     metadata?: Record<string, unknown>;
+    /** Abort the in-flight provider request when this signal fires. */
+    abortSignal?: AbortSignal;
 };
 /**
  * Shared transcription behavior, mirroring BaseClient for text generation:

package/dist/transcription/baseTranscriptionClient.js

@@ -1,5 +1,5 @@
-import { getModelForProvider, isSpeechToTextModel, } from "../models.js";
-import { calculateTranscriptionCost } from "../model.js";
+import { getModelForProvider, isSpeechToTextModel, modelSupportsInputModality, audioInputConstraints, } from "../models.js";
+import { Model, calculateTranscriptionCost } from "../model.js";
 import { success, failure } from "../types/result.js";
 import { loadBlob } from "../util/blobRef.js";
 import { audioFormatForMime, canonicalizeMime } from "../util/mime.js";
@@ -7,24 +7,24 @@ import { redactSecret } from "../util/redact.js";
 import { getLogger } from "../util/logger.js";
 export const DEFAULT_TRANSCRIBE_BYTES = 25 * 1024 * 1024;
 /** Validate the declarative STT constraint block once before consuming it. */
-function transcriptionConstraintError(model) {
-    const modelMaxBytes = model.maxBytes;
+function transcriptionConstraintError(modelName, c) {
+    const modelMaxBytes = c.maxBytes;
     if (modelMaxBytes !== undefined &&
         (typeof modelMaxBytes !== "number" || !Number.isFinite(modelMaxBytes) || modelMaxBytes <= 0)) {
-        return `Model "${model.modelName}" has an invalid maxBytes value.`;
+        return `Model "${modelName}" has an invalid maxBytes value.`;
     }
-    const supportedMimeTypes = model.supportedMimeTypes;
+    const supportedMimeTypes = c.supportedMimeTypes;
     if (supportedMimeTypes !== undefined &&
         (!Array.isArray(supportedMimeTypes) ||
             !supportedMimeTypes.every((mime) => typeof mime === "string"))) {
-        return `Model "${model.modelName}" has invalid supportedMimeTypes.`;
+        return `Model "${modelName}" has invalid supportedMimeTypes.`;
     }
     return null;
 }
 // The caller's maxBytes is a safety limit; the model's maxBytes is the
 // provider's hard cap. Take the smaller of whichever are present so a caller
 // can tighten the limit but never bypass the provider cap.
-function resolveTranscriptionMaxBytes(callerMaxBytes, model) {
+function resolveTranscriptionMaxBytes(callerMaxBytes, modelMaxBytes) {
     if (callerMaxBytes !== undefined &&
         (!Number.isFinite(callerMaxBytes) || callerMaxBytes <= 0)) {
         return failure(`maxBytes must be a positive finite number (got ${callerMaxBytes}).`);
@@ -33,8 +33,8 @@ function resolveTranscriptionMaxBytes(callerMaxBytes, model) {
     if (callerMaxBytes !== undefined) {
         limits.push(callerMaxBytes);
     }
-    if (model?.maxBytes !== undefined) {
-        limits.push(model.maxBytes);
+    if (modelMaxBytes !== undefined) {
+        limits.push(modelMaxBytes);
     }
     if (limits.length === 0) {
         return success(DEFAULT_TRANSCRIBE_BYTES);
@@ -53,18 +53,29 @@ export class BaseTranscriptionClient {
         this.config = config;
     }
     async transcribe(source) {
+        // Already-aborted signal: stop before doing any paid work.
+        if (this.config.abortSignal?.aborted) {
+            return failure("Request was aborted");
+        }
         try {
             const model = getModelForProvider(this.config.provider, this.config.model, this.config.modelData);
-            if (model !== undefined && !isSpeechToTextModel(model)) {
-                return failure(`Model "${this.config.model}" is not a speech-to-text model.`);
+            // A model is a valid transcription target if it is a dedicated STT model,
+            // or a multimodal text model that accepts audio input (e.g. Gemini). An
+            // unknown model (undefined) flows through — the provider is authority.
+            const acceptsAudio = model === undefined ||
+                isSpeechToTextModel(model) ||
+                modelSupportsInputModality(this.config.model, "audio", this.config.modelData, this.config.provider) === true;
+            if (!acceptsAudio) {
+                return failure(`Model "${this.config.model}" cannot accept audio input (not a transcription model).`);
             }
+            const constraints = model !== undefined ? audioInputConstraints(model) : {};
             if (model !== undefined) {
-                const constraintError = transcriptionConstraintError(model);
+                const constraintError = transcriptionConstraintError(this.config.model, constraints);
                 if (constraintError !== null) {
                     return failure(constraintError);
                 }
             }
-            const effectiveLimit = resolveTranscriptionMaxBytes(this.config.maxBytes, model);
+            const effectiveLimit = resolveTranscriptionMaxBytes(this.config.maxBytes, constraints.maxBytes);
             if (!effectiveLimit.success) {
                 return effectiveLimit;
             }
@@ -76,25 +87,40 @@ export class BaseTranscriptionClient {
                 return failure(`Failed to load audio for transcription: ${err.message}`);
             }
             const mimeType = loaded.mimeType ?? "application/octet-stream";
-            if (model !== undefined && model.supportedMimeTypes !== undefined) {
+            if (constraints.supportedMimeTypes !== undefined) {
                 const audioFormat = audioFormatForMime(mimeType);
                 const normalizedMime = audioFormat?.mimeType ?? canonicalizeMime(mimeType);
-                if (!model.supportedMimeTypes.includes(normalizedMime)) {
+                if (!constraints.supportedMimeTypes.includes(normalizedMime)) {
                     return failure(`Unsupported audio type "${mimeType}" for model "${this.config.model}". ` +
-                        `Supported: ${model.supportedMimeTypes.join(", ")}.`);
+                        `Supported: ${constraints.supportedMimeTypes.join(", ")}.`);
                 }
             }
+            // Re-check after the async preflight (blob load / MIME validation): the
+            // signal may have fired during it, and we must not dispatch a request.
+            if (this.config.abortSignal?.aborted) {
+                return failure("Request was aborted");
+            }
             const result = await this._transcribe(loaded.data, mimeType);
             if (!result.success) {
                 return result;
             }
-            const cost = calculateTranscriptionCost(model, result.value.durationSeconds);
+            let cost = calculateTranscriptionCost(model, result.value.durationSeconds);
+            if (cost === undefined && result.value.usage !== undefined) {
+                // Token-billed providers (Gemini) price through the shared cost engine.
+                cost =
+                    new Model(this.config.model, this.config.provider, this.config.modelData).calculateCost(result.value.usage) ?? undefined;
+            }
             if (cost !== undefined) {
                 result.value.cost = cost;
             }
             return result;
         }
         catch (err) {
+            // Caller-initiated cancellation surfaces as a distinguishable failure
+            // (matching the chat path), not a redacted provider error.
+            if (this.config.abortSignal?.aborted) {
+                return failure("Request was aborted");
+            }
             let msg = "transcribe() failed";
             if (err instanceof Error) {
                 msg = err.message;

package/dist/transcription/google.d.ts

@@ -0,0 +1,6 @@
+import { Result } from "../types/result.js";
+import { BaseTranscriptionClient } from "./baseTranscriptionClient.js";
+import type { TranscriptionResult } from "../transcription.js";
+export declare class GoogleTranscriptionClient extends BaseTranscriptionClient {
+    protected _transcribe(data: Uint8Array, mimeType: string): Promise<Result<TranscriptionResult>>;
+}

package/dist/transcription/google.js

@@ -0,0 +1,56 @@
+import { GoogleGenAI } from "@google/genai";
+import { success, failure } from "../types/result.js";
+import { BaseTranscriptionClient } from "./baseTranscriptionClient.js";
+import { normalizeGoogleAudioUsage } from "../util/googleAudioUsage.js";
+import { googleAudioWireMime } from "../util/audioMime.js";
+// Gemini's inline request cap is 20 MB for the *entire* request (audio bytes,
+// base64 expansion, prompt, and SDK envelope), not the raw file alone.
+const GOOGLE_INLINE_REQUEST_MAX_BYTES = 20_000_000;
+export class GoogleTranscriptionClient extends BaseTranscriptionClient {
+    // No try/catch: BaseTranscriptionClient.transcribe() is the exception boundary.
+    async _transcribe(data, mimeType) {
+        if (!this.config.apiKey) {
+            return failure("No Google API key provided. Set apiKey.google or GEMINI_API_KEY.");
+        }
+        if (this.config.timestampGranularity !== undefined) {
+            return failure("Gemini transcription does not support timestampGranularity.");
+        }
+        let instruction = "Transcribe the following audio verbatim. Output only the transcript text, with no commentary.";
+        if (this.config.language) {
+            instruction += ` The audio is in ${this.config.language}.`;
+        }
+        if (this.config.prompt) {
+            instruction += ` ${this.config.prompt}`;
+        }
+        const base64 = Buffer.from(data).toString("base64");
+        const request = {
+            model: this.config.model,
+            contents: [{
+                    role: "user",
+                    parts: [
+                        { inlineData: { mimeType: googleAudioWireMime(mimeType), data: base64 } },
+                        { text: instruction },
+                    ],
+                }],
+        };
+        if (Buffer.byteLength(JSON.stringify(request), "utf8") > GOOGLE_INLINE_REQUEST_MAX_BYTES) {
+            return failure("Audio and instructions exceed Gemini's 20 MB inline request limit; use a smaller source.");
+        }
+        const ai = new GoogleGenAI({ apiKey: this.config.apiKey });
+        // Signal is passed at call time (not part of `request`) so the encoded-size
+        // check above measures only the payload. Gemini's abortSignal is client-only:
+        // it tears down the request but does not stop server-side billing.
+        const res = await ai.models.generateContent({
+            ...request,
+            config: { abortSignal: this.config.abortSignal },
+        });
+        const usage = normalizeGoogleAudioUsage(res.usageMetadata, "input");
+        const result = {
+            text: res.text ?? "",
+            raw: res,
+        };
+        if (usage)
+            result.usage = usage;
+        return success(result);
+    }
+}

package/dist/transcription/groq.d.ts

@@ -0,0 +1,10 @@
+import OpenAI from "openai";
+import { OpenAITranscriptionClient } from "./openai.js";
+/**
+ * Groq exposes an OpenAI-compatible /audio/transcriptions endpoint (Whisper
+ * large-v3 / large-v3-turbo). Everything but the base URL is inherited.
+ */
+export declare class GroqTranscriptionClient extends OpenAITranscriptionClient {
+    protected makeClient(): OpenAI;
+    protected noKeyMessage(): string;
+}

package/dist/transcription/groq.js

@@ -0,0 +1,17 @@
+import OpenAI from "openai";
+import { OpenAITranscriptionClient } from "./openai.js";
+/**
+ * Groq exposes an OpenAI-compatible /audio/transcriptions endpoint (Whisper
+ * large-v3 / large-v3-turbo). Everything but the base URL is inherited.
+ */
+export class GroqTranscriptionClient extends OpenAITranscriptionClient {
+    makeClient() {
+        return new OpenAI({
+            apiKey: this.config.apiKey,
+            baseURL: "https://api.groq.com/openai/v1",
+        });
+    }
+    noKeyMessage() {
+        return "No Groq API key provided. Set apiKey.groq or GROQ_API_KEY.";
+    }
+}

package/dist/transcription/openai.d.ts

@@ -1,6 +1,11 @@
+import OpenAI from "openai";
 import { Result } from "../types/result.js";
 import { BaseTranscriptionClient } from "./baseTranscriptionClient.js";
 import type { TranscriptionResult } from "../transcription.js";
 export declare class OpenAITranscriptionClient extends BaseTranscriptionClient {
+    /** Build the OpenAI SDK client. Subclasses override to point at a compatible base URL. */
+    protected makeClient(): OpenAI;
+    /** Provider-specific diagnostic when no API key is resolved. Subclasses override. */
+    protected noKeyMessage(): string;
     protected _transcribe(data: Uint8Array, mimeType: string): Promise<Result<TranscriptionResult>>;
 }

package/dist/transcription/openai.js

@@ -3,16 +3,24 @@ import { success, failure } from "../types/result.js";
 import { transcriptionAudioType } from "../util/audioMime.js";
 import { BaseTranscriptionClient } from "./baseTranscriptionClient.js";
 export class OpenAITranscriptionClient extends BaseTranscriptionClient {
+    /** Build the OpenAI SDK client. Subclasses override to point at a compatible base URL. */
+    makeClient() {
+        return new OpenAI({ apiKey: this.config.apiKey });
+    }
+    /** Provider-specific diagnostic when no API key is resolved. Subclasses override. */
+    noKeyMessage() {
+        return "No OpenAI API key provided. Set apiKey.openAi or OPENAI_API_KEY.";
+    }
     // No try/catch here: BaseTranscriptionClient.transcribe() is the single
     // redacting/logging exception boundary.
     async _transcribe(data, mimeType) {
         if (!this.config.apiKey) {
-            return failure("No OpenAI API key provided. Set apiKey.openAi or OPENAI_API_KEY.");
+            return failure(this.noKeyMessage());
         }
         // Filename is an OpenAI upload detail, not part of the provider-neutral
         // operation contract. Derive the synthetic name from the normalized MIME.
         const filename = transcriptionAudioType(mimeType)?.filename ?? "audio.bin";
-        const client = new OpenAI({ apiKey: this.config.apiKey });
+        const client = this.makeClient();
         const file = await toFile(data, filename, { type: mimeType });
         const granularities = [];
         if (this.config.timestampGranularity) {
@@ -32,7 +40,7 @@ export class OpenAITranscriptionClient extends BaseTranscriptionClient {
         if (granularities.length > 0) {
             requestBody.timestamp_granularities = granularities;
         }
-        const res = (await client.audio.transcriptions.create(requestBody));
+        const res = (await client.audio.transcriptions.create(requestBody, { signal: this.config.abortSignal }));
         const result = { text: res.text, raw: res };
         if (res.language) {
             result.language = res.language;

package/dist/transcription/openaiCompat.d.ts

@@ -0,0 +1,13 @@
+import OpenAI from "openai";
+import { OpenAITranscriptionClient } from "./openai.js";
+/**
+ * Generic OpenAI-compatible transcription client. Point it at any provider
+ * exposing an OpenAI-shaped /audio/transcriptions endpoint via
+ * `config.baseUrl.openAiCompat` (or OPENAI_COMPAT_BASE_URL) and
+ * `config.apiKey.openAiCompat` (or OPENAI_COMPAT_API_KEY). Mirrors the chat
+ * `SmolOpenAiCompat` client.
+ */
+export declare class OpenAiCompatTranscriptionClient extends OpenAITranscriptionClient {
+    protected makeClient(): OpenAI;
+    protected noKeyMessage(): string;
+}

package/dist/transcription/openaiCompat.js

@@ -0,0 +1,22 @@
+import OpenAI from "openai";
+import { OpenAITranscriptionClient } from "./openai.js";
+import { resolveBaseUrl } from "../util/provider.js";
+/**
+ * Generic OpenAI-compatible transcription client. Point it at any provider
+ * exposing an OpenAI-shaped /audio/transcriptions endpoint via
+ * `config.baseUrl.openAiCompat` (or OPENAI_COMPAT_BASE_URL) and
+ * `config.apiKey.openAiCompat` (or OPENAI_COMPAT_API_KEY). Mirrors the chat
+ * `SmolOpenAiCompat` client.
+ */
+export class OpenAiCompatTranscriptionClient extends OpenAITranscriptionClient {
+    makeClient() {
+        const baseURL = resolveBaseUrl("openai-compat", { baseUrl: this.config.baseUrl });
+        if (!baseURL) {
+            throw new Error("openai-compat: base URL required (config.baseUrl.openAiCompat or OPENAI_COMPAT_BASE_URL).");
+        }
+        return new OpenAI({ apiKey: this.config.apiKey, baseURL });
+    }
+    noKeyMessage() {
+        return "No API key provided. Set apiKey.openAiCompat or OPENAI_COMPAT_API_KEY.";
+    }
+}

package/dist/transcription.d.ts

@@ -11,11 +11,14 @@ export type TranscribeOptions = {
     provider?: string;
     modelData?: ModelDataBlob;
     apiKey?: SmolConfig["apiKey"];
+    baseUrl?: SmolConfig["baseUrl"];
     language?: string;
     prompt?: string;
     timestampGranularity?: "segment" | "word";
     maxBytes?: number;
     metadata?: Record<string, unknown>;
+    /** Abort the in-flight provider request when this signal fires. */
+    abortSignal?: AbortSignal;
 };
 export type TranscriptionSegment = {
     start: number;

package/dist/transcription.js

@@ -3,10 +3,16 @@ import { redactSecret } from "./util/redact.js";
 import { getLogger } from "./util/logger.js";
 import { resolveProvider, resolveApiKey } from "./util/provider.js";
 import { OpenAITranscriptionClient } from "./transcription/openai.js";
+import { GroqTranscriptionClient } from "./transcription/groq.js";
+import { GoogleTranscriptionClient } from "./transcription/google.js";
+import { OpenAiCompatTranscriptionClient } from "./transcription/openaiCompat.js";
 export { DEFAULT_TRANSCRIBE_BYTES } from "./transcription/baseTranscriptionClient.js";
 // Checked before the user registry so a registered "openai" can't hijack the built-in.
 const builtinClients = Object.create(null);
 builtinClients["openai"] = OpenAITranscriptionClient;
+builtinClients["groq"] = GroqTranscriptionClient;
+builtinClients["google"] = GoogleTranscriptionClient;
+builtinClients["openai-compat"] = OpenAiCompatTranscriptionClient;
 // Null-prototype so provider names like "toString"/"__proto__" can't collide
 // with Object.prototype or pollute the registry.
 const registered = Object.create(null);

package/dist/types.d.ts

@@ -30,6 +30,7 @@ export type SmolConfig = {
         deepInfra?: string;
         liteLlm?: string;
         openAiCompat?: string;
+        groq?: string;
         /** Arbitrary provider names, for keys targeting a custom-registered provider
          *  (e.g. registerProvider("acme", ...)), keyed by the exact registered name. */
         [provider: string]: string | undefined;

package/dist/util/audioMime.d.ts

@@ -7,3 +7,20 @@ export declare function transcriptionAudioType(mime: string): TranscriptionAudio
 export declare function chatAudioFormat(mime: string): "mp3" | "wav" | null;
 export declare const SPEECH_FORMAT_TO_MIME: Record<SpeakFormat, string>;
 export declare function isSpeakFormat(value: string): value is SpeakFormat;
+/**
+ * Translate a canonical/alias audio MIME to the wire value Google expects.
+ * Google documents MP3 as `audio/mp3`, whereas this repo canonicalizes it to
+ * `audio/mpeg`; everything else passes through canonicalized.
+ */
+export declare function googleAudioWireMime(mimeType: string): string;
+export type PcmWavOptions = {
+    sampleRateHz: number;
+    channels: number;
+    bitsPerSample: number;
+};
+/**
+ * Wrap raw signed-integer little-endian PCM in a 44-byte RIFF/WAVE header so it
+ * becomes a directly-playable .wav. Pure function, no dependency. Used for
+ * Gemini TTS output, which is only ever raw PCM.
+ */
+export declare function pcmToWav(pcm: Uint8Array, opts: PcmWavOptions): Uint8Array;

package/dist/util/audioMime.js

@@ -1,4 +1,4 @@
-import { audioFormatForMime } from "./mime.js";
+import { audioFormatForMime, canonicalizeMime } from "./mime.js";
 export function transcriptionAudioType(mime) {
     const format = audioFormatForMime(mime);
     if (format === null) {
@@ -31,3 +31,44 @@ export const SPEECH_FORMAT_TO_MIME = {
 export function isSpeakFormat(value) {
     return Object.hasOwn(SPEECH_FORMAT_TO_MIME, value);
 }
+/**
+ * Translate a canonical/alias audio MIME to the wire value Google expects.
+ * Google documents MP3 as `audio/mp3`, whereas this repo canonicalizes it to
+ * `audio/mpeg`; everything else passes through canonicalized.
+ */
+export function googleAudioWireMime(mimeType) {
+    const format = audioFormatForMime(mimeType);
+    const canonical = format?.mimeType ?? canonicalizeMime(mimeType);
+    return canonical === "audio/mpeg" ? "audio/mp3" : canonical;
+}
+/**
+ * Wrap raw signed-integer little-endian PCM in a 44-byte RIFF/WAVE header so it
+ * becomes a directly-playable .wav. Pure function, no dependency. Used for
+ * Gemini TTS output, which is only ever raw PCM.
+ */
+export function pcmToWav(pcm, opts) {
+    const { sampleRateHz, channels, bitsPerSample } = opts;
+    const blockAlign = (channels * bitsPerSample) / 8;
+    const byteRate = sampleRateHz * blockAlign;
+    const out = new Uint8Array(44 + pcm.length);
+    const view = new DataView(out.buffer);
+    const writeAscii = (offset, s) => {
+        for (let i = 0; i < s.length; i++)
+            view.setUint8(offset + i, s.charCodeAt(i));
+    };
+    writeAscii(0, "RIFF");
+    view.setUint32(4, 36 + pcm.length, true);
+    writeAscii(8, "WAVE");
+    writeAscii(12, "fmt ");
+    view.setUint32(16, 16, true); // PCM fmt chunk size
+    view.setUint16(20, 1, true); // audio format = PCM
+    view.setUint16(22, channels, true);
+    view.setUint32(24, sampleRateHz, true);
+    view.setUint32(28, byteRate, true);
+    view.setUint16(32, blockAlign, true);
+    view.setUint16(34, bitsPerSample, true);
+    writeAscii(36, "data");
+    view.setUint32(40, pcm.length, true);
+    out.set(pcm, 44);
+    return out;
+}

package/dist/util/googleAudioUsage.d.ts

@@ -0,0 +1,14 @@
+import type { GenerateContentResponseUsageMetadata } from "@google/genai";
+import type { TokenUsage } from "../types/tokenUsage.js";
+export type GoogleAudioDirection = "input" | "output";
+/**
+ * Normalize Gemini's usage metadata into smoltalk's TokenUsage, separating the
+ * audio bucket for the given direction so the token-priced cost engine can bill
+ * it at the audio rate. STT sets audioDirection "input" (audio prompt tokens);
+ * TTS sets "output" (audio candidate tokens).
+ *
+ * Thinking tokens (`thoughtsTokenCount`) are reported separately from
+ * `candidatesTokenCount` and are always text — they go into the text-output
+ * bucket, never the audio-output bucket.
+ */
+export declare function normalizeGoogleAudioUsage(metadata: GenerateContentResponseUsageMetadata | undefined, audioDirection: GoogleAudioDirection): TokenUsage | undefined;

package/dist/util/googleAudioUsage.js

@@ -0,0 +1,52 @@
+function modalityTokens(details, modality) {
+    return (details ?? [])
+        // `modality` is a MediaModality enum; compare by its string value.
+        .filter((detail) => String(detail.modality) === modality)
+        .reduce((sum, detail) => sum + (detail.tokenCount ?? 0), 0);
+}
+/**
+ * Normalize Gemini's usage metadata into smoltalk's TokenUsage, separating the
+ * audio bucket for the given direction so the token-priced cost engine can bill
+ * it at the audio rate. STT sets audioDirection "input" (audio prompt tokens);
+ * TTS sets "output" (audio candidate tokens).
+ *
+ * Thinking tokens (`thoughtsTokenCount`) are reported separately from
+ * `candidatesTokenCount` and are always text — they go into the text-output
+ * bucket, never the audio-output bucket.
+ */
+export function normalizeGoogleAudioUsage(metadata, audioDirection) {
+    if (metadata === undefined) {
+        return undefined;
+    }
+    const prompt = metadata.promptTokenCount ?? 0;
+    const candidates = metadata.candidatesTokenCount ?? 0;
+    const thoughts = metadata.thoughtsTokenCount ?? 0;
+    if (audioDirection === "input") {
+        const audioIn = modalityTokens(metadata.promptTokensDetails, "AUDIO");
+        const usage = {
+            inputTokens: Math.max(0, prompt - audioIn),
+            outputTokens: candidates + thoughts,
+        };
+        if (audioIn > 0)
+            usage.inputAudioTokens = audioIn;
+        if (metadata.totalTokenCount !== undefined)
+            usage.totalTokens = metadata.totalTokenCount;
+        return usage;
+    }
+    // Output (TTS): split the audio bucket out of candidate tokens. Only assume
+    // "all candidates are audio" when the details array is missing/empty; when it
+    // is present, trust its AUDIO total even if that total is 0 (otherwise
+    // purely-text candidate tokens would be mispriced at the audio rate).
+    const details = metadata.candidatesTokensDetails;
+    const detailsPresent = Array.isArray(details) && details.length > 0;
+    const audioOut = detailsPresent ? modalityTokens(details, "AUDIO") : candidates;
+    const usage = {
+        inputTokens: prompt,
+        outputTokens: Math.max(0, candidates - audioOut) + thoughts,
+    };
+    if (audioOut > 0)
+        usage.outputAudioTokens = audioOut;
+    if (metadata.totalTokenCount !== undefined)
+        usage.totalTokens = metadata.totalTokenCount;
+    return usage;
+}

package/dist/util/mime.js

@@ -11,6 +11,8 @@ export const AUDIO_FORMATS = [
     { extension: "ogg", mimeType: "audio/ogg", aliasMimeTypes: [], aliasExtensions: [] },
     { extension: "flac", mimeType: "audio/flac", aliasMimeTypes: [], aliasExtensions: [] },
     { extension: "webm", mimeType: "audio/webm", aliasMimeTypes: [], aliasExtensions: [] },
+    { extension: "aac", mimeType: "audio/aac", aliasMimeTypes: [], aliasExtensions: [] },
+    { extension: "aiff", mimeType: "audio/aiff", aliasMimeTypes: ["audio/x-aiff"], aliasExtensions: ["aif"] },
 ];
 const IMAGE_AND_DOCUMENT_EXT_TO_MIME = {
     ".png": "image/png",

package/dist/util/provider.d.ts

@@ -19,6 +19,7 @@ type NestedKeyConfig = {
         deepInfra?: string;
         liteLlm?: string;
         openAiCompat?: string;
+        groq?: string;
         /** Arbitrary provider names, for keys targeting a custom-registered provider. */
         [provider: string]: string | undefined;
     };

package/dist/util/provider.js

@@ -37,6 +37,8 @@ export function resolveApiKey(provider, config) {
             return k?.liteLlm || process.env.LITELLM_API_KEY;
         case "openai-compat":
             return k?.openAiCompat || process.env.OPENAI_COMPAT_API_KEY;
+        case "groq":
+            return k?.groq || process.env.GROQ_API_KEY;
         default:
             return config.apiKey?.[provider];
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smoltalk",
-  "version": "0.10.0",
+  "version": "0.10.1",
   "description": "A common interface for LLM APIs",
   "homepage": "https://github.com/egonSchiele/smoltalk",
   "files": [