smoltalk 0.8.4 → 0.10.0

package/dist/transcription/openai.d.ts

@@ -0,0 +1,6 @@
+import { Result } from "../types/result.js";
+import { BaseTranscriptionClient } from "./baseTranscriptionClient.js";
+import type { TranscriptionResult } from "../transcription.js";
+export declare class OpenAITranscriptionClient extends BaseTranscriptionClient {
+    protected _transcribe(data: Uint8Array, mimeType: string): Promise<Result<TranscriptionResult>>;
+}

package/dist/transcription/openai.js

@@ -0,0 +1,59 @@
+import OpenAI, { toFile } from "openai";
+import { success, failure } from "../types/result.js";
+import { transcriptionAudioType } from "../util/audioMime.js";
+import { BaseTranscriptionClient } from "./baseTranscriptionClient.js";
+export class OpenAITranscriptionClient extends BaseTranscriptionClient {
+    // No try/catch here: BaseTranscriptionClient.transcribe() is the single
+    // redacting/logging exception boundary.
+    async _transcribe(data, mimeType) {
+        if (!this.config.apiKey) {
+            return failure("No OpenAI API key provided. Set apiKey.openAi or OPENAI_API_KEY.");
+        }
+        // Filename is an OpenAI upload detail, not part of the provider-neutral
+        // operation contract. Derive the synthetic name from the normalized MIME.
+        const filename = transcriptionAudioType(mimeType)?.filename ?? "audio.bin";
+        const client = new OpenAI({ apiKey: this.config.apiKey });
+        const file = await toFile(data, filename, { type: mimeType });
+        const granularities = [];
+        if (this.config.timestampGranularity) {
+            granularities.push(this.config.timestampGranularity);
+        }
+        const requestBody = {
+            file,
+            model: this.config.model,
+            response_format: "verbose_json",
+        };
+        if (this.config.language) {
+            requestBody.language = this.config.language;
+        }
+        if (this.config.prompt) {
+            requestBody.prompt = this.config.prompt;
+        }
+        if (granularities.length > 0) {
+            requestBody.timestamp_granularities = granularities;
+        }
+        const res = (await client.audio.transcriptions.create(requestBody));
+        const result = { text: res.text, raw: res };
+        if (res.language) {
+            result.language = res.language;
+        }
+        if (typeof res.duration === "number") {
+            result.durationSeconds = res.duration;
+        }
+        if (Array.isArray(res.segments)) {
+            result.segments = res.segments.map((segment) => ({
+                start: segment.start,
+                end: segment.end,
+                text: segment.text,
+            }));
+        }
+        if (Array.isArray(res.words)) {
+            result.words = res.words.map((word) => ({
+                start: word.start,
+                end: word.end,
+                word: word.word,
+            }));
+        }
+        return success(result);
+    }
+}

package/dist/transcription.d.ts

@@ -0,0 +1,51 @@
+import type { ModelDataBlob } from "./modelData.js";
+import type { SmolConfig } from "./types.js";
+import { Result } from "./types/result.js";
+import { TokenUsage } from "./types/tokenUsage.js";
+import { CostEstimate } from "./types/costEstimate.js";
+import { BlobRef } from "./util/blobRef.js";
+import { BaseTranscriptionClient, TranscriptionClientConfig } from "./transcription/baseTranscriptionClient.js";
+export { DEFAULT_TRANSCRIBE_BYTES } from "./transcription/baseTranscriptionClient.js";
+export type TranscribeOptions = {
+    model: string;
+    provider?: string;
+    modelData?: ModelDataBlob;
+    apiKey?: SmolConfig["apiKey"];
+    language?: string;
+    prompt?: string;
+    timestampGranularity?: "segment" | "word";
+    maxBytes?: number;
+    metadata?: Record<string, unknown>;
+};
+export type TranscriptionSegment = {
+    start: number;
+    end: number;
+    text: string;
+};
+export type TranscriptionWord = {
+    start: number;
+    end: number;
+    word: string;
+};
+export type TranscriptionResult = {
+    text: string;
+    language?: string;
+    durationSeconds?: number;
+    segments?: TranscriptionSegment[];
+    words?: TranscriptionWord[];
+    usage?: TokenUsage;
+    cost?: CostEstimate;
+    raw?: unknown;
+};
+export type TranscriptionClientClass = new (config: TranscriptionClientConfig) => BaseTranscriptionClient;
+export declare function registerTranscriptionProvider(name: string, cls: TranscriptionClientClass): void;
+/** Test-only: clear all registered custom providers so registrations don't leak across tests. */
+export declare function _resetForTests(): void;
+/**
+ * Resolve provider + API key and instantiate the matching transcription client
+ * for the declarative transcribe() operation. Never throws: a custom client
+ * class's constructor can throw, and this internal factory's catch redacts the
+ * resolved key so a constructor error cannot leak through the public wrapper.
+ */
+export declare function getTranscriptionClient(opts: TranscribeOptions): Result<BaseTranscriptionClient>;
+export declare function transcribe(source: BlobRef, opts: TranscribeOptions): Promise<Result<TranscriptionResult>>;

package/dist/transcription.js

@@ -0,0 +1,58 @@
+import { success, failure } from "./types/result.js";
+import { redactSecret } from "./util/redact.js";
+import { getLogger } from "./util/logger.js";
+import { resolveProvider, resolveApiKey } from "./util/provider.js";
+import { OpenAITranscriptionClient } from "./transcription/openai.js";
+export { DEFAULT_TRANSCRIBE_BYTES } from "./transcription/baseTranscriptionClient.js";
+// Checked before the user registry so a registered "openai" can't hijack the built-in.
+const builtinClients = Object.create(null);
+builtinClients["openai"] = OpenAITranscriptionClient;
+// Null-prototype so provider names like "toString"/"__proto__" can't collide
+// with Object.prototype or pollute the registry.
+const registered = Object.create(null);
+export function registerTranscriptionProvider(name, cls) {
+    registered[name] = cls;
+}
+/** Test-only: clear all registered custom providers so registrations don't leak across tests. */
+export function _resetForTests() {
+    for (const key of Object.keys(registered)) {
+        delete registered[key];
+    }
+}
+/**
+ * Resolve provider + API key and instantiate the matching transcription client
+ * for the declarative transcribe() operation. Never throws: a custom client
+ * class's constructor can throw, and this internal factory's catch redacts the
+ * resolved key so a constructor error cannot leak through the public wrapper.
+ */
+export function getTranscriptionClient(opts) {
+    let apiKeyForRedaction = "";
+    try {
+        const provider = resolveProvider(opts.model, opts.provider, opts.modelData);
+        const ClientClass = builtinClients[provider] ?? registered[provider];
+        if (ClientClass === undefined) {
+            return failure(`Provider "${provider}" has no transcription API. Register one with registerTranscriptionProvider(name, ClientClass).`);
+        }
+        const apiKey = resolveApiKey(provider, opts) ?? "";
+        apiKeyForRedaction = apiKey;
+        const { apiKey: _callerKeys, ...clientOpts } = opts;
+        const config = { ...clientOpts, provider, apiKey };
+        return success(new ClientClass(config));
+    }
+    catch (err) {
+        let msg = "getTranscriptionClient() failed";
+        if (err instanceof Error) {
+            msg = err.message;
+        }
+        const redacted = redactSecret(msg, apiKeyForRedaction);
+        getLogger().error("getTranscriptionClient() failed:", redacted);
+        return failure(redacted);
+    }
+}
+export async function transcribe(source, opts) {
+    const client = getTranscriptionClient(opts);
+    if (!client.success) {
+        return client;
+    }
+    return client.value.transcribe(source);
+}

package/dist/types/tokenUsage.d.ts

@@ -4,6 +4,8 @@ export type TokenUsage = {
     outputTokens: number;
     cachedInputTokens?: number;
     cacheCreationInputTokens?: number;
+    inputAudioTokens?: number;
+    outputAudioTokens?: number;
     totalTokens?: number;
 };
 export declare const TokenUsageSchema: z.ZodObject<{
@@ -11,6 +13,8 @@ export declare const TokenUsageSchema: z.ZodObject<{
     outputTokens: z.ZodNumber;
     cachedInputTokens: z.ZodOptional<z.ZodNumber>;
     cacheCreationInputTokens: z.ZodOptional<z.ZodNumber>;
+    inputAudioTokens: z.ZodOptional<z.ZodNumber>;
+    outputAudioTokens: z.ZodOptional<z.ZodNumber>;
     totalTokens: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 export declare function addTokenUsage(_a?: TokenUsage, _b?: TokenUsage): TokenUsage;

package/dist/types/tokenUsage.js

@@ -4,6 +4,8 @@ export const TokenUsageSchema = z.object({
     outputTokens: z.number(),
     cachedInputTokens: z.number().optional(),
     cacheCreationInputTokens: z.number().optional(),
+    inputAudioTokens: z.number().optional(),
+    outputAudioTokens: z.number().optional(),
     totalTokens: z.number().optional(),
 });
 export function addTokenUsage(_a, _b) {
@@ -22,6 +24,8 @@ export function addTokenUsage(_a, _b) {
         outputTokens: a.outputTokens + b.outputTokens,
         cachedInputTokens: (a.cachedInputTokens || 0) + (b.cachedInputTokens || 0),
         cacheCreationInputTokens: (a.cacheCreationInputTokens || 0) + (b.cacheCreationInputTokens || 0),
+        inputAudioTokens: (a.inputAudioTokens || 0) + (b.inputAudioTokens || 0),
+        outputAudioTokens: (a.outputAudioTokens || 0) + (b.outputAudioTokens || 0),
         totalTokens: (a.totalTokens || 0) + (b.totalTokens || 0),
     };
 }

package/dist/types.d.ts

@@ -30,6 +30,9 @@ export type SmolConfig = {
         deepInfra?: string;
         liteLlm?: string;
         openAiCompat?: string;
+        /** Arbitrary provider names, for keys targeting a custom-registered provider
+         *  (e.g. registerProvider("acme", ...)), keyed by the exact registered name. */
+        [provider: string]: string | undefined;
     };
     /** Custom base URLs, nested by provider. Defaults are baked in where applicable
      *  (e.g. openrouter, deepinfra); litellm and openai-compat require an explicit URL. */

package/dist/util/attachments.d.ts

@@ -1,4 +1,4 @@
-import type { ImageRef } from "./imageRef.js";
+import type { ImageRef } from "./blobRef.js";
 /** Filename sent to providers that require one when a file part omits its own. */
 export declare const DEFAULT_ATTACHMENT_FILENAME = "attachment.pdf";
 /** A file part's filename, falling back to the shared default when unset. */

package/dist/util/audioMime.d.ts

@@ -0,0 +1,9 @@
+export type SpeakFormat = "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm";
+export type TranscriptionAudioType = {
+    extension: string;
+    filename: string;
+};
+export declare function transcriptionAudioType(mime: string): TranscriptionAudioType | null;
+export declare function chatAudioFormat(mime: string): "mp3" | "wav" | null;
+export declare const SPEECH_FORMAT_TO_MIME: Record<SpeakFormat, string>;
+export declare function isSpeakFormat(value: string): value is SpeakFormat;

package/dist/util/audioMime.js

@@ -0,0 +1,33 @@
+import { audioFormatForMime } from "./mime.js";
+export function transcriptionAudioType(mime) {
+    const format = audioFormatForMime(mime);
+    if (format === null) {
+        return null;
+    }
+    return { extension: format.extension, filename: `audio.${format.extension}` };
+}
+export function chatAudioFormat(mime) {
+    const format = audioFormatForMime(mime);
+    if (format === null) {
+        return null;
+    }
+    if (format.extension === "mp3" || format.extension === "wav") {
+        return format.extension;
+    }
+    return null;
+}
+// PCM from OpenAI is headerless s16le / 24kHz / mono, which audio/L16 (big-endian
+// per RFC) would misdescribe — use octet-stream + structured metadata instead.
+export const SPEECH_FORMAT_TO_MIME = {
+    mp3: "audio/mpeg",
+    opus: "audio/ogg",
+    aac: "audio/aac",
+    flac: "audio/flac",
+    wav: "audio/wav",
+    pcm: "application/octet-stream",
+};
+// Object.hasOwn (not `in`) so prototype keys like "toString"/"__proto__"
+// never pass the guard.
+export function isSpeakFormat(value) {
+    return Object.hasOwn(SPEECH_FORMAT_TO_MIME, value);
+}

package/dist/util/{imageRef.d.ts → blobRef.d.ts}

@@ -1,4 +1,4 @@
-export type ImageRef = {
+export type BlobRef = {
     kind: "bytes";
     data: Uint8Array;
     mimeType: string;
@@ -20,23 +20,23 @@ export type ImageRef = {
      */
     timeoutMs?: number;
 };
-/** Neutral alias of the source union for non-image uses (uploads via {@link loadBlob}). */
-export type BlobRef = ImageRef;
-export type NormalizedImage = {
+/** Alias of {@link BlobRef} for image call sites, where it documents intent. */
+export type ImageRef = BlobRef;
+export type NormalizedBlob = {
     data: Uint8Array;
     mimeType: string;
 };
-/** Default timeout for fetching image URLs during normalization (60 seconds). */
+/** Default timeout for fetching attachment URLs during normalization (60 seconds). */
 export declare const DEFAULT_FETCH_TIMEOUT_MS = 60000;
-export declare function normalizeImageRef(ref: ImageRef, options?: {
+export declare function normalizeBlob(ref: BlobRef, options?: {
     allowedMimePrefixes?: string[];
     maxBytes?: number;
-}): Promise<NormalizedImage>;
+}): Promise<NormalizedBlob>;
 /**
- * Load a source to bytes WITHOUT the image MIME gate (used by file uploads,
+ * Load a source to bytes WITHOUT the MIME gate (used by file uploads,
  * which accept any type). Enforces `maxBytes` across all kinds.
  */
-export declare function loadBlob(ref: ImageRef, options?: {
+export declare function loadBlob(ref: BlobRef, options?: {
     maxBytes?: number;
 }): Promise<{
     data: Uint8Array;

package/dist/util/{imageRef.js → blobRef.js}

@@ -1,15 +1,8 @@
 import { readFile, stat } from "node:fs/promises";
 import { extname } from "node:path";
-/** Default timeout for fetching image URLs during normalization (60 seconds). */
+import { EXT_TO_MIME } from "./mime.js";
+/** Default timeout for fetching attachment URLs during normalization (60 seconds). */
 export const DEFAULT_FETCH_TIMEOUT_MS = 60_000;
-const EXT_TO_MIME = {
-    ".png": "image/png",
-    ".jpg": "image/jpeg",
-    ".jpeg": "image/jpeg",
-    ".webp": "image/webp",
-    ".gif": "image/gif",
-    ".pdf": "application/pdf",
-};
 function isAllowedMime(mimeType, allowedPrefixes) {
     for (const prefix of allowedPrefixes) {
         if (mimeType.startsWith(prefix)) {
@@ -18,7 +11,7 @@ function isAllowedMime(mimeType, allowedPrefixes) {
     }
     return false;
 }
-export async function normalizeImageRef(ref, options = {}) {
+export async function normalizeBlob(ref, options = {}) {
     const allowed = options.allowedMimePrefixes ?? ["image/"];
     const result = await loadRef(ref, allowed, options.maxBytes);
     if (options.maxBytes !== undefined && result.data.length > options.maxBytes) {
@@ -31,7 +24,7 @@ export async function normalizeImageRef(ref, options = {}) {
     return { data: result.data, mimeType: result.mimeType };
 }
 /**
- * Load a source to bytes WITHOUT the image MIME gate (used by file uploads,
+ * Load a source to bytes WITHOUT the MIME gate (used by file uploads,
  * which accept any type). Enforces `maxBytes` across all kinds.
  */
 export async function loadBlob(ref, options = {}) {
@@ -94,7 +87,7 @@ async function loadRef(ref, allowed, maxBytes) {
             const mimeType = ref.mimeType ?? inferred;
             if (allowed !== null && (!mimeType || !isAllowedMime(mimeType, allowed))) {
                 throw new Error(`Could not determine an allowed MIME type for path "${ref.path}". ` +
-                    `Allowed: ${allowed.join(", ")}. Pass an explicit mimeType on the ImageRef.`);
+                    `Allowed: ${allowed.join(", ")}. Pass an explicit mimeType on the BlobRef.`);
             }
             return { data: new Uint8Array(buf), mimeType };
         }
@@ -128,7 +121,7 @@ async function loadRef(ref, allowed, maxBytes) {
             if (allowed !== null && (!mimeType || !isAllowedMime(mimeType, allowed))) {
                 throw new Error(`Could not determine an allowed MIME type for URL "${ref.url}". ` +
                     `Response Content-Type was "${contentType ?? "missing"}". ` +
-                    `Allowed: ${allowed.join(", ")}. Pass an explicit mimeType on the ImageRef.`);
+                    `Allowed: ${allowed.join(", ")}. Pass an explicit mimeType on the BlobRef.`);
             }
             return { data: buf, mimeType };
         }

package/dist/util/mime.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Shared extension/MIME knowledge for attachments and audio. One table per
+ * format; every other ext↔MIME mapping in the package derives from these so
+ * the forward and inverse maps can't drift apart.
+ */
+export type AudioFormat = {
+    /** Primary file extension, without the dot. */
+    extension: string;
+    /** Canonical MIME type. */
+    mimeType: string;
+    /** Other MIME strings that identify the same container. */
+    aliasMimeTypes: readonly string[];
+    /** Other extensions that map to this format. */
+    aliasExtensions: readonly string[];
+};
+export declare const AUDIO_FORMATS: readonly AudioFormat[];
+/** Extension (with leading dot, lowercase) → MIME, across images, PDF, and audio. */
+export declare const EXT_TO_MIME: Record<string, string>;
+export declare function canonicalizeMime(mime: string): string;
+/** The audio format a MIME string identifies, or null when unrecognized. */
+export declare function audioFormatForMime(mime: string): AudioFormat | null;

package/dist/util/mime.js

@@ -0,0 +1,52 @@
+/**
+ * Shared extension/MIME knowledge for attachments and audio. One table per
+ * format; every other ext↔MIME mapping in the package derives from these so
+ * the forward and inverse maps can't drift apart.
+ */
+export const AUDIO_FORMATS = [
+    { extension: "mp3", mimeType: "audio/mpeg", aliasMimeTypes: ["audio/mp3"], aliasExtensions: ["mpeg", "mpga"] },
+    { extension: "wav", mimeType: "audio/wav", aliasMimeTypes: ["audio/x-wav"], aliasExtensions: [] },
+    { extension: "m4a", mimeType: "audio/m4a", aliasMimeTypes: ["audio/x-m4a"], aliasExtensions: [] },
+    { extension: "mp4", mimeType: "audio/mp4", aliasMimeTypes: ["video/mp4"], aliasExtensions: [] },
+    { extension: "ogg", mimeType: "audio/ogg", aliasMimeTypes: [], aliasExtensions: [] },
+    { extension: "flac", mimeType: "audio/flac", aliasMimeTypes: [], aliasExtensions: [] },
+    { extension: "webm", mimeType: "audio/webm", aliasMimeTypes: [], aliasExtensions: [] },
+];
+const IMAGE_AND_DOCUMENT_EXT_TO_MIME = {
+    ".png": "image/png",
+    ".jpg": "image/jpeg",
+    ".jpeg": "image/jpeg",
+    ".webp": "image/webp",
+    ".gif": "image/gif",
+    ".pdf": "application/pdf",
+};
+function buildExtToMime() {
+    const table = { ...IMAGE_AND_DOCUMENT_EXT_TO_MIME };
+    for (const format of AUDIO_FORMATS) {
+        table[`.${format.extension}`] = format.mimeType;
+        for (const alias of format.aliasExtensions) {
+            table[`.${alias}`] = format.mimeType;
+        }
+    }
+    return table;
+}
+/** Extension (with leading dot, lowercase) → MIME, across images, PDF, and audio. */
+export const EXT_TO_MIME = buildExtToMime();
+// Strips parameters (e.g. ";codecs=opus") and normalizes case, so MediaRecorder-
+// style MIME strings like "audio/webm;codecs=opus" or "AUDIO/MPEG" still match.
+export function canonicalizeMime(mime) {
+    return mime.split(";")[0].trim().toLowerCase();
+}
+/** The audio format a MIME string identifies, or null when unrecognized. */
+export function audioFormatForMime(mime) {
+    const canonical = canonicalizeMime(mime);
+    for (const format of AUDIO_FORMATS) {
+        if (format.mimeType === canonical) {
+            return format;
+        }
+        if (format.aliasMimeTypes.includes(canonical)) {
+            return format;
+        }
+    }
+    return null;
+}

package/dist/util/modalities.d.ts

@@ -1,2 +1,6 @@
-import { PromptResult, Result, SmolConfig } from "../types.js";
-export declare function validateModalities(config: SmolConfig): Result<PromptResult> | null;
+import { Message } from "../classes/message/index.js";
+/** Modalities a model must positively declare in its data block — for these,
+ *  "unknown" means "unsupported" (audio serialization is model-specific). */
+export declare const MODALITIES_REQUIRING_DECLARATION: ReadonlySet<string>;
+/** Which non-text input modalities the user messages actually use. */
+export declare function neededInputModalities(messages: Message[]): string[];

package/dist/util/modalities.js

@@ -1,10 +1,11 @@
 import { UserMessage } from "../classes/message/index.js";
-import { modelSupportsInputModality } from "../models.js";
-import { failure } from "../types.js";
-export function validateModalities(config) {
-    let needsImage = false;
-    let needsPdf = false;
-    for (const msg of config.messages) {
+/** Modalities a model must positively declare in its data block — for these,
+ *  "unknown" means "unsupported" (audio serialization is model-specific). */
+export const MODALITIES_REQUIRING_DECLARATION = new Set(["audio"]);
+/** Which non-text input modalities the user messages actually use. */
+export function neededInputModalities(messages) {
+    const needed = new Set();
+    for (const msg of messages) {
         if (!(msg instanceof UserMessage)) {
             continue;
         }
@@ -14,18 +15,15 @@ export function validateModalities(config) {
         }
         for (const part of parts) {
             if (part.type === "image") {
-                needsImage = true;
+                needed.add("image");
             }
             if (part.type === "file") {
-                needsPdf = true;
+                needed.add("pdf");
+            }
+            if (part.type === "audio") {
+                needed.add("audio");
             }
         }
     }
-    if (needsImage && modelSupportsInputModality(config.model, "image", config.modelData) === false) {
-        return failure(`Model ${config.model} does not support image input.`);
-    }
-    if (needsPdf && modelSupportsInputModality(config.model, "pdf", config.modelData) === false) {
-        return failure(`Model ${config.model} does not support PDF/document input.`);
-    }
-    return null;
+    return [...needed];
 }

package/dist/util/provider.d.ts

@@ -19,6 +19,8 @@ type NestedKeyConfig = {
         deepInfra?: string;
         liteLlm?: string;
         openAiCompat?: string;
+        /** Arbitrary provider names, for keys targeting a custom-registered provider. */
+        [provider: string]: string | undefined;
     };
     baseUrl?: {
         ollama?: string;

package/dist/util/provider.js

@@ -38,7 +38,7 @@ export function resolveApiKey(provider, config) {
         case "openai-compat":
             return k?.openAiCompat || process.env.OPENAI_COMPAT_API_KEY;
         default:
-            return undefined;
+            return config.apiKey?.[provider];
     }
 }
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smoltalk",
-  "version": "0.8.4",
+  "version": "0.10.0",
   "description": "A common interface for LLM APIs",
   "homepage": "https://github.com/egonSchiele/smoltalk",
   "files": [