smoltalk 0.8.4 → 0.10.0

package/README.md

@@ -8,15 +8,6 @@ Smoltalk exposes a common API to different LLM providers, with built-in cost tra
 pnpm install smoltalk
 ```
 
-> **Upgrading to 0.6.x?** The flat API-key/host fields on `SmolConfig` have
-> been removed in favor of nested `apiKey` and `baseUrl` maps. Migration:
-> ```diff
-> -{ openAiApiKey: "sk-...",   googleApiKey: "...", ollamaHost: "http://..." }
-> +{ apiKey: { openAi: "sk-...", google: "..." }, baseUrl: { ollama: "http://..." } }
-> ```
-> Env-var fallbacks are unchanged (`OPENAI_API_KEY`, `GEMINI_API_KEY`,
-> `ANTHROPIC_API_KEY`, `OLLAMA_HOST`).
-
 ## Hello world example
 
 ```typescript
@@ -346,7 +337,7 @@ tracking starts working.
 **One model — `registerTextModel` (recommended).** Register once at startup:
 
 ```ts
-import { registerTextModel, textSync } from "smoltalk";
+import { registerTextModel, textSync, userMessage } from "smoltalk";
 
 registerTextModel({
   modelName: "my-model",
@@ -359,6 +350,7 @@ registerTextModel({
   maxOutputTokens: 8192,
 });
 
+const messages = [userMessage("hello")];
 const res = await textSync({
   model: "my-model",
   provider: "openai-compat",
@@ -373,7 +365,7 @@ per-tenant rates), pass a minimal blob for a single call. It layers over the
 baseline exactly like a refresh blob:
 
 ```ts
-import { textSync, type ModelDataBlob } from "smoltalk";
+import { textSync, userMessage, type ModelDataBlob } from "smoltalk";
 
 const modelData: ModelDataBlob = {
   schemaVersion: 1,
@@ -392,6 +384,7 @@ const modelData: ModelDataBlob = {
   ],
 };
 
+const messages = [userMessage("hello")];
 await textSync({ model: "my-model", provider: "openai-compat", messages, modelData });
 ```
 
@@ -471,20 +464,26 @@ On Google, web search can't be combined with structured output in one call.
 
 ## Registering custom providers
 
-Smoltalk has three registration entry points — one per capability:
+Smoltalk has one registration entry point per capability:
 
 ```ts
 // example: skip-typecheck
 import {
-  success,                    // Result helper
-  registerProvider,           // text generation (a class extending BaseClient)
-  registerEmbeddingProvider,  // embeddings (a function)
-  registerImageProvider,      // images (a function)
+  success,                         // Result helper
+  registerProvider,                // text generation (a class extending BaseClient)
+  registerTranscriptionProvider,   // speech-to-text (a class extending BaseTranscriptionClient)
+  registerSpeechProvider,          // text-to-speech (a class extending BaseSpeechClient)
+  registerEmbeddingProvider,       // embeddings (a function)
+  registerImageProvider,           // images (a function)
 } from "smoltalk";
 
 // Text: a class extending BaseClient (implements _textSync / _textStream)
 registerProvider("my-llm", MyTextClient);
 
+// STT/TTS: classes extending the audio base clients (see "Audio (STT/TTS)")
+registerTranscriptionProvider("my-asr", MyTranscriptionClient);
+registerSpeechProvider("my-tts", MySpeechClient);
+
 // Embeddings: a function
 registerEmbeddingProvider("my-embed", async (inputs, config) => {
   // read credentials from config (e.g. config.metadata), call your service
@@ -504,8 +503,136 @@ precedence; a registered name that collides with a built-in is ignored. Custom
 providers receive the full `config` and read their own credentials from it
 (e.g. `config.metadata`).
 
-Text is a class (it needs retries, tool-loop detection, streaming); embeddings
-and images are one-shot functions.
+Text, transcription, and speech are classes: a base class owns the shared
+behavior (validation, cost, error handling) and the subclass implements only
+the provider call. Embeddings and images are one-shot functions.
+
+## Audio (STT/TTS)
+
+Three audio primitives, all OpenAI-only in v1. `transcribe()` (speech-to-text)
+and `speak()` (text-to-speech) are async and return `Result<T>` (never throw).
+`audioPart()` (attach audio to a chat message) is different: it's a
+synchronous plain-object constructor, not a `Result`-returning call — see
+"Audio in chat" below.
+
+### Speech-to-text
+
+```ts
+import { transcribe } from "smoltalk";
+
+const result = await transcribe(
+  { kind: "path", path: "./meeting.mp3" },
+  { model: "whisper-1" },
+);
+if (result.success) {
+  console.log(result.value.text);
+}
+```
+
+`whisper-1` is the only baked-in model in v1. Options: `language`, `prompt`,
+`timestampGranularity` (`"segment"` | `"word"`), `maxBytes` (a safety limit —
+the effective cap is the smaller of your limit and the model's declared upload
+cap, 25 MB for `whisper-1`). The result carries `text` plus optional
+`language`, `durationSeconds`, `segments`, `words`, `usage`, and `cost`.
+
+Model constraints (accepted MIME types, upload cap, per-minute price) live in
+the model registry, not in code — a model you add via `registerModelData` /
+`config.modelData` is validated against whatever its data block declares, and
+a model with no registry entry skips validation entirely (the provider is then
+the authority).
+
+Register a custom provider as a class:
+
+```ts
+// example: skip-typecheck
+import { BaseTranscriptionClient, registerTranscriptionProvider, success } from "smoltalk";
+
+class AcmeTranscription extends BaseTranscriptionClient {
+  protected async _transcribe(data: Uint8Array, mimeType: string) {
+    // call your API with this.config.apiKey; map the response
+    return success({ text: "..." });
+  }
+}
+registerTranscriptionProvider("acme", AcmeTranscription);
+
+// then: transcribe(source, { model: "acme-1", provider: "acme", apiKey: { acme: "..." } })
+```
+
+The base class owns blob loading, model-data validation, cost, and the
+redacting error boundary; `_transcribe()` is only the SDK call + response
+mapping.
+
+### Text-to-speech
+
+```ts
+import { speak } from "smoltalk";
+import { writeFile } from "node:fs/promises";
+
+const result = await speak("Hello from smoltalk.", {
+  model: "tts-1",
+  voice: "alloy",
+});
+if (result.success) {
+  await writeFile("out.mp3", result.value.audio); // caller owns the bytes
+}
+```
+
+`tts-1` and `tts-1-hd` are the only baked-in models in v1. `voice` is
+required. Options: `format` (OpenAI accepts `"mp3"` | `"opus"` | `"aac"` |
+`"flac"` | `"wav"` | `"pcm"`, default `"mp3"`; a custom provider may accept
+other strings) and `speed`. Limits are declared per model in the registry —
+for `tts-1`/`tts-1-hd` that's a 4096-code-point input cap, a 0.25–4.0 speed
+range, and the format list above; exceeding any of them returns a `Failure`
+before the request is sent. The returned `audio` is a `Uint8Array` you own —
+write it to disk, stream it, whatever you like. When `format` is `"pcm"`,
+`result.pcm` describes the raw stream (for OpenAI:
+`{ sampleRateHz: 24000, sampleFormat: "s16le", channels: 1 }`).
+
+Register a custom provider as a class, mirroring transcription:
+
+```ts
+// example: skip-typecheck
+import { BaseSpeechClient, registerSpeechProvider, success } from "smoltalk";
+
+class AcmeSpeech extends BaseSpeechClient {
+  protected async _speak(text: string) {
+    // call your API with this.config.apiKey / this.config.voice
+    return success({ audio: new Uint8Array(), mimeType: "audio/mpeg" });
+  }
+}
+registerSpeechProvider("acme", AcmeSpeech);
+```
+
+As with transcription, per-model constraints come from the model registry
+(`registerModelData` / `config.modelData`), so a custom model's caps, speed
+range, and formats are data, not code.
+
+### Audio in chat
+
+`audioPart()` attaches an audio clip to a `userMessage`, for models that
+accept audio input directly (as opposed to transcribing it first).
+`audioPart()` itself is a synchronous constructor that builds a content part
+— it always returns an `AudioPart`, never a `Result`, and it can't fail:
+
+```ts
+import { textSync, userMessage, audioPart } from "smoltalk";
+
+const messages = [
+  userMessage([
+    "What's being said in this clip?",
+    audioPart({ kind: "path", path: "./clip.wav" }),
+  ]),
+];
+
+const resp = await textSync({ messages, model: "gpt-audio-1.5" });
+```
+
+In v1 this only works with `gpt-audio-1.5` on the OpenAI Chat Completions
+provider (not `openai-responses`, and not other providers). Validation
+happens later, when the message is sent via `textSync`/`textStream` — an
+unsupported provider, a model without audio input, or audio that isn't
+`mp3`/`wav` surfaces as a `Failure` from that call, not from `audioPart()`
+itself. Audio is inlined as base64, not uploaded via the Files API.
 
 ## Limitations
 Smoltalk has support for a limited number of providers right now, and is mostly focused on the stateless APIs for text completion, though I plan to add support for more providers as well as image and speech models later. Smoltalk is also a personal project, and there are alternatives backed by companies:

package/dist/classes/ToolCall.js

@@ -79,17 +79,25 @@ export class ToolCall {
         };
     }
     toGoogle() {
-        return {
-            functionCall: {
-                name: this.name,
-                args: this.arguments,
-            },
-            // Gemini 3 requires the original thought signature echoed back on the
-            // function-call part; omitting it fails validation during tool use.
-            ...(this._thoughtSignature !== undefined && {
-                thoughtSignature: this._thoughtSignature,
-            }),
+        const functionCall = {
+            name: this.name,
+            args: this.arguments,
         };
+        // Echo the id when we have one: the Gemini API pairs a functionResponse
+        // back to its functionCall by id when present. Omit it when empty — current
+        // Gemini 3 preview models issue no ids, and an empty id is not a valid key.
+        if (this._id !== "") {
+            functionCall.id = this._id;
+        }
+        const result = {
+            functionCall,
+        };
+        // Gemini 3 requires the original thought signature echoed back on the
+        // function-call part; omitting it fails validation during tool use.
+        if (this._thoughtSignature !== undefined) {
+            result.thoughtSignature = this._thoughtSignature;
+        }
+        return result;
     }
     toOpenAIResponseInputItem() {
         return {

package/dist/classes/message/AssistantMessage.d.ts

@@ -31,6 +31,8 @@ export declare const AssistantMessageJSONSchema: z.ZodObject<{
         outputTokens: z.ZodNumber;
         cachedInputTokens: z.ZodOptional<z.ZodNumber>;
         cacheCreationInputTokens: z.ZodOptional<z.ZodNumber>;
+        inputAudioTokens: z.ZodOptional<z.ZodNumber>;
+        outputAudioTokens: z.ZodOptional<z.ZodNumber>;
         totalTokens: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>>;
     cost: z.ZodOptional<z.ZodObject<{

package/dist/classes/message/ToolMessage.js

@@ -91,18 +91,21 @@ export class ToolMessage extends BaseMessage {
         };
     }
     toGoogleMessage() {
+        const functionResponse = {
+            name: this.name,
+            response: {
+                result: this.content,
+            },
+        };
+        // Echo the id so Gemini can pair this response to its call by id — the
+        // documented matching mechanism. Only when non-empty: current Gemini 3
+        // preview models issue no ids, and an empty id is not a valid key.
+        if (this.tool_call_id !== "") {
+            functionResponse.id = this.tool_call_id;
+        }
         return {
             role: "user",
-            parts: [
-                {
-                    functionResponse: {
-                        name: this.name,
-                        response: {
-                            result: this.content,
-                        },
-                    },
-                },
-            ],
+            parts: [{ functionResponse }],
         };
     }
     toOllamaMessage() {

package/dist/classes/message/UserMessage.d.ts

@@ -65,6 +65,27 @@ export declare const UserMessageJSONSchema: z.ZodObject<{
             expiresAt: z.ZodOptional<z.ZodNumber>;
         }, z.core.$strip>], "kind">;
         filename: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        type: z.ZodLiteral<"audio">;
+        source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"bytes">;
+            data: z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>;
+            mimeType: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            kind: z.ZodLiteral<"base64">;
+            base64: z.ZodString;
+            mimeType: z.ZodString;
+        }, z.core.$strip>, z.ZodObject<{
+            kind: z.ZodLiteral<"path">;
+            path: z.ZodString;
+            mimeType: z.ZodOptional<z.ZodString>;
+        }, z.core.$strip>, z.ZodObject<{
+            kind: z.ZodLiteral<"url">;
+            url: z.ZodString;
+            mimeType: z.ZodOptional<z.ZodString>;
+            timeoutMs: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strip>], "kind">;
+        filename: z.ZodOptional<z.ZodString>;
     }, z.core.$strip>], "type">>]>;
     name: z.ZodOptional<z.ZodString>;
     rawData: z.ZodOptional<z.ZodAny>;

package/dist/classes/message/UserMessage.js

@@ -108,6 +108,9 @@ export class UserMessage extends BaseMessage {
                 images.push(refToBase64(part.source).base64);
                 continue;
             }
+            if (part.type === "audio") {
+                throw new Error("Ollama does not support audio input.");
+            }
             if (part.source.kind === "providerFile") {
                 throw new Error("Ollama does not support provider file references.");
             }

package/dist/classes/message/contentParts.d.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import type { ImageRef } from "../../util/imageRef.js";
+import type { ImageRef, BlobRef } from "../../util/blobRef.js";
 export type TextPart = {
     type: "text";
     text: string;
@@ -13,7 +13,12 @@ export type FilePart = {
     source: AttachmentSource;
     filename?: string;
 };
-export type UserContentPart = TextPart | ImagePart | FilePart;
+export type AudioPart = {
+    type: "audio";
+    source: BlobRef;
+    filename?: string;
+};
+export type UserContentPart = TextPart | ImagePart | FilePart | AudioPart;
 /** Normalized user-message content: a plain string or an array of typed parts. */
 export type UserContent = string | UserContentPart[];
 /** What callers may pass: a bare string element is sugar for a text part. */
@@ -139,6 +144,28 @@ export declare const FilePartSchema: z.ZodObject<{
     }, z.core.$strip>], "kind">;
     filename: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
+export declare const AudioPartSchema: z.ZodObject<{
+    type: z.ZodLiteral<"audio">;
+    source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"bytes">;
+        data: z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>;
+        mimeType: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"base64">;
+        base64: z.ZodString;
+        mimeType: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+        mimeType: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodString;
+        mimeType: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>], "kind">;
+    filename: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
 export declare const UserContentPartSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
     type: z.ZodLiteral<"text">;
     text: z.ZodString;
@@ -197,6 +224,27 @@ export declare const UserContentPartSchema: z.ZodDiscriminatedUnion<[z.ZodObject
         expiresAt: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>], "kind">;
     filename: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"audio">;
+    source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"bytes">;
+        data: z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>;
+        mimeType: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"base64">;
+        base64: z.ZodString;
+        mimeType: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+        mimeType: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodString;
+        mimeType: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>], "kind">;
+    filename: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>], "type">;
 export declare const UserContentSchema: z.ZodUnion<readonly [z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
     type: z.ZodLiteral<"text">;
@@ -256,4 +304,25 @@ export declare const UserContentSchema: z.ZodUnion<readonly [z.ZodString, z.ZodA
         expiresAt: z.ZodOptional<z.ZodNumber>;
     }, z.core.$strip>], "kind">;
     filename: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"audio">;
+    source: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"bytes">;
+        data: z.ZodCustom<Uint8Array<ArrayBuffer>, Uint8Array<ArrayBuffer>>;
+        mimeType: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"base64">;
+        base64: z.ZodString;
+        mimeType: z.ZodString;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"path">;
+        path: z.ZodString;
+        mimeType: z.ZodOptional<z.ZodString>;
+    }, z.core.$strip>, z.ZodObject<{
+        kind: z.ZodLiteral<"url">;
+        url: z.ZodString;
+        mimeType: z.ZodOptional<z.ZodString>;
+        timeoutMs: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>], "kind">;
+    filename: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>], "type">>]>;

package/dist/classes/message/contentParts.js

@@ -36,9 +36,15 @@ export const FilePartSchema = z.object({
     source: AttachmentSourceSchema,
     filename: z.string().optional(),
 });
+export const AudioPartSchema = z.object({
+    type: z.literal("audio"),
+    source: z.discriminatedUnion("kind", [...ImageRefSchema.options]),
+    filename: z.string().optional(),
+});
 export const UserContentPartSchema = z.discriminatedUnion("type", [
     TextPartSchema,
     ImagePartSchema,
     FilePartSchema,
+    AudioPartSchema,
 ]);
 export const UserContentSchema = z.union([z.string(), z.array(UserContentPartSchema)]);

package/dist/classes/message/index.d.ts

@@ -8,8 +8,8 @@ import type { AssistantMessageJSON } from "./AssistantMessage.js";
 import type { DeveloperMessageJSON } from "./DeveloperMessage.js";
 import type { SystemMessageJSON } from "./SystemMessage.js";
 import type { ToolMessageJSON } from "./ToolMessage.js";
-import { CostEstimate, TextPart, TokenUsage, UserContentInput, ImagePart, FilePart } from "../../types.js";
-import type { ImageRef } from "../../util/imageRef.js";
+import { CostEstimate, TextPart, TokenUsage, UserContentInput, ImagePart, FilePart, AudioPart } from "../../types.js";
+import type { ImageRef, BlobRef } from "../../util/blobRef.js";
 export * from "./AssistantMessage.js";
 export * from "./BaseMessage.js";
 export * from "./DeveloperMessage.js";
@@ -26,6 +26,9 @@ export declare function imagePart(source: ImageRef): ImagePart;
 export declare function filePart(source: ImageRef, options?: {
     filename?: string;
 }): FilePart;
+export declare function audioPart(source: BlobRef, options?: {
+    filename?: string;
+}): AudioPart;
 export declare function assistantMessage(content: string | Array<TextPart> | null, options?: {
     name?: string;
     audio?: any | null;

package/dist/classes/message/index.js

@@ -40,6 +40,13 @@ export function filePart(source, options = {}) {
     }
     return { type: "file", source, filename: options.filename };
 }
+export function audioPart(source, options = {}) {
+    const part = { type: "audio", source };
+    if (options.filename !== undefined) {
+        part.filename = options.filename;
+    }
+    return part;
+}
 export function assistantMessage(content, options = {}) {
     return new AssistantMessage(content, options);
 }

package/dist/classes/message/renderers/AnthropicRenderer.d.ts

@@ -1,5 +1,5 @@
 import type { PartRenderer } from "./PartRenderer.js";
-import type { TextPart, ImagePart, FilePart } from "../contentParts.js";
+import type { TextPart, ImagePart, FilePart, AudioPart } from "../contentParts.js";
 /** Renders parts for the Anthropic Messages API. */
 export declare class AnthropicRenderer implements PartRenderer<any> {
     text(part: TextPart): {
@@ -40,4 +40,5 @@ export declare class AnthropicRenderer implements PartRenderer<any> {
             data: string;
         };
     };
+    audio(_part: AudioPart): any;
 }

package/dist/classes/message/renderers/AnthropicRenderer.js

@@ -16,4 +16,7 @@ export class AnthropicRenderer {
         }
         return { type: "document", source: anthropicSource(part.source) };
     }
+    audio(_part) {
+        throw new Error("Audio input is not supported for this provider in v1.");
+    }
 }

package/dist/classes/message/renderers/GoogleRenderer.d.ts

@@ -1,5 +1,5 @@
 import type { PartRenderer } from "./PartRenderer.js";
-import type { TextPart, ImagePart, FilePart } from "../contentParts.js";
+import type { TextPart, ImagePart, FilePart, AudioPart } from "../contentParts.js";
 /** Renders parts for the Google Gemini API. */
 export declare class GoogleRenderer implements PartRenderer<any> {
     text(part: TextPart): {
@@ -31,5 +31,6 @@ export declare class GoogleRenderer implements PartRenderer<any> {
         };
         fileData?: undefined;
     };
+    audio(_part: AudioPart): any;
     private sourcePart;
 }

package/dist/classes/message/renderers/GoogleRenderer.js

@@ -10,6 +10,9 @@ export class GoogleRenderer {
     file(part) {
         return this.sourcePart(part.source);
     }
+    audio(_part) {
+        throw new Error("Audio input is not supported for this provider in v1.");
+    }
     sourcePart(source) {
         if (source.kind === "providerFile") {
             // `fileData` needs both a uri and a mimeType; a ref missing either (e.g. a

package/dist/classes/message/renderers/JSONRenderer.d.ts

@@ -1,8 +1,9 @@
 import type { PartRenderer } from "./PartRenderer.js";
-import type { TextPart, ImagePart, FilePart, UserContentPart } from "../contentParts.js";
+import type { TextPart, ImagePart, FilePart, AudioPart, UserContentPart } from "../contentParts.js";
 /** Renders parts back to a JSON-safe UserContentPart (for toJSON serialization). */
 export declare class JSONRenderer implements PartRenderer<UserContentPart> {
     text(part: TextPart): UserContentPart;
     image(part: ImagePart): UserContentPart;
     file(part: FilePart): UserContentPart;
+    audio(part: AudioPart): UserContentPart;
 }

package/dist/classes/message/renderers/JSONRenderer.js

@@ -18,4 +18,8 @@ export class JSONRenderer {
     file(part) {
         return { type: "file", source: bytesToBase64(part.source), filename: part.filename };
     }
+    audio(part) {
+        // bytesToBase64 only converts `bytes`; other kinds pass through, all within BlobRef.
+        return { type: "audio", source: bytesToBase64(part.source), filename: part.filename };
+    }
 }

package/dist/classes/message/renderers/OpenAIChatRenderer.d.ts

@@ -1,5 +1,5 @@
 import type { PartRenderer } from "./PartRenderer.js";
-import type { TextPart, ImagePart, FilePart } from "../contentParts.js";
+import type { TextPart, ImagePart, FilePart, AudioPart } from "../contentParts.js";
 /** Renders parts for the OpenAI Chat Completions API. */
 export declare class OpenAIChatRenderer implements PartRenderer<any> {
     text(part: TextPart): {
@@ -27,4 +27,11 @@ export declare class OpenAIChatRenderer implements PartRenderer<any> {
             file_id?: undefined;
         };
     };
+    audio(part: AudioPart): {
+        type: string;
+        input_audio: {
+            data: string;
+            format: "mp3" | "wav";
+        };
+    };
 }

package/dist/classes/message/renderers/OpenAIChatRenderer.js

@@ -1,4 +1,11 @@
 import { refToBase64, toDataUri, openAiImageUrl, attachmentFilename } from "../../../util/attachments.js";
+import { chatAudioFormat } from "../../../util/audioMime.js";
+function requirePreparedAudioPart(part) {
+    if (part.source.kind !== "base64") {
+        throw new Error("internal: audio source must be prepared as base64 before rendering");
+    }
+    return part;
+}
 /** Renders parts for the OpenAI Chat Completions API. */
 export class OpenAIChatRenderer {
     text(part) {
@@ -17,4 +24,15 @@ export class OpenAIChatRenderer {
         const { base64, mimeType } = refToBase64(part.source);
         return { type: "file", file: { file_data: toDataUri(base64, mimeType), filename: attachmentFilename(part.filename) } };
     }
+    audio(part) {
+        const prepared = requirePreparedAudioPart(part);
+        const format = chatAudioFormat(prepared.source.mimeType);
+        if (!format) {
+            throw new Error(`Chat audio supports only mp3/wav; got "${prepared.source.mimeType}".`);
+        }
+        return {
+            type: "input_audio",
+            input_audio: { data: prepared.source.base64, format },
+        };
+    }
 }

package/dist/classes/message/renderers/OpenAIResponsesRenderer.d.ts

@@ -1,5 +1,5 @@
 import type { PartRenderer } from "./PartRenderer.js";
-import type { TextPart, ImagePart, FilePart } from "../contentParts.js";
+import type { TextPart, ImagePart, FilePart, AudioPart } from "../contentParts.js";
 /** Renders parts for the OpenAI Responses API. */
 export declare class OpenAIResponsesRenderer implements PartRenderer<any> {
     text(part: TextPart): {
@@ -36,4 +36,5 @@ export declare class OpenAIResponsesRenderer implements PartRenderer<any> {
         file_id?: undefined;
         file_url?: undefined;
     };
+    audio(_part: AudioPart): any;
 }

package/dist/classes/message/renderers/OpenAIResponsesRenderer.js

@@ -20,4 +20,7 @@ export class OpenAIResponsesRenderer {
         const { base64, mimeType } = refToBase64(part.source);
         return { type: "input_file", file_data: toDataUri(base64, mimeType), filename: attachmentFilename(part.filename) };
     }
+    audio(_part) {
+        throw new Error("Audio input is not supported for this provider in v1.");
+    }
 }

package/dist/classes/message/renderers/PartRenderer.d.ts

@@ -1,14 +1,15 @@
-import type { TextPart, ImagePart, FilePart, UserContentPart } from "../contentParts.js";
+import type { TextPart, ImagePart, FilePart, AudioPart, UserContentPart } from "../contentParts.js";
 /**
  * A per-provider renderer for user-content parts. Each provider implements one
  * of these (one class per file, in this directory) so the message class stays
  * thin: the serializer picks a renderer and walks the parts with {@link renderParts}.
- * `text`/`image`/`file` return that provider's native representation of a part.
+ * `text`/`image`/`file`/`audio` return that provider's native representation of a part.
  */
 export interface PartRenderer<T> {
     text(part: TextPart): T;
     image(part: ImagePart): T;
     file(part: FilePart): T;
+    audio(part: AudioPart): T;
 }
 /** Walk content parts, dispatching each to the renderer's matching method. */
 export declare function renderParts<T>(parts: UserContentPart[], renderer: PartRenderer<T>): T[];

package/dist/classes/message/renderers/PartRenderer.js

@@ -8,6 +8,9 @@ export function renderParts(parts, renderer) {
         else if (part.type === "image") {
             out.push(renderer.image(part));
         }
+        else if (part.type === "audio") {
+            out.push(renderer.audio(part));
+        }
         else {
             out.push(renderer.file(part));
         }