smoltalk 0.9.0 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +175 -7
  2. package/dist/classes/message/AssistantMessage.d.ts +2 -0
  3. package/dist/classes/message/UserMessage.d.ts +21 -0
  4. package/dist/classes/message/UserMessage.js +3 -0
  5. package/dist/classes/message/contentParts.d.ts +71 -2
  6. package/dist/classes/message/contentParts.js +6 -0
  7. package/dist/classes/message/index.d.ts +5 -2
  8. package/dist/classes/message/index.js +7 -0
  9. package/dist/classes/message/renderers/AnthropicRenderer.d.ts +2 -1
  10. package/dist/classes/message/renderers/AnthropicRenderer.js +3 -0
  11. package/dist/classes/message/renderers/GoogleRenderer.d.ts +2 -1
  12. package/dist/classes/message/renderers/GoogleRenderer.js +3 -0
  13. package/dist/classes/message/renderers/JSONRenderer.d.ts +2 -1
  14. package/dist/classes/message/renderers/JSONRenderer.js +4 -0
  15. package/dist/classes/message/renderers/OpenAIChatRenderer.d.ts +8 -1
  16. package/dist/classes/message/renderers/OpenAIChatRenderer.js +18 -0
  17. package/dist/classes/message/renderers/OpenAIResponsesRenderer.d.ts +2 -1
  18. package/dist/classes/message/renderers/OpenAIResponsesRenderer.js +3 -0
  19. package/dist/classes/message/renderers/PartRenderer.d.ts +3 -2
  20. package/dist/classes/message/renderers/PartRenderer.js +3 -0
  21. package/dist/client.js +1 -0
  22. package/dist/clients/anthropic.js +1 -1
  23. package/dist/clients/baseClient.d.ts +13 -1
  24. package/dist/clients/baseClient.js +36 -7
  25. package/dist/clients/google.js +1 -1
  26. package/dist/clients/ollama.js +1 -1
  27. package/dist/clients/openai.d.ts +2 -1
  28. package/dist/clients/openai.js +15 -3
  29. package/dist/clients/openaiCompat.d.ts +2 -0
  30. package/dist/clients/openaiCompat.js +5 -0
  31. package/dist/clients/openaiResponses.js +1 -1
  32. package/dist/clients/resolveAttachments.d.ts +8 -4
  33. package/dist/clients/resolveAttachments.js +101 -50
  34. package/dist/embed.d.ts +4 -0
  35. package/dist/files.d.ts +1 -1
  36. package/dist/files.js +1 -1
  37. package/dist/image/google.js +2 -2
  38. package/dist/image/openai.js +3 -3
  39. package/dist/image.d.ts +1 -1
  40. package/dist/index.d.ts +10 -2
  41. package/dist/index.js +7 -1
  42. package/dist/model.d.ts +15 -4
  43. package/dist/model.js +72 -12
  44. package/dist/models.d.ts +204 -22
  45. package/dist/models.js +211 -30
  46. package/dist/speech/baseSpeechClient.d.ts +36 -0
  47. package/dist/speech/baseSpeechClient.js +117 -0
  48. package/dist/speech/google.d.ts +6 -0
  49. package/dist/speech/google.js +54 -0
  50. package/dist/speech/groq.d.ts +11 -0
  51. package/dist/speech/groq.js +19 -0
  52. package/dist/speech/openai.d.ts +14 -0
  53. package/dist/speech/openai.js +51 -0
  54. package/dist/speech/openaiCompat.d.ts +13 -0
  55. package/dist/speech/openaiCompat.js +22 -0
  56. package/dist/speech.d.ts +45 -0
  57. package/dist/speech.js +63 -0
  58. package/dist/transcription/baseTranscriptionClient.d.ts +36 -0
  59. package/dist/transcription/baseTranscriptionClient.js +133 -0
  60. package/dist/transcription/google.d.ts +6 -0
  61. package/dist/transcription/google.js +56 -0
  62. package/dist/transcription/groq.d.ts +10 -0
  63. package/dist/transcription/groq.js +17 -0
  64. package/dist/transcription/openai.d.ts +11 -0
  65. package/dist/transcription/openai.js +67 -0
  66. package/dist/transcription/openaiCompat.d.ts +13 -0
  67. package/dist/transcription/openaiCompat.js +22 -0
  68. package/dist/transcription.d.ts +54 -0
  69. package/dist/transcription.js +64 -0
  70. package/dist/types/tokenUsage.d.ts +4 -0
  71. package/dist/types/tokenUsage.js +4 -0
  72. package/dist/types.d.ts +4 -0
  73. package/dist/util/attachments.d.ts +1 -1
  74. package/dist/util/audioMime.d.ts +26 -0
  75. package/dist/util/audioMime.js +74 -0
  76. package/dist/util/{imageRef.d.ts → blobRef.d.ts} +9 -9
  77. package/dist/util/{imageRef.js → blobRef.js} +6 -13
  78. package/dist/util/googleAudioUsage.d.ts +14 -0
  79. package/dist/util/googleAudioUsage.js +52 -0
  80. package/dist/util/mime.d.ts +21 -0
  81. package/dist/util/mime.js +54 -0
  82. package/dist/util/modalities.d.ts +6 -2
  83. package/dist/util/modalities.js +13 -15
  84. package/dist/util/provider.d.ts +3 -0
  85. package/dist/util/provider.js +3 -1
  86. package/package.json +1 -1
@@ -6,11 +6,21 @@ import { stripCodeFence } from "../util/util.js";
6
6
  import { success, failure, } from "../types.js";
7
7
  import { validateHostedTools } from "../util/hostedTools.js";
8
8
  import { resolveMessageAttachments, messagesHaveAttachments, DEFAULT_MAX_ATTACHMENT_BYTES } from "./resolveAttachments.js";
9
- import { validateModalities } from "../util/modalities.js";
9
+ import { neededInputModalities, MODALITIES_REQUIRING_DECLARATION, } from "../util/modalities.js";
10
+ import { modelSupportsInputModality } from "../models.js";
10
11
  import { resolveProvider } from "../util/provider.js";
11
12
  import { isUnconstrainedSchema } from "../util/jsonSchema.js";
12
13
  import { z } from "zod";
13
14
  const DEFAULT_NUM_RETRIES = 2;
15
+ function clientSupportsAttachment(capabilities, modality) {
16
+ if (modality === "audio") {
17
+ return capabilities.audioFormats.length > 0;
18
+ }
19
+ if (modality === "image" || modality === "pdf") {
20
+ return capabilities.inputModalities.includes(modality);
21
+ }
22
+ return false;
23
+ }
14
24
  export class BaseClient {
15
25
  config;
16
26
  statelogClient;
@@ -57,24 +67,43 @@ export class BaseClient {
57
67
  return null;
58
68
  }
59
69
  /**
60
- * Gate on input modalities and resolve any image/PDF attachment refs
70
+ * What this client can accept as attachments. Subclasses override to declare
71
+ * more (or fewer). Checked against the messages, alongside the model's own
72
+ * declared modalities, before any serialization runs.
73
+ */
74
+ attachmentCapabilities() {
75
+ return { inputModalities: ["image", "pdf"], audioFormats: [] };
76
+ }
77
+ /**
78
+ * Gate on input modalities and resolve any image/PDF/audio attachment refs
61
79
  * (path/url/bytes → base64) before the synchronous serializers run. Returns
62
80
  * the (possibly rewritten) config on success, or a Failure to surface. Shared
63
81
  * by textSync and textStream so the two paths can't diverge.
64
82
  */
65
83
  async prepareAttachments(config) {
66
- const modalityResult = validateModalities(config);
67
- if (modalityResult && modalityResult.success === false) {
68
- return modalityResult;
69
- }
70
- if (!messagesHaveAttachments(config.messages)) {
84
+ const needed = neededInputModalities(config.messages);
85
+ if (needed.length === 0 && !messagesHaveAttachments(config.messages)) {
71
86
  return success(config);
72
87
  }
73
88
  const provider = resolveProvider(config.model, config.provider, config.modelData);
89
+ const capabilities = this.attachmentCapabilities();
90
+ for (const modality of needed) {
91
+ if (!clientSupportsAttachment(capabilities, modality)) {
92
+ return failure(`${modality[0].toUpperCase()}${modality.slice(1)} input is not supported by the "${provider}" provider.`);
93
+ }
94
+ const supported = modelSupportsInputModality(config.model, modality, config.modelData, provider);
95
+ if (supported === false) {
96
+ return failure(`Model ${config.model} does not support ${modality} input.`);
97
+ }
98
+ if (supported === undefined && MODALITIES_REQUIRING_DECLARATION.has(modality)) {
99
+ return failure(`Model ${config.model} does not support ${modality} input.`);
100
+ }
101
+ }
74
102
  const maxBytes = config.attachments?.maxBytes ?? DEFAULT_MAX_ATTACHMENT_BYTES;
75
103
  const resolved = await resolveMessageAttachments(config.messages, {
76
104
  provider,
77
105
  maxBytes,
106
+ audioFormats: capabilities.audioFormats,
78
107
  });
79
108
  if (!resolved.success) {
80
109
  return resolved;
@@ -209,7 +209,7 @@ export class SmolGoogle extends BaseClient {
209
209
  }
210
210
  this.client = new GoogleGenAI({ apiKey });
211
211
  this.logger = getLogger();
212
- this.model = new Model(config.model, undefined, config.modelData);
212
+ this.model = new Model(config.model, config.provider, config.modelData);
213
213
  }
214
214
  getClient() {
215
215
  return this.client;
@@ -20,7 +20,7 @@ export class SmolOllama extends BaseClient {
20
20
  constructor(config) {
21
21
  super(config);
22
22
  this.logger = getLogger();
23
- this.model = new Model(config.model, undefined, config.modelData);
23
+ this.model = new Model(config.model, config.provider, config.modelData);
24
24
  const apiKey = config.apiKey?.ollama;
25
25
  if (apiKey) {
26
26
  this.client = new Ollama({
@@ -1,7 +1,7 @@
1
1
  import OpenAI from "openai";
2
2
  import { PromptResult, Result, SmolClient, SmolConfig, StreamChunk, HostedToolResult } from "../types.js";
3
3
  import { EgonLog } from "../util/logger.js";
4
- import { BaseClient } from "./baseClient.js";
4
+ import { BaseClient, type ClientAttachmentCapabilities } from "./baseClient.js";
5
5
  import { ModelName } from "../models.js";
6
6
  import { Model } from "../model.js";
7
7
  import { CostEstimate, TokenUsage } from "../types.js";
@@ -11,6 +11,7 @@ export declare class SmolOpenAi extends BaseClient implements SmolClient {
11
11
  protected logger: EgonLog;
12
12
  protected model: Model;
13
13
  constructor(config: SmolOpenAiConfig);
14
+ protected attachmentCapabilities(): ClientAttachmentCapabilities;
14
15
  /**
15
16
  * Build the `new OpenAI({...})` options. Subclasses override to inject a
16
17
  * different baseURL or to pull the key from a different config field.
@@ -20,7 +20,11 @@ export class SmolOpenAi extends BaseClient {
20
20
  const options = this.resolveClientOptions(config);
21
21
  this.client = new OpenAI(options);
22
22
  this.logger = getLogger();
23
- this.model = new Model(config.model, undefined, config.modelData);
23
+ this.model = new Model(config.model, config.provider, config.modelData);
24
+ }
25
+ attachmentCapabilities() {
26
+ // Chat Completions input_audio accepts inline mp3/wav only.
27
+ return { inputModalities: ["image", "pdf"], audioFormats: ["mp3", "wav"] };
24
28
  }
25
29
  /**
26
30
  * Build the `new OpenAI({...})` options. Subclasses override to inject a
@@ -72,14 +76,22 @@ export class SmolOpenAi extends BaseClient {
72
76
  let cost;
73
77
  if (usageData) {
74
78
  const cached = usageData.prompt_tokens_details?.cached_tokens ?? 0;
79
+ const audioIn = usageData.prompt_tokens_details?.audio_tokens ?? 0;
80
+ const audioOut = usageData.completion_tokens_details?.audio_tokens ?? 0;
75
81
  usage = {
76
- inputTokens: Math.max(0, (usageData.prompt_tokens || 0) - cached),
77
- outputTokens: usageData.completion_tokens || 0,
82
+ inputTokens: Math.max(0, (usageData.prompt_tokens || 0) - cached - audioIn),
83
+ outputTokens: Math.max(0, (usageData.completion_tokens || 0) - audioOut),
78
84
  totalTokens: usageData.total_tokens,
79
85
  };
80
86
  if (cached > 0) {
81
87
  usage.cachedInputTokens = cached;
82
88
  }
89
+ if (audioIn > 0) {
90
+ usage.inputAudioTokens = audioIn;
91
+ }
92
+ if (audioOut > 0) {
93
+ usage.outputAudioTokens = audioOut;
94
+ }
83
95
  // Prefer provider-supplied cost when available (e.g. OpenRouter
84
96
  // usage.cost, DeepInfra usage.estimated_cost, LiteLLM header).
85
97
  // Fall back to the smoltalk model-registry calculation.
@@ -1,4 +1,5 @@
1
1
  import { SmolOpenAi } from "./openai.js";
2
+ import type { ClientAttachmentCapabilities } from "./baseClient.js";
2
3
  import type { SmolConfig } from "../types.js";
3
4
  /**
4
5
  * Generic OpenAI-compatible client. Use when pointing smoltalk at any
@@ -14,6 +15,7 @@ import type { SmolConfig } from "../types.js";
14
15
  * cost stays undefined — that's expected for arbitrary backends).
15
16
  */
16
17
  export declare class SmolOpenAiCompat extends SmolOpenAi {
18
+ protected attachmentCapabilities(): ClientAttachmentCapabilities;
17
19
  protected resolveClientOptions(config: SmolConfig): {
18
20
  apiKey: string;
19
21
  baseURL: string;
@@ -14,6 +14,11 @@ import { resolveApiKey, resolveBaseUrl } from "../util/provider.js";
14
14
  * cost stays undefined — that's expected for arbitrary backends).
15
15
  */
16
16
  export class SmolOpenAiCompat extends SmolOpenAi {
17
+ // Compat endpoints speak the Chat Completions wire format but do not get
18
+ // OpenAI's input_audio handling — declare no audio support.
19
+ attachmentCapabilities() {
20
+ return { inputModalities: ["image", "pdf"], audioFormats: [] };
21
+ }
17
22
  resolveClientOptions(config) {
18
23
  const apiKey = resolveApiKey("openai-compat", config);
19
24
  const baseURL = resolveBaseUrl("openai-compat", config);
@@ -62,7 +62,7 @@ export class SmolOpenAiResponses extends BaseClient {
62
62
  }
63
63
  this.client = new OpenAI({ apiKey });
64
64
  this.logger = getLogger();
65
- this.model = new Model(config.model, undefined, config.modelData);
65
+ this.model = new Model(config.model, config.provider, config.modelData);
66
66
  }
67
67
  getClient() {
68
68
  return this.client;
@@ -1,11 +1,15 @@
1
1
  import { Message } from "../classes/message/index.js";
2
2
  import { Result } from "../types.js";
3
3
  export declare const DEFAULT_MAX_ATTACHMENT_BYTES: number;
4
+ type ResolveOptions = {
5
+ provider: string;
6
+ maxBytes: number;
7
+ /** Audio containers (by primary extension) the target client accepts inline. */
8
+ audioFormats: readonly string[];
9
+ };
4
10
  /** Whether any user message carries an image/file attachment part. */
5
11
  export declare function messagesHaveAttachments(messages: Message[]): boolean;
6
12
  /** Whether `provider` accepts a remote URL directly for this part type. */
7
13
  export declare function acceptsRemoteUrl(provider: string, partType: "image" | "file"): boolean;
8
- export declare function resolveMessageAttachments(messages: Message[], options: {
9
- provider: string;
10
- maxBytes: number;
11
- }): Promise<Result<Message[]>>;
14
+ export declare function resolveMessageAttachments(messages: Message[], options: ResolveOptions): Promise<Result<Message[]>>;
15
+ export {};
@@ -1,6 +1,7 @@
1
1
  import { UserMessage } from "../classes/message/index.js";
2
- import { normalizeImageRef } from "../util/imageRef.js";
2
+ import { normalizeBlob } from "../util/blobRef.js";
3
3
  import { fileFamily } from "../util/attachments.js";
4
+ import { audioFormatForMime } from "../util/mime.js";
4
5
  import { success, failure } from "../types.js";
5
6
  export const DEFAULT_MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024;
6
7
  const URL_IMAGE_PROVIDERS = new Set([
@@ -24,7 +25,7 @@ export function messagesHaveAttachments(messages) {
24
25
  continue;
25
26
  }
26
27
  for (const part of parts) {
27
- if (part.type === "image" || part.type === "file") {
28
+ if (part.type === "image" || part.type === "file" || part.type === "audio") {
28
29
  return true;
29
30
  }
30
31
  }
@@ -38,6 +39,100 @@ export function acceptsRemoteUrl(provider, partType) {
38
39
  }
39
40
  return URL_PDF_PROVIDERS.has(provider);
40
41
  }
42
+ /** Load a ref to inline base64, gated to `allowed` MIME prefixes. Throws on failure. */
43
+ async function toBase64Source(source, allowed, maxBytes) {
44
+ const { data, mimeType } = await normalizeBlob(source, {
45
+ allowedMimePrefixes: allowed,
46
+ maxBytes,
47
+ });
48
+ return { kind: "base64", base64: Buffer.from(data).toString("base64"), mimeType };
49
+ }
50
+ /** Error message when a providerFile ref targets the wrong provider family, else null. */
51
+ function providerFileError(fileProvider, targetProvider) {
52
+ const family = fileFamily(targetProvider);
53
+ if (family === null || fileProvider !== family) {
54
+ return (`Attachment references a "${fileProvider}" file, but this call targets provider ` +
55
+ `"${targetProvider}" (file family ${family ?? "none"}).`);
56
+ }
57
+ return null;
58
+ }
59
+ // Audio has no providerFile/URL passthrough: Chat input_audio requires
60
+ // inline base64, so every audio source is normalized here.
61
+ async function resolveAudioPart(part, options) {
62
+ try {
63
+ const source = await toBase64Source(part.source, ["audio/"], options.maxBytes);
64
+ const audioFormat = audioFormatForMime(source.mimeType);
65
+ if (audioFormat === null || !options.audioFormats.includes(audioFormat.extension)) {
66
+ return failure(`Audio input for provider "${options.provider}" supports only ` +
67
+ `${options.audioFormats.join(", ")}; got "${source.mimeType}".`);
68
+ }
69
+ const resolved = { type: "audio", source };
70
+ if (part.filename !== undefined) {
71
+ resolved.filename = part.filename;
72
+ }
73
+ return success(resolved);
74
+ }
75
+ catch (err) {
76
+ return failure(`Failed to load audio attachment: ${err.message}`);
77
+ }
78
+ }
79
+ async function resolveImagePart(part, options) {
80
+ // Provider file references are validated and passed through (no download/cap).
81
+ if (part.source.kind === "providerFile") {
82
+ const mismatch = providerFileError(part.source.provider, options.provider);
83
+ if (mismatch !== null) {
84
+ return failure(mismatch);
85
+ }
86
+ if (options.provider === "openai") {
87
+ return failure("An image file reference requires the openai-responses provider (OpenAI Chat Completions has no image-by-file_id form).");
88
+ }
89
+ return success(part);
90
+ }
91
+ // Passthrough: keep a url ref when the target provider accepts a remote URL.
92
+ if (part.source.kind === "url" && acceptsRemoteUrl(options.provider, "image")) {
93
+ return success(part);
94
+ }
95
+ try {
96
+ const source = await toBase64Source(part.source, ["image/"], options.maxBytes);
97
+ return success({ type: "image", source });
98
+ }
99
+ catch (err) {
100
+ return failure(`Failed to load image attachment: ${err.message}`);
101
+ }
102
+ }
103
+ async function resolveFilePart(part, options) {
104
+ // Provider file references are validated and passed through (no download/cap).
105
+ if (part.source.kind === "providerFile") {
106
+ const mismatch = providerFileError(part.source.provider, options.provider);
107
+ if (mismatch !== null) {
108
+ return failure(mismatch);
109
+ }
110
+ return success(part);
111
+ }
112
+ // Passthrough: keep a url ref when the target provider accepts a remote URL.
113
+ if (part.source.kind === "url" && acceptsRemoteUrl(options.provider, "file")) {
114
+ return success(part);
115
+ }
116
+ try {
117
+ const source = await toBase64Source(part.source, ["application/pdf"], options.maxBytes);
118
+ return success({ type: "file", source, filename: part.filename });
119
+ }
120
+ catch (err) {
121
+ return failure(`Failed to load file attachment: ${err.message}`);
122
+ }
123
+ }
124
+ async function resolveUserPart(part, options) {
125
+ if (part.type === "text") {
126
+ return success(part);
127
+ }
128
+ if (part.type === "audio") {
129
+ return resolveAudioPart(part, options);
130
+ }
131
+ if (part.type === "image") {
132
+ return resolveImagePart(part, options);
133
+ }
134
+ return resolveFilePart(part, options);
135
+ }
41
136
  export async function resolveMessageAttachments(messages, options) {
42
137
  const out = [];
43
138
  for (const msg of messages) {
@@ -52,55 +147,11 @@ export async function resolveMessageAttachments(messages, options) {
52
147
  }
53
148
  const resolvedParts = [];
54
149
  for (const part of parts) {
55
- if (part.type === "text") {
56
- resolvedParts.push(part);
57
- continue;
58
- }
59
- // Provider file references are validated and passed through (no download/cap).
60
- if (part.source.kind === "providerFile") {
61
- const family = fileFamily(options.provider);
62
- if (family === null || part.source.provider !== family) {
63
- return failure(`Attachment references a "${part.source.provider}" file, but this call targets provider ` +
64
- `"${options.provider}" (file family ${family ?? "none"}).`);
65
- }
66
- if (part.type === "image" && options.provider === "openai") {
67
- return failure("An image file reference requires the openai-responses provider (OpenAI Chat Completions has no image-by-file_id form).");
68
- }
69
- resolvedParts.push(part);
70
- continue;
71
- }
72
- // Passthrough: keep a url ref when the target provider accepts a remote URL.
73
- if (part.source.kind === "url" && acceptsRemoteUrl(options.provider, part.type)) {
74
- resolvedParts.push(part);
75
- continue;
76
- }
77
- let allowed;
78
- if (part.type === "image") {
79
- allowed = ["image/"];
80
- }
81
- else {
82
- allowed = ["application/pdf"];
83
- }
84
- try {
85
- const { data, mimeType } = await normalizeImageRef(part.source, {
86
- allowedMimePrefixes: allowed,
87
- maxBytes: options.maxBytes,
88
- });
89
- const source = {
90
- kind: "base64",
91
- base64: Buffer.from(data).toString("base64"),
92
- mimeType,
93
- };
94
- if (part.type === "image") {
95
- resolvedParts.push({ type: "image", source });
96
- }
97
- else {
98
- resolvedParts.push({ type: "file", source, filename: part.filename });
99
- }
100
- }
101
- catch (err) {
102
- return failure(`Failed to load ${part.type} attachment: ${err.message}`);
150
+ const resolved = await resolveUserPart(part, options);
151
+ if (!resolved.success) {
152
+ return resolved;
103
153
  }
154
+ resolvedParts.push(resolved.value);
104
155
  }
105
156
  out.push(new UserMessage(resolvedParts, { name: msg.name, rawData: msg.rawData }));
106
157
  }
package/dist/embed.d.ts CHANGED
@@ -16,6 +16,8 @@ export type EmbedConfig = {
16
16
  deepInfra?: string;
17
17
  liteLlm?: string;
18
18
  openAiCompat?: string;
19
+ /** Arbitrary provider names, for keys targeting a custom-registered provider. */
20
+ [provider: string]: string | undefined;
19
21
  };
20
22
  /** Custom base URLs, nested by provider. */
21
23
  baseUrl?: {
@@ -23,6 +25,8 @@ export type EmbedConfig = {
23
25
  deepInfra?: string;
24
26
  liteLlm?: string;
25
27
  openAiCompat?: string;
28
+ /** Arbitrary provider names, for URLs targeting a custom-registered provider. */
29
+ [provider: string]: string | undefined;
26
30
  };
27
31
  metadata?: Record<string, unknown>;
28
32
  modelData?: ModelDataBlob;
package/dist/files.d.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { Result } from "./types/result.js";
2
2
  import type { SmolConfig } from "./types.js";
3
3
  import { ProviderFileRef } from "./classes/message/contentParts.js";
4
- import { BlobRef } from "./util/imageRef.js";
4
+ import { BlobRef } from "./util/blobRef.js";
5
5
  /** Default cap on a resolved upload's size. Callers can raise it via opts.maxBytes. */
6
6
  export declare const DEFAULT_UPLOAD_BYTES: number;
7
7
  /**
package/dist/files.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { failure } from "./types/result.js";
2
- import { loadBlob } from "./util/imageRef.js";
2
+ import { loadBlob } from "./util/blobRef.js";
3
3
  import { fileFamily } from "./util/attachments.js";
4
4
  import { resolveApiKey } from "./util/provider.js";
5
5
  import { openaiFileProvider } from "./files/openai.js";
@@ -1,7 +1,7 @@
1
1
  import { GoogleGenAI } from "@google/genai";
2
2
  import { success, failure } from "../types/result.js";
3
3
  import { getModel, isImageModel } from "../models.js";
4
- import { normalizeImageRef } from "../util/imageRef.js";
4
+ import { normalizeBlob } from "../util/blobRef.js";
5
5
  import { COST_DECIMAL_PLACES, round } from "../util/util.js";
6
6
  export async function googleImage(input, config, apiKey) {
7
7
  try {
@@ -9,7 +9,7 @@ export async function googleImage(input, config, apiKey) {
9
9
  const client = new GoogleGenAI({ apiKey });
10
10
  const parts = [{ text: normalized.prompt }];
11
11
  if (normalized.images && normalized.images.length > 0) {
12
- const normalizedImages = await Promise.all(normalized.images.map((ref) => normalizeImageRef(ref)));
12
+ const normalizedImages = await Promise.all(normalized.images.map((ref) => normalizeBlob(ref)));
13
13
  for (const img of normalizedImages) {
14
14
  parts.push({
15
15
  inlineData: {
@@ -2,7 +2,7 @@ import OpenAI from "openai";
2
2
  import { toFile } from "openai/uploads";
3
3
  import { success, failure } from "../types/result.js";
4
4
  import { getModel, isImageModel } from "../models.js";
5
- import { normalizeImageRef } from "../util/imageRef.js";
5
+ import { normalizeBlob } from "../util/blobRef.js";
6
6
  import { COST_DECIMAL_PLACES, omitUndefined, round, tokenCost, } from "../util/util.js";
7
7
  export async function openaiImage(input, config, apiKey) {
8
8
  try {
@@ -51,11 +51,11 @@ function buildBaseParams(config, prompt) {
51
51
  }
52
52
  async function callEdit(client, baseParams, normalized) {
53
53
  const imageFiles = await Promise.all((normalized.images ?? []).map(async (ref, i) => {
54
- const n = await normalizeImageRef(ref);
54
+ const n = await normalizeBlob(ref);
55
55
  return toFileFor(n, `image-${i}`);
56
56
  }));
57
57
  const maskFile = normalized.mask
58
- ? await toFileFor(await normalizeImageRef(normalized.mask), "mask")
58
+ ? await toFileFor(await normalizeBlob(normalized.mask), "mask")
59
59
  : undefined;
60
60
  return client.images.edit(omitUndefined({
61
61
  ...baseParams,
package/dist/image.d.ts CHANGED
@@ -2,7 +2,7 @@ import type { ModelDataBlob } from "./modelData.js";
2
2
  import { Result } from "./types/result.js";
3
3
  import { TokenUsage } from "./types/tokenUsage.js";
4
4
  import { CostEstimate } from "./types/costEstimate.js";
5
- import { ImageRef } from "./util/imageRef.js";
5
+ import { ImageRef } from "./util/blobRef.js";
6
6
  export { ImageRef };
7
7
  export type ImageInput = string | {
8
8
  prompt: string;
package/dist/index.d.ts CHANGED
@@ -13,8 +13,16 @@ export * from "./embed.js";
13
13
  export * from "./image.js";
14
14
  export { uploadFile, deleteFile, registerFileProvider, DEFAULT_UPLOAD_BYTES } from "./files.js";
15
15
  export type { UploadFileOptions, FileProviderContext, FileProvider } from "./files.js";
16
- export { normalizeImageRef, loadBlob } from "./util/imageRef.js";
17
- export type { BlobRef } from "./util/imageRef.js";
16
+ export { normalizeBlob, loadBlob } from "./util/blobRef.js";
17
+ export type { BlobRef } from "./util/blobRef.js";
18
18
  export { getLogger, EgonLog } from "./util/logger.js";
19
19
  export type { LogLevel } from "./util/logger.js";
20
20
  export { redactAttachments } from "./util/redact.js";
21
+ export { transcribe, registerTranscriptionProvider, DEFAULT_TRANSCRIBE_BYTES, } from "./transcription.js";
22
+ export type { TranscribeOptions, TranscriptionSegment, TranscriptionWord, TranscriptionResult, TranscriptionClientClass, } from "./transcription.js";
23
+ export { BaseTranscriptionClient } from "./transcription/baseTranscriptionClient.js";
24
+ export type { TranscriptionClientConfig } from "./transcription/baseTranscriptionClient.js";
25
+ export { speak, registerSpeechProvider, } from "./speech.js";
26
+ export type { SpeakOptions, PcmAudioMetadata, SpeechResult, SpeechClientClass, } from "./speech.js";
27
+ export { BaseSpeechClient } from "./speech/baseSpeechClient.js";
28
+ export type { SpeechClientConfig } from "./speech/baseSpeechClient.js";
package/dist/index.js CHANGED
@@ -13,6 +13,12 @@ export * from "./embed.js";
13
13
  export * from "./image.js";
14
14
  // Explicit (not `export *`) so the test-only `_resetForTests` stays off the public surface.
15
15
  export { uploadFile, deleteFile, registerFileProvider, DEFAULT_UPLOAD_BYTES } from "./files.js";
16
- export { normalizeImageRef, loadBlob } from "./util/imageRef.js";
16
+ export { normalizeBlob, loadBlob } from "./util/blobRef.js";
17
17
  export { getLogger, EgonLog } from "./util/logger.js";
18
18
  export { redactAttachments } from "./util/redact.js";
19
+ // Explicit (not `export *`) so internal factories and test helpers stay private.
20
+ export { transcribe, registerTranscriptionProvider, DEFAULT_TRANSCRIBE_BYTES, } from "./transcription.js";
21
+ export { BaseTranscriptionClient } from "./transcription/baseTranscriptionClient.js";
22
+ // Explicit (not `export *`) so internal factories and test helpers stay private.
23
+ export { speak, registerSpeechProvider, } from "./speech.js";
24
+ export { BaseSpeechClient } from "./speech/baseSpeechClient.js";
package/dist/model.d.ts CHANGED
@@ -1,19 +1,22 @@
1
- import { ModelName, Provider } from "./models.js";
1
+ import { ModelName, ModelType } from "./models.js";
2
2
  import { ModelLike } from "./types.js";
3
3
  import type { ModelDataBlob } from "./modelData.js";
4
+ import type { CostEstimate } from "./types/costEstimate.js";
4
5
  export declare class Model {
5
6
  private model;
6
7
  private provider?;
7
8
  private modelData?;
8
- constructor(model: ModelName, provider?: Provider, modelData?: ModelDataBlob);
9
+ constructor(model: ModelName, provider?: string, modelData?: ModelDataBlob);
9
10
  getModel(): ModelName;
10
- getProvider(): Provider | undefined;
11
+ getProvider(): string | undefined;
11
12
  private lookupProvider;
12
13
  calculateCost(usage: {
13
14
  inputTokens: number;
14
15
  outputTokens: number;
15
16
  cachedInputTokens?: number;
16
17
  cacheCreationInputTokens?: number;
18
+ inputAudioTokens?: number;
19
+ outputAudioTokens?: number;
17
20
  }): {
18
21
  inputCost: number;
19
22
  outputCost: number;
@@ -24,5 +27,13 @@ export declare class Model {
24
27
  } | null;
25
28
  toString(): string;
26
29
  toJSON(): ModelName;
27
- static create(model: ModelLike, provider?: Provider, modelData?: ModelDataBlob): Model;
30
+ static create(model: ModelLike, provider?: string, modelData?: ModelDataBlob): Model;
28
31
  }
32
+ /**
33
+ * Per-minute STT pricing from a registry entry. Returns undefined (cost
34
+ * omitted, no error) when the model, rate, or duration is unknown — a rate of
35
+ * 0 still yields a present zero cost.
36
+ */
37
+ export declare function calculateTranscriptionCost(model: ModelType | undefined, durationSeconds: number | undefined): CostEstimate | undefined;
38
+ /** Per-code-point TTS pricing from a registry entry; same omission semantics. */
39
+ export declare function calculateSpeechCost(model: ModelType | undefined, charCount: number): CostEstimate | undefined;