smoltalk 0.9.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +175 -7
- package/dist/classes/message/AssistantMessage.d.ts +2 -0
- package/dist/classes/message/UserMessage.d.ts +21 -0
- package/dist/classes/message/UserMessage.js +3 -0
- package/dist/classes/message/contentParts.d.ts +71 -2
- package/dist/classes/message/contentParts.js +6 -0
- package/dist/classes/message/index.d.ts +5 -2
- package/dist/classes/message/index.js +7 -0
- package/dist/classes/message/renderers/AnthropicRenderer.d.ts +2 -1
- package/dist/classes/message/renderers/AnthropicRenderer.js +3 -0
- package/dist/classes/message/renderers/GoogleRenderer.d.ts +2 -1
- package/dist/classes/message/renderers/GoogleRenderer.js +3 -0
- package/dist/classes/message/renderers/JSONRenderer.d.ts +2 -1
- package/dist/classes/message/renderers/JSONRenderer.js +4 -0
- package/dist/classes/message/renderers/OpenAIChatRenderer.d.ts +8 -1
- package/dist/classes/message/renderers/OpenAIChatRenderer.js +18 -0
- package/dist/classes/message/renderers/OpenAIResponsesRenderer.d.ts +2 -1
- package/dist/classes/message/renderers/OpenAIResponsesRenderer.js +3 -0
- package/dist/classes/message/renderers/PartRenderer.d.ts +3 -2
- package/dist/classes/message/renderers/PartRenderer.js +3 -0
- package/dist/client.js +1 -0
- package/dist/clients/anthropic.js +1 -1
- package/dist/clients/baseClient.d.ts +13 -1
- package/dist/clients/baseClient.js +36 -7
- package/dist/clients/google.js +1 -1
- package/dist/clients/ollama.js +1 -1
- package/dist/clients/openai.d.ts +2 -1
- package/dist/clients/openai.js +15 -3
- package/dist/clients/openaiCompat.d.ts +2 -0
- package/dist/clients/openaiCompat.js +5 -0
- package/dist/clients/openaiResponses.js +1 -1
- package/dist/clients/resolveAttachments.d.ts +8 -4
- package/dist/clients/resolveAttachments.js +101 -50
- package/dist/embed.d.ts +4 -0
- package/dist/files.d.ts +1 -1
- package/dist/files.js +1 -1
- package/dist/image/google.js +2 -2
- package/dist/image/openai.js +3 -3
- package/dist/image.d.ts +1 -1
- package/dist/index.d.ts +10 -2
- package/dist/index.js +7 -1
- package/dist/model.d.ts +15 -4
- package/dist/model.js +72 -12
- package/dist/models.d.ts +204 -22
- package/dist/models.js +211 -30
- package/dist/speech/baseSpeechClient.d.ts +36 -0
- package/dist/speech/baseSpeechClient.js +117 -0
- package/dist/speech/google.d.ts +6 -0
- package/dist/speech/google.js +54 -0
- package/dist/speech/groq.d.ts +11 -0
- package/dist/speech/groq.js +19 -0
- package/dist/speech/openai.d.ts +14 -0
- package/dist/speech/openai.js +51 -0
- package/dist/speech/openaiCompat.d.ts +13 -0
- package/dist/speech/openaiCompat.js +22 -0
- package/dist/speech.d.ts +45 -0
- package/dist/speech.js +63 -0
- package/dist/transcription/baseTranscriptionClient.d.ts +36 -0
- package/dist/transcription/baseTranscriptionClient.js +133 -0
- package/dist/transcription/google.d.ts +6 -0
- package/dist/transcription/google.js +56 -0
- package/dist/transcription/groq.d.ts +10 -0
- package/dist/transcription/groq.js +17 -0
- package/dist/transcription/openai.d.ts +11 -0
- package/dist/transcription/openai.js +67 -0
- package/dist/transcription/openaiCompat.d.ts +13 -0
- package/dist/transcription/openaiCompat.js +22 -0
- package/dist/transcription.d.ts +54 -0
- package/dist/transcription.js +64 -0
- package/dist/types/tokenUsage.d.ts +4 -0
- package/dist/types/tokenUsage.js +4 -0
- package/dist/types.d.ts +4 -0
- package/dist/util/attachments.d.ts +1 -1
- package/dist/util/audioMime.d.ts +26 -0
- package/dist/util/audioMime.js +74 -0
- package/dist/util/{imageRef.d.ts → blobRef.d.ts} +9 -9
- package/dist/util/{imageRef.js → blobRef.js} +6 -13
- package/dist/util/googleAudioUsage.d.ts +14 -0
- package/dist/util/googleAudioUsage.js +52 -0
- package/dist/util/mime.d.ts +21 -0
- package/dist/util/mime.js +54 -0
- package/dist/util/modalities.d.ts +6 -2
- package/dist/util/modalities.js +13 -15
- package/dist/util/provider.d.ts +3 -0
- package/dist/util/provider.js +3 -1
- package/package.json +1 -1
|
@@ -6,11 +6,21 @@ import { stripCodeFence } from "../util/util.js";
|
|
|
6
6
|
import { success, failure, } from "../types.js";
|
|
7
7
|
import { validateHostedTools } from "../util/hostedTools.js";
|
|
8
8
|
import { resolveMessageAttachments, messagesHaveAttachments, DEFAULT_MAX_ATTACHMENT_BYTES } from "./resolveAttachments.js";
|
|
9
|
-
import {
|
|
9
|
+
import { neededInputModalities, MODALITIES_REQUIRING_DECLARATION, } from "../util/modalities.js";
|
|
10
|
+
import { modelSupportsInputModality } from "../models.js";
|
|
10
11
|
import { resolveProvider } from "../util/provider.js";
|
|
11
12
|
import { isUnconstrainedSchema } from "../util/jsonSchema.js";
|
|
12
13
|
import { z } from "zod";
|
|
13
14
|
const DEFAULT_NUM_RETRIES = 2;
|
|
15
|
+
function clientSupportsAttachment(capabilities, modality) {
|
|
16
|
+
if (modality === "audio") {
|
|
17
|
+
return capabilities.audioFormats.length > 0;
|
|
18
|
+
}
|
|
19
|
+
if (modality === "image" || modality === "pdf") {
|
|
20
|
+
return capabilities.inputModalities.includes(modality);
|
|
21
|
+
}
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
14
24
|
export class BaseClient {
|
|
15
25
|
config;
|
|
16
26
|
statelogClient;
|
|
@@ -57,24 +67,43 @@ export class BaseClient {
|
|
|
57
67
|
return null;
|
|
58
68
|
}
|
|
59
69
|
/**
|
|
60
|
-
*
|
|
70
|
+
* What this client can accept as attachments. Subclasses override to declare
|
|
71
|
+
* more (or fewer). Checked against the messages, alongside the model's own
|
|
72
|
+
* declared modalities, before any serialization runs.
|
|
73
|
+
*/
|
|
74
|
+
attachmentCapabilities() {
|
|
75
|
+
return { inputModalities: ["image", "pdf"], audioFormats: [] };
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Gate on input modalities and resolve any image/PDF/audio attachment refs
|
|
61
79
|
* (path/url/bytes → base64) before the synchronous serializers run. Returns
|
|
62
80
|
* the (possibly rewritten) config on success, or a Failure to surface. Shared
|
|
63
81
|
* by textSync and textStream so the two paths can't diverge.
|
|
64
82
|
*/
|
|
65
83
|
async prepareAttachments(config) {
|
|
66
|
-
const
|
|
67
|
-
if (
|
|
68
|
-
return modalityResult;
|
|
69
|
-
}
|
|
70
|
-
if (!messagesHaveAttachments(config.messages)) {
|
|
84
|
+
const needed = neededInputModalities(config.messages);
|
|
85
|
+
if (needed.length === 0 && !messagesHaveAttachments(config.messages)) {
|
|
71
86
|
return success(config);
|
|
72
87
|
}
|
|
73
88
|
const provider = resolveProvider(config.model, config.provider, config.modelData);
|
|
89
|
+
const capabilities = this.attachmentCapabilities();
|
|
90
|
+
for (const modality of needed) {
|
|
91
|
+
if (!clientSupportsAttachment(capabilities, modality)) {
|
|
92
|
+
return failure(`${modality[0].toUpperCase()}${modality.slice(1)} input is not supported by the "${provider}" provider.`);
|
|
93
|
+
}
|
|
94
|
+
const supported = modelSupportsInputModality(config.model, modality, config.modelData, provider);
|
|
95
|
+
if (supported === false) {
|
|
96
|
+
return failure(`Model ${config.model} does not support ${modality} input.`);
|
|
97
|
+
}
|
|
98
|
+
if (supported === undefined && MODALITIES_REQUIRING_DECLARATION.has(modality)) {
|
|
99
|
+
return failure(`Model ${config.model} does not support ${modality} input.`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
74
102
|
const maxBytes = config.attachments?.maxBytes ?? DEFAULT_MAX_ATTACHMENT_BYTES;
|
|
75
103
|
const resolved = await resolveMessageAttachments(config.messages, {
|
|
76
104
|
provider,
|
|
77
105
|
maxBytes,
|
|
106
|
+
audioFormats: capabilities.audioFormats,
|
|
78
107
|
});
|
|
79
108
|
if (!resolved.success) {
|
|
80
109
|
return resolved;
|
package/dist/clients/google.js
CHANGED
|
@@ -209,7 +209,7 @@ export class SmolGoogle extends BaseClient {
|
|
|
209
209
|
}
|
|
210
210
|
this.client = new GoogleGenAI({ apiKey });
|
|
211
211
|
this.logger = getLogger();
|
|
212
|
-
this.model = new Model(config.model,
|
|
212
|
+
this.model = new Model(config.model, config.provider, config.modelData);
|
|
213
213
|
}
|
|
214
214
|
getClient() {
|
|
215
215
|
return this.client;
|
package/dist/clients/ollama.js
CHANGED
|
@@ -20,7 +20,7 @@ export class SmolOllama extends BaseClient {
|
|
|
20
20
|
constructor(config) {
|
|
21
21
|
super(config);
|
|
22
22
|
this.logger = getLogger();
|
|
23
|
-
this.model = new Model(config.model,
|
|
23
|
+
this.model = new Model(config.model, config.provider, config.modelData);
|
|
24
24
|
const apiKey = config.apiKey?.ollama;
|
|
25
25
|
if (apiKey) {
|
|
26
26
|
this.client = new Ollama({
|
package/dist/clients/openai.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import OpenAI from "openai";
|
|
2
2
|
import { PromptResult, Result, SmolClient, SmolConfig, StreamChunk, HostedToolResult } from "../types.js";
|
|
3
3
|
import { EgonLog } from "../util/logger.js";
|
|
4
|
-
import { BaseClient } from "./baseClient.js";
|
|
4
|
+
import { BaseClient, type ClientAttachmentCapabilities } from "./baseClient.js";
|
|
5
5
|
import { ModelName } from "../models.js";
|
|
6
6
|
import { Model } from "../model.js";
|
|
7
7
|
import { CostEstimate, TokenUsage } from "../types.js";
|
|
@@ -11,6 +11,7 @@ export declare class SmolOpenAi extends BaseClient implements SmolClient {
|
|
|
11
11
|
protected logger: EgonLog;
|
|
12
12
|
protected model: Model;
|
|
13
13
|
constructor(config: SmolOpenAiConfig);
|
|
14
|
+
protected attachmentCapabilities(): ClientAttachmentCapabilities;
|
|
14
15
|
/**
|
|
15
16
|
* Build the `new OpenAI({...})` options. Subclasses override to inject a
|
|
16
17
|
* different baseURL or to pull the key from a different config field.
|
package/dist/clients/openai.js
CHANGED
|
@@ -20,7 +20,11 @@ export class SmolOpenAi extends BaseClient {
|
|
|
20
20
|
const options = this.resolveClientOptions(config);
|
|
21
21
|
this.client = new OpenAI(options);
|
|
22
22
|
this.logger = getLogger();
|
|
23
|
-
this.model = new Model(config.model,
|
|
23
|
+
this.model = new Model(config.model, config.provider, config.modelData);
|
|
24
|
+
}
|
|
25
|
+
attachmentCapabilities() {
|
|
26
|
+
// Chat Completions input_audio accepts inline mp3/wav only.
|
|
27
|
+
return { inputModalities: ["image", "pdf"], audioFormats: ["mp3", "wav"] };
|
|
24
28
|
}
|
|
25
29
|
/**
|
|
26
30
|
* Build the `new OpenAI({...})` options. Subclasses override to inject a
|
|
@@ -72,14 +76,22 @@ export class SmolOpenAi extends BaseClient {
|
|
|
72
76
|
let cost;
|
|
73
77
|
if (usageData) {
|
|
74
78
|
const cached = usageData.prompt_tokens_details?.cached_tokens ?? 0;
|
|
79
|
+
const audioIn = usageData.prompt_tokens_details?.audio_tokens ?? 0;
|
|
80
|
+
const audioOut = usageData.completion_tokens_details?.audio_tokens ?? 0;
|
|
75
81
|
usage = {
|
|
76
|
-
inputTokens: Math.max(0, (usageData.prompt_tokens || 0) - cached),
|
|
77
|
-
outputTokens: usageData.completion_tokens || 0,
|
|
82
|
+
inputTokens: Math.max(0, (usageData.prompt_tokens || 0) - cached - audioIn),
|
|
83
|
+
outputTokens: Math.max(0, (usageData.completion_tokens || 0) - audioOut),
|
|
78
84
|
totalTokens: usageData.total_tokens,
|
|
79
85
|
};
|
|
80
86
|
if (cached > 0) {
|
|
81
87
|
usage.cachedInputTokens = cached;
|
|
82
88
|
}
|
|
89
|
+
if (audioIn > 0) {
|
|
90
|
+
usage.inputAudioTokens = audioIn;
|
|
91
|
+
}
|
|
92
|
+
if (audioOut > 0) {
|
|
93
|
+
usage.outputAudioTokens = audioOut;
|
|
94
|
+
}
|
|
83
95
|
// Prefer provider-supplied cost when available (e.g. OpenRouter
|
|
84
96
|
// usage.cost, DeepInfra usage.estimated_cost, LiteLLM header).
|
|
85
97
|
// Fall back to the smoltalk model-registry calculation.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { SmolOpenAi } from "./openai.js";
|
|
2
|
+
import type { ClientAttachmentCapabilities } from "./baseClient.js";
|
|
2
3
|
import type { SmolConfig } from "../types.js";
|
|
3
4
|
/**
|
|
4
5
|
* Generic OpenAI-compatible client. Use when pointing smoltalk at any
|
|
@@ -14,6 +15,7 @@ import type { SmolConfig } from "../types.js";
|
|
|
14
15
|
* cost stays undefined — that's expected for arbitrary backends).
|
|
15
16
|
*/
|
|
16
17
|
export declare class SmolOpenAiCompat extends SmolOpenAi {
|
|
18
|
+
protected attachmentCapabilities(): ClientAttachmentCapabilities;
|
|
17
19
|
protected resolveClientOptions(config: SmolConfig): {
|
|
18
20
|
apiKey: string;
|
|
19
21
|
baseURL: string;
|
|
@@ -14,6 +14,11 @@ import { resolveApiKey, resolveBaseUrl } from "../util/provider.js";
|
|
|
14
14
|
* cost stays undefined — that's expected for arbitrary backends).
|
|
15
15
|
*/
|
|
16
16
|
export class SmolOpenAiCompat extends SmolOpenAi {
|
|
17
|
+
// Compat endpoints speak the Chat Completions wire format but do not get
|
|
18
|
+
// OpenAI's input_audio handling — declare no audio support.
|
|
19
|
+
attachmentCapabilities() {
|
|
20
|
+
return { inputModalities: ["image", "pdf"], audioFormats: [] };
|
|
21
|
+
}
|
|
17
22
|
resolveClientOptions(config) {
|
|
18
23
|
const apiKey = resolveApiKey("openai-compat", config);
|
|
19
24
|
const baseURL = resolveBaseUrl("openai-compat", config);
|
|
@@ -62,7 +62,7 @@ export class SmolOpenAiResponses extends BaseClient {
|
|
|
62
62
|
}
|
|
63
63
|
this.client = new OpenAI({ apiKey });
|
|
64
64
|
this.logger = getLogger();
|
|
65
|
-
this.model = new Model(config.model,
|
|
65
|
+
this.model = new Model(config.model, config.provider, config.modelData);
|
|
66
66
|
}
|
|
67
67
|
getClient() {
|
|
68
68
|
return this.client;
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import { Message } from "../classes/message/index.js";
|
|
2
2
|
import { Result } from "../types.js";
|
|
3
3
|
export declare const DEFAULT_MAX_ATTACHMENT_BYTES: number;
|
|
4
|
+
type ResolveOptions = {
|
|
5
|
+
provider: string;
|
|
6
|
+
maxBytes: number;
|
|
7
|
+
/** Audio containers (by primary extension) the target client accepts inline. */
|
|
8
|
+
audioFormats: readonly string[];
|
|
9
|
+
};
|
|
4
10
|
/** Whether any user message carries an image/file attachment part. */
|
|
5
11
|
export declare function messagesHaveAttachments(messages: Message[]): boolean;
|
|
6
12
|
/** Whether `provider` accepts a remote URL directly for this part type. */
|
|
7
13
|
export declare function acceptsRemoteUrl(provider: string, partType: "image" | "file"): boolean;
|
|
8
|
-
export declare function resolveMessageAttachments(messages: Message[], options:
|
|
9
|
-
|
|
10
|
-
maxBytes: number;
|
|
11
|
-
}): Promise<Result<Message[]>>;
|
|
14
|
+
export declare function resolveMessageAttachments(messages: Message[], options: ResolveOptions): Promise<Result<Message[]>>;
|
|
15
|
+
export {};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { UserMessage } from "../classes/message/index.js";
|
|
2
|
-
import {
|
|
2
|
+
import { normalizeBlob } from "../util/blobRef.js";
|
|
3
3
|
import { fileFamily } from "../util/attachments.js";
|
|
4
|
+
import { audioFormatForMime } from "../util/mime.js";
|
|
4
5
|
import { success, failure } from "../types.js";
|
|
5
6
|
export const DEFAULT_MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024;
|
|
6
7
|
const URL_IMAGE_PROVIDERS = new Set([
|
|
@@ -24,7 +25,7 @@ export function messagesHaveAttachments(messages) {
|
|
|
24
25
|
continue;
|
|
25
26
|
}
|
|
26
27
|
for (const part of parts) {
|
|
27
|
-
if (part.type === "image" || part.type === "file") {
|
|
28
|
+
if (part.type === "image" || part.type === "file" || part.type === "audio") {
|
|
28
29
|
return true;
|
|
29
30
|
}
|
|
30
31
|
}
|
|
@@ -38,6 +39,100 @@ export function acceptsRemoteUrl(provider, partType) {
|
|
|
38
39
|
}
|
|
39
40
|
return URL_PDF_PROVIDERS.has(provider);
|
|
40
41
|
}
|
|
42
|
+
/** Load a ref to inline base64, gated to `allowed` MIME prefixes. Throws on failure. */
|
|
43
|
+
async function toBase64Source(source, allowed, maxBytes) {
|
|
44
|
+
const { data, mimeType } = await normalizeBlob(source, {
|
|
45
|
+
allowedMimePrefixes: allowed,
|
|
46
|
+
maxBytes,
|
|
47
|
+
});
|
|
48
|
+
return { kind: "base64", base64: Buffer.from(data).toString("base64"), mimeType };
|
|
49
|
+
}
|
|
50
|
+
/** Error message when a providerFile ref targets the wrong provider family, else null. */
|
|
51
|
+
function providerFileError(fileProvider, targetProvider) {
|
|
52
|
+
const family = fileFamily(targetProvider);
|
|
53
|
+
if (family === null || fileProvider !== family) {
|
|
54
|
+
return (`Attachment references a "${fileProvider}" file, but this call targets provider ` +
|
|
55
|
+
`"${targetProvider}" (file family ${family ?? "none"}).`);
|
|
56
|
+
}
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
// Audio has no providerFile/URL passthrough: Chat input_audio requires
|
|
60
|
+
// inline base64, so every audio source is normalized here.
|
|
61
|
+
async function resolveAudioPart(part, options) {
|
|
62
|
+
try {
|
|
63
|
+
const source = await toBase64Source(part.source, ["audio/"], options.maxBytes);
|
|
64
|
+
const audioFormat = audioFormatForMime(source.mimeType);
|
|
65
|
+
if (audioFormat === null || !options.audioFormats.includes(audioFormat.extension)) {
|
|
66
|
+
return failure(`Audio input for provider "${options.provider}" supports only ` +
|
|
67
|
+
`${options.audioFormats.join(", ")}; got "${source.mimeType}".`);
|
|
68
|
+
}
|
|
69
|
+
const resolved = { type: "audio", source };
|
|
70
|
+
if (part.filename !== undefined) {
|
|
71
|
+
resolved.filename = part.filename;
|
|
72
|
+
}
|
|
73
|
+
return success(resolved);
|
|
74
|
+
}
|
|
75
|
+
catch (err) {
|
|
76
|
+
return failure(`Failed to load audio attachment: ${err.message}`);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
async function resolveImagePart(part, options) {
|
|
80
|
+
// Provider file references are validated and passed through (no download/cap).
|
|
81
|
+
if (part.source.kind === "providerFile") {
|
|
82
|
+
const mismatch = providerFileError(part.source.provider, options.provider);
|
|
83
|
+
if (mismatch !== null) {
|
|
84
|
+
return failure(mismatch);
|
|
85
|
+
}
|
|
86
|
+
if (options.provider === "openai") {
|
|
87
|
+
return failure("An image file reference requires the openai-responses provider (OpenAI Chat Completions has no image-by-file_id form).");
|
|
88
|
+
}
|
|
89
|
+
return success(part);
|
|
90
|
+
}
|
|
91
|
+
// Passthrough: keep a url ref when the target provider accepts a remote URL.
|
|
92
|
+
if (part.source.kind === "url" && acceptsRemoteUrl(options.provider, "image")) {
|
|
93
|
+
return success(part);
|
|
94
|
+
}
|
|
95
|
+
try {
|
|
96
|
+
const source = await toBase64Source(part.source, ["image/"], options.maxBytes);
|
|
97
|
+
return success({ type: "image", source });
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
return failure(`Failed to load image attachment: ${err.message}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
async function resolveFilePart(part, options) {
|
|
104
|
+
// Provider file references are validated and passed through (no download/cap).
|
|
105
|
+
if (part.source.kind === "providerFile") {
|
|
106
|
+
const mismatch = providerFileError(part.source.provider, options.provider);
|
|
107
|
+
if (mismatch !== null) {
|
|
108
|
+
return failure(mismatch);
|
|
109
|
+
}
|
|
110
|
+
return success(part);
|
|
111
|
+
}
|
|
112
|
+
// Passthrough: keep a url ref when the target provider accepts a remote URL.
|
|
113
|
+
if (part.source.kind === "url" && acceptsRemoteUrl(options.provider, "file")) {
|
|
114
|
+
return success(part);
|
|
115
|
+
}
|
|
116
|
+
try {
|
|
117
|
+
const source = await toBase64Source(part.source, ["application/pdf"], options.maxBytes);
|
|
118
|
+
return success({ type: "file", source, filename: part.filename });
|
|
119
|
+
}
|
|
120
|
+
catch (err) {
|
|
121
|
+
return failure(`Failed to load file attachment: ${err.message}`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
async function resolveUserPart(part, options) {
|
|
125
|
+
if (part.type === "text") {
|
|
126
|
+
return success(part);
|
|
127
|
+
}
|
|
128
|
+
if (part.type === "audio") {
|
|
129
|
+
return resolveAudioPart(part, options);
|
|
130
|
+
}
|
|
131
|
+
if (part.type === "image") {
|
|
132
|
+
return resolveImagePart(part, options);
|
|
133
|
+
}
|
|
134
|
+
return resolveFilePart(part, options);
|
|
135
|
+
}
|
|
41
136
|
export async function resolveMessageAttachments(messages, options) {
|
|
42
137
|
const out = [];
|
|
43
138
|
for (const msg of messages) {
|
|
@@ -52,55 +147,11 @@ export async function resolveMessageAttachments(messages, options) {
|
|
|
52
147
|
}
|
|
53
148
|
const resolvedParts = [];
|
|
54
149
|
for (const part of parts) {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
}
|
|
59
|
-
// Provider file references are validated and passed through (no download/cap).
|
|
60
|
-
if (part.source.kind === "providerFile") {
|
|
61
|
-
const family = fileFamily(options.provider);
|
|
62
|
-
if (family === null || part.source.provider !== family) {
|
|
63
|
-
return failure(`Attachment references a "${part.source.provider}" file, but this call targets provider ` +
|
|
64
|
-
`"${options.provider}" (file family ${family ?? "none"}).`);
|
|
65
|
-
}
|
|
66
|
-
if (part.type === "image" && options.provider === "openai") {
|
|
67
|
-
return failure("An image file reference requires the openai-responses provider (OpenAI Chat Completions has no image-by-file_id form).");
|
|
68
|
-
}
|
|
69
|
-
resolvedParts.push(part);
|
|
70
|
-
continue;
|
|
71
|
-
}
|
|
72
|
-
// Passthrough: keep a url ref when the target provider accepts a remote URL.
|
|
73
|
-
if (part.source.kind === "url" && acceptsRemoteUrl(options.provider, part.type)) {
|
|
74
|
-
resolvedParts.push(part);
|
|
75
|
-
continue;
|
|
76
|
-
}
|
|
77
|
-
let allowed;
|
|
78
|
-
if (part.type === "image") {
|
|
79
|
-
allowed = ["image/"];
|
|
80
|
-
}
|
|
81
|
-
else {
|
|
82
|
-
allowed = ["application/pdf"];
|
|
83
|
-
}
|
|
84
|
-
try {
|
|
85
|
-
const { data, mimeType } = await normalizeImageRef(part.source, {
|
|
86
|
-
allowedMimePrefixes: allowed,
|
|
87
|
-
maxBytes: options.maxBytes,
|
|
88
|
-
});
|
|
89
|
-
const source = {
|
|
90
|
-
kind: "base64",
|
|
91
|
-
base64: Buffer.from(data).toString("base64"),
|
|
92
|
-
mimeType,
|
|
93
|
-
};
|
|
94
|
-
if (part.type === "image") {
|
|
95
|
-
resolvedParts.push({ type: "image", source });
|
|
96
|
-
}
|
|
97
|
-
else {
|
|
98
|
-
resolvedParts.push({ type: "file", source, filename: part.filename });
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
catch (err) {
|
|
102
|
-
return failure(`Failed to load ${part.type} attachment: ${err.message}`);
|
|
150
|
+
const resolved = await resolveUserPart(part, options);
|
|
151
|
+
if (!resolved.success) {
|
|
152
|
+
return resolved;
|
|
103
153
|
}
|
|
154
|
+
resolvedParts.push(resolved.value);
|
|
104
155
|
}
|
|
105
156
|
out.push(new UserMessage(resolvedParts, { name: msg.name, rawData: msg.rawData }));
|
|
106
157
|
}
|
package/dist/embed.d.ts
CHANGED
|
@@ -16,6 +16,8 @@ export type EmbedConfig = {
|
|
|
16
16
|
deepInfra?: string;
|
|
17
17
|
liteLlm?: string;
|
|
18
18
|
openAiCompat?: string;
|
|
19
|
+
/** Arbitrary provider names, for keys targeting a custom-registered provider. */
|
|
20
|
+
[provider: string]: string | undefined;
|
|
19
21
|
};
|
|
20
22
|
/** Custom base URLs, nested by provider. */
|
|
21
23
|
baseUrl?: {
|
|
@@ -23,6 +25,8 @@ export type EmbedConfig = {
|
|
|
23
25
|
deepInfra?: string;
|
|
24
26
|
liteLlm?: string;
|
|
25
27
|
openAiCompat?: string;
|
|
28
|
+
/** Arbitrary provider names, for URLs targeting a custom-registered provider. */
|
|
29
|
+
[provider: string]: string | undefined;
|
|
26
30
|
};
|
|
27
31
|
metadata?: Record<string, unknown>;
|
|
28
32
|
modelData?: ModelDataBlob;
|
package/dist/files.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { Result } from "./types/result.js";
|
|
2
2
|
import type { SmolConfig } from "./types.js";
|
|
3
3
|
import { ProviderFileRef } from "./classes/message/contentParts.js";
|
|
4
|
-
import { BlobRef } from "./util/
|
|
4
|
+
import { BlobRef } from "./util/blobRef.js";
|
|
5
5
|
/** Default cap on a resolved upload's size. Callers can raise it via opts.maxBytes. */
|
|
6
6
|
export declare const DEFAULT_UPLOAD_BYTES: number;
|
|
7
7
|
/**
|
package/dist/files.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { failure } from "./types/result.js";
|
|
2
|
-
import { loadBlob } from "./util/
|
|
2
|
+
import { loadBlob } from "./util/blobRef.js";
|
|
3
3
|
import { fileFamily } from "./util/attachments.js";
|
|
4
4
|
import { resolveApiKey } from "./util/provider.js";
|
|
5
5
|
import { openaiFileProvider } from "./files/openai.js";
|
package/dist/image/google.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { GoogleGenAI } from "@google/genai";
|
|
2
2
|
import { success, failure } from "../types/result.js";
|
|
3
3
|
import { getModel, isImageModel } from "../models.js";
|
|
4
|
-
import {
|
|
4
|
+
import { normalizeBlob } from "../util/blobRef.js";
|
|
5
5
|
import { COST_DECIMAL_PLACES, round } from "../util/util.js";
|
|
6
6
|
export async function googleImage(input, config, apiKey) {
|
|
7
7
|
try {
|
|
@@ -9,7 +9,7 @@ export async function googleImage(input, config, apiKey) {
|
|
|
9
9
|
const client = new GoogleGenAI({ apiKey });
|
|
10
10
|
const parts = [{ text: normalized.prompt }];
|
|
11
11
|
if (normalized.images && normalized.images.length > 0) {
|
|
12
|
-
const normalizedImages = await Promise.all(normalized.images.map((ref) =>
|
|
12
|
+
const normalizedImages = await Promise.all(normalized.images.map((ref) => normalizeBlob(ref)));
|
|
13
13
|
for (const img of normalizedImages) {
|
|
14
14
|
parts.push({
|
|
15
15
|
inlineData: {
|
package/dist/image/openai.js
CHANGED
|
@@ -2,7 +2,7 @@ import OpenAI from "openai";
|
|
|
2
2
|
import { toFile } from "openai/uploads";
|
|
3
3
|
import { success, failure } from "../types/result.js";
|
|
4
4
|
import { getModel, isImageModel } from "../models.js";
|
|
5
|
-
import {
|
|
5
|
+
import { normalizeBlob } from "../util/blobRef.js";
|
|
6
6
|
import { COST_DECIMAL_PLACES, omitUndefined, round, tokenCost, } from "../util/util.js";
|
|
7
7
|
export async function openaiImage(input, config, apiKey) {
|
|
8
8
|
try {
|
|
@@ -51,11 +51,11 @@ function buildBaseParams(config, prompt) {
|
|
|
51
51
|
}
|
|
52
52
|
async function callEdit(client, baseParams, normalized) {
|
|
53
53
|
const imageFiles = await Promise.all((normalized.images ?? []).map(async (ref, i) => {
|
|
54
|
-
const n = await
|
|
54
|
+
const n = await normalizeBlob(ref);
|
|
55
55
|
return toFileFor(n, `image-${i}`);
|
|
56
56
|
}));
|
|
57
57
|
const maskFile = normalized.mask
|
|
58
|
-
? await toFileFor(await
|
|
58
|
+
? await toFileFor(await normalizeBlob(normalized.mask), "mask")
|
|
59
59
|
: undefined;
|
|
60
60
|
return client.images.edit(omitUndefined({
|
|
61
61
|
...baseParams,
|
package/dist/image.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { ModelDataBlob } from "./modelData.js";
|
|
|
2
2
|
import { Result } from "./types/result.js";
|
|
3
3
|
import { TokenUsage } from "./types/tokenUsage.js";
|
|
4
4
|
import { CostEstimate } from "./types/costEstimate.js";
|
|
5
|
-
import { ImageRef } from "./util/
|
|
5
|
+
import { ImageRef } from "./util/blobRef.js";
|
|
6
6
|
export { ImageRef };
|
|
7
7
|
export type ImageInput = string | {
|
|
8
8
|
prompt: string;
|
package/dist/index.d.ts
CHANGED
|
@@ -13,8 +13,16 @@ export * from "./embed.js";
|
|
|
13
13
|
export * from "./image.js";
|
|
14
14
|
export { uploadFile, deleteFile, registerFileProvider, DEFAULT_UPLOAD_BYTES } from "./files.js";
|
|
15
15
|
export type { UploadFileOptions, FileProviderContext, FileProvider } from "./files.js";
|
|
16
|
-
export {
|
|
17
|
-
export type { BlobRef } from "./util/
|
|
16
|
+
export { normalizeBlob, loadBlob } from "./util/blobRef.js";
|
|
17
|
+
export type { BlobRef } from "./util/blobRef.js";
|
|
18
18
|
export { getLogger, EgonLog } from "./util/logger.js";
|
|
19
19
|
export type { LogLevel } from "./util/logger.js";
|
|
20
20
|
export { redactAttachments } from "./util/redact.js";
|
|
21
|
+
export { transcribe, registerTranscriptionProvider, DEFAULT_TRANSCRIBE_BYTES, } from "./transcription.js";
|
|
22
|
+
export type { TranscribeOptions, TranscriptionSegment, TranscriptionWord, TranscriptionResult, TranscriptionClientClass, } from "./transcription.js";
|
|
23
|
+
export { BaseTranscriptionClient } from "./transcription/baseTranscriptionClient.js";
|
|
24
|
+
export type { TranscriptionClientConfig } from "./transcription/baseTranscriptionClient.js";
|
|
25
|
+
export { speak, registerSpeechProvider, } from "./speech.js";
|
|
26
|
+
export type { SpeakOptions, PcmAudioMetadata, SpeechResult, SpeechClientClass, } from "./speech.js";
|
|
27
|
+
export { BaseSpeechClient } from "./speech/baseSpeechClient.js";
|
|
28
|
+
export type { SpeechClientConfig } from "./speech/baseSpeechClient.js";
|
package/dist/index.js
CHANGED
|
@@ -13,6 +13,12 @@ export * from "./embed.js";
|
|
|
13
13
|
export * from "./image.js";
|
|
14
14
|
// Explicit (not `export *`) so the test-only `_resetForTests` stays off the public surface.
|
|
15
15
|
export { uploadFile, deleteFile, registerFileProvider, DEFAULT_UPLOAD_BYTES } from "./files.js";
|
|
16
|
-
export {
|
|
16
|
+
export { normalizeBlob, loadBlob } from "./util/blobRef.js";
|
|
17
17
|
export { getLogger, EgonLog } from "./util/logger.js";
|
|
18
18
|
export { redactAttachments } from "./util/redact.js";
|
|
19
|
+
// Explicit (not `export *`) so internal factories and test helpers stay private.
|
|
20
|
+
export { transcribe, registerTranscriptionProvider, DEFAULT_TRANSCRIBE_BYTES, } from "./transcription.js";
|
|
21
|
+
export { BaseTranscriptionClient } from "./transcription/baseTranscriptionClient.js";
|
|
22
|
+
// Explicit (not `export *`) so internal factories and test helpers stay private.
|
|
23
|
+
export { speak, registerSpeechProvider, } from "./speech.js";
|
|
24
|
+
export { BaseSpeechClient } from "./speech/baseSpeechClient.js";
|
package/dist/model.d.ts
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
|
-
import { ModelName,
|
|
1
|
+
import { ModelName, ModelType } from "./models.js";
|
|
2
2
|
import { ModelLike } from "./types.js";
|
|
3
3
|
import type { ModelDataBlob } from "./modelData.js";
|
|
4
|
+
import type { CostEstimate } from "./types/costEstimate.js";
|
|
4
5
|
export declare class Model {
|
|
5
6
|
private model;
|
|
6
7
|
private provider?;
|
|
7
8
|
private modelData?;
|
|
8
|
-
constructor(model: ModelName, provider?:
|
|
9
|
+
constructor(model: ModelName, provider?: string, modelData?: ModelDataBlob);
|
|
9
10
|
getModel(): ModelName;
|
|
10
|
-
getProvider():
|
|
11
|
+
getProvider(): string | undefined;
|
|
11
12
|
private lookupProvider;
|
|
12
13
|
calculateCost(usage: {
|
|
13
14
|
inputTokens: number;
|
|
14
15
|
outputTokens: number;
|
|
15
16
|
cachedInputTokens?: number;
|
|
16
17
|
cacheCreationInputTokens?: number;
|
|
18
|
+
inputAudioTokens?: number;
|
|
19
|
+
outputAudioTokens?: number;
|
|
17
20
|
}): {
|
|
18
21
|
inputCost: number;
|
|
19
22
|
outputCost: number;
|
|
@@ -24,5 +27,13 @@ export declare class Model {
|
|
|
24
27
|
} | null;
|
|
25
28
|
toString(): string;
|
|
26
29
|
toJSON(): ModelName;
|
|
27
|
-
static create(model: ModelLike, provider?:
|
|
30
|
+
static create(model: ModelLike, provider?: string, modelData?: ModelDataBlob): Model;
|
|
28
31
|
}
|
|
32
|
+
/**
|
|
33
|
+
* Per-minute STT pricing from a registry entry. Returns undefined (cost
|
|
34
|
+
* omitted, no error) when the model, rate, or duration is unknown — a rate of
|
|
35
|
+
* 0 still yields a present zero cost.
|
|
36
|
+
*/
|
|
37
|
+
export declare function calculateTranscriptionCost(model: ModelType | undefined, durationSeconds: number | undefined): CostEstimate | undefined;
|
|
38
|
+
/** Per-code-point TTS pricing from a registry entry; same omission semantics. */
|
|
39
|
+
export declare function calculateSpeechCost(model: ModelType | undefined, charCount: number): CostEstimate | undefined;
|