smoltalk 0.2.2 → 0.3.0

package/dist/client.js

@@ -11,6 +11,7 @@ import { SmolOpenAi } from "./clients/openai.js";
 import { SmolOpenAiResponses } from "./clients/openaiResponses.js";
 import { getModel, isTextModel } from "./models.js";
 import { SmolError } from "./smolError.js";
+import { resolveApiKey, resolveProvider } from "./util/provider.js";
 const registeredProviders = {};
 export function registerProvider(providerName, clientClass) {
     registeredProviders[providerName] = clientClass;
@@ -23,22 +24,20 @@ export function unregisterProvider(providerName) {
     return false;
 }
 export function getClient(config) {
-    let provider = config.provider;
     const modelName = config.model;
-    if (!provider) {
+    const provider = resolveProvider(modelName, config.provider);
+    // For getClient, validate that the model is a text model when no explicit
+    // provider is given (since this factory only returns text-generation clients).
+    if (!config.provider) {
         const model = getModel(modelName);
-        if (model === undefined) {
-            throw new SmolError(`Model ${modelName} is not recognized. Please specify a known model, or explicitly set the provider option in the config.`);
-        }
-        if (!isTextModel(model)) {
+        if (model && !isTextModel(model)) {
             throw new SmolError(`Only text models are supported currently. ${modelName} is a ${model?.type} model.`);
         }
-        provider = model.provider;
     }
     const resolvedKeys = {
-        openAiApiKey: config.openAiApiKey || process.env.OPENAI_API_KEY,
-        googleApiKey: config.googleApiKey || process.env.GEMINI_API_KEY,
-        anthropicApiKey: config.anthropicApiKey || process.env.ANTHROPIC_API_KEY,
+        openAiApiKey: resolveApiKey("openai", config),
+        googleApiKey: resolveApiKey("google", config),
+        anthropicApiKey: resolveApiKey("anthropic", config),
     };
     const clientConfig = {
         messages: [],

package/dist/clients/baseClient.js

@@ -2,6 +2,7 @@ import { userMessage, assistantMessage } from "../classes/message/index.js";
 import { getLogger } from "../util/logger.js";
 import { SmolStructuredOutputError } from "../smolError.js";
 import { getStatelogClient } from "../statelogClient.js";
+import { stripCodeFence } from "../util/util.js";
 import { success, } from "../types.js";
 import { z } from "zod";
 const DEFAULT_NUM_RETRIES = 2;
@@ -143,10 +144,7 @@ export class BaseClient {
         }
         // 2. String → try JSON.parse, then recurse
         if (typeof rawValue === "string") {
-            const stripped = rawValue
-                .trim()
-                .replace(/^```json\s*/, "")
-                .replace(/```\s*$/, "");
+            const stripped = stripCodeFence(rawValue);
             try {
                 return this.extractResponse(promptConfig, JSON.parse(stripped), schema, depth + 1);
             }

package/dist/embed/google.d.ts

@@ -0,0 +1,3 @@
+import { EmbedConfig, EmbedResult } from "../embed.js";
+import { Result } from "../types/result.js";
+export declare function googleEmbed(inputs: string[], config: EmbedConfig, apiKey: string): Promise<Result<EmbedResult>>;

package/dist/embed/google.js

@@ -0,0 +1,25 @@
+import { GoogleGenAI } from "@google/genai";
+import { success, failure } from "../types/result.js";
+export async function googleEmbed(inputs, config, apiKey) {
+    try {
+        const client = new GoogleGenAI({ apiKey });
+        const response = await client.models.embedContent({
+            model: config.model,
+            contents: inputs,
+            ...(config.dimensions !== undefined
+                ? { config: { outputDimensionality: config.dimensions } }
+                : {}),
+        });
+        const embeddings = (response.embeddings ?? []).map((e) => e.values ?? []);
+        return success({
+            embeddings,
+            model: config.model,
+            // Google Gemini embeddings API does not return token usage in the
+            // response. Cost cannot be auto-computed without a separate
+            // countTokens() call.
+        });
+    }
+    catch (err) {
+        return failure(err instanceof Error ? err.message : "Google embedding request failed");
+    }
+}

package/dist/embed/ollama.d.ts

@@ -0,0 +1,3 @@
+import { EmbedConfig, EmbedResult } from "../embed.js";
+import { Result } from "../types/result.js";
+export declare function ollamaEmbed(inputs: string[], config: EmbedConfig, apiKey?: string, ollamaHost?: string): Promise<Result<EmbedResult>>;

package/dist/embed/ollama.js

@@ -0,0 +1,35 @@
+import { Ollama } from "ollama";
+import { success, failure } from "../types/result.js";
+import { DEFAULT_OLLAMA_HOST } from "../clients/ollama.js";
+export async function ollamaEmbed(inputs, config, apiKey, ollamaHost) {
+    try {
+        let client;
+        if (apiKey) {
+            client = new Ollama({
+                host: "https://cloud.ollama.com",
+                headers: { Authorization: "Bearer " + apiKey },
+            });
+        }
+        else {
+            client = new Ollama({ host: ollamaHost || DEFAULT_OLLAMA_HOST });
+        }
+        const response = await client.embed({
+            model: config.model,
+            input: inputs,
+            ...(config.dimensions !== undefined
+                ? { dimensions: config.dimensions }
+                : {}),
+        });
+        return success({
+            embeddings: response.embeddings,
+            model: response.model,
+            tokenUsage: {
+                inputTokens: response.prompt_eval_count ?? 0,
+                outputTokens: 0,
+            },
+        });
+    }
+    catch (err) {
+        return failure(err instanceof Error ? err.message : "Ollama embedding request failed");
+    }
+}

package/dist/embed/openai.d.ts

@@ -0,0 +1,3 @@
+import { EmbedConfig, EmbedResult } from "../embed.js";
+import { Result } from "../types/result.js";
+export declare function openaiEmbed(inputs: string[], config: EmbedConfig, apiKey: string): Promise<Result<EmbedResult>>;

package/dist/embed/openai.js

@@ -0,0 +1,42 @@
+import OpenAI from "openai";
+import { success, failure } from "../types/result.js";
+import { getModel, isEmbeddingsModel } from "../models.js";
+import { round } from "../util/util.js";
+export async function openaiEmbed(inputs, config, apiKey) {
+    try {
+        const client = new OpenAI({ apiKey });
+        const response = await client.embeddings.create({
+            model: config.model,
+            input: inputs,
+            ...(config.dimensions !== undefined
+                ? { dimensions: config.dimensions }
+                : {}),
+        });
+        const embeddings = [...response.data]
+            .sort((a, b) => a.index - b.index)
+            .map((d) => d.embedding);
+        const inputTokens = response.usage.prompt_tokens;
+        const costEstimate = calculateEmbeddingCost(config.model, inputTokens);
+        return success({
+            embeddings,
+            model: response.model,
+            tokenUsage: { inputTokens, outputTokens: 0 },
+            costEstimate,
+        });
+    }
+    catch (err) {
+        return failure(err instanceof Error ? err.message : "OpenAI embedding request failed");
+    }
+}
+function calculateEmbeddingCost(modelName, inputTokens) {
+    const model = getModel(modelName);
+    if (!model || !isEmbeddingsModel(model) || !model.tokenCost)
+        return undefined;
+    const inputCost = round((inputTokens * model.tokenCost) / 1_000_000, 6);
+    return {
+        inputCost,
+        outputCost: 0,
+        totalCost: inputCost,
+        currency: "USD",
+    };
+}

package/dist/embed.d.ts

@@ -0,0 +1,21 @@
+import { Provider } from "./models.js";
+import { Result } from "./types/result.js";
+import { TokenUsage } from "./types/tokenUsage.js";
+import { CostEstimate } from "./types/costEstimate.js";
+export type EmbedConfig = {
+    model: string;
+    provider?: Provider;
+    dimensions?: number;
+    openAiApiKey?: string;
+    googleApiKey?: string;
+    ollamaApiKey?: string;
+    ollamaHost?: string;
+    metadata?: Record<string, unknown>;
+};
+export type EmbedResult = {
+    embeddings: number[][];
+    model: string;
+    tokenUsage?: TokenUsage;
+    costEstimate?: CostEstimate;
+};
+export declare function embed(input: string | string[], config: EmbedConfig): Promise<Result<EmbedResult>>;

package/dist/embed.js

@@ -0,0 +1,35 @@
+import { failure } from "./types/result.js";
+import { resolveProvider, resolveApiKey } from "./util/provider.js";
+import { openaiEmbed } from "./embed/openai.js";
+import { googleEmbed } from "./embed/google.js";
+import { ollamaEmbed } from "./embed/ollama.js";
+export async function embed(input, config) {
+    const inputs = Array.isArray(input) ? input : [input];
+    let provider;
+    try {
+        provider = resolveProvider(config.model, config.provider);
+    }
+    catch (err) {
+        return failure(err instanceof Error ? err.message : "Failed to resolve provider");
+    }
+    const apiKey = resolveApiKey(provider, config);
+    switch (provider) {
+        case "openai":
+        case "openai-responses": {
+            if (!apiKey) {
+                return failure("No OpenAI API key provided. Set openAiApiKey in config or the OPENAI_API_KEY environment variable.");
+            }
+            return openaiEmbed(inputs, config, apiKey);
+        }
+        case "google": {
+            if (!apiKey) {
+                return failure("No Google API key provided. Set googleApiKey in config or the GEMINI_API_KEY environment variable.");
+            }
+            return googleEmbed(inputs, config, apiKey);
+        }
+        case "ollama":
+            return ollamaEmbed(inputs, config, apiKey, config.ollamaHost);
+        default:
+            return failure(`Provider "${provider}" does not support embeddings`);
+    }
+}

package/dist/image/google.d.ts

@@ -0,0 +1,3 @@
+import { ImageInput, ImageConfig, ImageGenResult } from "../image.js";
+import { Result } from "../types/result.js";
+export declare function googleImage(input: ImageInput, config: ImageConfig, apiKey: string): Promise<Result<ImageGenResult>>;

package/dist/image/google.js

@@ -0,0 +1,57 @@
+import { GoogleGenAI } from "@google/genai";
+import { success, failure } from "../types/result.js";
+import { getModel, isImageModel } from "../models.js";
+import { normalizeImageRef } from "../util/imageRef.js";
+import { COST_DECIMAL_PLACES, round } from "../util/util.js";
+export async function googleImage(input, config, apiKey) {
+    try {
+        const normalized = typeof input === "string" ? { prompt: input } : input;
+        const client = new GoogleGenAI({ apiKey });
+        const parts = [{ text: normalized.prompt }];
+        if (normalized.images && normalized.images.length > 0) {
+            const normalizedImages = await Promise.all(normalized.images.map((ref) => normalizeImageRef(ref)));
+            for (const img of normalizedImages) {
+                parts.push({
+                    inlineData: {
+                        mimeType: img.mimeType,
+                        data: Buffer.from(img.data).toString("base64"),
+                    },
+                });
+            }
+        }
+        const response = await client.models.generateContent({
+            model: config.model,
+            contents: [{ role: "user", parts }],
+            config: {
+                responseModalities: ["IMAGE", "TEXT"],
+                ...(config.metadata ?? {}),
+            },
+        });
+        const images = [];
+        for (const candidate of response.candidates ?? []) {
+            for (const part of candidate.content?.parts ?? []) {
+                if (part.inlineData?.data && part.inlineData?.mimeType) {
+                    images.push({
+                        data: new Uint8Array(Buffer.from(part.inlineData.data, "base64")),
+                        mimeType: part.inlineData.mimeType,
+                    });
+                }
+            }
+        }
+        return success({
+            images,
+            model: config.model,
+            costEstimate: calculateGoogleImageCost(config.model, images.length),
+        });
+    }
+    catch (err) {
+        return failure(err instanceof Error ? err.message : "Google image request failed");
+    }
+}
+function calculateGoogleImageCost(modelName, imageCount) {
+    const model = getModel(modelName);
+    if (!model || !isImageModel(model) || !model.costPerImage)
+        return undefined;
+    const totalCost = round(model.costPerImage * imageCount, COST_DECIMAL_PLACES);
+    return { inputCost: 0, outputCost: totalCost, totalCost, currency: "USD" };
+}

package/dist/image/openai.d.ts

@@ -0,0 +1,3 @@
+import { ImageInput, ImageConfig, ImageGenResult } from "../image.js";
+import { Result } from "../types/result.js";
+export declare function openaiImage(input: ImageInput, config: ImageConfig, apiKey: string): Promise<Result<ImageGenResult>>;

package/dist/image/openai.js

@@ -0,0 +1,140 @@
+import OpenAI from "openai";
+import { toFile } from "openai/uploads";
+import { success, failure } from "../types/result.js";
+import { getModel, isImageModel } from "../models.js";
+import { normalizeImageRef } from "../util/imageRef.js";
+import { COST_DECIMAL_PLACES, omitUndefined, round, tokenCost, } from "../util/util.js";
+export async function openaiImage(input, config, apiKey) {
+    try {
+        const normalized = typeof input === "string" ? { prompt: input } : input;
+        const hasImages = !!(normalized.images && normalized.images.length > 0);
+        if (normalized.mask && !hasImages) {
+            return failure("A mask was provided without any input images. Masks are only valid for image edits — pass at least one entry in `images` alongside the mask.");
+        }
+        const client = new OpenAI({ apiKey });
+        const baseParams = buildBaseParams(config, normalized.prompt);
+        const response = hasImages
+            ? await callEdit(client, baseParams, normalized)
+            : await client.images.generate(baseParams);
+        const mimeType = mimeFromFormat(config.outputFormat) ?? "image/png";
+        const images = (response.data ?? []).map((d) => ({
+            data: new Uint8Array(Buffer.from(d.b64_json, "base64")),
+            mimeType,
+            revisedPrompt: d.revised_prompt,
+        }));
+        const tokenUsage = extractUsage(response);
+        const costEstimate = tokenUsage
+            ? calculateImageCost(config.model, tokenUsage)
+            : calculatePerImageCost(config.model, images.length);
+        return success({
+            images,
+            model: config.model,
+            tokenUsage,
+            costEstimate,
+        });
+    }
+    catch (err) {
+        return failure(err instanceof Error ? err.message : "OpenAI image request failed");
+    }
+}
+function buildBaseParams(config, prompt) {
+    return omitUndefined({
+        model: config.model,
+        prompt,
+        n: config.n,
+        size: config.size,
+        quality: config.quality,
+        output_format: config.outputFormat,
+        background: config.background,
+        ...(config.metadata ?? {}),
+    });
+}
+async function callEdit(client, baseParams, normalized) {
+    const imageFiles = await Promise.all((normalized.images ?? []).map(async (ref, i) => {
+        const n = await normalizeImageRef(ref);
+        return toFileFor(n, `image-${i}`);
+    }));
+    const maskFile = normalized.mask
+        ? await toFileFor(await normalizeImageRef(normalized.mask), "mask")
+        : undefined;
+    return client.images.edit(omitUndefined({
+        ...baseParams,
+        image: imageFiles.length === 1 ? imageFiles[0] : imageFiles,
+        mask: maskFile,
+    }));
+}
+async function toFileFor(img, baseName) {
+    return toFile(img.data, `${baseName}.${extFromMime(img.mimeType)}`, {
+        type: img.mimeType,
+    });
+}
+function extFromMime(mime) {
+    if (mime === "image/jpeg")
+        return "jpg";
+    if (mime === "image/webp")
+        return "webp";
+    return "png";
+}
+function mimeFromFormat(fmt) {
+    if (!fmt)
+        return undefined;
+    if (fmt === "jpeg")
+        return "image/jpeg";
+    if (fmt === "webp")
+        return "image/webp";
+    return "image/png";
+}
+function extractUsage(response) {
+    const u = response?.usage;
+    if (!u)
+        return undefined;
+    return {
+        inputTokens: u.input_tokens ?? 0,
+        outputTokens: u.output_tokens ?? 0,
+        cachedInputTokens: u.input_tokens_details?.cached_tokens,
+        inputImageTokens: u.input_tokens_details?.image_tokens,
+        inputTextTokens: u.input_tokens_details?.text_tokens,
+        totalTokens: u.total_tokens,
+    };
+}
+function calculateImageCost(modelName, usage) {
+    const model = getModel(modelName);
+    if (!model || !isImageModel(model))
+        return undefined;
+    const totalIn = usage.inputTokens ?? 0;
+    const cachedIn = usage.cachedInputTokens ?? 0;
+    const imgOut = usage.outputTokens ?? 0;
+    // Prefer the detailed breakdown if the API returned it; otherwise treat
+    // all non-cached input tokens as text input.
+    let textIn;
+    let imageIn;
+    if (usage.inputTextTokens !== undefined ||
+        usage.inputImageTokens !== undefined) {
+        textIn = Math.max(0, (usage.inputTextTokens ?? 0) - cachedIn);
+        imageIn = usage.inputImageTokens ?? 0;
+    }
+    else {
+        textIn = Math.max(0, totalIn - cachedIn);
+        imageIn = 0;
+    }
+    const textInputCost = tokenCost(textIn, model.inputTokenCost);
+    const imageInputCost = tokenCost(imageIn, model.inputImageTokenCost);
+    const cachedCost = tokenCost(cachedIn, model.cachedInputTokenCost);
+    const outputCost = tokenCost(imgOut, model.outputImageTokenCost);
+    const inputCost = round(textInputCost + imageInputCost, COST_DECIMAL_PLACES);
+    const totalCost = round(inputCost + cachedCost + outputCost, COST_DECIMAL_PLACES);
+    return {
+        inputCost,
+        outputCost,
+        cachedInputCost: cachedCost,
+        totalCost,
+        currency: "USD",
+    };
+}
+function calculatePerImageCost(modelName, imageCount) {
+    const model = getModel(modelName);
+    if (!model || !isImageModel(model) || !model.costPerImage)
+        return undefined;
+    const totalCost = round(model.costPerImage * imageCount, COST_DECIMAL_PLACES);
+    return { inputCost: 0, outputCost: totalCost, totalCost, currency: "USD" };
+}

package/dist/image.d.ts

@@ -0,0 +1,35 @@
+import { Provider } from "./models.js";
+import { Result } from "./types/result.js";
+import { TokenUsage } from "./types/tokenUsage.js";
+import { CostEstimate } from "./types/costEstimate.js";
+import { ImageRef } from "./util/imageRef.js";
+export { ImageRef };
+export type ImageInput = string | {
+    prompt: string;
+    images?: ImageRef[];
+    mask?: ImageRef;
+};
+export type ImageConfig = {
+    model: string;
+    provider?: Provider;
+    n?: number;
+    size?: string;
+    quality?: "low" | "medium" | "high" | "auto";
+    outputFormat?: "png" | "jpeg" | "webp";
+    background?: "transparent" | "opaque" | "auto";
+    openAiApiKey?: string;
+    googleApiKey?: string;
+    metadata?: Record<string, unknown>;
+};
+export type GeneratedImage = {
+    data: Uint8Array;
+    mimeType: string;
+    revisedPrompt?: string;
+};
+export type ImageGenResult = {
+    images: GeneratedImage[];
+    model: string;
+    tokenUsage?: TokenUsage;
+    costEstimate?: CostEstimate;
+};
+export declare function image(input: ImageInput, config: ImageConfig): Promise<Result<ImageGenResult>>;

package/dist/image.js

@@ -0,0 +1,37 @@
+import { failure } from "./types/result.js";
+import { resolveProvider, resolveApiKey } from "./util/provider.js";
+import { openaiImage } from "./image/openai.js";
+import { googleImage } from "./image/google.js";
+export async function image(input, config) {
+    let provider;
+    try {
+        provider = resolveProvider(config.model, config.provider);
+    }
+    catch (err) {
+        return failure(err instanceof Error ? err.message : "Failed to resolve provider");
+    }
+    const apiKey = resolveApiKey(provider, config);
+    // `mask` is only meaningful for OpenAI inpainting. Reject up front so
+    // other providers don't silently drop it.
+    const hasMask = typeof input !== "string" && !!input.mask;
+    if (hasMask && provider !== "openai" && provider !== "openai-responses") {
+        return failure(`\`mask\` is only supported by the OpenAI image edit endpoint; provider "${provider}" cannot use it.`);
+    }
+    switch (provider) {
+        case "openai":
+        case "openai-responses": {
+            if (!apiKey) {
+                return failure("No OpenAI API key provided. Set openAiApiKey in config or the OPENAI_API_KEY environment variable.");
+            }
+            return openaiImage(input, config, apiKey);
+        }
+        case "google": {
+            if (!apiKey) {
+                return failure("No Google API key provided. Set googleApiKey in config or the GEMINI_API_KEY environment variable.");
+            }
+            return googleImage(input, config, apiKey);
+        }
+        default:
+            return failure(`Provider "${provider}" does not support image generation`);
+    }
+}

package/dist/index.d.ts

@@ -7,5 +7,7 @@ export * from "./util/util.js";
 export * from "./classes/message/index.js";
 export * from "./functions.js";
 export * from "./classes/ToolCall.js";
+export * from "./embed.js";
+export * from "./image.js";
 export { getLogger, EgonLog } from "./util/logger.js";
 export type { LogLevel } from "./util/logger.js";

package/dist/index.js

@@ -7,4 +7,6 @@ export * from "./util/util.js";
 export * from "./classes/message/index.js";
 export * from "./functions.js";
 export * from "./classes/ToolCall.js";
+export * from "./embed.js";
+export * from "./image.js";
 export { getLogger, EgonLog } from "./util/logger.js";