smoltalk 0.13.2 → 0.14.0

package/README.md

@@ -276,7 +276,7 @@ const r = await textSync({
 | `deepinfra`     | ✅   | ✅         | ❌ (uses per-model endpoints, not OpenAI shape) | ❌ |
 | `litellm`       | ✅   | ✅         | ✅ (if the upstream model supports it) | ✅ (if upstream supports it) |
 | `openai-compat` | ✅   | ✅         | ✅ (backend-dependent) | depends on backend |
-| `mlx`           | ✅   | ❌         | ❌               | ❌ |
+| `mlx`           | ✅   | ✅         | ❌               | ❌ |
 
 Smoltalk surfaces a clear `failure(...)` from `embed()`/`image()` for the
 unsupported combinations rather than silently dropping the call.
@@ -543,6 +543,11 @@ const { resolveModel } = await loadLlamaCpp({
 const modelPath = await resolveModel("hf:org/repo/model.gguf", "/models/cache");
 ```
 
+`embed()` with `provider: "llama-cpp"` and a local `.gguf` path works the
+same way and needs `smoltalk-llama-cpp` >= 0.5.0; the vector is computed in
+process. See that package's README for the embedding caveats (dimension
+truncation, one model file per role).
+
 ## Audio (STT/TTS)
 
 Three audio primitives. `transcribe()` (speech-to-text) and `speak()`

package/dist/clients/llamaCppLoader.d.ts

@@ -1,4 +1,5 @@
 import type { BaseClient } from "./baseClient.js";
+import type { EmbedProvider } from "../embed.js";
 /**
  * Minimal structural view of smoltalk-llama-cpp's module. Declared here (not
  * imported from the plugin) so smoltalk compiles without the plugin installed
@@ -7,6 +8,9 @@ import type { BaseClient } from "./baseClient.js";
 export type LlamaCppModule = {
     LlamaCPP: typeof BaseClient;
     resolveModel: (uriOrPath: string, cacheDir: string) => Promise<string>;
+    /** Present from smoltalk-llama-cpp 0.5.0. Absent on older plugins, which
+     *  then serve chat only. */
+    embed?: EmbedProvider;
 };
 type ImportFn = (specifier: string) => Promise<Record<string, unknown>>;
 /**

package/dist/embed/mlx.d.ts

@@ -0,0 +1,15 @@
+import { EmbedConfig, EmbedResult } from "../embed.js";
+import { Result } from "../types/result.js";
+/**
+ * Embeddings from an MLX server on localhost (`agency local serve
+ * --embedding …`, or any server with an OpenAI-shaped /v1/embeddings
+ * route). The same call the OpenAI helper makes, with the two things the
+ * chat client also fixes: the key is a placeholder the server ignores, and
+ * the cost is zero. `dimensions` is passed through as-is; whether the
+ * server honours it depends on the server and model.
+ *
+ * Float encoding is requested because a base URL is given (see
+ * openaiEmbed): a local server that ignores the SDK's base64 default and
+ * returns float arrays would otherwise yield empty vectors.
+ */
+export declare function mlxEmbed(inputs: string[], config: EmbedConfig, baseURL: string): Promise<Result<EmbedResult>>;

package/dist/embed/mlx.js

@@ -0,0 +1,24 @@
+import { success } from "../types/result.js";
+import { openaiEmbed } from "./openai.js";
+/**
+ * Embeddings from an MLX server on localhost (`agency local serve
+ * --embedding …`, or any server with an OpenAI-shaped /v1/embeddings
+ * route). The same call the OpenAI helper makes, with the two things the
+ * chat client also fixes: the key is a placeholder the server ignores, and
+ * the cost is zero. `dimensions` is passed through as-is; whether the
+ * server honours it depends on the server and model.
+ *
+ * Float encoding is requested because a base URL is given (see
+ * openaiEmbed): a local server that ignores the SDK's base64 default and
+ * returns float arrays would otherwise yield empty vectors.
+ */
+export async function mlxEmbed(inputs, config, baseURL) {
+    const result = await openaiEmbed(inputs, config, "mlx-local", baseURL);
+    if (!result.success) {
+        return result;
+    }
+    return success({
+        ...result.value,
+        costEstimate: { inputCost: 0, outputCost: 0, totalCost: 0, currency: "USD" },
+    });
+}

package/dist/embed/openai.d.ts

@@ -1,9 +1,19 @@
 import { EmbedConfig, EmbedResult } from "../embed.js";
 import { Result } from "../types/result.js";
+export type OpenAiEmbedOptions = {
+    /**
+     * Wire encoding to ask the server for. Left unset, real OpenAI (no
+     * baseURL) keeps the SDK's base64 default, and every other backend gets
+     * "float": the SDK decodes an unrequested reply as base64 no matter what
+     * came back, so a server that ignores the field and returns float arrays
+     * would yield empty vectors. Set explicitly to override either default.
+     */
+    encodingFormat?: "float" | "base64";
+};
 /**
  * OpenAI-compatible embedding call. Used by openai directly and by other
  * OpenAI-shape backends (deepinfra, litellm, openai-compat) which pass a
  * custom `baseURL`. Cost comes from the smoltalk model registry; provider-
  * returned cost fields aren't standardized on this endpoint.
  */
-export declare function openaiEmbed(inputs: string[], config: EmbedConfig, apiKey: string, baseURL?: string): Promise<Result<EmbedResult>>;
+export declare function openaiEmbed(inputs: string[], config: EmbedConfig, apiKey: string, baseURL?: string, options?: OpenAiEmbedOptions): Promise<Result<EmbedResult>>;

package/dist/embed/openai.js

@@ -2,25 +2,47 @@ import OpenAI from "openai";
 import { success, failure } from "../types/result.js";
 import { getModel, isEmbeddingsModel } from "../models.js";
 import { round } from "../util/util.js";
+/**
+ * A server may answer a float request with base64 anyway (some always
+ * encode). Once we name an encoding the SDK returns the body untouched, so
+ * handle both shapes here. Little-endian float32, the same layout the SDK's
+ * own decoder assumes.
+ */
+function toFloats(embedding) {
+    if (typeof embedding === "string") {
+        const bytes = Buffer.from(embedding, "base64");
+        const view = new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
+        return Array.from(view);
+    }
+    return embedding;
+}
 /**
  * OpenAI-compatible embedding call. Used by openai directly and by other
  * OpenAI-shape backends (deepinfra, litellm, openai-compat) which pass a
  * custom `baseURL`. Cost comes from the smoltalk model registry; provider-
  * returned cost fields aren't standardized on this endpoint.
  */
-export async function openaiEmbed(inputs, config, apiKey, baseURL) {
+export async function openaiEmbed(inputs, config, apiKey, baseURL, options) {
     try {
         const client = new OpenAI({ apiKey, ...(baseURL ? { baseURL } : {}) });
-        const response = await client.embeddings.create({
+        const body = {
             model: config.model,
             input: inputs,
-            ...(config.dimensions !== undefined
-                ? { dimensions: config.dimensions }
-                : {}),
-        });
+        };
+        if (config.dimensions !== undefined) {
+            body.dimensions = config.dimensions;
+        }
+        let encodingFormat = options?.encodingFormat;
+        if (encodingFormat === undefined && baseURL !== undefined) {
+            encodingFormat = "float";
+        }
+        if (encodingFormat !== undefined) {
+            body.encoding_format = encodingFormat;
+        }
+        const response = await client.embeddings.create(body);
         const embeddings = [...response.data]
             .sort((a, b) => a.index - b.index)
-            .map((d) => d.embedding);
+            .map((d) => toFloats(d.embedding));
         const inputTokens = response.usage.prompt_tokens;
         const costEstimate = calculateEmbeddingCost(config.model, inputTokens, config.modelData);
         return success({

package/dist/embed.d.ts

@@ -25,6 +25,7 @@ export type EmbedConfig = {
         deepInfra?: string;
         liteLlm?: string;
         openAiCompat?: string;
+        mlx?: string;
         /** Arbitrary provider names, for URLs targeting a custom-registered provider. */
         [provider: string]: string | undefined;
     };
@@ -39,4 +40,9 @@ export type EmbedResult = {
 };
 export type EmbedProvider = (inputs: string[], config: EmbedConfig) => Promise<Result<EmbedResult>>;
 export declare function registerEmbeddingProvider(name: string, fn: EmbedProvider): void;
+/** True when `name` has an embed provider registered through
+ *  registerEmbeddingProvider. The built-in cases in embed() are not its
+ *  concern, the same as hasProvider in client.ts. */
+export declare function hasEmbeddingProvider(name: string): boolean;
+export declare function unregisterEmbeddingProvider(name: string): boolean;
 export declare function embed(input: string | string[], config: EmbedConfig): Promise<Result<EmbedResult>>;

package/dist/embed.js

@@ -3,12 +3,33 @@ import { resolveProvider, resolveApiKey, resolveBaseUrl } from "./util/provider.
 import { openaiEmbed } from "./embed/openai.js";
 import { googleEmbed } from "./embed/google.js";
 import { ollamaEmbed } from "./embed/ollama.js";
+import { mlxEmbed } from "./embed/mlx.js";
+import { loadLlamaCpp } from "./clients/llamaCppLoader.js";
+function errorMessage(err) {
+    if (err instanceof Error) {
+        return err.message;
+    }
+    return String(err);
+}
 // Null-prototype so provider names like "toString"/"__proto__" can't collide
 // with Object.prototype or pollute the registry.
 const registeredEmbedProviders = Object.create(null);
 export function registerEmbeddingProvider(name, fn) {
     registeredEmbedProviders[name] = fn;
 }
+/** True when `name` has an embed provider registered through
+ *  registerEmbeddingProvider. The built-in cases in embed() are not its
+ *  concern, the same as hasProvider in client.ts. */
+export function hasEmbeddingProvider(name) {
+    return name in registeredEmbedProviders;
+}
+export function unregisterEmbeddingProvider(name) {
+    if (name in registeredEmbedProviders) {
+        delete registeredEmbedProviders[name];
+        return true;
+    }
+    return false;
+}
 export async function embed(input, config) {
     const inputs = Array.isArray(input) ? input : [input];
     let provider;
@@ -63,6 +84,31 @@ export async function embed(input, config) {
             }
             return openaiEmbed(inputs, config, apiKey, baseURL);
         }
+        case "mlx": {
+            // resolveBaseUrl always returns a value for "mlx" (it has a default).
+            return mlxEmbed(inputs, config, resolveBaseUrl("mlx", config));
+        }
+        case "llama-cpp": {
+            // A hand-registered provider wins, the same rule loadLlamaCpp applies
+            // to the chat class. Otherwise load the plugin the way text() does;
+            // the loader caches its import.
+            const custom = registeredEmbedProviders[provider];
+            if (custom) {
+                return custom(inputs, config);
+            }
+            let plugin;
+            try {
+                plugin = await loadLlamaCpp();
+            }
+            catch (err) {
+                return failure(errorMessage(err));
+            }
+            if (typeof plugin.embed !== "function") {
+                return failure("Your installed smoltalk-llama-cpp has no embeddings support. " +
+                    "Upgrade it (npm i smoltalk-llama-cpp@latest; >=0.5.0 required).");
+            }
+            return plugin.embed(inputs, config);
+        }
         default: {
             const custom = registeredEmbedProviders[provider];
             if (custom) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smoltalk",
-  "version": "0.13.2",
+  "version": "0.14.0",
   "description": "A common interface for LLM APIs",
   "homepage": "https://github.com/egonSchiele/smoltalk",
   "files": [