smoltalk 0.15.3 → 0.17.0

package/README.md

@@ -229,6 +229,31 @@ Used with `responseFormat` to control validation behavior (currently OpenAI only
 | `strict` | `boolean` | | Whether to use strict schema validation. |
 | `numRetries` | `number` | `2` | How many times to retry if the response fails schema validation. |
 | `allowExtraKeys` | `boolean` | | If `true`, strips unexpected keys instead of failing validation. |
+| `separateFromTools` | `boolean` | `true` | Local providers only (`mlx`, `llama-cpp`). See below. |
+
+#### Tools and a response format in one call
+
+Some providers cannot hold a reply to a schema while the model may still call
+a tool. Google rejects the combination. A local server (`mlx`, `llama-cpp`)
+accepts it but leaves the schema unenforced, because a tool call is not JSON,
+and a local model then tends to answer in prose.
+
+For these providers smoltalk makes the call as two requests. The first
+carries the tools and no schema. If the model calls a tool, that is the
+result, and your tool loop carries on as usual. Once the model answers in
+words, a second request asks for that answer in the schema, and that is the
+`output` you get. Usage and cost cover both requests.
+
+On the local providers the second request keeps the whole conversation, so a
+field such as "did this cover everything the user asked?" can be answered.
+`mlx` also keeps the tool list and sends `tool_choice: "none"`, so the server
+can reuse its cached prompt. `responseFormatOptions.separateFromTools: false`
+sends one request instead and leaves the schema unenforced. A streamed call
+is never split.
+
+With `strict: true`, a reply that fails validation is retried as usual. The
+retry repeats only the second request, so the model cannot answer "fix this
+JSON" with another tool call.
 
 ### `toolLoopDetection`
 
@@ -813,6 +838,43 @@ unsupported provider, a model without audio input, or audio that isn't
 `mp3`/`wav` surfaces as a `Failure` from that call, not from `audioPart()`
 itself. Audio is inlined as base64, not uploaded via the Files API.
 
+## In the browser
+
+smoltalk bundles for the browser. A bundler run for the browser (Vite,
+webpack with `target: "web"`, esbuild with `--platform=browser`) picks the
+browser side of smoltalk's `#platform` and `#ollama` imports on its own;
+there is no separate entry point and no flag. With esbuild, mark Node
+builtins external, because `@anthropic-ai/sdk` lazily imports `node:fs` on
+the path it takes when no API key is passed (Vite does this for you):
+
+```sh
+esbuild app.ts --bundle --platform=browser --format=esm --external:node:*
+```
+
+Differences from Node:
+
+- There are no environment variables. `env` reads as unset, nothing
+  throws, and a client built without a key fails with the SDK's own
+  missing-key error. Pass keys in `apiKey`.
+- A `{ kind: "path" }` attachment throws: "readFileBytes is not available
+  in a browser build of smoltalk."
+- `refreshModels` accepts `https:` URLs only; a `file:` URL fails the
+  same way, from `readTextFromFileUrl`.
+- `loadLlamaCpp` fails: it loads a native module. Without `entryPath` the
+  error is the loader's "Failed to load smoltalk-llama-cpp", wrapping the
+  browser's refusal to resolve the bare specifier; with `entryPath` it is
+  `pathToFileUrl`'s. Local inference in a browser is the `smoltalk-webllm`
+  plugin.
+
+The `openai` and `@anthropic-ai/sdk` clients are constructed with
+`dangerouslyAllowBrowser: true`. That flag exists because a key sent to a
+browser is visible to anyone who can open the page. For a production app,
+keep the key on a server. The built-in OpenAI, Anthropic, and Google
+clients take no base URL, so the browser side reaches such a server as an
+OpenAI-compatible endpoint: `provider: "openai-compat"` with
+`baseUrl.openAiCompat` set to the proxy, and the proxy holding the real
+key.
+
 ## Limitations
 Smoltalk has support for a limited number of providers right now, and is mostly focused on the stateless APIs for text completion, though I plan to add support for more providers as well as image and speech models later. Smoltalk is also a personal project, and there are alternatives backed by companies:
 

package/dist/clients/anthropic.js

@@ -1,3 +1,4 @@
+import { env } from "#platform";
 import Anthropic from "@anthropic-ai/sdk";
 import { ToolCall } from "../classes/ToolCall.js";
 import { SystemMessage, DeveloperMessage } from "../classes/message/index.js";
@@ -219,11 +220,11 @@ export class SmolAnthropic extends BaseClient {
     model;
     constructor(config) {
         super(config);
-        const apiKey = config.apiKey?.anthropic || process.env.ANTHROPIC_API_KEY;
+        const apiKey = config.apiKey?.anthropic || env("ANTHROPIC_API_KEY");
         if (!apiKey) {
             throw new Error("Anthropic API key is required for SmolAnthropic client.");
         }
-        this.client = new Anthropic({ apiKey });
+        this.client = new Anthropic({ dangerouslyAllowBrowser: true, apiKey });
         this.logger = getLogger();
         this.model = new Model(config.model, config.provider, config.modelData);
     }

package/dist/clients/baseClient.js

@@ -1,4 +1,5 @@
-import { userMessage, assistantMessage } from "../classes/message/index.js";
+import { assistantMessage } from "../classes/message/index.js";
+import { failedValidationMessage, returnedUndefinedMessage, } from "./validationRetry.js";
 import { getLogger } from "../util/logger.js";
 import { SmolStructuredOutputError } from "../smolError.js";
 import { getStatelogClient } from "../statelogClient.js";
@@ -295,7 +296,7 @@ export class BaseClient {
             if (!("output" in result.value)) {
                 const retryMessages = [
                     ...promptConfig.messages,
-                    userMessage(`You returned "undefined" instead of a valid response. Please provide a valid response.`),
+                    returnedUndefinedMessage(),
                 ];
                 return this.textWithRetry({ ...promptConfig, messages: retryMessages }, retries - 1);
             }
@@ -332,7 +333,7 @@ export class BaseClient {
                     const retryMessages = [
                         ...promptConfig.messages,
                         assistantMessage(output),
-                        userMessage(`Your previous response failed validation. Please fix the following errors and try again:\n${errorMessage}`),
+                        failedValidationMessage(errorMessage),
                     ];
                     return this.textWithRetry({ ...promptConfig, messages: retryMessages }, retries - 1);
                 }

package/dist/clients/google.js

@@ -1,3 +1,4 @@
+import { env } from "#platform";
 import { GoogleGenAI } from "@google/genai";
 import { ToolCall } from "../classes/ToolCall.js";
 import { getLogger } from "../util/logger.js";
@@ -203,7 +204,7 @@ export class SmolGoogle extends BaseClient {
     model;
     constructor(config) {
         super(config);
-        const apiKey = config.apiKey?.google || process.env.GEMINI_API_KEY;
+        const apiKey = config.apiKey?.google || env("GEMINI_API_KEY");
         if (!apiKey) {
             throw new Error("Google API key is required for SmolGoogle client.");
         }

package/dist/clients/llamaCppLoader.js

@@ -1,4 +1,4 @@
-import { pathToFileURL } from "url";
+import { pathToFileUrl } from "#platform";
 import { hasProvider, registerProvider } from "../client.js";
 import { SmolError } from "../smolError.js";
 const realImport = (specifier) => import(specifier);
@@ -63,7 +63,7 @@ export function loadLlamaCpp(options) {
 async function doLoad(entryPath) {
     let importSource;
     if (entryPath) {
-        importSource = pathToFileURL(entryPath).href;
+        importSource = pathToFileUrl(entryPath);
     }
     else {
         importSource = "smoltalk-llama-cpp";

package/dist/clients/mlx.d.ts

@@ -1,10 +1,20 @@
 import { SmolOpenAiCompat } from "./openaiCompat.js";
-import type { SmolConfig } from "../types.js";
+import type { PromptResult, SmolConfig } from "../types.js";
+import type { Result } from "../types/result.js";
 /**
  * Client for an MLX server on localhost, usually started with `mlx_lm.server`
  * (or `agency local serve`). The server speaks the OpenAI chat format, so this
  * is the openai-compat client with three things fixed: the base URL has a
  * default, no API key is needed, and the cost is always zero.
+ *
+ * A call with both tools and a `responseFormat` is made as two requests (see
+ * structuredAfterTools.ts). The second keeps the tool list and sends
+ * `tool_choice: "none"`. Keeping the list matters: the chat template renders
+ * the tools at the start of the prompt, so dropping them would make the
+ * server read the whole prompt again instead of reusing its cache. A server
+ * that enforces a schema when `tool_choice` is `"none"` (`agency local
+ * serve`) then holds the reply to the format. Streaming calls are not split:
+ * as on Google, a streamed call with tools leaves the format unenforced.
  */
 export declare class SmolMlx extends SmolOpenAiCompat {
     protected resolveClientOptions(config: SmolConfig): {
@@ -12,4 +22,5 @@ export declare class SmolMlx extends SmolOpenAiCompat {
         baseURL: string;
     };
     protected resolveCostUsd(): number;
+    _textSync(config: SmolConfig): Promise<Result<PromptResult>>;
 }

package/dist/clients/mlx.js

@@ -1,10 +1,20 @@
 import { SmolOpenAiCompat } from "./openaiCompat.js";
 import { resolveBaseUrl } from "../util/provider.js";
+import { separatesStructuredOutput, toolsThenStructuredOutput, } from "./structuredAfterTools.js";
 /**
  * Client for an MLX server on localhost, usually started with `mlx_lm.server`
  * (or `agency local serve`). The server speaks the OpenAI chat format, so this
  * is the openai-compat client with three things fixed: the base URL has a
  * default, no API key is needed, and the cost is always zero.
+ *
+ * A call with both tools and a `responseFormat` is made as two requests (see
+ * structuredAfterTools.ts). The second keeps the tool list and sends
+ * `tool_choice: "none"`. Keeping the list matters: the chat template renders
+ * the tools at the start of the prompt, so dropping them would make the
+ * server read the whole prompt again instead of reusing its cache. A server
+ * that enforces a schema when `tool_choice` is `"none"` (`agency local
+ * serve`) then holds the reply to the format. Streaming calls are not split:
+ * as on Google, a streamed call with tools leaves the format unenforced.
  */
 export class SmolMlx extends SmolOpenAiCompat {
     resolveClientOptions(config) {
@@ -16,4 +26,17 @@ export class SmolMlx extends SmolOpenAiCompat {
     resolveCostUsd() {
         return 0;
     }
+    async _textSync(config) {
+        if (!separatesStructuredOutput(config)) {
+            return super._textSync(config);
+        }
+        return toolsThenStructuredOutput({
+            config,
+            run: (one) => super._textSync(one),
+            formatConfig: (followUp) => ({
+                ...followUp,
+                rawAttributes: { ...followUp.rawAttributes, tool_choice: "none" },
+            }),
+        });
+    }
 }

package/dist/clients/ollama.d.ts

@@ -1,4 +1,4 @@
-import { Ollama } from "ollama";
+import { Ollama } from "#ollama";
 import { PromptResult, Result, SmolClient, SmolConfig, StreamChunk } from "../types.js";
 import { BaseClient } from "./baseClient.js";
 import { ModelName } from "../models.js";

package/dist/clients/ollama.js

@@ -1,4 +1,4 @@
-import { Ollama } from "ollama";
+import { Ollama } from "#ollama";
 import { ToolCall } from "../classes/ToolCall.js";
 import { getLogger } from "../util/logger.js";
 import { redactAttachments } from "../util/redact.js";

package/dist/clients/openai.d.ts

@@ -14,7 +14,8 @@ export declare class SmolOpenAi extends BaseClient implements SmolClient {
     protected attachmentCapabilities(): ClientAttachmentCapabilities;
     /**
      * Build the `new OpenAI({...})` options. Subclasses override to inject a
-     * different baseURL or to pull the key from a different config field.
+     * different baseURL or to pull the key from a different config field. The
+     * constructor adds `dangerouslyAllowBrowser` itself.
      *
      * Default behavior: read OPENAI_API_KEY (or config.apiKey.openAi). Throws
      * if the key is missing — subclasses with required keys do their own check.

package/dist/clients/openai.js

@@ -1,3 +1,4 @@
+import { env } from "#platform";
 import OpenAI from "openai";
 import { success, } from "../types.js";
 import { ToolCall } from "../classes/ToolCall.js";
@@ -19,7 +20,7 @@ export class SmolOpenAi extends BaseClient {
     constructor(config) {
         super(config);
         const options = this.resolveClientOptions(config);
-        this.client = new OpenAI(options);
+        this.client = new OpenAI({ ...options, dangerouslyAllowBrowser: true });
         this.logger = getLogger();
         this.model = new Model(config.model, config.provider, config.modelData);
     }
@@ -29,13 +30,14 @@ export class SmolOpenAi extends BaseClient {
     }
     /**
      * Build the `new OpenAI({...})` options. Subclasses override to inject a
-     * different baseURL or to pull the key from a different config field.
+     * different baseURL or to pull the key from a different config field. The
+     * constructor adds `dangerouslyAllowBrowser` itself.
      *
      * Default behavior: read OPENAI_API_KEY (or config.apiKey.openAi). Throws
      * if the key is missing — subclasses with required keys do their own check.
      */
     resolveClientOptions(config) {
-        const apiKey = config.apiKey?.openAi || process.env.OPENAI_API_KEY;
+        const apiKey = config.apiKey?.openAi || env("OPENAI_API_KEY");
         if (!apiKey) {
             throw new Error("OpenAI API key is required for SmolOpenAi client.");
         }

package/dist/clients/openaiResponses.js

@@ -1,3 +1,4 @@
+import { env } from "#platform";
 import OpenAI from "openai";
 import { success, } from "../types.js";
 import { ToolCall } from "../classes/ToolCall.js";
@@ -57,11 +58,11 @@ export class SmolOpenAiResponses extends BaseClient {
     model;
     constructor(config) {
         super(config);
-        const apiKey = config.apiKey?.openAi || process.env.OPENAI_API_KEY;
+        const apiKey = config.apiKey?.openAi || env("OPENAI_API_KEY");
         if (!apiKey) {
             throw new Error("OpenAI API key is required for SmolOpenAiResponses client.");
         }
-        this.client = new OpenAI({ apiKey });
+        this.client = new OpenAI({ dangerouslyAllowBrowser: true, apiKey });
         this.logger = getLogger();
         this.model = new Model(config.model, config.provider, config.modelData);
     }

package/dist/clients/resolveAttachments.js

@@ -1,3 +1,4 @@
+import { bytesToBase64 } from "../util/base64.js";
 import { UserMessage } from "../classes/message/index.js";
 import { normalizeBlob } from "../util/blobRef.js";
 import { fileFamily } from "../util/attachments.js";
@@ -45,7 +46,7 @@ async function toBase64Source(source, allowed, maxBytes) {
         allowedMimePrefixes: allowed,
         maxBytes,
     });
-    return { kind: "base64", base64: Buffer.from(data).toString("base64"), mimeType };
+    return { kind: "base64", base64: bytesToBase64(data), mimeType };
 }
 /** Error message when a providerFile ref targets the wrong provider family, else null. */
 function providerFileError(fileProvider, targetProvider) {

package/dist/clients/structuredAfterTools.d.ts

@@ -0,0 +1,36 @@
+import type { PromptResult, SmolConfig } from "../types.js";
+import { type Result } from "../types/result.js";
+/**
+ * A local model cannot be held to a JSON schema while it may still call a
+ * tool: a tool call is not JSON, so the servers leave the schema unenforced
+ * on any request that carries tools, and the model is free to answer in
+ * prose. A call with both tools and a `responseFormat` is therefore made as
+ * two requests, the way the Google client does it. The first carries the
+ * tools and no schema. If the model calls a tool, that is the result. If it
+ * answers, a second request asks for that answer in the schema, with tool
+ * calls ruled out so the schema can be enforced.
+ *
+ * Unlike the Google client, the second request keeps the whole conversation.
+ * A schema field such as "did this answer everything the user asked?" cannot
+ * be filled in from the answer alone.
+ */
+/** What the second request says, after the model's own answer. */
+export declare const STRUCTURED_FOLLOW_UP = "Give that same reply again in the required JSON format. Do not call a tool.";
+/** Whether a call's typed reply is made as a second request. */
+export declare function separatesStructuredOutput(config: SmolConfig): boolean;
+/**
+ * Makes the two requests. `run` sends one request. `formatConfig` turns the
+ * follow-up config into one whose schema the provider will enforce: the MLX
+ * client sets `tool_choice: "none"`, the llama.cpp client drops the tools.
+ *
+ * When the reply fails the caller's validation, the caller sends the
+ * conversation back with a request to fix it. The model has already answered
+ * by then, so only the format request is made. A tool round there would let
+ * the model answer "fix this JSON" with a tool call, and a tool that has
+ * already run would run again.
+ */
+export declare function toolsThenStructuredOutput(args: {
+    config: SmolConfig;
+    run: (config: SmolConfig) => Promise<Result<PromptResult>>;
+    formatConfig: (config: SmolConfig) => SmolConfig;
+}): Promise<Result<PromptResult>>;

package/dist/clients/structuredAfterTools.js

@@ -0,0 +1,87 @@
+import { assistantMessage, userMessage } from "../classes/message/index.js";
+import { addCosts } from "../types/costEstimate.js";
+import { addTokenUsage } from "../types/tokenUsage.js";
+import { success } from "../types/result.js";
+import { endsInValidationRetry } from "./validationRetry.js";
+/**
+ * A local model cannot be held to a JSON schema while it may still call a
+ * tool: a tool call is not JSON, so the servers leave the schema unenforced
+ * on any request that carries tools, and the model is free to answer in
+ * prose. A call with both tools and a `responseFormat` is therefore made as
+ * two requests, the way the Google client does it. The first carries the
+ * tools and no schema. If the model calls a tool, that is the result. If it
+ * answers, a second request asks for that answer in the schema, with tool
+ * calls ruled out so the schema can be enforced.
+ *
+ * Unlike the Google client, the second request keeps the whole conversation.
+ * A schema field such as "did this answer everything the user asked?" cannot
+ * be filled in from the answer alone.
+ */
+/** What the second request says, after the model's own answer. */
+export const STRUCTURED_FOLLOW_UP = "Give that same reply again in the required JSON format. Do not call a tool.";
+/** Whether a call's typed reply is made as a second request. */
+export function separatesStructuredOutput(config) {
+    const hasTools = (config.tools?.length ?? 0) > 0;
+    if (!hasTools || !config.responseFormat) {
+        return false;
+    }
+    return config.responseFormatOptions?.separateFromTools !== false;
+}
+/**
+ * The format request's result as the caller sees it. The request rules tool
+ * calls out, but a server too old to honour that could still return one. It
+ * must not be run: the tool round is over. Such a reply has no text either,
+ * so `fallback` stands in for it. The caller's validation then fails on the
+ * fallback and asks again, instead of finding nothing to validate.
+ */
+function formatReply(result, fallback) {
+    return { ...result, toolCalls: [], output: result.output || fallback };
+}
+/**
+ * Makes the two requests. `run` sends one request. `formatConfig` turns the
+ * follow-up config into one whose schema the provider will enforce: the MLX
+ * client sets `tool_choice: "none"`, the llama.cpp client drops the tools.
+ *
+ * When the reply fails the caller's validation, the caller sends the
+ * conversation back with a request to fix it. The model has already answered
+ * by then, so only the format request is made. A tool round there would let
+ * the model answer "fix this JSON" with a tool call, and a tool that has
+ * already run would run again.
+ */
+export async function toolsThenStructuredOutput(args) {
+    const { config, run, formatConfig } = args;
+    if (endsInValidationRetry(config.messages)) {
+        const only = await run(formatConfig(config));
+        return only.success ? success(formatReply(only.value, "")) : only;
+    }
+    const first = await run({ ...config, responseFormat: undefined });
+    if (!first.success) {
+        return first;
+    }
+    // A tool call is the whole reply. An empty reply has nothing to put in
+    // the format, so it goes back as it is and fails validation as before.
+    if (first.value.toolCalls.length > 0 || !first.value.output) {
+        return first;
+    }
+    const answer = first.value.output;
+    const second = await run(formatConfig({
+        ...config,
+        messages: [...config.messages, assistantMessage(answer), userMessage(STRUCTURED_FOLLOW_UP)],
+    }));
+    if (!second.success) {
+        return second;
+    }
+    const thinkingBlocks = [
+        ...(first.value.thinkingBlocks ?? []),
+        ...(second.value.thinkingBlocks ?? []),
+    ];
+    const merged = {
+        ...formatReply(second.value, answer),
+        usage: addTokenUsage(first.value.usage, second.value.usage),
+        cost: addCosts(first.value.cost, second.value.cost),
+    };
+    if (thinkingBlocks.length > 0) {
+        merged.thinkingBlocks = thinkingBlocks;
+    }
+    return success(merged);
+}

package/dist/clients/validationRetry.d.ts

@@ -0,0 +1,5 @@
+import type { Message } from "../classes/message/index.js";
+export declare function failedValidationMessage(errorMessage: string): import("../classes/message/UserMessage.js").UserMessage;
+export declare function returnedUndefinedMessage(): import("../classes/message/UserMessage.js").UserMessage;
+/** Whether the last message asks the model to correct its previous reply. */
+export declare function endsInValidationRetry(messages: Message[]): boolean;

package/dist/clients/validationRetry.js

@@ -0,0 +1,24 @@
+import { userMessage } from "../classes/message/index.js";
+/**
+ * The messages `textWithRetry` adds when a reply does not fit the response
+ * format, and the test for them. A client that makes a tool round before the
+ * format request (see structuredAfterTools.ts) uses the test to skip the tool
+ * round on a retry.
+ */
+const FAILED_VALIDATION = "Your previous response failed validation.";
+const RETURNED_UNDEFINED = `You returned "undefined" instead of a valid response.`;
+export function failedValidationMessage(errorMessage) {
+    return userMessage(`${FAILED_VALIDATION} Please fix the following errors and try again:\n${errorMessage}`);
+}
+export function returnedUndefinedMessage() {
+    return userMessage(`${RETURNED_UNDEFINED} Please provide a valid response.`);
+}
+/** Whether the last message asks the model to correct its previous reply. */
+export function endsInValidationRetry(messages) {
+    const last = messages[messages.length - 1];
+    if (!last || last.role !== "user") {
+        return false;
+    }
+    const text = last.content;
+    return text.startsWith(FAILED_VALIDATION) || text.startsWith(RETURNED_UNDEFINED);
+}

package/dist/embed/openai.js

@@ -1,3 +1,4 @@
+import { base64ToBytes } from "../util/base64.js";
 import OpenAI from "openai";
 import { success, failure } from "../types/result.js";
 import { getModel, isEmbeddingsModel } from "../models.js";
@@ -10,7 +11,7 @@ import { round } from "../util/util.js";
  */
 function toFloats(embedding) {
     if (typeof embedding === "string") {
-        const bytes = Buffer.from(embedding, "base64");
+        const bytes = base64ToBytes(embedding);
         const view = new Float32Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 4));
         return Array.from(view);
     }
@@ -24,7 +25,7 @@ function toFloats(embedding) {
  */
 export async function openaiEmbed(inputs, config, apiKey, baseURL, options) {
     try {
-        const client = new OpenAI({ apiKey, ...(baseURL ? { baseURL } : {}) });
+        const client = new OpenAI({ dangerouslyAllowBrowser: true, apiKey, ...(baseURL ? { baseURL } : {}) });
         const body = {
             model: config.model,
             input: inputs,

package/dist/files/anthropic.js

@@ -7,13 +7,13 @@ export function anthropicFileRef(uploaded, mimeType) {
 class AnthropicFileProvider extends BaseFileProvider {
     label = "Anthropic";
     async doUpload(data, mimeType, ctx) {
-        const client = new Anthropic({ apiKey: ctx.apiKey });
+        const client = new Anthropic({ dangerouslyAllowBrowser: true, apiKey: ctx.apiKey });
         const file = await toFile(data, ctx.filename ?? "upload", { type: mimeType });
         const uploaded = await client.beta.files.upload({ file, betas: [FILES_BETA] });
         return anthropicFileRef(uploaded, mimeType);
     }
     async doDelete(ref, ctx) {
-        const client = new Anthropic({ apiKey: ctx.apiKey });
+        const client = new Anthropic({ dangerouslyAllowBrowser: true, apiKey: ctx.apiKey });
         await client.beta.files.delete(ref.id, { betas: [FILES_BETA] });
     }
 }

package/dist/files/openai.js

@@ -8,7 +8,7 @@ export function openaiFileRef(created, mimeType) {
 class OpenAIFileProvider extends BaseFileProvider {
     label = "OpenAI";
     async doUpload(data, mimeType, ctx) {
-        const client = new OpenAI({ apiKey: ctx.apiKey });
+        const client = new OpenAI({ dangerouslyAllowBrowser: true, apiKey: ctx.apiKey });
         const file = await toFile(data, ctx.filename ?? "upload", { type: mimeType });
         // S5: the mapper below is trivial and won't throw post-create, so no orphan
         // is left here; a provider whose mapper does real work should best-effort delete.
@@ -16,7 +16,7 @@ class OpenAIFileProvider extends BaseFileProvider {
         return openaiFileRef(created, mimeType);
     }
     async doDelete(ref, ctx) {
-        const client = new OpenAI({ apiKey: ctx.apiKey });
+        const client = new OpenAI({ dangerouslyAllowBrowser: true, apiKey: ctx.apiKey });
         await client.files.delete(ref.id);
     }
 }

package/dist/image/google.js

@@ -1,3 +1,4 @@
+import { base64ToBytes, bytesToBase64 } from "../util/base64.js";
 import { GoogleGenAI } from "@google/genai";
 import { success, failure } from "../types/result.js";
 import { getModel, isImageModel } from "../models.js";
@@ -14,7 +15,7 @@ export async function googleImage(input, config, apiKey) {
                 parts.push({
                     inlineData: {
                         mimeType: img.mimeType,
-                        data: Buffer.from(img.data).toString("base64"),
+                        data: bytesToBase64(img.data),
                     },
                 });
             }
@@ -32,7 +33,7 @@ export async function googleImage(input, config, apiKey) {
             for (const part of candidate.content?.parts ?? []) {
                 if (part.inlineData?.data && part.inlineData?.mimeType) {
                     images.push({
-                        data: new Uint8Array(Buffer.from(part.inlineData.data, "base64")),
+                        data: base64ToBytes(part.inlineData.data),
                         mimeType: part.inlineData.mimeType,
                     });
                 }

package/dist/image/openai.js

@@ -1,3 +1,4 @@
+import { base64ToBytes } from "../util/base64.js";
 import OpenAI from "openai";
 import { toFile } from "openai/uploads";
 import { success, failure } from "../types/result.js";
@@ -11,14 +12,14 @@ export async function openaiImage(input, config, apiKey) {
         if (normalized.mask && !hasImages) {
             return failure("A mask was provided without any input images. Masks are only valid for image edits — pass at least one entry in `images` alongside the mask.");
         }
-        const client = new OpenAI({ apiKey });
+        const client = new OpenAI({ dangerouslyAllowBrowser: true, apiKey });
         const baseParams = buildBaseParams(config, normalized.prompt);
         const response = hasImages
             ? await callEdit(client, baseParams, normalized)
             : await client.images.generate(baseParams);
         const mimeType = mimeFromFormat(config.outputFormat) ?? "image/png";
         const images = (response.data ?? []).map((d) => ({
-            data: new Uint8Array(Buffer.from(d.b64_json, "base64")),
+            data: base64ToBytes(d.b64_json),
             mimeType,
             revisedPrompt: d.revised_prompt,
         }));

package/dist/image/openaiCompat.js

@@ -1,3 +1,4 @@
+import { base64ToBytes } from "../util/base64.js";
 import OpenAI from "openai";
 import { success, failure } from "../types/result.js";
 import { omitUndefined } from "../util/util.js";
@@ -17,7 +18,7 @@ export async function openaiCompatImage(input, config, apiKey, baseURL) {
         if (normalized.mask) {
             return failure("openai-compat: image masks are not supported on this generic adapter.");
         }
-        const client = new OpenAI({ apiKey, baseURL });
+        const client = new OpenAI({ dangerouslyAllowBrowser: true, apiKey, baseURL });
         const params = omitUndefined({
             model: config.model,
             prompt: normalized.prompt,
@@ -31,7 +32,7 @@ export async function openaiCompatImage(input, config, apiKey, baseURL) {
         const response = await client.images.generate(params);
         const mimeType = mimeFromFormat(config.outputFormat) ?? "image/png";
         const images = (response.data ?? []).map((d) => ({
-            data: new Uint8Array(Buffer.from(d.b64_json, "base64")),
+            data: base64ToBytes(d.b64_json),
             mimeType,
             revisedPrompt: d.revised_prompt,
         }));

package/dist/index.d.ts

@@ -23,6 +23,7 @@ export type { BlobRef } from "./util/blobRef.js";
 export { getLogger, EgonLog } from "./util/logger.js";
 export type { LogLevel } from "./util/logger.js";
 export { redactAttachments } from "./util/redact.js";
+export { separatesStructuredOutput, toolsThenStructuredOutput, STRUCTURED_FOLLOW_UP, } from "./clients/structuredAfterTools.js";
 export { transcribe, registerTranscriptionProvider, DEFAULT_TRANSCRIBE_BYTES, } from "./transcription.js";
 export type { TranscribeOptions, TranscriptionSegment, TranscriptionWord, TranscriptionResult, TranscriptionClientClass, } from "./transcription.js";
 export { BaseTranscriptionClient } from "./transcription/baseTranscriptionClient.js";

package/dist/index.js

@@ -21,6 +21,8 @@ export { uploadFile, deleteFile, registerFileProvider, DEFAULT_UPLOAD_BYTES } fr
 export { normalizeBlob, loadBlob } from "./util/blobRef.js";
 export { getLogger, EgonLog } from "./util/logger.js";
 export { redactAttachments } from "./util/redact.js";
+// For provider plugins whose backend cannot enforce a schema alongside tools.
+export { separatesStructuredOutput, toolsThenStructuredOutput, STRUCTURED_FOLLOW_UP, } from "./clients/structuredAfterTools.js";
 // Explicit (not `export *`) so internal factories and test helpers stay private.
 export { transcribe, registerTranscriptionProvider, DEFAULT_TRANSCRIBE_BYTES, } from "./transcription.js";
 export { BaseTranscriptionClient } from "./transcription/baseTranscriptionClient.js";