smoltalk 0.15.2 → 0.16.0

package/README.md

@@ -229,6 +229,31 @@ Used with `responseFormat` to control validation behavior (currently OpenAI only
 | `strict` | `boolean` | | Whether to use strict schema validation. |
 | `numRetries` | `number` | `2` | How many times to retry if the response fails schema validation. |
 | `allowExtraKeys` | `boolean` | | If `true`, strips unexpected keys instead of failing validation. |
+| `separateFromTools` | `boolean` | `true` | Local providers only (`mlx`, `llama-cpp`). See below. |
+
+#### Tools and a response format in one call
+
+Some providers cannot hold a reply to a schema while the model may still call
+a tool. Google rejects the combination. A local server (`mlx`, `llama-cpp`)
+accepts it but leaves the schema unenforced, because a tool call is not JSON,
+and a local model then tends to answer in prose.
+
+For these providers smoltalk makes the call as two requests. The first
+carries the tools and no schema. If the model calls a tool, that is the
+result, and your tool loop carries on as usual. Once the model answers in
+words, a second request asks for that answer in the schema, and that is the
+`output` you get. Usage and cost cover both requests.
+
+On the local providers the second request keeps the whole conversation, so a
+field such as "did this cover everything the user asked?" can be answered.
+`mlx` also keeps the tool list and sends `tool_choice: "none"`, so the server
+can reuse its cached prompt. `responseFormatOptions.separateFromTools: false`
+sends one request instead and leaves the schema unenforced. A streamed call
+is never split.
+
+With `strict: true`, a reply that fails validation is retried as usual. The
+retry repeats only the second request, so the model cannot answer "fix this
+JSON" with another tool call.
 
 ### `toolLoopDetection`
 
@@ -615,12 +640,12 @@ const r = await decide(
 );
 ```
 
-`provider: "typesafe"` is required for any model name the registry does not
-know. It says "this endpoint speaks the decision protocol".
+For a Laya model name the registry does not know, set `provider: "typesafe"`
+to select its decision protocol.
 
 OpenRouter and Vercel AI Gateway also serve Jev over this protocol, so no
-TypeSafe account is needed. OpenRouter's model name, `jev-1.13`, is in the
-registry:
+TypeSafe account is needed. To use OpenRouter, set `provider: "openrouter"`
+and provide `OPENROUTER_API_KEY` or `config.apiKey.openRouter`:
 
 ```typescript
 import { decide } from "smoltalk";
@@ -630,12 +655,17 @@ const r = await decide(
   { refund: { type: "noul", instructions: "Is the customer asking for money back?" } },
   {
     model: "jev-1.13",
-    apiKey: { typesafe: process.env.OPENROUTER_API_KEY },
-    baseUrl: { typesafe: "https://openrouter.ai/api" },
+    provider: "openrouter",
+    apiKey: { openRouter: process.env.OPENROUTER_API_KEY },
   },
 );
 ```
 
+OpenRouter requests go to `https://openrouter.ai/api/v1/systemone`. Set
+`baseUrl.openRouter` to override the API base URL, including its version
+path; `decide()` appends `/systemone`. Without an explicit provider,
+`jev-1.13` still defaults to TypeSafe.
+
 Vercel's base URL is `https://ai-gateway.vercel.sh/typesafe` and its model
 name is `typesafe-ai/jev`, which needs `provider: "typesafe"`.
 

package/dist/clients/baseClient.js

@@ -1,4 +1,5 @@
-import { userMessage, assistantMessage } from "../classes/message/index.js";
+import { assistantMessage } from "../classes/message/index.js";
+import { failedValidationMessage, returnedUndefinedMessage, } from "./validationRetry.js";
 import { getLogger } from "../util/logger.js";
 import { SmolStructuredOutputError } from "../smolError.js";
 import { getStatelogClient } from "../statelogClient.js";
@@ -295,7 +296,7 @@ export class BaseClient {
             if (!("output" in result.value)) {
                 const retryMessages = [
                     ...promptConfig.messages,
-                    userMessage(`You returned "undefined" instead of a valid response. Please provide a valid response.`),
+                    returnedUndefinedMessage(),
                 ];
                 return this.textWithRetry({ ...promptConfig, messages: retryMessages }, retries - 1);
             }
@@ -332,7 +333,7 @@ export class BaseClient {
                     const retryMessages = [
                         ...promptConfig.messages,
                         assistantMessage(output),
-                        userMessage(`Your previous response failed validation. Please fix the following errors and try again:\n${errorMessage}`),
+                        failedValidationMessage(errorMessage),
                     ];
                     return this.textWithRetry({ ...promptConfig, messages: retryMessages }, retries - 1);
                 }

package/dist/clients/mlx.d.ts

@@ -1,10 +1,20 @@
 import { SmolOpenAiCompat } from "./openaiCompat.js";
-import type { SmolConfig } from "../types.js";
+import type { PromptResult, SmolConfig } from "../types.js";
+import type { Result } from "../types/result.js";
 /**
  * Client for an MLX server on localhost, usually started with `mlx_lm.server`
  * (or `agency local serve`). The server speaks the OpenAI chat format, so this
  * is the openai-compat client with three things fixed: the base URL has a
  * default, no API key is needed, and the cost is always zero.
+ *
+ * A call with both tools and a `responseFormat` is made as two requests (see
+ * structuredAfterTools.ts). The second keeps the tool list and sends
+ * `tool_choice: "none"`. Keeping the list matters: the chat template renders
+ * the tools at the start of the prompt, so dropping them would make the
+ * server read the whole prompt again instead of reusing its cache. A server
+ * that enforces a schema when `tool_choice` is `"none"` (`agency local
+ * serve`) then holds the reply to the format. Streaming calls are not split:
+ * as on Google, a streamed call with tools leaves the format unenforced.
  */
 export declare class SmolMlx extends SmolOpenAiCompat {
     protected resolveClientOptions(config: SmolConfig): {
@@ -12,4 +22,5 @@ export declare class SmolMlx extends SmolOpenAiCompat {
         baseURL: string;
     };
     protected resolveCostUsd(): number;
+    _textSync(config: SmolConfig): Promise<Result<PromptResult>>;
 }

package/dist/clients/mlx.js

@@ -1,10 +1,20 @@
 import { SmolOpenAiCompat } from "./openaiCompat.js";
 import { resolveBaseUrl } from "../util/provider.js";
+import { separatesStructuredOutput, toolsThenStructuredOutput, } from "./structuredAfterTools.js";
 /**
  * Client for an MLX server on localhost, usually started with `mlx_lm.server`
  * (or `agency local serve`). The server speaks the OpenAI chat format, so this
  * is the openai-compat client with three things fixed: the base URL has a
  * default, no API key is needed, and the cost is always zero.
+ *
+ * A call with both tools and a `responseFormat` is made as two requests (see
+ * structuredAfterTools.ts). The second keeps the tool list and sends
+ * `tool_choice: "none"`. Keeping the list matters: the chat template renders
+ * the tools at the start of the prompt, so dropping them would make the
+ * server read the whole prompt again instead of reusing its cache. A server
+ * that enforces a schema when `tool_choice` is `"none"` (`agency local
+ * serve`) then holds the reply to the format. Streaming calls are not split:
+ * as on Google, a streamed call with tools leaves the format unenforced.
  */
 export class SmolMlx extends SmolOpenAiCompat {
     resolveClientOptions(config) {
@@ -16,4 +26,17 @@ export class SmolMlx extends SmolOpenAiCompat {
     resolveCostUsd() {
         return 0;
     }
+    async _textSync(config) {
+        if (!separatesStructuredOutput(config)) {
+            return super._textSync(config);
+        }
+        return toolsThenStructuredOutput({
+            config,
+            run: (one) => super._textSync(one),
+            formatConfig: (followUp) => ({
+                ...followUp,
+                rawAttributes: { ...followUp.rawAttributes, tool_choice: "none" },
+            }),
+        });
+    }
 }

package/dist/clients/structuredAfterTools.d.ts

@@ -0,0 +1,36 @@
+import type { PromptResult, SmolConfig } from "../types.js";
+import { type Result } from "../types/result.js";
+/**
+ * A local model cannot be held to a JSON schema while it may still call a
+ * tool: a tool call is not JSON, so the servers leave the schema unenforced
+ * on any request that carries tools, and the model is free to answer in
+ * prose. A call with both tools and a `responseFormat` is therefore made as
+ * two requests, the way the Google client does it. The first carries the
+ * tools and no schema. If the model calls a tool, that is the result. If it
+ * answers, a second request asks for that answer in the schema, with tool
+ * calls ruled out so the schema can be enforced.
+ *
+ * Unlike the Google client, the second request keeps the whole conversation.
+ * A schema field such as "did this answer everything the user asked?" cannot
+ * be filled in from the answer alone.
+ */
+/** What the second request says, after the model's own answer. */
+export declare const STRUCTURED_FOLLOW_UP = "Give that same reply again in the required JSON format. Do not call a tool.";
+/** Whether a call's typed reply is made as a second request. */
+export declare function separatesStructuredOutput(config: SmolConfig): boolean;
+/**
+ * Makes the two requests. `run` sends one request. `formatConfig` turns the
+ * follow-up config into one whose schema the provider will enforce: the MLX
+ * client sets `tool_choice: "none"`, the llama.cpp client drops the tools.
+ *
+ * When the reply fails the caller's validation, the caller sends the
+ * conversation back with a request to fix it. The model has already answered
+ * by then, so only the format request is made. A tool round there would let
+ * the model answer "fix this JSON" with a tool call, and a tool that has
+ * already run would run again.
+ */
+export declare function toolsThenStructuredOutput(args: {
+    config: SmolConfig;
+    run: (config: SmolConfig) => Promise<Result<PromptResult>>;
+    formatConfig: (config: SmolConfig) => SmolConfig;
+}): Promise<Result<PromptResult>>;

package/dist/clients/structuredAfterTools.js

@@ -0,0 +1,87 @@
+import { assistantMessage, userMessage } from "../classes/message/index.js";
+import { addCosts } from "../types/costEstimate.js";
+import { addTokenUsage } from "../types/tokenUsage.js";
+import { success } from "../types/result.js";
+import { endsInValidationRetry } from "./validationRetry.js";
+/**
+ * A local model cannot be held to a JSON schema while it may still call a
+ * tool: a tool call is not JSON, so the servers leave the schema unenforced
+ * on any request that carries tools, and the model is free to answer in
+ * prose. A call with both tools and a `responseFormat` is therefore made as
+ * two requests, the way the Google client does it. The first carries the
+ * tools and no schema. If the model calls a tool, that is the result. If it
+ * answers, a second request asks for that answer in the schema, with tool
+ * calls ruled out so the schema can be enforced.
+ *
+ * Unlike the Google client, the second request keeps the whole conversation.
+ * A schema field such as "did this answer everything the user asked?" cannot
+ * be filled in from the answer alone.
+ */
+/** What the second request says, after the model's own answer. */
+export const STRUCTURED_FOLLOW_UP = "Give that same reply again in the required JSON format. Do not call a tool.";
+/** Whether a call's typed reply is made as a second request. */
+export function separatesStructuredOutput(config) {
+    const hasTools = (config.tools?.length ?? 0) > 0;
+    if (!hasTools || !config.responseFormat) {
+        return false;
+    }
+    return config.responseFormatOptions?.separateFromTools !== false;
+}
+/**
+ * The format request's result as the caller sees it. The request rules tool
+ * calls out, but a server too old to honour that could still return one. It
+ * must not be run: the tool round is over. Such a reply has no text either,
+ * so `fallback` stands in for it. The caller's validation then fails on the
+ * fallback and asks again, instead of finding nothing to validate.
+ */
+function formatReply(result, fallback) {
+    return { ...result, toolCalls: [], output: result.output || fallback };
+}
+/**
+ * Makes the two requests. `run` sends one request. `formatConfig` turns the
+ * follow-up config into one whose schema the provider will enforce: the MLX
+ * client sets `tool_choice: "none"`, the llama.cpp client drops the tools.
+ *
+ * When the reply fails the caller's validation, the caller sends the
+ * conversation back with a request to fix it. The model has already answered
+ * by then, so only the format request is made. A tool round there would let
+ * the model answer "fix this JSON" with a tool call, and a tool that has
+ * already run would run again.
+ */
+export async function toolsThenStructuredOutput(args) {
+    const { config, run, formatConfig } = args;
+    if (endsInValidationRetry(config.messages)) {
+        const only = await run(formatConfig(config));
+        return only.success ? success(formatReply(only.value, "")) : only;
+    }
+    const first = await run({ ...config, responseFormat: undefined });
+    if (!first.success) {
+        return first;
+    }
+    // A tool call is the whole reply. An empty reply has nothing to put in
+    // the format, so it goes back as it is and fails validation as before.
+    if (first.value.toolCalls.length > 0 || !first.value.output) {
+        return first;
+    }
+    const answer = first.value.output;
+    const second = await run(formatConfig({
+        ...config,
+        messages: [...config.messages, assistantMessage(answer), userMessage(STRUCTURED_FOLLOW_UP)],
+    }));
+    if (!second.success) {
+        return second;
+    }
+    const thinkingBlocks = [
+        ...(first.value.thinkingBlocks ?? []),
+        ...(second.value.thinkingBlocks ?? []),
+    ];
+    const merged = {
+        ...formatReply(second.value, answer),
+        usage: addTokenUsage(first.value.usage, second.value.usage),
+        cost: addCosts(first.value.cost, second.value.cost),
+    };
+    if (thinkingBlocks.length > 0) {
+        merged.thinkingBlocks = thinkingBlocks;
+    }
+    return success(merged);
+}

package/dist/clients/validationRetry.d.ts

@@ -0,0 +1,5 @@
+import type { Message } from "../classes/message/index.js";
+export declare function failedValidationMessage(errorMessage: string): import("../classes/message/UserMessage.js").UserMessage;
+export declare function returnedUndefinedMessage(): import("../classes/message/UserMessage.js").UserMessage;
+/** Whether the last message asks the model to correct its previous reply. */
+export declare function endsInValidationRetry(messages: Message[]): boolean;

package/dist/clients/validationRetry.js

@@ -0,0 +1,24 @@
+import { userMessage } from "../classes/message/index.js";
+/**
+ * The messages `textWithRetry` adds when a reply does not fit the response
+ * format, and the test for them. A client that makes a tool round before the
+ * format request (see structuredAfterTools.ts) uses the test to skip the tool
+ * round on a retry.
+ */
+const FAILED_VALIDATION = "Your previous response failed validation.";
+const RETURNED_UNDEFINED = `You returned "undefined" instead of a valid response.`;
+export function failedValidationMessage(errorMessage) {
+    return userMessage(`${FAILED_VALIDATION} Please fix the following errors and try again:\n${errorMessage}`);
+}
+export function returnedUndefinedMessage() {
+    return userMessage(`${RETURNED_UNDEFINED} Please provide a valid response.`);
+}
+/** Whether the last message asks the model to correct its previous reply. */
+export function endsInValidationRetry(messages) {
+    const last = messages[messages.length - 1];
+    if (!last || last.role !== "user") {
+        return false;
+    }
+    const text = last.content;
+    return text.startsWith(FAILED_VALIDATION) || text.startsWith(RETURNED_UNDEFINED);
+}

package/dist/decide.d.ts

@@ -53,15 +53,18 @@ export type DecideConfig = {
     model: string;
     /** Required when the model is not in the registry. */
     provider?: string;
-    /** API keys, nested by provider. Falls back to TYPESAFE_API_KEY. */
+    /** API keys, nested by provider. Falls back to TYPESAFE_API_KEY or OPENROUTER_API_KEY. */
     apiKey?: {
         typesafe?: string;
+        openRouter?: string;
         [provider: string]: string | undefined;
     };
-    /** Custom base URLs, nested by provider. Falls back to TYPESAFE_BASE_URL,
-     *  then TypeSafe's host. Point `typesafe` at a Laya server to use Laya. */
+    /** Custom base URLs, nested by provider. TypeSafe falls back to TYPESAFE_BASE_URL,
+     *  then its host. OpenRouter defaults to https://openrouter.ai/api/v1.
+     *  Point `typesafe` at a Laya server to use Laya. */
     baseUrl?: {
         typesafe?: string;
+        openRouter?: string;
         [provider: string]: string | undefined;
     };
     /** Refreshed model data to layer over the baked-in registry. */

package/dist/decide.js

@@ -9,8 +9,8 @@
  *
  * `decide()` follows the shape of `embed()`: payload first, config last,
  * provider and key and base URL resolved through the shared helpers. There
- * is one provider, `typesafe`, which is the wire protocol. A Laya server
- * speaks the same protocol, so it is reached by setting `baseUrl.typesafe`.
+ * are two providers: `typesafe` and `openrouter`. A Laya server speaks the
+ * TypeSafe protocol, so it is reached by setting `baseUrl.typesafe`.
  */
 import { z } from "zod";
 import { success, failure } from "./types/result.js";
@@ -151,8 +151,8 @@ export async function decide(state, questions, config) {
     catch (err) {
         return failure(errorMessage(err));
     }
-    if (provider !== DECISION_PROVIDER) {
-        return failure(`Provider "${provider}" does not answer decisions. Only "${DECISION_PROVIDER}" does; set config.provider to it for a model the registry does not know.`);
+    if (provider !== DECISION_PROVIDER && provider !== "openrouter") {
+        return failure(`Provider "${provider}" does not answer decisions. Use "${DECISION_PROVIDER}" or "openrouter"; set config.provider for a model the registry does not know.`);
     }
     // The registry entry for the requested name, when there is one. It sets
     // the question cap and the price. A Laya model has no entry and no price.
@@ -164,15 +164,18 @@ export async function decide(state, questions, config) {
     }
     const apiKey = resolveApiKey(provider, config);
     if (!apiKey) {
-        return failure("No TypeSafe API key provided. Set config.apiKey.typesafe or the TYPESAFE_API_KEY environment variable.");
+        return failure(provider === "openrouter"
+            ? "No OpenRouter API key provided. Set config.apiKey.openRouter or the OPENROUTER_API_KEY environment variable."
+            : "No TypeSafe API key provided. Set config.apiKey.typesafe or the TYPESAFE_API_KEY environment variable.");
     }
     const baseUrl = resolveBaseUrl(provider, config).replace(/\/+$/, "");
+    const endpoint = provider === "openrouter" ? `${baseUrl}/systemone` : `${baseUrl}/v1/systemone`;
     if (config.abortSignal?.aborted) {
         return failure("Request was aborted");
     }
     let response;
     try {
-        response = await fetch(`${baseUrl}/v1/systemone`, {
+        response = await fetch(endpoint, {
             method: "POST",
             headers: {
                 Authorization: `Bearer ${apiKey}`,

package/dist/index.d.ts

@@ -23,6 +23,7 @@ export type { BlobRef } from "./util/blobRef.js";
 export { getLogger, EgonLog } from "./util/logger.js";
 export type { LogLevel } from "./util/logger.js";
 export { redactAttachments } from "./util/redact.js";
+export { separatesStructuredOutput, toolsThenStructuredOutput, STRUCTURED_FOLLOW_UP, } from "./clients/structuredAfterTools.js";
 export { transcribe, registerTranscriptionProvider, DEFAULT_TRANSCRIBE_BYTES, } from "./transcription.js";
 export type { TranscribeOptions, TranscriptionSegment, TranscriptionWord, TranscriptionResult, TranscriptionClientClass, } from "./transcription.js";
 export { BaseTranscriptionClient } from "./transcription/baseTranscriptionClient.js";

package/dist/index.js

@@ -21,6 +21,8 @@ export { uploadFile, deleteFile, registerFileProvider, DEFAULT_UPLOAD_BYTES } fr
 export { normalizeBlob, loadBlob } from "./util/blobRef.js";
 export { getLogger, EgonLog } from "./util/logger.js";
 export { redactAttachments } from "./util/redact.js";
+// For provider plugins whose backend cannot enforce a schema alongside tools.
+export { separatesStructuredOutput, toolsThenStructuredOutput, STRUCTURED_FOLLOW_UP, } from "./clients/structuredAfterTools.js";
 // Explicit (not `export *`) so internal factories and test helpers stay private.
 export { transcribe, registerTranscriptionProvider, DEFAULT_TRANSCRIBE_BYTES, } from "./transcription.js";
 export { BaseTranscriptionClient } from "./transcription/baseTranscriptionClient.js";