smoltalk 0.15.3 → 0.16.0

package/README.md

@@ -229,6 +229,31 @@ Used with `responseFormat` to control validation behavior (currently OpenAI only
 | `strict` | `boolean` | | Whether to use strict schema validation. |
 | `numRetries` | `number` | `2` | How many times to retry if the response fails schema validation. |
 | `allowExtraKeys` | `boolean` | | If `true`, strips unexpected keys instead of failing validation. |
+| `separateFromTools` | `boolean` | `true` | Local providers only (`mlx`, `llama-cpp`). See below. |
+
+#### Tools and a response format in one call
+
+Some providers cannot hold a reply to a schema while the model may still call
+a tool. Google rejects the combination. A local server (`mlx`, `llama-cpp`)
+accepts it but leaves the schema unenforced, because a tool call is not JSON,
+and a local model then tends to answer in prose.
+
+For these providers smoltalk makes the call as two requests. The first
+carries the tools and no schema. If the model calls a tool, that is the
+result, and your tool loop carries on as usual. Once the model answers in
+words, a second request asks for that answer in the schema, and that is the
+`output` you get. Usage and cost cover both requests.
+
+On the local providers the second request keeps the whole conversation, so a
+field such as "did this cover everything the user asked?" can be answered.
+`mlx` also keeps the tool list and sends `tool_choice: "none"`, so the server
+can reuse its cached prompt. `responseFormatOptions.separateFromTools: false`
+sends one request instead and leaves the schema unenforced. A streamed call
+is never split.
+
+With `strict: true`, a reply that fails validation is retried as usual. The
+retry repeats only the second request, so the model cannot answer "fix this
+JSON" with another tool call.
 
 ### `toolLoopDetection`
 

package/dist/clients/baseClient.js

@@ -1,4 +1,5 @@
-import { userMessage, assistantMessage } from "../classes/message/index.js";
+import { assistantMessage } from "../classes/message/index.js";
+import { failedValidationMessage, returnedUndefinedMessage, } from "./validationRetry.js";
 import { getLogger } from "../util/logger.js";
 import { SmolStructuredOutputError } from "../smolError.js";
 import { getStatelogClient } from "../statelogClient.js";
@@ -295,7 +296,7 @@ export class BaseClient {
             if (!("output" in result.value)) {
                 const retryMessages = [
                     ...promptConfig.messages,
-                    userMessage(`You returned "undefined" instead of a valid response. Please provide a valid response.`),
+                    returnedUndefinedMessage(),
                 ];
                 return this.textWithRetry({ ...promptConfig, messages: retryMessages }, retries - 1);
             }
@@ -332,7 +333,7 @@ export class BaseClient {
                     const retryMessages = [
                         ...promptConfig.messages,
                         assistantMessage(output),
-                        userMessage(`Your previous response failed validation. Please fix the following errors and try again:\n${errorMessage}`),
+                        failedValidationMessage(errorMessage),
                     ];
                     return this.textWithRetry({ ...promptConfig, messages: retryMessages }, retries - 1);
                 }

package/dist/clients/mlx.d.ts

@@ -1,10 +1,20 @@
 import { SmolOpenAiCompat } from "./openaiCompat.js";
-import type { SmolConfig } from "../types.js";
+import type { PromptResult, SmolConfig } from "../types.js";
+import type { Result } from "../types/result.js";
 /**
  * Client for an MLX server on localhost, usually started with `mlx_lm.server`
  * (or `agency local serve`). The server speaks the OpenAI chat format, so this
  * is the openai-compat client with three things fixed: the base URL has a
  * default, no API key is needed, and the cost is always zero.
+ *
+ * A call with both tools and a `responseFormat` is made as two requests (see
+ * structuredAfterTools.ts). The second keeps the tool list and sends
+ * `tool_choice: "none"`. Keeping the list matters: the chat template renders
+ * the tools at the start of the prompt, so dropping them would make the
+ * server read the whole prompt again instead of reusing its cache. A server
+ * that enforces a schema when `tool_choice` is `"none"` (`agency local
+ * serve`) then holds the reply to the format. Streaming calls are not split:
+ * as on Google, a streamed call with tools leaves the format unenforced.
  */
 export declare class SmolMlx extends SmolOpenAiCompat {
     protected resolveClientOptions(config: SmolConfig): {
@@ -12,4 +22,5 @@ export declare class SmolMlx extends SmolOpenAiCompat {
         baseURL: string;
     };
     protected resolveCostUsd(): number;
+    _textSync(config: SmolConfig): Promise<Result<PromptResult>>;
 }

package/dist/clients/mlx.js

@@ -1,10 +1,20 @@
 import { SmolOpenAiCompat } from "./openaiCompat.js";
 import { resolveBaseUrl } from "../util/provider.js";
+import { separatesStructuredOutput, toolsThenStructuredOutput, } from "./structuredAfterTools.js";
 /**
  * Client for an MLX server on localhost, usually started with `mlx_lm.server`
  * (or `agency local serve`). The server speaks the OpenAI chat format, so this
  * is the openai-compat client with three things fixed: the base URL has a
  * default, no API key is needed, and the cost is always zero.
+ *
+ * A call with both tools and a `responseFormat` is made as two requests (see
+ * structuredAfterTools.ts). The second keeps the tool list and sends
+ * `tool_choice: "none"`. Keeping the list matters: the chat template renders
+ * the tools at the start of the prompt, so dropping them would make the
+ * server read the whole prompt again instead of reusing its cache. A server
+ * that enforces a schema when `tool_choice` is `"none"` (`agency local
+ * serve`) then holds the reply to the format. Streaming calls are not split:
+ * as on Google, a streamed call with tools leaves the format unenforced.
  */
 export class SmolMlx extends SmolOpenAiCompat {
     resolveClientOptions(config) {
@@ -16,4 +26,17 @@ export class SmolMlx extends SmolOpenAiCompat {
     resolveCostUsd() {
         return 0;
     }
+    async _textSync(config) {
+        if (!separatesStructuredOutput(config)) {
+            return super._textSync(config);
+        }
+        return toolsThenStructuredOutput({
+            config,
+            run: (one) => super._textSync(one),
+            formatConfig: (followUp) => ({
+                ...followUp,
+                rawAttributes: { ...followUp.rawAttributes, tool_choice: "none" },
+            }),
+        });
+    }
 }

package/dist/clients/structuredAfterTools.d.ts

@@ -0,0 +1,36 @@
+import type { PromptResult, SmolConfig } from "../types.js";
+import { type Result } from "../types/result.js";
+/**
+ * A local model cannot be held to a JSON schema while it may still call a
+ * tool: a tool call is not JSON, so the servers leave the schema unenforced
+ * on any request that carries tools, and the model is free to answer in
+ * prose. A call with both tools and a `responseFormat` is therefore made as
+ * two requests, the way the Google client does it. The first carries the
+ * tools and no schema. If the model calls a tool, that is the result. If it
+ * answers, a second request asks for that answer in the schema, with tool
+ * calls ruled out so the schema can be enforced.
+ *
+ * Unlike the Google client, the second request keeps the whole conversation.
+ * A schema field such as "did this answer everything the user asked?" cannot
+ * be filled in from the answer alone.
+ */
+/** What the second request says, after the model's own answer. */
+export declare const STRUCTURED_FOLLOW_UP = "Give that same reply again in the required JSON format. Do not call a tool.";
+/** Whether a call's typed reply is made as a second request. */
+export declare function separatesStructuredOutput(config: SmolConfig): boolean;
+/**
+ * Makes the two requests. `run` sends one request. `formatConfig` turns the
+ * follow-up config into one whose schema the provider will enforce: the MLX
+ * client sets `tool_choice: "none"`, the llama.cpp client drops the tools.
+ *
+ * When the reply fails the caller's validation, the caller sends the
+ * conversation back with a request to fix it. The model has already answered
+ * by then, so only the format request is made. A tool round there would let
+ * the model answer "fix this JSON" with a tool call, and a tool that has
+ * already run would run again.
+ */
+export declare function toolsThenStructuredOutput(args: {
+    config: SmolConfig;
+    run: (config: SmolConfig) => Promise<Result<PromptResult>>;
+    formatConfig: (config: SmolConfig) => SmolConfig;
+}): Promise<Result<PromptResult>>;

package/dist/clients/structuredAfterTools.js

@@ -0,0 +1,87 @@
+import { assistantMessage, userMessage } from "../classes/message/index.js";
+import { addCosts } from "../types/costEstimate.js";
+import { addTokenUsage } from "../types/tokenUsage.js";
+import { success } from "../types/result.js";
+import { endsInValidationRetry } from "./validationRetry.js";
+/**
+ * A local model cannot be held to a JSON schema while it may still call a
+ * tool: a tool call is not JSON, so the servers leave the schema unenforced
+ * on any request that carries tools, and the model is free to answer in
+ * prose. A call with both tools and a `responseFormat` is therefore made as
+ * two requests, the way the Google client does it. The first carries the
+ * tools and no schema. If the model calls a tool, that is the result. If it
+ * answers, a second request asks for that answer in the schema, with tool
+ * calls ruled out so the schema can be enforced.
+ *
+ * Unlike the Google client, the second request keeps the whole conversation.
+ * A schema field such as "did this answer everything the user asked?" cannot
+ * be filled in from the answer alone.
+ */
+/** What the second request says, after the model's own answer. */
+export const STRUCTURED_FOLLOW_UP = "Give that same reply again in the required JSON format. Do not call a tool.";
+/** Whether a call's typed reply is made as a second request. */
+export function separatesStructuredOutput(config) {
+    const hasTools = (config.tools?.length ?? 0) > 0;
+    if (!hasTools || !config.responseFormat) {
+        return false;
+    }
+    return config.responseFormatOptions?.separateFromTools !== false;
+}
+/**
+ * The format request's result as the caller sees it. The request rules tool
+ * calls out, but a server too old to honour that could still return one. It
+ * must not be run: the tool round is over. Such a reply has no text either,
+ * so `fallback` stands in for it. The caller's validation then fails on the
+ * fallback and asks again, instead of finding nothing to validate.
+ */
+function formatReply(result, fallback) {
+    return { ...result, toolCalls: [], output: result.output || fallback };
+}
+/**
+ * Makes the two requests. `run` sends one request. `formatConfig` turns the
+ * follow-up config into one whose schema the provider will enforce: the MLX
+ * client sets `tool_choice: "none"`, the llama.cpp client drops the tools.
+ *
+ * When the reply fails the caller's validation, the caller sends the
+ * conversation back with a request to fix it. The model has already answered
+ * by then, so only the format request is made. A tool round there would let
+ * the model answer "fix this JSON" with a tool call, and a tool that has
+ * already run would run again.
+ */
+export async function toolsThenStructuredOutput(args) {
+    const { config, run, formatConfig } = args;
+    if (endsInValidationRetry(config.messages)) {
+        const only = await run(formatConfig(config));
+        return only.success ? success(formatReply(only.value, "")) : only;
+    }
+    const first = await run({ ...config, responseFormat: undefined });
+    if (!first.success) {
+        return first;
+    }
+    // A tool call is the whole reply. An empty reply has nothing to put in
+    // the format, so it goes back as it is and fails validation as before.
+    if (first.value.toolCalls.length > 0 || !first.value.output) {
+        return first;
+    }
+    const answer = first.value.output;
+    const second = await run(formatConfig({
+        ...config,
+        messages: [...config.messages, assistantMessage(answer), userMessage(STRUCTURED_FOLLOW_UP)],
+    }));
+    if (!second.success) {
+        return second;
+    }
+    const thinkingBlocks = [
+        ...(first.value.thinkingBlocks ?? []),
+        ...(second.value.thinkingBlocks ?? []),
+    ];
+    const merged = {
+        ...formatReply(second.value, answer),
+        usage: addTokenUsage(first.value.usage, second.value.usage),
+        cost: addCosts(first.value.cost, second.value.cost),
+    };
+    if (thinkingBlocks.length > 0) {
+        merged.thinkingBlocks = thinkingBlocks;
+    }
+    return success(merged);
+}

package/dist/clients/validationRetry.d.ts

@@ -0,0 +1,5 @@
+import type { Message } from "../classes/message/index.js";
+export declare function failedValidationMessage(errorMessage: string): import("../classes/message/UserMessage.js").UserMessage;
+export declare function returnedUndefinedMessage(): import("../classes/message/UserMessage.js").UserMessage;
+/** Whether the last message asks the model to correct its previous reply. */
+export declare function endsInValidationRetry(messages: Message[]): boolean;

package/dist/clients/validationRetry.js

@@ -0,0 +1,24 @@
+import { userMessage } from "../classes/message/index.js";
+/**
+ * The messages `textWithRetry` adds when a reply does not fit the response
+ * format, and the test for them. A client that makes a tool round before the
+ * format request (see structuredAfterTools.ts) uses the test to skip the tool
+ * round on a retry.
+ */
+const FAILED_VALIDATION = "Your previous response failed validation.";
+const RETURNED_UNDEFINED = `You returned "undefined" instead of a valid response.`;
+export function failedValidationMessage(errorMessage) {
+    return userMessage(`${FAILED_VALIDATION} Please fix the following errors and try again:\n${errorMessage}`);
+}
+export function returnedUndefinedMessage() {
+    return userMessage(`${RETURNED_UNDEFINED} Please provide a valid response.`);
+}
+/** Whether the last message asks the model to correct its previous reply. */
+export function endsInValidationRetry(messages) {
+    const last = messages[messages.length - 1];
+    if (!last || last.role !== "user") {
+        return false;
+    }
+    const text = last.content;
+    return text.startsWith(FAILED_VALIDATION) || text.startsWith(RETURNED_UNDEFINED);
+}

package/dist/index.d.ts

@@ -23,6 +23,7 @@ export type { BlobRef } from "./util/blobRef.js";
 export { getLogger, EgonLog } from "./util/logger.js";
 export type { LogLevel } from "./util/logger.js";
 export { redactAttachments } from "./util/redact.js";
+export { separatesStructuredOutput, toolsThenStructuredOutput, STRUCTURED_FOLLOW_UP, } from "./clients/structuredAfterTools.js";
 export { transcribe, registerTranscriptionProvider, DEFAULT_TRANSCRIBE_BYTES, } from "./transcription.js";
 export type { TranscribeOptions, TranscriptionSegment, TranscriptionWord, TranscriptionResult, TranscriptionClientClass, } from "./transcription.js";
 export { BaseTranscriptionClient } from "./transcription/baseTranscriptionClient.js";

package/dist/index.js

@@ -21,6 +21,8 @@ export { uploadFile, deleteFile, registerFileProvider, DEFAULT_UPLOAD_BYTES } fr
 export { normalizeBlob, loadBlob } from "./util/blobRef.js";
 export { getLogger, EgonLog } from "./util/logger.js";
 export { redactAttachments } from "./util/redact.js";
+// For provider plugins whose backend cannot enforce a schema alongside tools.
+export { separatesStructuredOutput, toolsThenStructuredOutput, STRUCTURED_FOLLOW_UP, } from "./clients/structuredAfterTools.js";
 // Explicit (not `export *`) so internal factories and test helpers stay private.
 export { transcribe, registerTranscriptionProvider, DEFAULT_TRANSCRIBE_BYTES, } from "./transcription.js";
 export { BaseTranscriptionClient } from "./transcription/baseTranscriptionClient.js";