smoltalk 0.0.54 → 0.0.56

package/README.md

@@ -1,6 +1,6 @@
 # Smoltalk
 
-Smoltalk exposes a common API to different LLM providers. There are other packages that do this, but Smoltalk allows you to build strategies on top of it. Here is a simple example. Hello world, this is functionality that other packages allow.
+Smoltalk exposes a common API to different LLM providers. There are other packages that do this, but Smoltalk allows you to build strategies on top of it. Here is a simple example.
 
 ## Install
 
@@ -11,23 +11,80 @@ pnpm install smoltalk
 ## Hello world example
 
 ```typescript
-import { getClient } from "smoltalk";
-
-const client = getClient({
-  openAiApiKey: process.env.OPENAI_API_KEY || "",
-  googleApiKey: process.env.GEMINI_API_KEY || "",
-  logLevel: "debug",
-  model: "gemini-2.0-flash-lite",
-});
+import { text, userMessage } from "smoltalk";
 
 async function main() {
-  const resp = await client.prompt("Hello, how are you?");
-  console.log(resp);
+  const messages = [userMessage("Write me a 10 word story.")];
+  const response = await text({
+    messages,
+    model: "gpt-5.4",
+  });
+  console.log(response);
 }
 
 main();
 ```
 
+This is functionality that other packages allow.
+<details>
+  <summary>Response</summary>
+
+```
+{
+  success: true,
+  value: {
+    output: 'Clock stopped; everyone smiled as tomorrow finally arrived before yesterday.',
+    toolCalls: [],
+    usage: {
+      inputTokens: 14,
+      outputTokens: 15,
+      cachedInputTokens: 0,
+      totalTokens: 29
+    },
+    cost: {
+      inputCost: 0.000035,
+      outputCost: 0.000225,
+      cachedInputCost: undefined,
+      totalCost: 0.00026,
+      currency: 'USD'
+    },
+    model: 'gpt-5.4'
+  }
+}
+```
+</details>
+
+What if you wanted to have fallbacks in case the OpenAI API was down? Just change the `model` field:
+
+```ts
+  const response = await text({
+    messages,
+    model: fallback("gpt-5.4", "gemini-2.5-flash-lite"),
+    // or multiple fallbacks:
+    // model: fallback("gpt-5.4", ["gemini-2.5-flash-lite", "gemini-3-flash-preview"]),
+  });
+```
+
+Or what if you wanted to try a couple of models and take the first response?
+
+```ts
+  const response = await text({
+    messages,
+    model: race("gpt-5.4", "gemini-2.5-flash-lite", "o4-mini"),
+  });
+```
+
+Or combine them:
+
+```ts
+  const response = await text({
+    messages,
+    model: race(fallback("gpt-5.4", "gemini-2.5-flash-lite"), "o4-mini"),
+  });
+```
+
+You get the idea.
+
 ## Longer tutorial
 To use Smoltak, you first create a client:
 
@@ -157,20 +214,15 @@ Detects when the model is stuck in a repetitive tool-call loop.
 | `intervention` | `string` | Action to take: `"remove-tool"`, `"remove-all-tools"`, `"throw-error"`, or `"halt-execution"`. |
 | `excludeTools` | `string[]` | Tool names to ignore when counting consecutive calls. |
 
-## Prior art
+## Limitations
+Smoltalk has support for a limited number of providers right now, and is mostly focused on the stateless APIs for text completion, though I plan to add support for more providers as well as image and speech models later. Smoltalk is also a personal project, and there are alternatives backed by companies:
+
 - Langchain
-OpenRouter
+- OpenRouter
 - Vercel AI
 
-These are all good options, but they are quite heavy, and I wanted a lighter option. That said, you may be better off with one of the above alternatives:
-- They are backed by a business and are more likely to be responsive.
-- They support way more functionality and providers. Smoltalk currently supports just a subset of functionality for OpenAI and Google.
-
-## Functionality
-Smoltalk pretty much lets you generate text using an OpenAI or Google model, with support for function calling and structured output, and that's it. I will add functionality and providers sporadically when I have time and need.
-
 ## Contributing
-This repo could use some help! Any of the following contributions would be helpful:
+Contributions are welcome. Any of the following contributions would be helpful:
 - Adding support for API parameters or endpoints
 - Adding support for different providers
-- Updating the list of models
+- Updating the list of models

package/dist/clients/anthropic.d.ts

@@ -12,6 +12,7 @@ export declare class SmolAnthropic extends BaseClient implements SmolClient {
     getModel(): ModelName;
     private calculateUsageAndCost;
     private buildRequest;
+    private rethrowAsSmolError;
     _textSync(config: PromptConfig): Promise<Result<PromptResult>>;
     _textStream(config: PromptConfig): AsyncGenerator<StreamChunk>;
 }

package/dist/clients/anthropic.js

@@ -4,6 +4,7 @@ import { SystemMessage, DeveloperMessage } from "../classes/message/index.js";
 import { getLogger } from "../logger.js";
 import { success, } from "../types.js";
 import { zodToAnthropicTool } from "../util/tool.js";
+import { SmolContentPolicyError, SmolContextWindowExceededError, } from "../smolError.js";
 import { BaseClient } from "./baseClient.js";
 import { Model } from "../model.js";
 const DEFAULT_MAX_TOKENS = 4096;
@@ -82,6 +83,24 @@ export class SmolAnthropic extends BaseClient {
                 : undefined;
         return { system, messages: anthropicMessages, tools, thinking };
     }
+    rethrowAsSmolError(error) {
+        if (error instanceof Anthropic.APIError) {
+            const msg = error.message.toLowerCase();
+            if (msg.includes("prompt is too long") ||
+                msg.includes("context length") ||
+                msg.includes("context window") ||
+                msg.includes("too many tokens")) {
+                throw new SmolContextWindowExceededError(error.message);
+            }
+            if (msg.includes("content policy") ||
+                msg.includes("usage policies") ||
+                msg.includes("content filtering") ||
+                msg.includes("violates our")) {
+                throw new SmolContentPolicyError(error.message);
+            }
+        }
+        throw error;
+    }
     async _textSync(config) {
         const { system, messages, tools, thinking } = this.buildRequest(config);
         let debugData = {
@@ -95,19 +114,25 @@ export class SmolAnthropic extends BaseClient {
         this.logger.debug("Sending request to Anthropic:", debugData);
         this.statelogClient?.promptRequest(debugData);
         const signal = this.getAbortSignal(config);
-        const response = await this.client.messages.create({
-            model: this.getModel(),
-            max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,
-            messages,
-            ...(system && { system }),
-            ...(tools && { tools }),
-            ...(thinking && { thinking }),
-            ...(config.temperature !== undefined && {
-                temperature: config.temperature,
-            }),
-            ...(config.rawAttributes || {}),
-            stream: false,
-        }, { ...(signal && { signal }) });
+        let response;
+        try {
+            response = await this.client.messages.create({
+                model: this.getModel(),
+                max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,
+                messages,
+                ...(system && { system }),
+                ...(tools && { tools }),
+                ...(thinking && { thinking }),
+                ...(config.temperature !== undefined && {
+                    temperature: config.temperature,
+                }),
+                ...(config.rawAttributes || {}),
+                stream: false,
+            }, { ...(signal && { signal }) });
+        }
+        catch (error) {
+            this.rethrowAsSmolError(error);
+        }
         this.logger.debug("Response from Anthropic:", response);
         this.statelogClient?.promptResponse(response);
         let output = null;
@@ -148,19 +173,25 @@ export class SmolAnthropic extends BaseClient {
         this.logger.debug("Sending streaming request to Anthropic:", streamDebugData);
         this.statelogClient?.promptRequest(streamDebugData);
         const signal = this.getAbortSignal(config);
-        const stream = await this.client.messages.create({
-            model: this.model,
-            max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,
-            messages,
-            ...(system && { system }),
-            ...(tools && { tools }),
-            ...(thinking && { thinking }),
-            ...(config.temperature !== undefined && {
-                temperature: config.temperature,
-            }),
-            ...(config.rawAttributes || {}),
-            stream: true,
-        }, { ...(signal && { signal }) });
+        let stream;
+        try {
+            stream = await this.client.messages.create({
+                model: this.model,
+                max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,
+                messages,
+                ...(system && { system }),
+                ...(tools && { tools }),
+                ...(thinking && { thinking }),
+                ...(config.temperature !== undefined && {
+                    temperature: config.temperature,
+                }),
+                ...(config.rawAttributes || {}),
+                stream: true,
+            }, { ...(signal && { signal }) });
+        }
+        catch (error) {
+            this.rethrowAsSmolError(error);
+        }
         let content = "";
         // Track tool blocks by index: index -> { id, name, arguments (partial JSON) }
         const toolBlocks = new Map();

package/dist/clients/baseClient.d.ts

@@ -24,6 +24,7 @@ export declare class BaseClient implements SmolClient {
         continue: boolean;
         newPromptConfig: PromptConfig;
     };
+    private recordLatency;
     extractResponse(promptConfig: PromptConfig, rawValue: any, schema: any, depth?: number): any;
     textWithRetry(promptConfig: PromptConfig, retries: number): Promise<Result<PromptResult>>;
     _textSync(promptConfig: PromptConfig): Promise<Result<PromptResult>>;

package/dist/clients/baseClient.js

@@ -1,4 +1,5 @@
 import { AssistantMessage, userMessage, assistantMessage, } from "../classes/message/index.js";
+import { latencyTracker } from "../latencyTracker.js";
 import { getLogger } from "../logger.js";
 import { getModel, isTextModel } from "../models.js";
 import { SmolStructuredOutputError } from "../smolError.js";
@@ -146,9 +147,11 @@ export class BaseClient {
                 value: { output: null, toolCalls: [], model: this.config.model },
             };
         }
+        const startTime = performance.now();
         try {
             const result = await this.textWithRetry(newPromptConfig, newPromptConfig.responseFormatOptions?.numRetries ||
                 DEFAULT_NUM_RETRIES);
+            this.recordLatency(startTime, result);
             return result;
         }
         catch (err) {
@@ -210,6 +213,15 @@ export class BaseClient {
         }
         return { continue: true, newPromptConfig: promptConfig };
     }
+    recordLatency(startTime, result) {
+        if (!result.success)
+            return;
+        const outputTokens = result.value.usage?.outputTokens;
+        if (!outputTokens || outputTokens <= 0)
+            return;
+        const elapsedMs = performance.now() - startTime;
+        latencyTracker.record(this.config.model, elapsedMs, outputTokens);
+    }
     extractResponse(promptConfig, rawValue, schema, depth = 0) {
         const MAX_DEPTH = 5;
         if (depth > MAX_DEPTH) {
@@ -374,8 +386,18 @@ export class BaseClient {
             };
             return;
         }
+        const startTime = performance.now();
         try {
-            yield* this._textStream(newPromptConfig);
+            for await (const chunk of this._textStream(newPromptConfig)) {
+                if (chunk.type === "done") {
+                    const outputTokens = chunk.result.usage?.outputTokens;
+                    if (outputTokens && outputTokens > 0) {
+                        const elapsedMs = performance.now() - startTime;
+                        latencyTracker.record(this.config.model, elapsedMs, outputTokens);
+                    }
+                }
+                yield chunk;
+            }
         }
         catch (err) {
             if (this.isAbortError(err)) {

package/dist/clients/google.js

@@ -3,6 +3,7 @@ import { ToolCall } from "../classes/ToolCall.js";
 import { getLogger } from "../logger.js";
 import { addCosts, addTokenUsage, success, } from "../types.js";
 import { zodToGoogleTool } from "../util/tool.js";
+import { SmolContentPolicyError, SmolContextWindowExceededError, } from "../smolError.js";
 import { sanitizeAttributes } from "../util.js";
 import { BaseClient } from "./baseClient.js";
 import { Model } from "../model.js";
@@ -171,10 +172,28 @@ export class SmolGoogle extends BaseClient {
     async __textSync(request) {
         this.logger.debug("Sending request to Google Gemini:", JSON.stringify(request, null, 2));
         this.statelogClient?.promptRequest(request);
-        // Send the prompt as the latest message
-        const result = await this.client.models.generateContent(request);
+        let result;
+        try {
+            result = await this.client.models.generateContent(request);
+        }
+        catch (error) {
+            const msg = (error.message || "").toLowerCase();
+            if (msg.includes("token") &&
+                (msg.includes("exceed") ||
+                    msg.includes("too long") ||
+                    msg.includes("limit"))) {
+                throw new SmolContextWindowExceededError(error.message);
+            }
+            throw error;
+        }
         this.logger.debug("Response from Google Gemini:", JSON.stringify(result, null, 2));
         this.statelogClient?.promptResponse(result);
+        for (const candidate of result.candidates || []) {
+            const finishReason = candidate.finishReason;
+            if (finishReason === "SAFETY" || finishReason === "PROHIBITED_CONTENT") {
+                throw new SmolContentPolicyError(`Content blocked by Google safety filter: ${finishReason}`);
+            }
+        }
         const toolCalls = [];
         const thinkingBlocks = [];
         let textContent = "";
@@ -230,7 +249,20 @@ export class SmolGoogle extends BaseClient {
         }
         this.logger.debug("Sending streaming request to Google Gemini:", JSON.stringify(request, null, 2));
         this.statelogClient?.promptRequest(request);
-        const stream = await this.client.models.generateContentStream(request);
+        let stream;
+        try {
+            stream = await this.client.models.generateContentStream(request);
+        }
+        catch (error) {
+            const msg = (error.message || "").toLowerCase();
+            if (msg.includes("token") &&
+                (msg.includes("exceed") ||
+                    msg.includes("too long") ||
+                    msg.includes("limit"))) {
+                throw new SmolContextWindowExceededError(error.message);
+            }
+            throw error;
+        }
         let content = "";
         const toolCallsMap = new Map();
         const thinkingBlocks = [];

package/dist/clients/ollama.js

@@ -5,6 +5,7 @@ import { success, } from "../types.js";
 import { zodToGoogleTool } from "../util/tool.js";
 import { sanitizeAttributes } from "../util.js";
 import { BaseClient } from "./baseClient.js";
+import { SmolContextWindowExceededError } from "../smolError.js";
 import { Model } from "../model.js";
 export const DEFAULT_OLLAMA_HOST = "http://localhost:11434";
 export class SmolOllama extends BaseClient {
@@ -80,6 +81,13 @@ export class SmolOllama extends BaseClient {
             // @ts-ignore
             result = await this.client.chat(request);
         }
+        catch (error) {
+            const msg = (error.message || "").toLowerCase();
+            if (msg.includes("context length") || msg.includes("context window")) {
+                throw new SmolContextWindowExceededError(error.message);
+            }
+            throw error;
+        }
         finally {
             if (signal && abortHandler) {
                 signal.removeEventListener("abort", abortHandler);

package/dist/clients/openai.d.ts

@@ -12,6 +12,7 @@ export declare class SmolOpenAi extends BaseClient implements SmolClient {
     getModel(): ModelName;
     private calculateUsageAndCost;
     private buildRequest;
+    private rethrowAsSmolError;
     _textSync(config: PromptConfig): Promise<Result<PromptResult>>;
     _textStream(config: PromptConfig): AsyncGenerator<StreamChunk>;
 }

package/dist/clients/openai.js

@@ -4,6 +4,7 @@ import { ToolCall } from "../classes/ToolCall.js";
 import { isFunctionToolCall, sanitizeAttributes } from "../util.js";
 import { getLogger } from "../logger.js";
 import { BaseClient } from "./baseClient.js";
+import { SmolContentPolicyError, SmolContextWindowExceededError, } from "../smolError.js";
 import { zodToOpenAITool } from "../util/tool.js";
 import { Model } from "../model.js";
 export class SmolOpenAi extends BaseClient {
@@ -68,17 +69,37 @@ export class SmolOpenAi extends BaseClient {
         }
         return request;
     }
+    rethrowAsSmolError(error) {
+        if (error instanceof OpenAI.APIError) {
+            if (error.code === "context_length_exceeded") {
+                throw new SmolContextWindowExceededError(error.message);
+            }
+            if (error.code === "content_policy_violation") {
+                throw new SmolContentPolicyError(error.message);
+            }
+        }
+        throw error;
+    }
     async _textSync(config) {
         const request = this.buildRequest(config);
         this.logger.debug("Sending request to OpenAI:", JSON.stringify(request, null, 2));
         this.statelogClient?.promptRequest(request);
         const signal = this.getAbortSignal(config);
-        const completion = await this.client.chat.completions.create({
-            ...request,
-            stream: false,
-        }, { ...(signal && { signal }) });
+        let completion;
+        try {
+            completion = await this.client.chat.completions.create({
+                ...request,
+                stream: false,
+            }, { ...(signal && { signal }) });
+        }
+        catch (error) {
+            this.rethrowAsSmolError(error);
+        }
         this.logger.debug("Response from OpenAI:", JSON.stringify(completion, null, 2));
         this.statelogClient?.promptResponse(completion);
+        if (completion.choices[0]?.finish_reason === "content_filter") {
+            throw new SmolContentPolicyError("Content blocked by OpenAI content filter");
+        }
         const message = completion.choices[0].message;
         const output = message.content;
         const _toolCalls = message.tool_calls;
@@ -109,11 +130,17 @@ export class SmolOpenAi extends BaseClient {
         this.logger.debug("Sending streaming request to OpenAI:", JSON.stringify(request, null, 2));
         this.statelogClient?.promptRequest(request);
         const signal = this.getAbortSignal(config);
-        const completion = await this.client.chat.completions.create({
-            ...request,
-            stream: true,
-            stream_options: { include_usage: true },
-        }, { ...(signal && { signal }) });
+        let completion;
+        try {
+            completion = await this.client.chat.completions.create({
+                ...request,
+                stream: true,
+                stream_options: { include_usage: true },
+            }, { ...(signal && { signal }) });
+        }
+        catch (error) {
+            this.rethrowAsSmolError(error);
+        }
         let content = "";
         const toolCallsMap = new Map();
         let usage;
@@ -127,6 +154,9 @@ export class SmolOpenAi extends BaseClient {
             }
             if (!chunk.choices || chunk.choices.length === 0)
                 continue;
+            if (chunk.choices[0]?.finish_reason === "content_filter") {
+                throw new SmolContentPolicyError("Content blocked by OpenAI content filter");
+            }
             const delta = chunk.choices[0]?.delta;
             if (!delta)
                 continue;

package/dist/clients/openaiResponses.d.ts

@@ -13,6 +13,7 @@ export declare class SmolOpenAiResponses extends BaseClient implements SmolClien
     private convertMessages;
     private buildRequest;
     private calculateUsageAndCost;
+    private rethrowAsSmolError;
     _textSync(config: PromptConfig): Promise<Result<PromptResult>>;
     _textStream(config: PromptConfig): AsyncGenerator<StreamChunk>;
 }

package/dist/clients/openaiResponses.js

@@ -6,6 +6,7 @@ import { BaseClient } from "./baseClient.js";
 import { zodToOpenAIResponsesTool } from "../util/tool.js";
 import { sanitizeAttributes } from "../util.js";
 import { Model } from "../model.js";
+import { SmolContentPolicyError, SmolContextWindowExceededError, } from "../smolError.js";
 export class SmolOpenAiResponses extends BaseClient {
     client;
     logger;
@@ -101,15 +102,32 @@ export class SmolOpenAiResponses extends BaseClient {
         }
         return { usage, cost };
     }
+    rethrowAsSmolError(error) {
+        if (error instanceof OpenAI.APIError) {
+            if (error.code === "context_length_exceeded") {
+                throw new SmolContextWindowExceededError(error.message);
+            }
+            if (error.code === "content_policy_violation") {
+                throw new SmolContentPolicyError(error.message);
+            }
+        }
+        throw error;
+    }
     async _textSync(config) {
         const request = this.buildRequest(config);
         this.logger.debug("Sending request to OpenAI Responses API:", JSON.stringify(request, null, 2));
         this.statelogClient?.promptRequest(request);
         const signal = this.getAbortSignal(config);
-        const response = await this.client.responses.create({
-            ...request,
-            stream: false,
-        }, { ...(signal && { signal }) });
+        let response;
+        try {
+            response = await this.client.responses.create({
+                ...request,
+                stream: false,
+            }, { ...(signal && { signal }) });
+        }
+        catch (error) {
+            this.rethrowAsSmolError(error);
+        }
         this.logger.debug("Response from OpenAI Responses API:", JSON.stringify(response, null, 2));
         this.statelogClient?.promptResponse(response);
         const output = response.output_text || null;
@@ -133,9 +151,15 @@ export class SmolOpenAiResponses extends BaseClient {
         this.logger.debug("Sending streaming request to OpenAI Responses API:", JSON.stringify(request, null, 2));
         this.statelogClient?.promptRequest(request);
         const signal = this.getAbortSignal(config);
-        const stream = this.client.responses.stream(request, {
-            ...(signal && { signal }),
-        });
+        let stream;
+        try {
+            stream = this.client.responses.stream(request, {
+                ...(signal && { signal }),
+            });
+        }
+        catch (error) {
+            this.rethrowAsSmolError(error);
+        }
         let content = "";
         const functionCalls = new Map();
         let usage;

package/dist/index.d.ts

@@ -8,3 +8,5 @@ export * from "./classes/message/index.js";
 export * from "./functions.js";
 export * from "./classes/ToolCall.js";
 export * from "./strategies/index.js";
+export { latencyTracker } from "./latencyTracker.js";
+export type { LatencySample } from "./latencyTracker.js";

package/dist/index.js

@@ -8,3 +8,4 @@ export * from "./classes/message/index.js";
 export * from "./functions.js";
 export * from "./classes/ToolCall.js";
 export * from "./strategies/index.js";
+export { latencyTracker } from "./latencyTracker.js";

package/dist/latencyTracker.d.ts

@@ -0,0 +1,32 @@
+export type LatencySample = {
+    /** Milliseconds per output token */
+    msPerToken: number;
+    /** Timestamp when sample was recorded */
+    timestamp: number;
+};
+declare class LatencyTracker {
+    private samples;
+    private windowSize;
+    constructor(windowSize?: number);
+    /** Record a latency sample for a model. */
+    record(model: string, elapsedMs: number, outputTokens: number): void;
+    /** Get the windowed mean ms-per-token for a model, or null if no samples. */
+    getMeanMsPerToken(model: string): number | null;
+    /**
+     * Get estimated output tokens per second for a model based on tracked latency.
+     * Returns null if no samples exist or if the number of samples is below the minimum required.
+     */
+    getTokensPerSecond(model: string, minSamples?: number): number | null;
+    /** Get the number of samples recorded for a model. */
+    getSampleCount(model: string): number;
+    /** Get all samples for a model (defensive copy). */
+    getSamples(model: string): LatencySample[];
+    /** Clear all samples for a model. */
+    clear(model?: string): void;
+    /** Update the window size. Existing samples beyond the new size are trimmed. */
+    setWindowSize(size: number): void;
+    getWindowSize(): number;
+}
+/** Global singleton latency tracker. */
+export declare const latencyTracker: LatencyTracker;
+export {};