smoltalk 0.0.58 → 0.0.60

package/dist/classes/ToolCall.js

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { getLogger } from "../logger.js";
+import { getLogger } from "../util/logger.js";
 export const ToolCallJSONSchema = z.object({
     id: z.string().default(""),
     name: z.string(),

package/dist/client.d.ts

@@ -3,6 +3,8 @@ export * from "./clients/google.js";
 export * from "./clients/openai.js";
 export * from "./clients/openaiResponses.js";
 export * from "./clients/baseClient.js";
+export * from "./clients/ollama.js";
+export * from "./clients/llamaCpp.js";
 import { BaseClient } from "./clients/baseClient.js";
 import { ResolvedSmolConfig } from "./types.js";
 export declare function registerProvider(providerName: string, clientClass: typeof BaseClient): void;

package/dist/client.js

@@ -3,8 +3,11 @@ export * from "./clients/google.js";
 export * from "./clients/openai.js";
 export * from "./clients/openaiResponses.js";
 export * from "./clients/baseClient.js";
+export * from "./clients/ollama.js";
+export * from "./clients/llamaCpp.js";
 import { SmolAnthropic } from "./clients/anthropic.js";
 import { SmolGoogle } from "./clients/google.js";
+import { LlamaCPP } from "./clients/llamaCpp.js";
 import { SmolOllama } from "./clients/ollama.js";
 import { SmolOpenAi } from "./clients/openai.js";
 import { SmolOpenAiResponses } from "./clients/openaiResponses.js";
@@ -63,6 +66,8 @@ export function getClient(config) {
             return new SmolGoogle(clientConfig);
         case "ollama":
             return new SmolOllama(clientConfig);
+        case "llama-cpp":
+            return new LlamaCPP(clientConfig);
         default:
             if (provider in registeredProviders) {
                 const ClientClass = registeredProviders[provider];

package/dist/clients/anthropic.js

@@ -1,7 +1,7 @@
 import Anthropic from "@anthropic-ai/sdk";
 import { ToolCall } from "../classes/ToolCall.js";
 import { SystemMessage, DeveloperMessage } from "../classes/message/index.js";
-import { getLogger } from "../logger.js";
+import { getLogger } from "../util/logger.js";
 import { success, } from "../types.js";
 import { zodToAnthropicTool } from "../util/tool.js";
 import { SmolContentPolicyError, SmolContextWindowExceededError, } from "../smolError.js";
@@ -255,7 +255,10 @@ export class SmolAnthropic extends BaseClient {
             }
         }
         this.logger.debug("Streaming response completed from Anthropic");
-        this.statelogClient?.promptResponse({ content, usage: { inputTokens, outputTokens } });
+        this.statelogClient?.promptResponse({
+            content,
+            usage: { inputTokens, outputTokens },
+        });
         const toolCalls = [];
         for (const block of toolBlocks.values()) {
             const toolCall = new ToolCall(block.id, block.name, block.arguments);

package/dist/clients/baseClient.d.ts

@@ -15,10 +15,6 @@ export declare class BaseClient implements SmolClient {
     }): AsyncGenerator<StreamChunk>;
     text(promptConfig: PromptConfig): Promise<Result<PromptResult>> | AsyncGenerator<StreamChunk>;
     checkMessageLimit(promptConfig: PromptConfig): Result<PromptResult> | null;
-    applyBudget(promptConfig: PromptConfig): {
-        config: PromptConfig;
-        failure?: Result<PromptResult>;
-    };
     textSync(promptConfig: PromptConfig): Promise<Result<PromptResult>>;
     checkForToolLoops(promptConfig: PromptConfig): {
         continue: boolean;

package/dist/clients/baseClient.js

@@ -1,7 +1,6 @@
-import { AssistantMessage, userMessage, assistantMessage, } from "../classes/message/index.js";
+import { userMessage, assistantMessage, } from "../classes/message/index.js";
 import { latencyTracker } from "../latencyTracker.js";
-import { getLogger } from "../logger.js";
-import { getModel, isTextModel } from "../models.js";
+import { getLogger } from "../util/logger.js";
 import { SmolStructuredOutputError } from "../smolError.js";
 import { getStatelogClient } from "../statelogClient.js";
 import { success, } from "../types.js";
@@ -20,19 +19,7 @@ export class BaseClient {
         }
     }
     getAbortSignal(promptConfig) {
-        const signals = [];
-        if (promptConfig.abortSignal) {
-            signals.push(promptConfig.abortSignal);
-        }
-        const timeBudgetMs = promptConfig.budget?.timeBudgetMs;
-        if (timeBudgetMs !== undefined) {
-            signals.push(AbortSignal.timeout(timeBudgetMs));
-        }
-        if (signals.length === 0)
-            return undefined;
-        if (signals.length === 1)
-            return signals[0];
-        return AbortSignal.any(signals);
+        return promptConfig.abortSignal;
     }
     isAbortError(err) {
         return ((err instanceof DOMException && err.name === "AbortError") ||
@@ -64,83 +51,11 @@ export class BaseClient {
         }
         return null;
     }
-    applyBudget(promptConfig) {
-        const budget = promptConfig.budget;
-        if (!budget)
-            return { config: promptConfig };
-        let config = { ...promptConfig };
-        // Auto-compute used values from message history when not explicitly provided
-        const assistantMessages = config.messages.filter((m) => m instanceof AssistantMessage);
-        const tokensUsed = budget.tokensUsed ??
-            assistantMessages.reduce((sum, m) => sum + (m.usage?.outputTokens ?? 0), 0);
-        const costUsed = budget.costUsed ??
-            assistantMessages.reduce((sum, m) => sum + (m.cost?.totalCost ?? 0), 0);
-        const requestsUsed = budget.requestsUsed ?? assistantMessages.length;
-        // Request budget check
-        if (budget.requestBudget !== undefined &&
-            requestsUsed >= budget.requestBudget) {
-            this.statelogClient?.debug("Request budget exhausted", {
-                requestsUsed,
-                requestBudget: budget.requestBudget,
-            });
-            return {
-                config,
-                failure: {
-                    success: false,
-                    error: `Request budget exhausted: ${requestsUsed} requests used, budget is ${budget.requestBudget}`,
-                },
-            };
-        }
-        // Token budget check
-        if (budget.tokenBudget !== undefined) {
-            const remaining = budget.tokenBudget - tokensUsed;
-            if (remaining <= 0) {
-                this.statelogClient?.debug("Token budget exhausted", {
-                    tokensUsed,
-                    tokenBudget: budget.tokenBudget,
-                });
-                return {
-                    config,
-                    failure: {
-                        success: false,
-                        error: `Token budget exhausted: ${tokensUsed} output tokens used, budget is ${budget.tokenBudget}`,
-                    },
-                };
-            }
-            config.maxTokens = Math.min(config.maxTokens ?? Infinity, remaining);
-        }
-        // Cost budget check
-        if (budget.costBudget !== undefined) {
-            const remainingUSD = budget.costBudget - costUsed;
-            if (remainingUSD <= 0) {
-                this.statelogClient?.debug("Cost budget exhausted", {
-                    costUsed,
-                    costBudget: budget.costBudget,
-                });
-                return {
-                    config,
-                    failure: {
-                        success: false,
-                        error: `Cost budget exhausted: $${costUsed.toFixed(4)} spent, budget is $${budget.costBudget.toFixed(4)}`,
-                    },
-                };
-            }
-            const model = getModel(this.config.model);
-            if (model && isTextModel(model) && model.outputTokenCost) {
-                const remainingTokens = Math.floor((remainingUSD / model.outputTokenCost) * 1_000_000);
-                config.maxTokens = Math.min(config.maxTokens ?? Infinity, remainingTokens);
-            }
-        }
-        return { config };
-    }
     async textSync(promptConfig) {
         const messageLimitResult = this.checkMessageLimit(promptConfig);
         if (messageLimitResult)
             return messageLimitResult;
-        const { config: budgetedConfig, failure: budgetFailure } = this.applyBudget(promptConfig);
-        if (budgetFailure)
-            return budgetFailure;
-        const { continue: shouldContinue, newPromptConfig } = this.checkForToolLoops(budgetedConfig);
+        const { continue: shouldContinue, newPromptConfig } = this.checkForToolLoops(promptConfig);
         if (!shouldContinue) {
             return {
                 success: true,
@@ -156,16 +71,11 @@ export class BaseClient {
         }
         catch (err) {
             if (this.isAbortError(err)) {
-                const timeBudgetMs = promptConfig.budget?.timeBudgetMs;
-                const message = timeBudgetMs
-                    ? `Request timed out after ${timeBudgetMs}ms`
-                    : "Request was aborted";
                 this.statelogClient?.debug("Request aborted or timed out", {
-                    reason: message,
-                    timeBudgetMs,
+                    reason: "Request was aborted",
                     promptConfig,
                 });
-                return { success: false, error: message };
+                return { success: false, error: "Request was aborted" };
             }
             throw err;
         }
@@ -364,17 +274,7 @@ export class BaseClient {
             };
             return;
         }
-        const { config: budgetedConfig, failure: budgetFailure } = this.applyBudget(config);
-        if (budgetFailure) {
-            yield {
-                type: "error",
-                error: budgetFailure.success === false
-                    ? budgetFailure.error
-                    : "Budget exceeded",
-            };
-            return;
-        }
-        const { continue: shouldContinue, newPromptConfig } = this.checkForToolLoops(budgetedConfig);
+        const { continue: shouldContinue, newPromptConfig } = this.checkForToolLoops(config);
         if (!shouldContinue) {
             yield {
                 type: "done",
@@ -401,16 +301,11 @@ export class BaseClient {
         }
         catch (err) {
             if (this.isAbortError(err)) {
-                const timeBudgetMs = config.budget?.timeBudgetMs;
-                const message = timeBudgetMs
-                    ? `Request timed out after ${timeBudgetMs}ms`
-                    : "Request was aborted";
                 this.statelogClient?.debug("Streaming request aborted or timed out", {
-                    reason: message,
-                    timeBudgetMs,
+                    reason: "Request was aborted",
                     newPromptConfig,
                 });
-                yield { type: "timeout", error: message };
+                yield { type: "timeout", error: "Request was aborted" };
             }
             else {
                 throw err;

package/dist/clients/google.js

@@ -1,10 +1,10 @@
 import { GoogleGenAI } from "@google/genai";
 import { ToolCall } from "../classes/ToolCall.js";
-import { getLogger } from "../logger.js";
+import { getLogger } from "../util/logger.js";
 import { addCosts, addTokenUsage, success, } from "../types.js";
 import { zodToGoogleTool } from "../util/tool.js";
 import { SmolContentPolicyError, SmolContextWindowExceededError, } from "../smolError.js";
-import { sanitizeAttributes } from "../util.js";
+import { sanitizeAttributes } from "../util/util.js";
 import { BaseClient } from "./baseClient.js";
 import { Model } from "../model.js";
 import { userMessage } from "../classes/message/index.js";

package/dist/clients/llamaCpp.d.ts

@@ -0,0 +1,28 @@
+import { BaseClient } from "./baseClient.js";
+import { BaseClientConfig, PromptConfig, PromptResult, Result, StreamChunk } from "../types.js";
+export declare class LlamaCPP extends BaseClient {
+    private llama;
+    private llamaModel;
+    private modelDir;
+    private model;
+    private logger;
+    constructor(config: BaseClientConfig);
+    setup(): Promise<void>;
+    private getModelName;
+    /**
+     * Converts smoltalk messages to node-llama-cpp's ChatHistoryItem format.
+     * Builds the full history including the last user message (LlamaChat.generateResponse
+     * expects the complete history, unlike LlamaChatSession which takes the last message separately).
+     */
+    private convertMessages;
+    /**
+     * Builds node-llama-cpp function definitions from smoltalk tool configs.
+     * Uses ChatModelFunctions (no handler) — LlamaChat.generateResponse() returns
+     * function calls without executing them, which matches smoltalk's tool loop model.
+     */
+    private buildFunctions;
+    private calculateUsageAndCost;
+    private extractToolCalls;
+    _textSync(config: PromptConfig): Promise<Result<PromptResult>>;
+    _textStream(config: PromptConfig): AsyncGenerator<StreamChunk>;
+}

package/dist/clients/llamaCpp.js

@@ -0,0 +1,316 @@
+import { getLlama, LlamaChat, LlamaLogLevel } from "node-llama-cpp";
+import { BaseClient } from "./baseClient.js";
+import { ToolCall } from "../classes/ToolCall.js";
+import { getLogger } from "../util/logger.js";
+import { Model } from "../model.js";
+import { sanitizeAttributes } from "../util/util.js";
+import { success, } from "../types.js";
+import path from "path";
+export class LlamaCPP extends BaseClient {
+    llama = null;
+    llamaModel = null;
+    modelDir;
+    model;
+    logger;
+    constructor(config) {
+        super(config);
+        if (!config.llamaCppModelDir) {
+            throw new Error("llamaCppModelDir is required in the config when using the LlamaCPP client.");
+        }
+        this.model = new Model(config.model);
+        this.modelDir = config.llamaCppModelDir;
+        this.logger = getLogger();
+    }
+    async setup() {
+        this.llama = await getLlama({ logLevel: LlamaLogLevel.error });
+        this.llamaModel = await this.llama.loadModel({
+            modelPath: path.join(this.modelDir, this.config.model),
+        });
+    }
+    getModelName() {
+        return this.model.getResolvedModel();
+    }
+    /**
+     * Converts smoltalk messages to node-llama-cpp's ChatHistoryItem format.
+     * Builds the full history including the last user message (LlamaChat.generateResponse
+     * expects the complete history, unlike LlamaChatSession which takes the last message separately).
+     */
+    convertMessages(messages) {
+        let systemPrompt;
+        const chatHistory = [];
+        for (let i = 0; i < messages.length; i++) {
+            const msg = messages[i];
+            if (msg.role === "system" || msg.role === "developer") {
+                if (!systemPrompt) {
+                    systemPrompt = msg.content;
+                }
+                else {
+                    systemPrompt += "\n" + msg.content;
+                }
+            }
+            else if (msg.role === "user") {
+                chatHistory.push({ type: "user", text: msg.content });
+            }
+            else if (msg.role === "assistant") {
+                const assistantMsg = msg;
+                const response = [];
+                if (assistantMsg.content) {
+                    response.push(assistantMsg.content);
+                }
+                // Handle tool calls: pair them with their results from subsequent tool messages
+                if (assistantMsg.toolCalls?.length) {
+                    for (const tc of assistantMsg.toolCalls) {
+                        // Find the corresponding tool result message
+                        const toolResultMsg = messages
+                            .slice(i + 1)
+                            .find((m) => m.role === "tool" &&
+                            m.tool_call_id === tc.id);
+                        response.push({
+                            type: "functionCall",
+                            name: tc.name,
+                            params: tc.arguments,
+                            result: toolResultMsg ? toolResultMsg.content : undefined,
+                        });
+                    }
+                }
+                chatHistory.push({ type: "model", response });
+            }
+            // Tool messages are handled as part of assistant messages above
+        }
+        // Prepend system message if present
+        if (systemPrompt) {
+            chatHistory.unshift({ type: "system", text: systemPrompt });
+        }
+        return { systemPrompt, chatHistory };
+    }
+    /**
+     * Builds node-llama-cpp function definitions from smoltalk tool configs.
+     * Uses ChatModelFunctions (no handler) — LlamaChat.generateResponse() returns
+     * function calls without executing them, which matches smoltalk's tool loop model.
+     */
+    buildFunctions(tools) {
+        if (!tools)
+            return undefined;
+        const functions = {};
+        for (const tool of tools) {
+            const jsonSchema = tool.schema.toJSONSchema();
+            functions[tool.name] = {
+                description: tool.description,
+                params: jsonSchema,
+            };
+        }
+        return functions;
+    }
+    calculateUsageAndCost(meterBefore, meterAfter) {
+        const inputTokens = meterAfter.usedInputTokens - meterBefore.usedInputTokens;
+        const outputTokens = meterAfter.usedOutputTokens - meterBefore.usedOutputTokens;
+        const usage = {
+            inputTokens,
+            outputTokens,
+            totalTokens: inputTokens + outputTokens,
+        };
+        const cost = this.model.calculateCost(usage) ?? undefined;
+        return { usage, cost };
+    }
+    extractToolCalls(functionCalls) {
+        if (!functionCalls?.length)
+            return [];
+        return functionCalls.map((fc) => new ToolCall(fc.functionName, fc.functionName, (fc.params ?? {})));
+    }
+    async _textSync(config) {
+        if (!this.llama || !this.llamaModel) {
+            await this.setup();
+        }
+        const setupLlama = this.llama;
+        const setupModel = this.llamaModel;
+        const { chatHistory } = this.convertMessages(config.messages);
+        if (chatHistory.length === 0) {
+            return success({
+                output: "",
+                toolCalls: [],
+                model: this.getModelName(),
+            });
+        }
+        // Create grammar for response format
+        let grammar;
+        if (config.responseFormat) {
+            grammar = await setupLlama.createGrammarForJsonSchema(config.responseFormat.toJSONSchema());
+        }
+        // Create context and LlamaChat
+        const context = await setupModel.createContext();
+        const sequence = context.getSequence();
+        const chat = new LlamaChat({
+            contextSequence: sequence,
+        });
+        // Build tools if provided
+        const functions = this.buildFunctions(config.tools);
+        // Track token usage
+        const meterBefore = sequence.tokenMeter.getState();
+        // Build options
+        const options = {};
+        if (config.maxTokens !== undefined) {
+            options.maxTokens = config.maxTokens;
+        }
+        if (config.temperature !== undefined) {
+            options.temperature = config.temperature;
+        }
+        if (config.abortSignal) {
+            options.signal = config.abortSignal;
+            options.stopOnAbortSignal = true;
+        }
+        if (grammar && !functions) {
+            options.grammar = grammar;
+        }
+        if (functions) {
+            options.functions = functions;
+        }
+        // Apply raw attributes
+        Object.assign(options, sanitizeAttributes(config.rawAttributes));
+        this.logger.debug("Sending request to llama.cpp");
+        this.statelogClient?.promptRequest({
+            model: this.getModelName(),
+            messageCount: config.messages.length,
+        });
+        let result;
+        let meterAfter;
+        try {
+            result = await chat.generateResponse(chatHistory, options);
+            meterAfter = sequence.tokenMeter.getState();
+        }
+        finally {
+            chat.dispose();
+            await context.dispose();
+        }
+        // Extract text output
+        const output = result.response || null;
+        // Extract tool calls — generateResponse returns them without executing handlers
+        const toolCalls = this.extractToolCalls(result.functionCalls);
+        // Calculate usage and cost
+        const { usage, cost } = this.calculateUsageAndCost(meterBefore, meterAfter);
+        this.logger.debug("Response from llama.cpp:", output);
+        this.statelogClient?.promptResponse({ output, usage, cost });
+        return success({
+            output,
+            toolCalls,
+            usage,
+            cost,
+            model: this.getModelName(),
+        });
+    }
+    async *_textStream(config) {
+        if (!this.llama || !this.llamaModel) {
+            await this.setup();
+        }
+        const setupLlama = this.llama;
+        const setupModel = this.llamaModel;
+        const { chatHistory } = this.convertMessages(config.messages);
+        if (chatHistory.length === 0) {
+            yield {
+                type: "done",
+                result: { output: null, toolCalls: [], model: this.getModelName() },
+            };
+            return;
+        }
+        // Create grammar for response format
+        let grammar;
+        if (config.responseFormat) {
+            grammar = await setupLlama.createGrammarForJsonSchema(config.responseFormat.toJSONSchema());
+        }
+        // Create context and LlamaChat
+        const context = await setupModel.createContext();
+        const sequence = context.getSequence();
+        const chat = new LlamaChat({
+            contextSequence: sequence,
+        });
+        const functions = this.buildFunctions(config.tools);
+        const meterBefore = sequence.tokenMeter.getState();
+        // Bridge callback-based streaming to async generator using a queue
+        const chunks = [];
+        let resolveWaiter = null;
+        let done = false;
+        const pushChunk = (chunk) => {
+            chunks.push(chunk);
+            if (resolveWaiter) {
+                resolveWaiter();
+                resolveWaiter = null;
+            }
+        };
+        // Build options
+        const options = {
+            onTextChunk: (text) => {
+                pushChunk({ type: "text", text });
+            },
+        };
+        if (config.maxTokens !== undefined) {
+            options.maxTokens = config.maxTokens;
+        }
+        if (config.temperature !== undefined) {
+            options.temperature = config.temperature;
+        }
+        if (config.abortSignal) {
+            options.signal = config.abortSignal;
+            options.stopOnAbortSignal = true;
+        }
+        if (grammar && !functions) {
+            options.grammar = grammar;
+        }
+        if (functions) {
+            options.functions = functions;
+        }
+        Object.assign(options, sanitizeAttributes(config.rawAttributes));
+        this.logger.debug("Sending streaming request to llama.cpp");
+        this.statelogClient?.promptRequest({
+            model: this.getModelName(),
+            messageCount: config.messages.length,
+        });
+        // Run generateResponse in background, push chunks as they arrive
+        const promptPromise = chat
+            .generateResponse(chatHistory, options)
+            .then((result) => {
+            const meterAfter = sequence.tokenMeter.getState();
+            const toolCalls = this.extractToolCalls(result.functionCalls);
+            for (const tc of toolCalls) {
+                pushChunk({ type: "tool_call", toolCall: tc });
+            }
+            const { usage, cost } = this.calculateUsageAndCost(meterBefore, meterAfter);
+            const output = result.response || null;
+            this.logger.debug("Streaming response completed from llama.cpp");
+            this.statelogClient?.promptResponse({ output, usage, cost });
+            pushChunk({
+                type: "done",
+                result: {
+                    output,
+                    toolCalls,
+                    usage,
+                    cost,
+                    model: this.getModelName(),
+                },
+            });
+        })
+            .catch((error) => {
+            pushChunk({ type: "error", error: error.message });
+        })
+            .finally(() => {
+            done = true;
+            chat.dispose();
+            context.dispose();
+            // Wake up the generator if it's waiting
+            if (resolveWaiter) {
+                resolveWaiter();
+                resolveWaiter = null;
+            }
+        });
+        // Yield chunks as they arrive
+        while (!done || chunks.length > 0) {
+            if (chunks.length > 0) {
+                yield chunks.shift();
+            }
+            else if (!done) {
+                await new Promise((resolve) => {
+                    resolveWaiter = resolve;
+                });
+            }
+        }
+        await promptPromise;
+    }
+}

package/dist/clients/ollama.js

@@ -1,9 +1,9 @@
 import { Ollama } from "ollama";
 import { ToolCall } from "../classes/ToolCall.js";
-import { getLogger } from "../logger.js";
+import { getLogger } from "../util/logger.js";
 import { success, } from "../types.js";
 import { zodToGoogleTool } from "../util/tool.js";
-import { sanitizeAttributes } from "../util.js";
+import { sanitizeAttributes } from "../util/util.js";
 import { BaseClient } from "./baseClient.js";
 import { SmolContextWindowExceededError } from "../smolError.js";
 import { Model } from "../model.js";

package/dist/clients/openai.js

@@ -1,8 +1,8 @@
 import OpenAI from "openai";
 import { success, } from "../types.js";
 import { ToolCall } from "../classes/ToolCall.js";
-import { isFunctionToolCall, sanitizeAttributes } from "../util.js";
-import { getLogger } from "../logger.js";
+import { isFunctionToolCall, sanitizeAttributes } from "../util/util.js";
+import { getLogger } from "../util/logger.js";
 import { BaseClient } from "./baseClient.js";
 import { SmolContentPolicyError, SmolContextWindowExceededError, } from "../smolError.js";
 import { zodToOpenAITool } from "../util/tool.js";
@@ -111,7 +111,9 @@ export class SmolOpenAi extends BaseClient {
                 }
                 else {
                     this.logger.warn(`Unsupported tool call type: ${tc.type} for tool call ID: ${tc.id}`);
-                    this.statelogClient?.debug(`Unsupported tool call type: ${tc.type}`, { toolCallId: tc.id });
+                    this.statelogClient?.debug(`Unsupported tool call type: ${tc.type}`, {
+                        toolCallId: tc.id,
+                    });
                 }
             }
         }