veryfront 0.1.13 → 0.1.15

package/src/src/agent/react/use-chat/browser-inference/worker-client.ts

@@ -0,0 +1,89 @@
+/**
+ * BrowserInferenceClient — manages Web Worker lifecycle for browser-side inference.
+ *
+ * Singleton per session. Lazily creates Worker on first generate() call.
+ * Uses inline Blob URL approach — no separate build entry point needed.
+ */
+import * as dntShim from "../../../../../_dnt.shims.js";
+
+
+import type { WorkerRequest, WorkerResponse } from "./types.js";
+import { WORKER_SCRIPT } from "./worker-script.js";
+
+export interface GenerateCallbacks {
+  onStatus?: (status: "loading-runtime" | "downloading-model" | "ready" | "generating") => void;
+  onDownloadProgress?: (progress: number, file?: string) => void;
+  onToken?: (token: string) => void;
+  onDone?: (text: string) => void;
+  onError?: (error: string) => void;
+}
+
+let instance: BrowserInferenceClient | null = null;
+
+export class BrowserInferenceClient {
+  private worker: Worker | null = null;
+  private blobUrl: string | null = null;
+
+  static getInstance(): BrowserInferenceClient {
+    if (!instance) instance = new BrowserInferenceClient();
+    return instance;
+  }
+
+  private ensureWorker(): Worker {
+    if (this.worker) return this.worker;
+
+    const blob = new dntShim.Blob([WORKER_SCRIPT], { type: "application/javascript" });
+    this.blobUrl = URL.createObjectURL(blob);
+    this.worker = new Worker(this.blobUrl, { type: "module" });
+
+    return this.worker;
+  }
+
+  generate(
+    id: string,
+    messages: Array<{ role: string; content: string }>,
+    options: { maxNewTokens?: number; temperature?: number; systemPrompt?: string },
+    callbacks: GenerateCallbacks,
+  ): void {
+    const worker = this.ensureWorker();
+
+    worker.onmessage = (event: MessageEvent<WorkerResponse>) => {
+      const msg = event.data;
+      switch (msg.type) {
+        case "status":
+          callbacks.onStatus?.(msg.status);
+          break;
+        case "download-progress":
+          callbacks.onDownloadProgress?.(msg.progress, msg.file);
+          break;
+        case "token":
+          if (msg.id === id) callbacks.onToken?.(msg.token);
+          break;
+        case "done":
+          if (msg.id === id) callbacks.onDone?.(msg.text);
+          break;
+        case "error":
+          if (msg.id === id) callbacks.onError?.(msg.error);
+          break;
+      }
+    };
+
+    worker.onerror = (event) => {
+      callbacks.onError?.(event.message || "Worker error");
+    };
+
+    const request: WorkerRequest = { type: "generate", id, messages, options };
+    worker.postMessage(request);
+  }
+
+  stop(): void {
+    if (this.worker) {
+      this.worker.terminate();
+      this.worker = null;
+    }
+    if (this.blobUrl) {
+      URL.revokeObjectURL(this.blobUrl);
+      this.blobUrl = null;
+    }
+  }
+}

package/src/src/agent/react/use-chat/browser-inference/worker-script.ts

@@ -0,0 +1,98 @@
+/**
+ * Inline Worker script as a string.
+ *
+ * Loaded via Blob URL — no separate build entry point needed.
+ * Dynamically imports @huggingface/transformers from CDN inside the Worker.
+ */
+
+export const WORKER_SCRIPT = /* js */ `
+let pipeline = null;
+let generating = false;
+
+async function loadPipeline(callbacks) {
+  if (pipeline) return pipeline;
+
+  callbacks.onStatus("loading-runtime");
+
+  const { pipeline: createPipeline, env } =
+    await import("https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.4.2");
+
+  env.useBrowserCache = true;
+  env.allowLocalModels = false;
+
+  callbacks.onStatus("downloading-model");
+
+  pipeline = await createPipeline(
+    "text-generation",
+    "HuggingFaceTB/SmolLM2-135M-Instruct",
+    {
+      dtype: "q4",
+      device: "wasm",
+      progress_callback: (progress) => {
+        if (progress.status === "progress" && progress.total) {
+          callbacks.onProgress(Math.round((progress.loaded / progress.total) * 100), progress.file);
+        }
+      },
+    },
+  );
+
+  callbacks.onStatus("ready");
+  return pipeline;
+}
+
+self.onmessage = async (event) => {
+  const request = event.data;
+  if (request.type !== "generate") return;
+
+  const { id, messages, options } = request;
+
+  try {
+    const pipe = await loadPipeline({
+      onStatus: (status) => self.postMessage({ type: "status", status }),
+      onProgress: (progress, file) => self.postMessage({ type: "download-progress", progress, file }),
+    });
+
+    self.postMessage({ type: "status", status: "generating" });
+    generating = true;
+
+    const chatMessages = [];
+    if (options?.systemPrompt) {
+      chatMessages.push({ role: "system", content: options.systemPrompt });
+    }
+    chatMessages.push(...messages);
+
+    // Helper: generated_text is a plain string for raw prompts but an array
+    // of {role, content} message objects when using chat format. Extract the
+    // last assistant message's content in either case.
+    function extractText(generated) {
+      if (typeof generated === "string") return generated;
+      if (Array.isArray(generated)) {
+        const last = generated[generated.length - 1];
+        return last?.content ?? "";
+      }
+      return "";
+    }
+
+    const result = await pipe(chatMessages, {
+      max_new_tokens: options?.maxNewTokens ?? 512,
+      temperature: options?.temperature ?? 0.7,
+      do_sample: true,
+      return_full_text: false,
+      callback_function: (output) => {
+        if (!generating) return;
+        const text = extractText(output?.[0]?.generated_text);
+        if (text) {
+          self.postMessage({ type: "token", id, token: text });
+        }
+      },
+    });
+
+    generating = false;
+    const finalText = extractText(result?.[0]?.generated_text);
+    self.postMessage({ type: "done", id, text: finalText });
+  } catch (error) {
+    generating = false;
+    self.postMessage({ type: "error", id, error: error?.message ?? String(error) });
+  }
+};
+`;

package/src/src/agent/react/use-chat/index.ts

@@ -7,7 +7,9 @@
 
 export { useChat } from "./use-chat.js";
 export type {
+  BrowserInferenceStatus,
   DynamicToolUIPart,
+  InferenceMode,
   OnToolCallArg,
   ReasoningUIPart,
   TextUIPart,

package/src/src/agent/react/use-chat/types.ts

@@ -1,6 +1,18 @@
 import * as dntShim from "../../../../_dnt.shims.js";
 export type StreamState = "streaming" | "done";
 
+/** Where inference is happening */
+export type InferenceMode = "cloud" | "server-local" | "browser";
+
+/** Browser-side model loading and inference status */
+export type BrowserInferenceStatus =
+  | "idle"
+  | "loading-runtime"
+  | "downloading-model"
+  | "ready"
+  | "generating"
+  | "error";
+
 export interface TextUIPart {
   type: "text";
   text: string;
@@ -90,6 +102,10 @@ export interface UseChatOptions {
   credentials?: RequestCredentials;
   /** Override model at runtime (e.g. "openai/gpt-4o", "anthropic/claude-sonnet-4-5-20250929") */
   model?: string;
+  /** System prompt for browser-side inference (server uses agent config) */
+  systemPrompt?: string;
+  /** Enable/disable browser fallback when server can't provide AI. Default: true */
+  browserFallback?: boolean;
   onResponse?: (response: dntShim.Response) => void;
   onFinish?: (message: UIMessage) => void;
   onError?: (error: Error) => void;
@@ -103,6 +119,10 @@ export interface UseChatResult {
   error: Error | null;
   /** Current model override (undefined = use agent default) */
   model: string | undefined;
+  /** Where inference is currently happening */
+  inferenceMode: InferenceMode;
+  /** Browser-side model loading/inference status (null when not using browser fallback) */
+  browserStatus: BrowserInferenceStatus | null;
   setInput: (input: string) => void;
   /** Change the model for subsequent requests */
   setModel: (model: string | undefined) => void;

package/src/src/agent/react/use-chat/use-chat.ts

@@ -4,6 +4,11 @@
  * Complete chat state management with zero UI.
  * Consumes the veryfront streaming protocol
  * (message-start/message-finish + step-start/step-end).
+ *
+ * Supports three inference modes:
+ * - cloud: API key present, normal server-side inference
+ * - server-local: No API key, server runs local model via ONNX
+ * - browser: Server can't run ONNX (compiled binary), falls back to browser Worker
  */
 import * as dntShim from "../../../../_dnt.shims.js";
 
@@ -12,7 +17,15 @@ import { useCallback, useRef, useState } from "react";
 import { createError, ensureError, toError } from "../../../errors/veryfront-error.js";
 
 import { handleStreamingResponse } from "./streaming/index.js";
-import type { ToolOutput, UIMessage, UseChatOptions, UseChatResult } from "./types.js";
+import type {
+  BrowserInferenceStatus,
+  InferenceMode,
+  ToolOutput,
+  UIMessage,
+  UIMessagePart,
+  UseChatOptions,
+  UseChatResult,
+} from "./types.js";
 import { generateClientId } from "./utils.js";
 
 /**
@@ -25,7 +38,16 @@ export function useChat(options: UseChatOptions): UseChatResult {
   const [error, setError] = useState<Error | null>(null);
   const [data, setData] = useState<unknown>(null);
   const [model, setModel] = useState<string | undefined>(options.model);
+  const [inferenceMode, setInferenceMode] = useState<InferenceMode>("cloud");
+  const [browserStatus, setBrowserStatus] = useState<BrowserInferenceStatus | null>(null);
   const abortControllerRef = useRef<AbortController | null>(null);
+  const browserInferenceActiveRef = useRef(false);
+  const browserInferenceRejectRef = useRef<((reason: Error) => void) | null>(null);
+
+  // System prompt for browser fallback (from 503 response or options)
+  const systemPromptRef = useRef<string>(
+    options.systemPrompt ?? "You are a helpful AI assistant.",
+  );
 
   // Track pending tool outputs for addToolOutput
   const pendingToolOutputsRef = useRef<Map<string, ToolOutput>>(new Map());
@@ -57,6 +79,64 @@ export function useChat(options: UseChatOptions): UseChatResult {
     );
   }, []);
 
+  /**
+   * Run inference in the browser via Web Worker.
+   * Lazily imports the browser-inference module to avoid bundling it
+   * when server-side inference works fine.
+   */
+  const doBrowserInference = useCallback(
+    async (allMessages: UIMessage[]) => {
+      browserInferenceActiveRef.current = true;
+
+      try {
+        const { runBrowserInference } = await import(
+          "./browser-inference/browser-engine.js"
+        );
+
+        await new Promise<void>((resolve, reject) => {
+          browserInferenceRejectRef.current = reject;
+          let hasAddedMessage = false;
+
+          runBrowserInference(allMessages, systemPromptRef.current, {
+            onUpdate: (parts: UIMessagePart[], messageId: string) => {
+              if (!hasAddedMessage) {
+                hasAddedMessage = true;
+                setMessages((prev) => [
+                  ...prev,
+                  { id: messageId, role: "assistant", parts },
+                ]);
+                return;
+              }
+              setMessages((prev) => prev.map((m) => (m.id === messageId ? { ...m, parts } : m)));
+            },
+            onMessage: (assistantMessage: UIMessage) => {
+              setMessages((prev) => {
+                if (!hasAddedMessage) return [...prev, assistantMessage];
+                return prev.map((m) => m.id === assistantMessage.id ? assistantMessage : m);
+              });
+              options.onFinish?.(assistantMessage);
+              browserInferenceRejectRef.current = null;
+              resolve();
+            },
+            onStatusChange: (status: BrowserInferenceStatus) => {
+              setBrowserStatus(status);
+            },
+            onDownloadProgress: () => {
+              // Progress is tracked via onStatusChange("downloading-model")
+            },
+            onError: (err: Error) => {
+              browserInferenceRejectRef.current = null;
+              reject(err);
+            },
+          });
+        });
+      } finally {
+        browserInferenceActiveRef.current = false;
+      }
+    },
+    [options],
+  );
+
   /**
    * Send a message and stream assistant updates.
    */
@@ -72,10 +152,18 @@ export function useChat(options: UseChatOptions): UseChatResult {
       setIsLoading(true);
       setError(null);
 
-      const abortController = new AbortController();
-      abortControllerRef.current = abortController;
-
       try {
+        const allMessages = [...messages, userMessage];
+
+        // If already in browser mode, skip fetch entirely
+        if (inferenceMode === "browser") {
+          await doBrowserInference(allMessages);
+          return;
+        }
+
+        const abortController = new AbortController();
+        abortControllerRef.current = abortController;
+
         const response = await dntShim.fetch(options.api, {
           method: "POST",
           headers: {
@@ -84,13 +172,31 @@ export function useChat(options: UseChatOptions): UseChatResult {
           },
           credentials: options.credentials,
           body: JSON.stringify({
-            messages: [...messages, userMessage],
+            messages: allMessages,
             ...(model ? { model } : {}),
             ...options.body,
           }),
           signal: abortController.signal,
         });
 
+        // Handle 503 — server can't provide AI, fall back to browser
+        if (response.status === 503 && (options.browserFallback ?? true)) {
+          try {
+            const body = await response.json();
+            if (body.code === "NO_AI_AVAILABLE") {
+              if (body.systemPrompt) {
+                systemPromptRef.current = body.systemPrompt;
+              }
+              setInferenceMode("browser");
+              setBrowserStatus("idle");
+              await doBrowserInference(allMessages);
+              return;
+            }
+          } catch {
+            // If parsing fails, fall through to normal error handling
+          }
+        }
+
         if (!response.ok) {
           throw toError(
             createError({
@@ -116,7 +222,20 @@ export function useChat(options: UseChatOptions): UseChatResult {
             });
             options.onFinish?.(assistantMessage);
           },
-          onData: setData,
+          onData: (eventData) => {
+            setData(eventData);
+            // Detect inference mode from server metadata
+            if (
+              eventData &&
+              typeof eventData === "object" &&
+              "inferenceMode" in eventData
+            ) {
+              const mode = (eventData as { inferenceMode: string }).inferenceMode;
+              if (mode === "server-local" || mode === "cloud") {
+                setInferenceMode(mode);
+              }
+            }
+          },
           onUpdate: (parts, messageId) => {
             const id = messageId ?? streamingMessageId;
 
@@ -153,7 +272,7 @@ export function useChat(options: UseChatOptions): UseChatResult {
         abortControllerRef.current = null;
       }
     },
-    [messages, model, options],
+    [messages, model, options, inferenceMode, doBrowserInference],
   );
 
   /**
@@ -176,9 +295,28 @@ export function useChat(options: UseChatOptions): UseChatResult {
   /**
    * Stop generation
    */
-  const stop = useCallback(() => {
+  const stop = useCallback(async () => {
     abortControllerRef.current?.abort();
     abortControllerRef.current = null;
+
+    // Also stop browser inference Worker if active
+    if (browserInferenceActiveRef.current) {
+      // Settle the pending doBrowserInference promise before terminating the Worker
+      browserInferenceRejectRef.current?.(new Error("Generation stopped by user"));
+      browserInferenceRejectRef.current = null;
+
+      try {
+        const { stopBrowserInference } = await import(
+          "./browser-inference/browser-engine.js"
+        );
+        stopBrowserInference();
+      } catch {
+        // Worker module may already be terminated or unavailable
+      }
+      browserInferenceActiveRef.current = false;
+      setBrowserStatus("ready");
+    }
+
     setIsLoading(false);
   }, []);
 
@@ -215,6 +353,8 @@ export function useChat(options: UseChatOptions): UseChatResult {
     isLoading,
     error,
     model,
+    inferenceMode,
+    browserStatus,
     setInput,
     setModel,
     sendMessage,

package/src/src/agent/runtime/index.ts

@@ -21,7 +21,7 @@ import {
   type MessagePart,
   type ToolCall,
 } from "../types.js";
-import { resolveModel } from "../../provider/index.js";
+import { ensureModelReady, resolveModel } from "../../provider/index.js";
 import { executeTool } from "../../tool/index.js";
 import { generateId } from "../../utils/id.js";
 import { detectPlatform, getPlatformCapabilities } from "../../platform/core-platform.js";
@@ -36,7 +36,7 @@ import { convertToModelMessages } from "./model-message-converter.js";
 import { convertToolsToAISDK } from "./model-tool-converter.js";
 import { createStreamState, processStream } from "./ai-stream-handler.js";
 import { MiddlewareChain } from "../middleware/chain.js";
-import { generateText, streamText } from "ai";
+import { generateText, type LanguageModel, streamText } from "ai";
 
 // Re-export from submodules
 export { generateMessageId, sendSSE } from "./sse-utils.js";
@@ -59,6 +59,27 @@ import { accumulateUsage, getMaxSteps, normalizeInput } from "./input-utils.js";
 
 const logger = serverLogger.component("agent");
 
+/**
+ * Detect whether the resolved model is local inference.
+ * Handles both explicit "local/*" requests and cloud->local auto-fallback.
+ */
+function isLocalInferenceModel(model: LanguageModel, requestedModel: string): boolean {
+  if (requestedModel.startsWith("local/")) return true;
+
+  // LanguageModel is a union that includes string, so we need to narrow first
+  if (typeof model === "string") return model.startsWith("local/");
+
+  if ("provider" in model && model.provider === "local") return true;
+
+  if (
+    "modelId" in model && typeof model.modelId === "string" && model.modelId.startsWith("local/")
+  ) {
+    return true;
+  }
+
+  return false;
+}
+
 export class AgentRuntime {
   private id: string;
   private config: AgentConfig;
@@ -125,6 +146,7 @@ export class AgentRuntime {
     modelOverride?: string,
   ): Promise<ReadableStream<Uint8Array>> {
     const modelString = modelOverride || this.config.model;
+    const requestedModel = modelString || this.config.model;
 
     for (const msg of messages) await this.memory.add(msg);
 
@@ -135,6 +157,18 @@ export class AgentRuntime {
     const toolContext = { agentId: this.id, ...context };
     const textPartId = generateId("text");
 
+    // Resolve model BEFORE creating the ReadableStream — if this throws
+    // (e.g., no_ai_available), the error propagates to the caller who can
+    // return a proper error response (503) instead of a 200 with an error event.
+    const languageModel = resolveModel(requestedModel);
+
+    // Eagerly verify the model runtime is available. For local models this
+    // checks that @huggingface/transformers can be imported. Must happen
+    // BEFORE creating the ReadableStream so no_ai_available errors propagate
+    // to the caller (createChatHandler) who returns a 503 with browser fallback
+    // info, instead of being swallowed as an in-band SSE error in a 200 response.
+    await ensureModelReady(languageModel);
+
     return new ReadableStream<Uint8Array>({
       start: async (controller) => {
         try {
@@ -142,6 +176,14 @@ export class AgentRuntime {
 
           const messageId = generateMessageId();
           sendSSE(controller, encoder, { type: "message-start", messageId });
+          sendSSE(controller, encoder, {
+            type: "data",
+            data: {
+              inferenceMode: isLocalInferenceModel(languageModel, requestedModel)
+                ? "server-local"
+                : "cloud",
+            },
+          });
           sendSSE(controller, encoder, { type: "text-start", id: textPartId });
 
           await this.executeAgentLoopStreaming(
@@ -153,6 +195,7 @@ export class AgentRuntime {
             textPartId,
             toolContext,
             modelString,
+            languageModel,
           );
 
           sendSSE(controller, encoder, { type: "text-end", id: textPartId });
@@ -181,17 +224,28 @@ export class AgentRuntime {
     return withSpan("agent.execution_loop", async (loopSpan) => {
       const { maxAgentSteps } = getPlatformCapabilities();
       const maxSteps = this.computeMaxSteps(maxAgentSteps);
-      const languageModel = resolveModel(modelString || this.config.model);
+      const requestedModel = modelString || this.config.model;
+      const languageModel = resolveModel(requestedModel);
 
       const toolCalls: ToolCall[] = [];
       const currentMessages = [...messages];
       const totalUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
 
+      // Local models can't reliably do function calling — skip tools gracefully.
+      const isLocal = isLocalInferenceModel(languageModel, requestedModel);
+      if (isLocal && this.config.tools) {
+        logger.warn(
+          `Agent "${this.id}" has tools configured but is using local model "${requestedModel}". ` +
+            "Local models don't support tool calling — tools will be skipped. " +
+            "Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or GOOGLE_API_KEY for full tool support.",
+        );
+      }
+
       for (let step = 0; step < maxSteps; step++) {
         this.status = "thinking";
         addSpanEvent(loopSpan, "step_start", { step });
 
-        const tools = getAvailableTools(this.config.tools);
+        const tools = isLocal ? [] : getAvailableTools(this.config.tools);
 
         const response = await withSpan("agent.generate_text", async (span) => {
           setSpanAttributes(span, {
@@ -350,19 +404,31 @@ export class AgentRuntime {
     textPartId?: string,
     toolContext?: Record<string, unknown>,
     modelString?: string,
+    resolvedModel?: LanguageModel,
   ): Promise<AgentResponse> {
     const { maxAgentSteps } = getPlatformCapabilities();
     const maxSteps = this.computeMaxSteps(maxAgentSteps);
-    const languageModel = resolveModel(modelString || this.config.model);
+    const requestedModel = modelString || this.config.model;
+    const languageModel = resolvedModel ?? resolveModel(requestedModel);
 
     const toolCalls: ToolCall[] = [];
     const currentMessages = [...messages];
     const totalUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
 
+    // Local models can't reliably do function calling — skip tools gracefully.
+    const isLocalStreaming = isLocalInferenceModel(languageModel, requestedModel);
+    if (isLocalStreaming && this.config.tools) {
+      logger.warn(
+        `Agent "${this.id}" has tools configured but is using local model "${requestedModel}". ` +
+          "Local models don't support tool calling — tools will be skipped. " +
+          "Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or GOOGLE_API_KEY for full tool support.",
+      );
+    }
+
     for (let step = 0; step < maxSteps; step++) {
       sendSSE(controller, encoder, { type: "step-start" });
 
-      const tools = getAvailableTools(this.config.tools);
+      const tools = isLocalStreaming ? [] : getAvailableTools(this.config.tools);
       const result = streamText({
         model: languageModel,
         system: systemPrompt,