veryfront 0.1.13 → 0.1.14

package/src/src/agent/react/use-chat/browser-inference/browser-engine.ts

@@ -0,0 +1,81 @@
+/**
+ * Bridge between BrowserInferenceClient (Worker) and useChat's streaming interface.
+ *
+ * Converts UIMessage[] to simple {role, content}[] for the Worker,
+ * and synthesizes the same streaming updates that handleStreamingResponse produces.
+ */
+
+import type { BrowserInferenceStatus } from "../types.js";
+import type { UIMessage, UIMessagePart } from "../types.js";
+import { generateClientId } from "../utils.js";
+import { BrowserInferenceClient } from "./worker-client.js";
+
+export interface BrowserInferenceCallbacks {
+  onUpdate: (parts: UIMessagePart[], messageId: string) => void;
+  onMessage: (message: UIMessage) => void;
+  onStatusChange: (status: BrowserInferenceStatus) => void;
+  onDownloadProgress?: (progress: number) => void;
+  onError: (error: Error) => void;
+}
+
+function extractTextFromMessages(
+  messages: UIMessage[],
+): Array<{ role: string; content: string }> {
+  return messages
+    .filter((m) => m.role === "user" || m.role === "assistant")
+    .map((m) => {
+      const textPart = m.parts.find((p) => p.type === "text");
+      const content = textPart && "text" in textPart ? textPart.text : "";
+      return { role: m.role, content };
+    })
+    .filter((m) => m.content.length > 0);
+}
+
+export function runBrowserInference(
+  messages: UIMessage[],
+  systemPrompt: string,
+  callbacks: BrowserInferenceCallbacks,
+): void {
+  const client = BrowserInferenceClient.getInstance();
+  const messageId = generateClientId("msg");
+  const simpleMessages = extractTextFromMessages(messages);
+
+  let accumulated = "";
+
+  client.generate(
+    messageId,
+    simpleMessages,
+    { systemPrompt, maxNewTokens: 512, temperature: 0.7 },
+    {
+      onStatus: (status) => callbacks.onStatusChange(status),
+      onDownloadProgress: (progress) => callbacks.onDownloadProgress?.(progress),
+      onToken: (token) => {
+        accumulated = token; // transformers.js callback_function sends full text each time
+        const parts: UIMessagePart[] = [
+          { type: "text", text: accumulated, state: "streaming" },
+        ];
+        callbacks.onUpdate(parts, messageId);
+      },
+      onDone: (text) => {
+        const finalParts: UIMessagePart[] = [
+          { type: "text", text: text || accumulated, state: "done" },
+        ];
+        const assistantMessage: UIMessage = {
+          id: messageId,
+          role: "assistant",
+          parts: finalParts,
+        };
+        callbacks.onMessage(assistantMessage);
+        callbacks.onStatusChange("ready");
+      },
+      onError: (errorMsg) => {
+        callbacks.onStatusChange("error");
+        callbacks.onError(new Error(errorMsg));
+      },
+    },
+  );
+}
+
+export function stopBrowserInference(): void {
+  BrowserInferenceClient.getInstance().stop();
+}

package/src/src/agent/react/use-chat/browser-inference/types.ts

@@ -0,0 +1,52 @@
+/**
+ * Worker ↔ main thread message protocol for browser-side inference.
+ */
+
+export interface WorkerGenerateRequest {
+  type: "generate";
+  id: string;
+  messages: Array<{ role: string; content: string }>;
+  options?: {
+    maxNewTokens?: number;
+    temperature?: number;
+    systemPrompt?: string;
+  };
+}
+
+export type WorkerRequest = WorkerGenerateRequest;
+
+export interface WorkerStatusResponse {
+  type: "status";
+  status: "loading-runtime" | "downloading-model" | "ready" | "generating";
+}
+
+export interface WorkerDownloadProgressResponse {
+  type: "download-progress";
+  progress: number; // 0–100
+  file?: string;
+}
+
+export interface WorkerTokenResponse {
+  type: "token";
+  id: string;
+  token: string;
+}
+
+export interface WorkerDoneResponse {
+  type: "done";
+  id: string;
+  text: string;
+}
+
+export interface WorkerErrorResponse {
+  type: "error";
+  id: string;
+  error: string;
+}
+
+export type WorkerResponse =
+  | WorkerStatusResponse
+  | WorkerDownloadProgressResponse
+  | WorkerTokenResponse
+  | WorkerDoneResponse
+  | WorkerErrorResponse;

package/src/src/agent/react/use-chat/browser-inference/worker-client.ts

@@ -0,0 +1,89 @@
+/**
+ * BrowserInferenceClient — manages Web Worker lifecycle for browser-side inference.
+ *
+ * Singleton per session. Lazily creates Worker on first generate() call.
+ * Uses inline Blob URL approach — no separate build entry point needed.
+ */
+import * as dntShim from "../../../../../_dnt.shims.js";
+
+
+import type { WorkerRequest, WorkerResponse } from "./types.js";
+import { WORKER_SCRIPT } from "./worker-script.js";
+
+export interface GenerateCallbacks {
+  onStatus?: (status: "loading-runtime" | "downloading-model" | "ready" | "generating") => void;
+  onDownloadProgress?: (progress: number, file?: string) => void;
+  onToken?: (token: string) => void;
+  onDone?: (text: string) => void;
+  onError?: (error: string) => void;
+}
+
+let instance: BrowserInferenceClient | null = null;
+
+export class BrowserInferenceClient {
+  private worker: Worker | null = null;
+  private blobUrl: string | null = null;
+
+  static getInstance(): BrowserInferenceClient {
+    if (!instance) instance = new BrowserInferenceClient();
+    return instance;
+  }
+
+  private ensureWorker(): Worker {
+    if (this.worker) return this.worker;
+
+    const blob = new dntShim.Blob([WORKER_SCRIPT], { type: "application/javascript" });
+    this.blobUrl = URL.createObjectURL(blob);
+    this.worker = new Worker(this.blobUrl, { type: "module" });
+
+    return this.worker;
+  }
+
+  generate(
+    id: string,
+    messages: Array<{ role: string; content: string }>,
+    options: { maxNewTokens?: number; temperature?: number; systemPrompt?: string },
+    callbacks: GenerateCallbacks,
+  ): void {
+    const worker = this.ensureWorker();
+
+    worker.onmessage = (event: MessageEvent<WorkerResponse>) => {
+      const msg = event.data;
+      switch (msg.type) {
+        case "status":
+          callbacks.onStatus?.(msg.status);
+          break;
+        case "download-progress":
+          callbacks.onDownloadProgress?.(msg.progress, msg.file);
+          break;
+        case "token":
+          if (msg.id === id) callbacks.onToken?.(msg.token);
+          break;
+        case "done":
+          if (msg.id === id) callbacks.onDone?.(msg.text);
+          break;
+        case "error":
+          if (msg.id === id) callbacks.onError?.(msg.error);
+          break;
+      }
+    };
+
+    worker.onerror = (event) => {
+      callbacks.onError?.(event.message || "Worker error");
+    };
+
+    const request: WorkerRequest = { type: "generate", id, messages, options };
+    worker.postMessage(request);
+  }
+
+  stop(): void {
+    if (this.worker) {
+      this.worker.terminate();
+      this.worker = null;
+    }
+    if (this.blobUrl) {
+      URL.revokeObjectURL(this.blobUrl);
+      this.blobUrl = null;
+    }
+  }
+}

package/src/src/agent/react/use-chat/browser-inference/worker-script.ts

@@ -0,0 +1,98 @@
+/**
+ * Inline Worker script as a string.
+ *
+ * Loaded via Blob URL — no separate build entry point needed.
+ * Dynamically imports @huggingface/transformers from CDN inside the Worker.
+ */
+
+export const WORKER_SCRIPT = /* js */ `
+let pipeline = null;
+let generating = false;
+
+async function loadPipeline(callbacks) {
+  if (pipeline) return pipeline;
+
+  callbacks.onStatus("loading-runtime");
+
+  const { pipeline: createPipeline, env } =
+    await import("https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.4.2");
+
+  env.useBrowserCache = true;
+  env.allowLocalModels = false;
+
+  callbacks.onStatus("downloading-model");
+
+  pipeline = await createPipeline(
+    "text-generation",
+    "HuggingFaceTB/SmolLM2-135M-Instruct",
+    {
+      dtype: "q4",
+      device: "wasm",
+      progress_callback: (progress) => {
+        if (progress.status === "progress" && progress.total) {
+          callbacks.onProgress(Math.round((progress.loaded / progress.total) * 100), progress.file);
+        }
+      },
+    },
+  );
+
+  callbacks.onStatus("ready");
+  return pipeline;
+}
+
+self.onmessage = async (event) => {
+  const request = event.data;
+  if (request.type !== "generate") return;
+
+  const { id, messages, options } = request;
+
+  try {
+    const pipe = await loadPipeline({
+      onStatus: (status) => self.postMessage({ type: "status", status }),
+      onProgress: (progress, file) => self.postMessage({ type: "download-progress", progress, file }),
+    });
+
+    self.postMessage({ type: "status", status: "generating" });
+    generating = true;
+
+    const chatMessages = [];
+    if (options?.systemPrompt) {
+      chatMessages.push({ role: "system", content: options.systemPrompt });
+    }
+    chatMessages.push(...messages);
+
+    // Helper: generated_text is a plain string for raw prompts but an array
+    // of {role, content} message objects when using chat format. Extract the
+    // last assistant message's content in either case.
+    function extractText(generated) {
+      if (typeof generated === "string") return generated;
+      if (Array.isArray(generated)) {
+        const last = generated[generated.length - 1];
+        return last?.content ?? "";
+      }
+      return "";
+    }
+
+    const result = await pipe(chatMessages, {
+      max_new_tokens: options?.maxNewTokens ?? 512,
+      temperature: options?.temperature ?? 0.7,
+      do_sample: true,
+      return_full_text: false,
+      callback_function: (output) => {
+        if (!generating) return;
+        const text = extractText(output?.[0]?.generated_text);
+        if (text) {
+          self.postMessage({ type: "token", id, token: text });
+        }
+      },
+    });
+
+    generating = false;
+    const finalText = extractText(result?.[0]?.generated_text);
+    self.postMessage({ type: "done", id, text: finalText });
+  } catch (error) {
+    generating = false;
+    self.postMessage({ type: "error", id, error: error?.message ?? String(error) });
+  }
+};
+`;

package/src/src/agent/react/use-chat/index.ts

@@ -7,7 +7,9 @@
 
 export { useChat } from "./use-chat.js";
 export type {
+  BrowserInferenceStatus,
   DynamicToolUIPart,
+  InferenceMode,
   OnToolCallArg,
   ReasoningUIPart,
   TextUIPart,

package/src/src/agent/react/use-chat/types.ts

@@ -1,6 +1,18 @@
 import * as dntShim from "../../../../_dnt.shims.js";
 export type StreamState = "streaming" | "done";
 
+/** Where inference is happening */
+export type InferenceMode = "cloud" | "server-local" | "browser";
+
+/** Browser-side model loading and inference status */
+export type BrowserInferenceStatus =
+  | "idle"
+  | "loading-runtime"
+  | "downloading-model"
+  | "ready"
+  | "generating"
+  | "error";
+
 export interface TextUIPart {
   type: "text";
   text: string;
@@ -90,6 +102,10 @@ export interface UseChatOptions {
   credentials?: RequestCredentials;
   /** Override model at runtime (e.g. "openai/gpt-4o", "anthropic/claude-sonnet-4-5-20250929") */
   model?: string;
+  /** System prompt for browser-side inference (server uses agent config) */
+  systemPrompt?: string;
+  /** Enable/disable browser fallback when server can't provide AI. Default: true */
+  browserFallback?: boolean;
   onResponse?: (response: dntShim.Response) => void;
   onFinish?: (message: UIMessage) => void;
   onError?: (error: Error) => void;
@@ -103,6 +119,10 @@ export interface UseChatResult {
   error: Error | null;
   /** Current model override (undefined = use agent default) */
   model: string | undefined;
+  /** Where inference is currently happening */
+  inferenceMode: InferenceMode;
+  /** Browser-side model loading/inference status (null when not using browser fallback) */
+  browserStatus: BrowserInferenceStatus | null;
   setInput: (input: string) => void;
   /** Change the model for subsequent requests */
   setModel: (model: string | undefined) => void;

package/src/src/agent/react/use-chat/use-chat.ts

@@ -4,6 +4,11 @@
  * Complete chat state management with zero UI.
  * Consumes the veryfront streaming protocol
  * (message-start/message-finish + step-start/step-end).
+ *
+ * Supports three inference modes:
+ * - cloud: API key present, normal server-side inference
+ * - server-local: No API key, server runs local model via ONNX
+ * - browser: Server can't run ONNX (compiled binary), falls back to browser Worker
  */
 import * as dntShim from "../../../../_dnt.shims.js";
 
@@ -12,7 +17,15 @@ import { useCallback, useRef, useState } from "react";
 import { createError, ensureError, toError } from "../../../errors/veryfront-error.js";
 
 import { handleStreamingResponse } from "./streaming/index.js";
-import type { ToolOutput, UIMessage, UseChatOptions, UseChatResult } from "./types.js";
+import type {
+  BrowserInferenceStatus,
+  InferenceMode,
+  ToolOutput,
+  UIMessage,
+  UIMessagePart,
+  UseChatOptions,
+  UseChatResult,
+} from "./types.js";
 import { generateClientId } from "./utils.js";
 
 /**
@@ -25,7 +38,16 @@ export function useChat(options: UseChatOptions): UseChatResult {
   const [error, setError] = useState<Error | null>(null);
   const [data, setData] = useState<unknown>(null);
   const [model, setModel] = useState<string | undefined>(options.model);
+  const [inferenceMode, setInferenceMode] = useState<InferenceMode>("cloud");
+  const [browserStatus, setBrowserStatus] = useState<BrowserInferenceStatus | null>(null);
   const abortControllerRef = useRef<AbortController | null>(null);
+  const browserInferenceActiveRef = useRef(false);
+  const browserInferenceRejectRef = useRef<((reason: Error) => void) | null>(null);
+
+  // System prompt for browser fallback (from 503 response or options)
+  const systemPromptRef = useRef<string>(
+    options.systemPrompt ?? "You are a helpful AI assistant.",
+  );
 
   // Track pending tool outputs for addToolOutput
   const pendingToolOutputsRef = useRef<Map<string, ToolOutput>>(new Map());
@@ -57,6 +79,64 @@ export function useChat(options: UseChatOptions): UseChatResult {
     );
   }, []);
 
+  /**
+   * Run inference in the browser via Web Worker.
+   * Lazily imports the browser-inference module to avoid bundling it
+   * when server-side inference works fine.
+   */
+  const doBrowserInference = useCallback(
+    async (allMessages: UIMessage[]) => {
+      browserInferenceActiveRef.current = true;
+
+      try {
+        const { runBrowserInference } = await import(
+          "./browser-inference/browser-engine.js"
+        );
+
+        await new Promise<void>((resolve, reject) => {
+          browserInferenceRejectRef.current = reject;
+          let hasAddedMessage = false;
+
+          runBrowserInference(allMessages, systemPromptRef.current, {
+            onUpdate: (parts: UIMessagePart[], messageId: string) => {
+              if (!hasAddedMessage) {
+                hasAddedMessage = true;
+                setMessages((prev) => [
+                  ...prev,
+                  { id: messageId, role: "assistant", parts },
+                ]);
+                return;
+              }
+              setMessages((prev) => prev.map((m) => (m.id === messageId ? { ...m, parts } : m)));
+            },
+            onMessage: (assistantMessage: UIMessage) => {
+              setMessages((prev) => {
+                if (!hasAddedMessage) return [...prev, assistantMessage];
+                return prev.map((m) => m.id === assistantMessage.id ? assistantMessage : m);
+              });
+              options.onFinish?.(assistantMessage);
+              browserInferenceRejectRef.current = null;
+              resolve();
+            },
+            onStatusChange: (status: BrowserInferenceStatus) => {
+              setBrowserStatus(status);
+            },
+            onDownloadProgress: () => {
+              // Progress is tracked via onStatusChange("downloading-model")
+            },
+            onError: (err: Error) => {
+              browserInferenceRejectRef.current = null;
+              reject(err);
+            },
+          });
+        });
+      } finally {
+        browserInferenceActiveRef.current = false;
+      }
+    },
+    [options],
+  );
+
   /**
    * Send a message and stream assistant updates.
    */
@@ -72,10 +152,18 @@ export function useChat(options: UseChatOptions): UseChatResult {
       setIsLoading(true);
       setError(null);
 
-      const abortController = new AbortController();
-      abortControllerRef.current = abortController;
-
       try {
+        const allMessages = [...messages, userMessage];
+
+        // If already in browser mode, skip fetch entirely
+        if (inferenceMode === "browser") {
+          await doBrowserInference(allMessages);
+          return;
+        }
+
+        const abortController = new AbortController();
+        abortControllerRef.current = abortController;
+
         const response = await dntShim.fetch(options.api, {
           method: "POST",
           headers: {
@@ -84,13 +172,31 @@ export function useChat(options: UseChatOptions): UseChatResult {
           },
           credentials: options.credentials,
           body: JSON.stringify({
-            messages: [...messages, userMessage],
+            messages: allMessages,
             ...(model ? { model } : {}),
             ...options.body,
           }),
           signal: abortController.signal,
         });
 
+        // Handle 503 — server can't provide AI, fall back to browser
+        if (response.status === 503 && (options.browserFallback ?? true)) {
+          try {
+            const body = await response.json();
+            if (body.code === "NO_AI_AVAILABLE") {
+              if (body.systemPrompt) {
+                systemPromptRef.current = body.systemPrompt;
+              }
+              setInferenceMode("browser");
+              setBrowserStatus("idle");
+              await doBrowserInference(allMessages);
+              return;
+            }
+          } catch {
+            // If parsing fails, fall through to normal error handling
+          }
+        }
+
         if (!response.ok) {
           throw toError(
             createError({
@@ -116,7 +222,20 @@ export function useChat(options: UseChatOptions): UseChatResult {
             });
             options.onFinish?.(assistantMessage);
           },
-          onData: setData,
+          onData: (eventData) => {
+            setData(eventData);
+            // Detect inference mode from server metadata
+            if (
+              eventData &&
+              typeof eventData === "object" &&
+              "inferenceMode" in eventData
+            ) {
+              const mode = (eventData as { inferenceMode: string }).inferenceMode;
+              if (mode === "server-local" || mode === "cloud") {
+                setInferenceMode(mode);
+              }
+            }
+          },
           onUpdate: (parts, messageId) => {
             const id = messageId ?? streamingMessageId;
 
@@ -153,7 +272,7 @@ export function useChat(options: UseChatOptions): UseChatResult {
         abortControllerRef.current = null;
       }
     },
-    [messages, model, options],
+    [messages, model, options, inferenceMode, doBrowserInference],
   );
 
   /**
@@ -176,9 +295,28 @@ export function useChat(options: UseChatOptions): UseChatResult {
   /**
    * Stop generation
    */
-  const stop = useCallback(() => {
+  const stop = useCallback(async () => {
     abortControllerRef.current?.abort();
     abortControllerRef.current = null;
+
+    // Also stop browser inference Worker if active
+    if (browserInferenceActiveRef.current) {
+      // Settle the pending doBrowserInference promise before terminating the Worker
+      browserInferenceRejectRef.current?.(new Error("Generation stopped by user"));
+      browserInferenceRejectRef.current = null;
+
+      try {
+        const { stopBrowserInference } = await import(
+          "./browser-inference/browser-engine.js"
+        );
+        stopBrowserInference();
+      } catch {
+        // Worker module may already be terminated or unavailable
+      }
+      browserInferenceActiveRef.current = false;
+      setBrowserStatus("ready");
+    }
+
     setIsLoading(false);
   }, []);
 
@@ -215,6 +353,8 @@ export function useChat(options: UseChatOptions): UseChatResult {
     isLoading,
     error,
     model,
+    inferenceMode,
+    browserStatus,
     setInput,
     setModel,
     sendMessage,