veryfront 0.1.13 → 0.1.14

package/src/src/agent/runtime/index.ts

@@ -21,7 +21,7 @@ import {
   type MessagePart,
   type ToolCall,
 } from "../types.js";
-import { resolveModel } from "../../provider/index.js";
+import { ensureModelReady, resolveModel } from "../../provider/index.js";
 import { executeTool } from "../../tool/index.js";
 import { generateId } from "../../utils/id.js";
 import { detectPlatform, getPlatformCapabilities } from "../../platform/core-platform.js";
@@ -36,7 +36,7 @@ import { convertToModelMessages } from "./model-message-converter.js";
 import { convertToolsToAISDK } from "./model-tool-converter.js";
 import { createStreamState, processStream } from "./ai-stream-handler.js";
 import { MiddlewareChain } from "../middleware/chain.js";
-import { generateText, streamText } from "ai";
+import { generateText, type LanguageModel, streamText } from "ai";
 
 // Re-export from submodules
 export { generateMessageId, sendSSE } from "./sse-utils.js";
@@ -59,6 +59,27 @@ import { accumulateUsage, getMaxSteps, normalizeInput } from "./input-utils.js";
 
 const logger = serverLogger.component("agent");
 
+/**
+ * Detect whether the resolved model is local inference.
+ * Handles both explicit "local/*" requests and cloud->local auto-fallback.
+ */
+function isLocalInferenceModel(model: LanguageModel, requestedModel: string): boolean {
+  if (requestedModel.startsWith("local/")) return true;
+
+  // LanguageModel is a union that includes string, so we need to narrow first
+  if (typeof model === "string") return model.startsWith("local/");
+
+  if ("provider" in model && model.provider === "local") return true;
+
+  if (
+    "modelId" in model && typeof model.modelId === "string" && model.modelId.startsWith("local/")
+  ) {
+    return true;
+  }
+
+  return false;
+}
+
 export class AgentRuntime {
   private id: string;
   private config: AgentConfig;
@@ -125,6 +146,7 @@ export class AgentRuntime {
     modelOverride?: string,
   ): Promise<ReadableStream<Uint8Array>> {
     const modelString = modelOverride || this.config.model;
+    const requestedModel = modelString || this.config.model;
 
     for (const msg of messages) await this.memory.add(msg);
 
@@ -135,6 +157,18 @@ export class AgentRuntime {
     const toolContext = { agentId: this.id, ...context };
     const textPartId = generateId("text");
 
+    // Resolve model BEFORE creating the ReadableStream — if this throws
+    // (e.g., no_ai_available), the error propagates to the caller who can
+    // return a proper error response (503) instead of a 200 with an error event.
+    const languageModel = resolveModel(requestedModel);
+
+    // Eagerly verify the model runtime is available. For local models this
+    // checks that @huggingface/transformers can be imported. Must happen
+    // BEFORE creating the ReadableStream so no_ai_available errors propagate
+    // to the caller (createChatHandler) who returns a 503 with browser fallback
+    // info, instead of being swallowed as an in-band SSE error in a 200 response.
+    await ensureModelReady(languageModel);
+
     return new ReadableStream<Uint8Array>({
       start: async (controller) => {
         try {
@@ -142,6 +176,14 @@ export class AgentRuntime {
 
           const messageId = generateMessageId();
           sendSSE(controller, encoder, { type: "message-start", messageId });
+          sendSSE(controller, encoder, {
+            type: "data",
+            data: {
+              inferenceMode: isLocalInferenceModel(languageModel, requestedModel)
+                ? "server-local"
+                : "cloud",
+            },
+          });
           sendSSE(controller, encoder, { type: "text-start", id: textPartId });
 
           await this.executeAgentLoopStreaming(
@@ -153,6 +195,7 @@ export class AgentRuntime {
             textPartId,
             toolContext,
             modelString,
+            languageModel,
           );
 
           sendSSE(controller, encoder, { type: "text-end", id: textPartId });
@@ -181,17 +224,28 @@ export class AgentRuntime {
     return withSpan("agent.execution_loop", async (loopSpan) => {
       const { maxAgentSteps } = getPlatformCapabilities();
       const maxSteps = this.computeMaxSteps(maxAgentSteps);
-      const languageModel = resolveModel(modelString || this.config.model);
+      const requestedModel = modelString || this.config.model;
+      const languageModel = resolveModel(requestedModel);
 
       const toolCalls: ToolCall[] = [];
       const currentMessages = [...messages];
       const totalUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
 
+      // Local models can't reliably do function calling — skip tools gracefully.
+      const isLocal = isLocalInferenceModel(languageModel, requestedModel);
+      if (isLocal && this.config.tools) {
+        logger.warn(
+          `Agent "${this.id}" has tools configured but is using local model "${requestedModel}". ` +
+            "Local models don't support tool calling — tools will be skipped. " +
+            "Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or GOOGLE_API_KEY for full tool support.",
+        );
+      }
+
       for (let step = 0; step < maxSteps; step++) {
         this.status = "thinking";
         addSpanEvent(loopSpan, "step_start", { step });
 
-        const tools = getAvailableTools(this.config.tools);
+        const tools = isLocal ? [] : getAvailableTools(this.config.tools);
 
         const response = await withSpan("agent.generate_text", async (span) => {
           setSpanAttributes(span, {
@@ -350,19 +404,31 @@ export class AgentRuntime {
     textPartId?: string,
     toolContext?: Record<string, unknown>,
     modelString?: string,
+    resolvedModel?: LanguageModel,
   ): Promise<AgentResponse> {
     const { maxAgentSteps } = getPlatformCapabilities();
     const maxSteps = this.computeMaxSteps(maxAgentSteps);
-    const languageModel = resolveModel(modelString || this.config.model);
+    const requestedModel = modelString || this.config.model;
+    const languageModel = resolvedModel ?? resolveModel(requestedModel);
 
     const toolCalls: ToolCall[] = [];
     const currentMessages = [...messages];
     const totalUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
 
+    // Local models can't reliably do function calling — skip tools gracefully.
+    const isLocalStreaming = isLocalInferenceModel(languageModel, requestedModel);
+    if (isLocalStreaming && this.config.tools) {
+      logger.warn(
+        `Agent "${this.id}" has tools configured but is using local model "${requestedModel}". ` +
+          "Local models don't support tool calling — tools will be skipped. " +
+          "Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or GOOGLE_API_KEY for full tool support.",
+      );
+    }
+
     for (let step = 0; step < maxSteps; step++) {
       sendSSE(controller, encoder, { type: "step-start" });
 
-      const tools = getAvailableTools(this.config.tools);
+      const tools = isLocalStreaming ? [] : getAvailableTools(this.config.tools);
       const result = streamText({
         model: languageModel,
         system: systemPrompt,

package/src/src/build/production-build/templates.ts

@@ -5,39 +5,9 @@
  */
 
 /**
- * Client-side CSS styles for loading states, error display, and prose formatting
+ * Client-side CSS styles for error display in production builds
  */
-export const CLIENT_STYLES = `body {
-  margin: 0;
-  font-family:
-    -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
-  line-height: 1.5;
-}
-
-.loading-container {
-  display: flex;
-  justify-content: center;
-  align-items: center;
-  min-height: 100vh;
-  background: #f9fafb;
-}
-
-.loading-spinner {
-  width: 40px;
-  height: 40px;
-  border: 3px solid #e5e7eb;
-  border-top-color: #3b82f6;
-  border-radius: 50%;
-  animation: spin 1s linear infinite;
-}
-
-@keyframes spin {
-  to {
-    transform: rotate(360deg);
-  }
-}
-
-.error-container {
+export const CLIENT_STYLES = `.error-container {
   max-width: 600px;
   margin: 2rem auto;
   padding: 2rem;
@@ -45,42 +15,6 @@ export const CLIENT_STYLES = `body {
   border: 1px solid #fcc;
   border-radius: 8px;
   color: #c00;
-}
-
-.prose {
-  max-width: 65ch;
-  margin: 0 auto;
-  padding: 2rem;
-}
-
-.prose h1, .prose h2, .prose h3 {
-  margin-top: 2em;
-  margin-bottom: 1em;
-}
-
-.prose p {
-  margin-bottom: 1.5em;
-}
-
-.prose code {
-  background: #f3f4f6;
-  padding: 0.2em 0.4em;
-  border-radius: 3px;
-  font-size: 0.875em;
-}
-
-.prose pre {
-  background: #1f2937;
-  color: #f9fafb;
-  padding: 1em;
-  border-radius: 8px;
-  overflow-x: auto;
-}
-
-.prose pre code {
-  background: transparent;
-  padding: 0;
-  color: inherit;
 }`;
 
 /**

package/src/src/chat/index.ts

@@ -98,7 +98,9 @@ export type { AgentTheme, ChatTheme } from "../react/components/ai/theme.js";
 
 export { useChat } from "../agent/react/use-chat/index.js";
 export type {
+  BrowserInferenceStatus,
   DynamicToolUIPart,
+  InferenceMode,
   OnToolCallArg,
   ReasoningUIPart,
   TextUIPart,

package/src/src/errors/veryfront-error.ts

@@ -75,7 +75,8 @@ export type VeryfrontErrorData =
   | { type: "file"; message: string; context?: FileContext }
   | { type: "network"; message: string; context?: NetworkContext }
   | { type: "permission"; message: string; context?: FileContext }
-  | { type: "not_supported"; message: string; feature?: string };
+  | { type: "not_supported"; message: string; feature?: string }
+  | { type: "no_ai_available"; message: string };
 
 export function createError(error: VeryfrontErrorData): VeryfrontErrorData {
   return error;

package/src/src/platform/adapters/runtime/deno/adapter.ts

@@ -393,7 +393,11 @@ export class DenoAdapter implements RuntimeAdapter {
       }
       : handler;
 
-    const server = dntShim.Deno.serve({
+    // Access native Deno.serve via `self` to bypass dnt shim transform.
+    // dnt rewrites both `Deno.*` and `globalThis.*` to use @deno/shim-deno which lacks .serve.
+    // `self` is not shimmed by dnt and equals `globalThis` in Deno.
+    const nativeDeno = (self as unknown as Record<string, typeof dntShim.Deno>)["Deno"]!;
+    const server = nativeDeno.serve({
       port,
       hostname,
       signal,

package/src/src/platform/compat/http/deno-server.ts

@@ -13,7 +13,9 @@ export class DenoHttpServer implements HttpServer {
 
     onListen?.({ hostname, port });
 
-    await dntShim.Deno.serve({ port, hostname, signal: serveSignal }, handler);
+    // Access native Deno.serve via `self` to bypass dnt shim transform.
+    const nativeDeno = (self as unknown as Record<string, typeof dntShim.Deno>)["Deno"]!;
+    await nativeDeno.serve({ port, hostname, signal: serveSignal }, handler);
   }
 
   close(): Promise<void> {

package/src/src/provider/index.ts

@@ -20,6 +20,7 @@ import "../../_dnt.polyfills.js";
 
 export {
   clearModelProviders,
+  ensureModelReady,
   getRegisteredModelProviders,
   hasModelProvider,
   registerModelProvider,

package/src/src/provider/local/ai-sdk-adapter.ts

@@ -0,0 +1,207 @@
+/**
+ * AI SDK Adapter for Local Models
+ *
+ * Bridges `@huggingface/transformers` local inference to the AI SDK
+ * `LanguageModelV2` interface. This allows `streamText()` and
+ * `generateText()` to work with local models seamlessly.
+ *
+ * @module provider/local
+ */
+
+import type { LanguageModel } from "ai";
+import { generate, generateStream } from "./local-engine.js";
+import type { ChatMessage, GenerateOptions } from "./local-engine.js";
+import { DEFAULT_LOCAL_MODEL } from "./model-catalog.js";
+import { serverLogger } from "../../utils/index.js";
+import { createError, fromError, toError } from "../../errors/veryfront-error.js";
+import { isLocalAIDisabled } from "./env.js";
+
+const logger = serverLogger.component("local-llm");
+
+/**
+ * Convert AI SDK LanguageModelV2 prompt format to simple ChatMessage array.
+ *
+ * The AI SDK prompt is an array of message objects with role and content arrays.
+ * We extract text content for the local model.
+ */
+// deno-lint-ignore no-explicit-any
+function convertPrompt(prompt: any[]): ChatMessage[] {
+  const messages: ChatMessage[] = [];
+
+  for (const msg of prompt) {
+    const role = msg.role as "system" | "user" | "assistant" | "tool";
+    // Skip tool messages — local models don't support tool calling
+    if (role === "tool") continue;
+
+    const mappedRole = role === "system" ? "system" : role === "user" ? "user" : "assistant";
+
+    // Extract text content from content array
+    let text = "";
+    if (typeof msg.content === "string") {
+      text = msg.content;
+    } else if (Array.isArray(msg.content)) {
+      for (const part of msg.content) {
+        if (part.type === "text" && typeof part.text === "string") {
+          text += part.text;
+        }
+      }
+    }
+
+    if (text) {
+      messages.push({ role: mappedRole, content: text });
+    }
+  }
+
+  return messages;
+}
+
+/**
+ * Create a local AI SDK LanguageModel for the given model ID.
+ *
+ * The returned object implements the LanguageModelV2 interface, making it
+ * compatible with all AI SDK functions (`streamText`, `generateText`, etc.)
+ * and all VeryFront hooks (`useChat`).
+ */
+export function createLocalModel(modelId?: string): LanguageModel {
+  const resolvedId = modelId || DEFAULT_LOCAL_MODEL;
+
+  const model = {
+    /** Marker so ensureModelReady() can distinguish real local-engine models
+     *  from mock/custom providers that happen to use provider:"local". */
+    _isVfLocalModel: true as const,
+    specificationVersion: "v2" as const,
+    provider: "local",
+    modelId: `local/${resolvedId}`,
+
+    supportedUrls: {},
+
+    async doGenerate(options: {
+      prompt: unknown[];
+      maxOutputTokens?: number;
+      temperature?: number;
+      topP?: number;
+      topK?: number;
+      stopSequences?: string[];
+    }) {
+      const messages = convertPrompt(options.prompt as unknown[]);
+      const genOptions: GenerateOptions = {
+        maxNewTokens: options.maxOutputTokens ?? 512,
+        temperature: options.temperature ?? 0.7,
+        topP: options.topP,
+        topK: options.topK,
+        stopSequences: options.stopSequences,
+      };
+
+      logger.debug(`[local] doGenerate: ${messages.length} messages → ${resolvedId}`);
+
+      const text = await generate(resolvedId, messages, genOptions);
+
+      return {
+        content: [{ type: "text" as const, text }],
+        finishReason: "stop" as const,
+        usage: {
+          inputTokens: undefined,
+          outputTokens: undefined,
+          totalTokens: undefined,
+        },
+        warnings: [],
+      };
+    },
+
+    async doStream(options: {
+      prompt: unknown[];
+      maxOutputTokens?: number;
+      temperature?: number;
+      topP?: number;
+      topK?: number;
+      stopSequences?: string[];
+    }) {
+      // Eagerly check if local AI is disabled — must throw before creating the
+      // ReadableStream, otherwise the 200 response headers are already committed.
+      // Note: getTransformers() in local-engine.ts also checks this, but we need
+      // the check here too because doStream creates a ReadableStream wrapper and
+      // errors inside it would be swallowed as in-band stream errors.
+      if (isLocalAIDisabled()) {
+        throw toError(
+          createError({
+            type: "no_ai_available",
+            message: "Local AI disabled via VERYFRONT_DISABLE_LOCAL_AI environment variable.",
+          }),
+        );
+      }
+
+      const messages = convertPrompt(options.prompt as unknown[]);
+      const genOptions: GenerateOptions = {
+        maxNewTokens: options.maxOutputTokens ?? 512,
+        temperature: options.temperature ?? 0.7,
+        topP: options.topP,
+        topK: options.topK,
+        stopSequences: options.stopSequences,
+      };
+
+      logger.debug(`[local] doStream: ${messages.length} messages → ${resolvedId}`);
+
+      const textId = `text-${Date.now()}`;
+
+      const stream = new ReadableStream({
+        async start(controller) {
+          try {
+            // Emit stream-start
+            controller.enqueue({ type: "stream-start", warnings: [] });
+
+            // Emit response metadata
+            controller.enqueue({
+              type: "response-metadata",
+              id: `local-${Date.now()}`,
+              timestamp: new Date(),
+              modelId: `local/${resolvedId}`,
+            });
+
+            // Emit text-start
+            controller.enqueue({ type: "text-start", id: textId });
+
+            // Stream tokens
+            for await (const token of generateStream(resolvedId, messages, genOptions)) {
+              controller.enqueue({
+                type: "text-delta",
+                id: textId,
+                delta: token,
+              });
+            }
+
+            // Emit text-end
+            controller.enqueue({ type: "text-end", id: textId });
+
+            // Emit finish
+            controller.enqueue({
+              type: "finish",
+              finishReason: "stop",
+              usage: {
+                inputTokens: undefined,
+                outputTokens: undefined,
+                totalTokens: undefined,
+              },
+            });
+
+            controller.close();
+          } catch (error) {
+            // Let no_ai_available propagate — the chat handler needs it
+            // for a proper 503 response instead of a 200 with in-band error.
+            const vfError = fromError(error);
+            if (vfError?.type === "no_ai_available") throw error;
+
+            controller.enqueue({
+              type: "error",
+              error: error instanceof Error ? error : new Error(String(error)),
+            });
+            controller.close();
+          }
+        },
+      });
+
+      return { stream };
+    },
+  };
+
+  return model as LanguageModel;
+}

package/src/src/provider/local/env.ts

@@ -0,0 +1,26 @@
+/**
+ * Cross-platform environment helpers for local AI provider.
+ *
+ * Abstracts Deno/Node env access so all local-AI checks go through
+ * a single function — no duplicated `(globalThis as any).Deno?.env` patterns.
+ *
+ * @module provider/local
+ */
+
+/**
+ * Check whether local AI is explicitly disabled via environment variable.
+ * Works in Deno, Node, and compiled binaries.
+ */
+import * as dntShim from "../../../_dnt.shims.js";
+
+export function isLocalAIDisabled(): boolean {
+  // deno-lint-ignore no-explicit-any
+  const denoVal = (dntShim.dntGlobalThis as any).Deno?.env?.get?.("VERYFRONT_DISABLE_LOCAL_AI");
+  if (denoVal === "1") return true;
+
+  if (typeof process !== "undefined" && process.env?.VERYFRONT_DISABLE_LOCAL_AI === "1") {
+    return true;
+  }
+
+  return false;
+}