veryfront 0.1.13 → 0.1.14

package/src/src/provider/local/local-engine.ts

@@ -0,0 +1,288 @@
+/**
+ * Local Model Engine
+ *
+ * Singleton wrapper around `@huggingface/transformers` for server-side
+ * local LLM inference. Provides lazy model loading and streaming text
+ * generation via async generators.
+ *
+ * Uses ONNX Runtime for inference with q4 quantization — NOT q4f16
+ * due to a known ONNX bug with f16 LayerNorm on CPU.
+ *
+ * @module provider/local
+ */
+
+import { serverLogger } from "../../utils/index.js";
+import { createError, toError } from "../../errors/veryfront-error.js";
+import { DEFAULT_LOCAL_MODEL, type ModelInfo, resolveLocalModel } from "./model-catalog.js";
+import { isLocalAIDisabled } from "./env.js";
+
+const logger = serverLogger.component("local-llm");
+
+/** Chat message format expected by Transformers.js */
+export interface ChatMessage {
+  role: "system" | "user" | "assistant";
+  content: string;
+}
+
+/** Options for text generation */
+export interface GenerateOptions {
+  maxNewTokens?: number;
+  temperature?: number;
+  topP?: number;
+  topK?: number;
+  stopSequences?: string[];
+}
+
+// deno-lint-ignore no-explicit-any
+type TransformersModule = any;
+// deno-lint-ignore no-explicit-any
+type Pipeline = any;
+
+/** Cached pipeline instances keyed by HuggingFace model ID */
+const pipelineCache = new Map<string, Pipeline>();
+
+/** Whether a model is currently being loaded (prevents concurrent loads) */
+const loadingLocks = new Map<string, Promise<Pipeline>>();
+
+/** Lazily loaded @huggingface/transformers module */
+let transformersModule: TransformersModule | null = null;
+
+/**
+ * Lazily import @huggingface/transformers.
+ * Only loads when actually needed, keeping startup fast when API keys are present.
+ */
+async function getTransformers(): Promise<TransformersModule> {
+  if (transformersModule) return transformersModule;
+
+  if (isLocalAIDisabled()) {
+    throw toError(
+      createError({
+        type: "no_ai_available",
+        message: "Local AI disabled via VERYFRONT_DISABLE_LOCAL_AI environment variable.",
+      }),
+    );
+  }
+
+  logger.info("Loading @huggingface/transformers...");
+
+  try {
+    transformersModule = await import("@huggingface/transformers");
+  } catch {
+    throw toError(
+      createError({
+        type: "no_ai_available",
+        message:
+          "Local AI model unavailable — native ONNX Runtime is not supported in this environment " +
+          "(e.g. compiled binaries). Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or GOOGLE_API_KEY " +
+          "in your .env file to use a cloud provider instead.",
+      }),
+    );
+  }
+
+  // Configure cache directory for model files
+  transformersModule.env.cacheDir = "./.cache/models";
+  // Disable browser-specific features in Node/Deno
+  transformersModule.env.useBrowserCache = false;
+
+  return transformersModule;
+}
+
+/**
+ * Load a text-generation pipeline for the given model.
+ * Returns a cached pipeline if already loaded.
+ */
+async function loadPipeline(modelInfo: ModelInfo): Promise<Pipeline> {
+  const cacheKey = modelInfo.hfId;
+
+  // Return cached pipeline
+  const cached = pipelineCache.get(cacheKey);
+  if (cached) return cached;
+
+  // Wait for existing load if in progress
+  const existingLock = loadingLocks.get(cacheKey);
+  if (existingLock) return existingLock;
+
+  // Start loading
+  const loadPromise = (async () => {
+    const transformers = await getTransformers();
+
+    logger.info(
+      `Loading local model: ${modelInfo.hfId} (${modelInfo.dtype}, ~${modelInfo.sizeMB}MB)...`,
+    );
+
+    const pipe = await transformers.pipeline(
+      "text-generation",
+      modelInfo.hfId,
+      {
+        dtype: modelInfo.dtype,
+        device: "cpu",
+      },
+    );
+
+    logger.info(`Model loaded: ${modelInfo.hfId}`);
+    pipelineCache.set(cacheKey, pipe);
+    loadingLocks.delete(cacheKey);
+    return pipe;
+  })();
+
+  loadingLocks.set(cacheKey, loadPromise);
+
+  try {
+    return await loadPromise;
+  } catch (error) {
+    loadingLocks.delete(cacheKey);
+
+    // Convert ONNX / native-addon errors to no_ai_available so they propagate
+    // correctly through the chat handler (503) instead of being swallowed as
+    // in-band SSE errors inside a 200 response stream.
+    const msg = error instanceof Error ? error.message : String(error);
+    if (
+      msg.includes("onnx") || msg.includes("ONNX") ||
+      msg.includes("dlopen") || msg.includes("dynamic linking") ||
+      msg.includes("native module") || msg.includes("SharedArrayBuffer")
+    ) {
+      transformersModule = null;
+      throw toError(
+        createError({
+          type: "no_ai_available",
+          message:
+            "Local AI model unavailable — native ONNX Runtime is not supported in this environment " +
+            "(e.g. compiled binaries). Set OPENAI_API_KEY, ANTHROPIC_API_KEY, or GOOGLE_API_KEY " +
+            "in your .env file to use a cloud provider instead.",
+        }),
+      );
+    }
+    throw error;
+  }
+}
+
+/**
+ * Eagerly verify that the local AI runtime (@huggingface/transformers + ONNX)
+ * is available by loading the default model pipeline.
+ *
+ * Call this *before* creating the HTTP response stream so that failures surface
+ * as a thrown error (→ 503) rather than being swallowed inside a ReadableStream
+ * (→ 200 with in-band SSE error).
+ *
+ * In compiled binaries, `import("@huggingface/transformers")` itself fails
+ * because `onnxruntime-node` eagerly `require()`s a native `.node` addon at
+ * import time and the addon isn't embedded in the binary.  In dev mode (Deno)
+ * the native addon exists on disk so the import succeeds, but `pipeline()` can
+ * still fail if the ONNX model files are missing.  Either way this function
+ * surfaces the error before the response stream is created.  The pipeline is
+ * cached after the first successful call, so subsequent checks are instant.
+ */
+export async function verifyLocalRuntime(modelId?: string): Promise<void> {
+  const modelInfo = resolveLocalModel(modelId || DEFAULT_LOCAL_MODEL);
+  await loadPipeline(modelInfo);
+}
+
+/**
+ * Generate text in a streaming fashion using an async generator.
+ *
+ * Yields individual tokens as they are generated by the model.
+ */
+export async function* generateStream(
+  modelId: string,
+  messages: ChatMessage[],
+  options: GenerateOptions = {},
+): AsyncGenerator<string, void, undefined> {
+  const modelInfo = resolveLocalModel(modelId);
+  const pipe = await loadPipeline(modelInfo);
+  const transformers = await getTransformers();
+
+  const {
+    maxNewTokens = 512,
+    temperature = 0.7,
+    topP,
+    topK,
+  } = options;
+
+  // Use a queue to bridge TextStreamer callbacks → async generator
+  const tokenQueue: string[] = [];
+  let resolveWaiting: (() => void) | null = null;
+  let done = false;
+
+  const streamer = new transformers.TextStreamer(pipe.tokenizer, {
+    skip_prompt: true,
+    skip_special_tokens: true,
+    callback_function: (text: string) => {
+      tokenQueue.push(text);
+      if (resolveWaiting) {
+        resolveWaiting();
+        resolveWaiting = null;
+      }
+    },
+  });
+
+  // Start generation in the background
+  const generatePromise = pipe(messages, {
+    max_new_tokens: maxNewTokens,
+    temperature,
+    top_p: topP,
+    top_k: topK,
+    do_sample: temperature > 0,
+    streamer,
+  }).then(() => {
+    done = true;
+    if (resolveWaiting) {
+      resolveWaiting();
+      resolveWaiting = null;
+    }
+  }).catch((error: Error) => {
+    done = true;
+    if (resolveWaiting) {
+      resolveWaiting();
+      resolveWaiting = null;
+    }
+    throw error;
+  });
+
+  // Yield tokens as they arrive
+  while (true) {
+    while (tokenQueue.length > 0) {
+      yield tokenQueue.shift()!;
+    }
+
+    if (done) break;
+
+    // Wait for more tokens
+    await new Promise<void>((resolve) => {
+      resolveWaiting = resolve;
+    });
+  }
+
+  // Ensure generation has completed
+  await generatePromise;
+}
+
+/**
+ * Generate text without streaming (full completion).
+ */
+export async function generate(
+  modelId: string,
+  messages: ChatMessage[],
+  options: GenerateOptions = {},
+): Promise<string> {
+  const chunks: string[] = [];
+  for await (const token of generateStream(modelId, messages, options)) {
+    chunks.push(token);
+  }
+  return chunks.join("");
+}
+
+/**
+ * Preload a model into memory. Useful for warming up on server start.
+ */
+export async function preloadModel(modelId: string): Promise<void> {
+  const modelInfo = resolveLocalModel(modelId);
+  await loadPipeline(modelInfo);
+}
+
+/**
+ * Check if a model is currently loaded in memory.
+ */
+export function isModelLoaded(modelId: string): boolean {
+  const modelInfo = resolveLocalModel(modelId);
+  return pipelineCache.has(modelInfo.hfId);
+}

package/src/src/provider/local/model-catalog.ts

@@ -0,0 +1,73 @@
+/**
+ * Local Model Catalog
+ *
+ * Maps friendly model IDs to HuggingFace model repository IDs.
+ * Used by the local inference engine to resolve model names.
+ *
+ * @module provider/local
+ */
+
+export interface ModelInfo {
+  /** HuggingFace model repository ID */
+  hfId: string;
+  /** Quantization dtype for ONNX Runtime */
+  dtype: "q4" | "q8" | "fp32";
+  /** Approximate download size in MB */
+  sizeMB: number;
+  /** Human-readable description */
+  description: string;
+}
+
+/**
+ * Catalog of supported local models.
+ *
+ * **Important:** Only `q4` quantization is used — `q4f16` has a known
+ * ONNX Runtime bug with LayerNorm on CPU that produces NaN outputs.
+ */
+const MODEL_CATALOG: Record<string, ModelInfo> = {
+  "smollm2-135m": {
+    hfId: "HuggingFaceTB/SmolLM2-135M-Instruct",
+    dtype: "q4",
+    sizeMB: 100,
+    description: "SmolLM2 135M — fast, lightweight chat model",
+  },
+  "smollm2-360m": {
+    hfId: "HuggingFaceTB/SmolLM2-360M-Instruct",
+    dtype: "q4",
+    sizeMB: 250,
+    description: "SmolLM2 360M — better quality, still fast",
+  },
+  "smollm2-1.7b": {
+    hfId: "HuggingFaceTB/SmolLM2-1.7B-Instruct",
+    dtype: "q4",
+    sizeMB: 1000,
+    description: "SmolLM2 1.7B — highest quality local model",
+  },
+};
+
+/** Default model used when no specific model ID is provided */
+export const DEFAULT_LOCAL_MODEL = "smollm2-135m";
+
+/**
+ * Resolve a friendly model ID to its HuggingFace model info.
+ * Falls back to treating the ID as a raw HuggingFace repository ID.
+ */
+export function resolveLocalModel(modelId: string): ModelInfo {
+  const catalogEntry = MODEL_CATALOG[modelId];
+  if (catalogEntry) return catalogEntry;
+
+  // Treat as raw HuggingFace model ID (e.g. "HuggingFaceTB/SmolLM2-135M-Instruct")
+  return {
+    hfId: modelId,
+    dtype: "q4",
+    sizeMB: 0,
+    description: `Custom model: ${modelId}`,
+  };
+}
+
+/**
+ * Get all available local model IDs.
+ */
+export function getLocalModelIds(): string[] {
+  return Object.keys(MODEL_CATALOG);
+}

package/src/src/provider/model-registry.ts

@@ -14,7 +14,7 @@
  */
 
 import type { LanguageModel } from "ai";
-import { createError, toError } from "../errors/veryfront-error.js";
+import { createError, fromError, toError } from "../errors/veryfront-error.js";
 import { createOpenAI } from "@ai-sdk/openai";
 import { createAnthropic } from "@ai-sdk/anthropic";
 import { createGoogleGenerativeAI } from "@ai-sdk/google";
@@ -24,6 +24,13 @@ import {
   getOpenAIEnvConfig,
 } from "../config/env.js";
 import { ProjectScopedRegistryManager } from "../ai/registry-manager.js";
+import { serverLogger } from "../utils/index.js";
+import { DEFAULT_LOCAL_MODEL } from "./local/model-catalog.js";
+import { createLocalModel } from "./local/ai-sdk-adapter.js";
+import { isLocalAIDisabled } from "./local/env.js";
+import { verifyLocalRuntime } from "./local/local-engine.js";
+
+const localLogger = serverLogger.component("local-llm");
 
 export type ModelProviderFactory = (modelId: string) => LanguageModel;
 
@@ -118,6 +125,16 @@ function autoInitializeFromEnv(): void {
       return createGoogleGenerativeAI({ apiKey: config.apiKey })(id);
     });
   }
+
+  // Register the local provider (always available, no API key needed).
+  // createLocalModel is a lightweight synchronous constructor — the actual
+  // @huggingface/transformers import and model loading happen lazily on
+  // the first doGenerate/doStream call, so this doesn't add startup overhead.
+  if (!manager.has("local")) {
+    manager.registerShared("local", (id) => {
+      return createLocalModel(id);
+    });
+  }
 }
 
 /**
@@ -168,7 +185,31 @@ export function resolveModel(modelString: string): LanguageModel {
     );
   }
 
-  return factory(modelId);
+  try {
+    return factory(modelId);
+  } catch (error) {
+    // Auto-fallback: when a cloud provider fails due to missing API key,
+    // transparently switch to the local model so chat works out of the box.
+    const errorData = fromError(error);
+    if (errorData?.type === "config" && providerName !== "local" && manager.has("local")) {
+      // Check if local AI is explicitly disabled (e.g., for testing)
+      if (isLocalAIDisabled()) {
+        throw toError(
+          createError({
+            type: "no_ai_available",
+            message: "Local AI disabled via VERYFRONT_DISABLE_LOCAL_AI environment variable.",
+          }),
+        );
+      }
+
+      localLogger.info(
+        `⚡ "${providerName}" unavailable (missing API key). Falling back to local model.`,
+      );
+      const localFactory = manager.get("local")!;
+      return localFactory(DEFAULT_LOCAL_MODEL);
+    }
+    throw error;
+  }
 }
 
 /**
@@ -187,6 +228,29 @@ export function getRegisteredModelProviders(): string[] {
   return manager.getAllIds();
 }
 
+/**
+ * Eagerly verify that the resolved model's runtime is available.
+ *
+ * For real local-engine models (created by `createLocalModel()`) this
+ * eagerly loads the ONNX pipeline to surface `no_ai_available` errors
+ * **before** the HTTP response stream is created. Must happen before the
+ * ReadableStream so the chat handler can return a proper 503 (with
+ * browser-fallback info) rather than a 200 with an in-band SSE error.
+ *
+ * Uses the `_isVfLocalModel` marker set by `createLocalModel()` to
+ * distinguish real local-engine models from mock/custom providers that
+ * happen to use `provider: "local"`.
+ */
+export async function ensureModelReady(
+  model: LanguageModel,
+): Promise<void> {
+  const m = model as Record<string, unknown>;
+  if (!m._isVfLocalModel) return;
+  // modelId is "local/<id>" — strip the prefix to get the catalog id.
+  const catalogId = typeof m.modelId === "string" ? m.modelId.replace(/^local\//, "") : undefined;
+  await verifyLocalRuntime(catalogId);
+}
+
 /**
  * Clear all registered model providers (for testing).
  */

package/src/src/proxy/main.ts

@@ -15,6 +15,9 @@
  * - LOCAL_PROJECTS: JSON map of slug → filesystem path (for dev)
  * - CACHE_TYPE: "memory" (default) or "redis"
  * - REDIS_URL: Redis connection URL (required if CACHE_TYPE=redis)
+ * - VERYFRONT_API_INTERNAL_URL: API URL for internal endpoints (falls back to VERYFRONT_PROXY_API_BASE_URL)
+ * - VERYFRONT_API_INTERNAL_USER: Basic auth user for internal API
+ * - VERYFRONT_API_INTERNAL_PASS: Basic auth pass for internal API
  */
 import * as dntShim from "../../_dnt.shims.js";
 
@@ -36,6 +39,7 @@ import {
 import { proxyLogger, runWithProxyRequestContext } from "./logger.js";
 import { ErrorPages } from "../server/utils/error-html.js";
 import { RendererRouter } from "./renderer-router.js";
+import { ServerResolver } from "./server-resolver.js";
 import { parseProjectDomain } from "../server/utils/domain-parser.js";
 import { exit, getEnv, onSignal } from "../platform/compat/process.js";
 import { createHttpServer, upgradeWebSocket } from "../platform/compat/http/index.js";
@@ -83,6 +87,13 @@ const rendererRouter = (discoveryHost || staticTargets)
     parseInt(getEnv("VERYFRONT_SERVER_DISCOVERY_INTERVAL_MS") || "15000") || 15000,
   )
   : null;
+
+// Dedicated server resolver: routes environments to their dedicated server if assigned
+const apiInternalUrl = getEnv("VERYFRONT_API_INTERNAL_URL") || config.apiBaseUrl;
+const apiInternalUser = getEnv("VERYFRONT_API_INTERNAL_USER") || "";
+const apiInternalPass = getEnv("VERYFRONT_API_INTERNAL_PASS") || "";
+const serverResolver = new ServerResolver(apiInternalUrl, apiInternalUser, apiInternalPass);
+
 const { hostname: HOST, port: PORT } = resolveProxyBinding();
 const WS_CONNECT_TIMEOUT_MS = 30000;
 // Timeout for forwarding requests to production server (SSR can take time on cold start)
@@ -340,10 +351,17 @@ function forwardToServer(req: dntShim.Request): Promise<dntShim.Response> {
           const isIdempotent = ["GET", "HEAD", "OPTIONS"].includes(req.method);
           const maxRetries = isIdempotent ? VERYFRONT_SERVER_RETRY_COUNT : 0;
           let lastError: Error | null = null;
+          // After a retryable connection error to a dedicated server, fall back to shared pool
+          let skipDedicated = false;
 
           for (let attempt = 0; attempt <= maxRetries; attempt++) {
-            // Re-resolve on each attempt so retries can pick a different pod
-            const baseUrl = rendererRouter?.resolve(ctx.projectSlug) ?? PRODUCTION_SERVER_URL;
+            // Resolve dedicated server per attempt so retries can fall back to shared pool
+            const dedicatedServerUrl = skipDedicated
+              ? null
+              : await serverResolver.resolve(ctx.environmentId);
+            const baseUrl = dedicatedServerUrl ??
+              rendererRouter?.resolve(ctx.projectSlug) ??
+              PRODUCTION_SERVER_URL;
             const serverUrl = new URL(url.pathname + url.search, baseUrl);
             // Delay before retry (not on first attempt)
             if (attempt > 0) {
@@ -425,10 +443,26 @@ function forwardToServer(req: dntShim.Request): Promise<dntShim.Response> {
 
               // Check if this is a retryable error and we have retries left
               if (isRetryableConnectionError(error) && attempt < maxRetries) {
-                proxyLogger.warn(`[Retry] Retryable connection error on attempt ${attempt + 1}`, {
-                  pathname: url.pathname,
-                  error: error instanceof Error ? error.message : String(error),
-                });
+                // If we were targeting a dedicated server, fall back to shared pool on retry
+                if (dedicatedServerUrl) {
+                  skipDedicated = true;
+                  proxyLogger.warn(
+                    `[Retry] Dedicated server unreachable, falling back to shared pool`,
+                    {
+                      pathname: url.pathname,
+                      dedicatedServerUrl,
+                      error: error instanceof Error ? error.message : String(error),
+                    },
+                  );
+                } else {
+                  proxyLogger.warn(
+                    `[Retry] Retryable connection error on attempt ${attempt + 1}`,
+                    {
+                      pathname: url.pathname,
+                      error: error instanceof Error ? error.message : String(error),
+                    },
+                  );
+                }
                 continue; // Try again
               }
 
@@ -551,6 +585,7 @@ function router(req: dntShim.Request): Promise<dntShim.Response> {
 async function shutdown(): Promise<void> {
   proxyLogger.info("Shutting down");
   rendererRouter?.close();
+  serverResolver.close();
   await proxyHandler.close();
   await shutdownOTLP();
   proxyLogger.info("Closed connections");