turbollm 0.5.0 → 0.6.1

package/dist/cli.js

@@ -87,7 +87,8 @@ function defaultConfig() {
     hf: { token: "" },
     modelDefaults: { ctx: 8192, ngl: 99, imageMaxTokens: 0, maxTokens: 0 },
     featuredOverrideUrl: "",
-    comfyui: { enabled: false, gatePath: "", url: "", reverseGate: false, cachePersist: false }
+    comfyui: { enabled: false, gatePath: "", url: "", reverseGate: false, cachePersist: false },
+    gateway: { autoSwap: true, keepN: 1 }
   };
 }
 var ConfigStore = class _ConfigStore {
@@ -228,6 +229,11 @@ function normalize(c) {
     // Reseated like the other known fields so it isn't dropped on every load.
     cachePersist: !!cu.cachePersist
   };
+  const gw = c.gateway ?? {};
+  c.gateway = {
+    autoSwap: gw.autoSwap !== false,
+    keepN: typeof gw.keepN === "number" && gw.keepN >= 1 ? Math.min(Math.floor(gw.keepN), 4) : 1
+  };
   c.telemetry.level = normalizeTelemetryLevel(c.telemetry.level);
   for (const e of c.engines) {
     e.capabilities ??= { kvTypes: [], flags: [] };
@@ -699,6 +705,11 @@ function vllmServerCommand(python, model, port2, host2, tensorParallelSize = 1)
     "vllm.entrypoints.openai.api_server",
     "--model",
     model,
+    // Serve under a fixed alias so requests can address the model by a stable name
+    // (TurboLLM's internal key is a display string with spaces). Mirrors mlx-lm's
+    // built-in `default_model` alias; see engineModelAlias() in compat.ts.
+    "--served-model-name",
+    "default_model",
     "--host",
     host2,
     "--port",
@@ -775,7 +786,7 @@ var Manager = class {
       }
     }
     const { cmd, args } = engineCommand(opts, port2, slotSavePath);
-    const child = spawn(cmd, args, { cwd: dirname2(cmd), windowsHide: true });
+    const child = spawn(cmd, args, { cwd: dirname2(cmd), windowsHide: true, env: pyEngineEnv(opts.engine.kind, this.store.dir()) });
     child.stdout?.pipe(logStream, { end: false });
     child.stderr?.pipe(logStream, { end: false });
     this.state = "starting";
@@ -947,10 +958,22 @@ var Manager = class {
     this.resolveExited?.();
   }
   async readiness(child, port2) {
-    const deadline = Date.now() + readinessTimeoutMs(this.opts?.engine.kind ?? "llama-server");
+    const kind = this.opts?.engine.kind ?? "llama-server";
+    const deadline = Date.now() + readinessTimeoutMs(kind);
     for (; ; ) {
       await sleep(500);
       if (this.child !== child || this.state !== "starting") return;
+      if (kind === "mlx" || kind === "vllm") {
+        const loadErr = detectPyLoadFailure(readTail(this.logPathStr, 200));
+        if (loadErr) {
+          if (this.child === child && this.state === "starting") {
+            this.state = "error";
+            this.errInfo = { code: "model_load_failed", message: loadErr, exitCode: -1, logTail: readTail(this.logPathStr, 20) };
+            child.kill("SIGKILL");
+          }
+          return;
+        }
+      }
       if (await probeReady(port2)) {
         if (this.child === child && this.state === "starting") {
           this.state = "running";
@@ -992,6 +1015,27 @@ function engineCommand(opts, port2, slotSavePath) {
 function readinessTimeoutMs(kind) {
   return kind === "vllm" ? 6e5 : 12e4;
 }
+function pyEngineEnv(kind, dataDir) {
+  if (kind !== "mlx" && kind !== "vllm") return void 0;
+  const hfHome = join6(dataDir, "hf-cache");
+  const hubCache = join6(hfHome, "hub");
+  mkdirSync4(hubCache, { recursive: true });
+  return {
+    ...process.env,
+    HF_HUB_OFFLINE: "1",
+    TRANSFORMERS_OFFLINE: "1",
+    HF_HOME: hfHome,
+    HF_HUB_CACHE: hubCache
+  };
+}
+function detectPyLoadFailure(lines) {
+  const text = lines.join("\n");
+  const isLoadCrash = /Exception in thread[^\n]*_generate/.test(text) || /in load_default\b/.test(text) || /in load_model\b/.test(text) || /load_weights/.test(text);
+  if (!isLoadCrash) return null;
+  const errLine = [...lines].reverse().find((l) => /^[A-Za-z_][\w.]*(Error|Exception):/.test(l.trim()));
+  const detail = (errLine ? errLine.trim() : "the model failed to load").slice(0, 200);
+  return `MLX could not load this model \u2014 ${detail} This usually means the installed mlx-lm version does not support this model's architecture or quantization.`;
+}
 function buildArgs(opts, port2, slotSavePath) {
   const args = ["-m", opts.modelPath, "--host", "127.0.0.1", "--port", String(port2)];
   const flags = opts.engine.capabilities.flags;
@@ -1709,6 +1753,10 @@ function engineAcceptsFormat(engineKind, format) {
   if (engineKind === "vllm") return format === "mlx";
   return format === "gguf";
 }
+var ENGINE_MODEL_ALIAS = "default_model";
+function engineModelAlias(engineKind) {
+  return engineKind === "mlx" || engineKind === "vllm" ? ENGINE_MODEL_ALIAS : null;
+}
 
 // src/models/scanner.ts
 import { existsSync as existsSync9, lstatSync, readdirSync as readdirSync3, readFileSync as readFileSync3, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
@@ -1966,6 +2014,20 @@ function quantFromName(filename) {
 // src/models/scanner.ts
 var CACHE_VERSION = 2;
 var SPLIT_RE = /^(.*)-(\d{5})-of-(\d{5})\.gguf$/i;
+var EMBED_ARCHS = /* @__PURE__ */ new Set([
+  "bert",
+  "nomic-bert",
+  "jina-bert-v3-base",
+  "jina-bert",
+  "distilbert",
+  "roberta",
+  "xlm-roberta",
+  "electra"
+]);
+var EMBED_FILE_RE = /\b(bge[-_]|nomic[-_]embed|all[-_]minilm|e5[-_]|gte[-_]|stella[-_]embed|jina[-_]embed|mxbai[-_]embed)\b/i;
+function isEmbeddingModel(arch2, name) {
+  return EMBED_ARCHS.has(arch2.toLowerCase()) || EMBED_FILE_RE.test(name);
+}
 var ScannerError = class extends Error {
   constructor(code, message) {
     super(message);
@@ -2111,6 +2173,7 @@ var Scanner = class {
     const quant = meta?.quant || quantFromName(fileName);
     const name = meta?.name || cleanName(fileName);
     const vision = mmprojPath !== null;
+    const arch2 = meta?.arch ?? "unknown";
     return {
       key: `${name.toLowerCase()}|${quant}|${sizeBytes}`,
       name,
@@ -2119,7 +2182,7 @@ var Scanner = class {
       format: "gguf",
       sizeBytes,
       sizeLabel: meta?.sizeLabel ?? "",
-      arch: meta?.arch ?? "unknown",
+      arch: arch2,
       quant,
       nativeCtx: meta?.nativeCtx ?? 0,
       blockCount: meta?.blockCount ?? 0,
@@ -2130,6 +2193,7 @@ var Scanner = class {
       vision,
       mmprojPath: vision ? mmprojPath : null,
       hasChatTemplate: meta?.hasChatTemplate ?? false,
+      embedding: isEmbeddingModel(arch2, fileName),
       incomplete,
       parseError,
       loaded: false,
@@ -2240,6 +2304,7 @@ function mlxEntryFor(dir) {
     vision: false,
     mmprojPath: null,
     hasChatTemplate,
+    embedding: isEmbeddingModel(arch2, basename(dir)),
     incomplete: false,
     parseError,
     loaded: false,
@@ -2404,6 +2469,7 @@ function deriveDefault(m, sys) {
     ropeFreqBase: 0,
     ropeFreqScale: 0,
     gpu: defaultGpu(),
+    grammar: "",
     extraArgs: []
   };
   if (m.moe && hasGpu && m.blockCount > 0) {
@@ -2493,6 +2559,8 @@ function profileToArgs(p, m, caps, cores = 0) {
     if (p.ropeFreqBase > 0 && has("--rope-freq-base")) a.push("--rope-freq-base", String(p.ropeFreqBase));
     if (p.ropeFreqScale > 0 && has("--rope-freq-scale")) a.push("--rope-freq-scale", String(p.ropeFreqScale));
   }
+  if (m.embedding && has("--embeddings")) a.push("--embeddings");
+  if (p.grammar && has("--grammar")) a.push("--grammar", p.grammar);
   a.push(...p.extraArgs);
   return a;
 }
@@ -2703,10 +2771,15 @@ var HfClient = class {
   tokenFn;
   version;
   cache = /* @__PURE__ */ new Map();
-  /** Search GGUF repos (spec 10 §2). Returns up to 30 rows sorted by downloads. */
-  async searchModels(query) {
+  /** Search repos (spec 10 §2). Returns up to 30 rows sorted by downloads.
+   *  Format filter adapts to the active engine kind:
+   *  - llama-server / TurboQuant → filter=gguf
+   *  - mlx                       → filter=mlx (HF library tag)
+   *  - vllm                      → no format filter (searches all HF repos) */
+  async searchModels(query, engineKind) {
     const q = query.trim();
-    const url = `${BASE}/api/models?search=${encodeURIComponent(q)}&filter=gguf&sort=downloads&direction=-1&limit=30&full=false`;
+    const formatFilter = engineKind === "mlx" ? "&filter=mlx" : engineKind === "vllm" ? "" : "&filter=gguf";
+    const url = `${BASE}/api/models?search=${encodeURIComponent(q)}${formatFilter}&sort=downloads&direction=-1&limit=30&full=false`;
     const raw = await this.getJson(url);
     return raw.map((m) => ({
       repo: m.id ?? m.modelId ?? "",
@@ -2722,8 +2795,30 @@ var HfClient = class {
   async getRepo(repo) {
     const info = await this.getJson(`${BASE}/api/models/${repo}`);
     const tree = await this.getJson(`${BASE}/api/models/${repo}/tree/main?recursive=true`);
-    const gguf = tree.filter((e) => e.type === "file" && /\.gguf$/i.test(e.path));
-    const files = groupFiles(repo, gguf);
+    const ggufEntries = tree.filter((e) => e.type === "file" && /\.gguf$/i.test(e.path));
+    const safetensorsEntries = tree.filter((e) => e.type === "file" && /\.safetensors$/i.test(e.path));
+    const isSafetensors = ggufEntries.length === 0 && safetensorsEntries.length > 0;
+    let files;
+    let safetensors;
+    if (isSafetensors) {
+      safetensors = true;
+      const components = tree.filter(
+        (e) => e.type === "file" && (/\.safetensors$/i.test(e.path) || /\.json$/i.test(e.path)) && !e.path.includes("/")
+        // root-level only — no nested model card assets
+      );
+      files = components.map((e) => ({
+        name: e.path,
+        quant: "mlx",
+        sizeBytes: e.lfs?.size ?? e.size ?? 0,
+        parts: 1,
+        mmproj: false,
+        safetensors: true,
+        sha256: e.lfs?.oid,
+        url: this.fileUrl(repo, e.path)
+      }));
+    } else {
+      files = groupFiles(repo, ggufEntries);
+    }
     const gated = info.gated === true || info.gated === "auto" || info.gated === "manual";
     const license = info.cardData?.license ?? (info.tags?.find((t) => t.startsWith("license:"))?.slice("license:".length) || "");
     return {
@@ -2733,7 +2828,8 @@ var HfClient = class {
       downloads: info.downloads ?? 0,
       likes: info.likes ?? 0,
       card: await this.getCard(repo),
-      files
+      files,
+      ...safetensors ? { safetensors } : {}
     };
   }
   /** Fetch the repo README (the model card), strip its YAML frontmatter, and cap the
@@ -2928,27 +3024,30 @@ var DownloadManager = class {
     const dir = this.primaryDir();
     if (!dir) throw new DownloadError("no_model_dir", "Add a model folder in Settings before downloading.");
     let repo = (input.repo ?? "").trim();
+    const subdir = (input.subdir ?? "").trim();
     let url;
     let filename;
     if (input.url) {
       const u = input.url.trim();
       if (!/^https?:\/\//i.test(u)) throw new DownloadError("invalid_url", "URL must start with http:// or https://.");
       const path = safePathname(u);
-      if (!/\.gguf$/i.test(path) && !HF_BLOB_RE.test(u)) {
+      if (!subdir && !/\.gguf$/i.test(path) && !HF_BLOB_RE.test(u)) {
         throw new DownloadError("invalid_url", "URL must point to a .gguf file.");
       }
       filename = basename2(path);
-      if (!/\.gguf$/i.test(filename)) throw new DownloadError("invalid_url", "Could not derive a .gguf filename from that URL.");
+      if (!subdir && !/\.gguf$/i.test(filename)) throw new DownloadError("invalid_url", "Could not derive a .gguf filename from that URL.");
       url = u;
       repo = "";
     } else {
       const rfilename = (input.rfilename ?? "").trim();
       if (!repo || !rfilename) throw new DownloadError("invalid_request", "repo and rfilename are required.");
-      if (!/\.gguf$/i.test(rfilename)) throw new DownloadError("invalid_url", "The file must be a .gguf.");
+      if (!subdir && !/\.gguf$/i.test(rfilename)) throw new DownloadError("invalid_url", "The file must be a .gguf.");
       filename = basename2(rfilename);
       url = `https://huggingface.co/${repo}/resolve/main/${rfilename}`;
     }
     const total = input.size ?? 0;
+    const destDir = subdir ? join10(dir, subdir) : dir;
+    if (subdir) mkdirSync6(destDir, { recursive: true });
     if (total > 0) this.assertDisk(dir, total);
     const id = `dl-${Date.now().toString(36)}-${(this.nextSeq++).toString(36)}`;
     const rec = {
@@ -2956,7 +3055,7 @@ var DownloadManager = class {
       name: filename,
       repo,
       url,
-      dest: join10(dir, filename),
+      dest: join10(destDir, filename),
       total,
       received: 0,
       status: "queued",
@@ -3526,6 +3625,219 @@ function sleep2(ms) {
   return new Promise((r) => setTimeout(r, ms));
 }
 
+// src/gateway/model-router.ts
+var ModelRouter = class {
+  constructor(store2, registry2, manager2, scanner2, comfy2) {
+    this.store = store2;
+    this.registry = registry2;
+    this.manager = manager2;
+    this.scanner = scanner2;
+    this.comfy = comfy2;
+  }
+  store;
+  registry;
+  manager;
+  scanner;
+  comfy;
+  /** Extra pool slots beyond the primary manager. Only populated when keepN > 1. */
+  extraSlots = /* @__PURE__ */ new Map();
+  /** Last-used timestamp for the primary manager slot (for LRU eviction). */
+  primaryLastUsed = 0;
+  /** Promise chain that serialises swap operations so concurrent requests for
+   *  different models queue rather than race. */
+  swapChain = Promise.resolve();
+  /** Route a request to the correct model target URL.
+   *  - If autoSwap is off: returns whatever the primary manager has loaded.
+   *  - If the requested model is already loaded: returns its target immediately.
+   *  - Otherwise: loads the model (swapping / evicting LRU as needed) and waits. */
+  async route(requestedModel) {
+    const cfg2 = this.store.snapshot();
+    if (!cfg2.gateway.autoSwap || !requestedModel.trim()) {
+      const t = this.manager.target();
+      return t ? { target: t } : { status: 503, message: "No model loaded. Load one in TurboLLM." };
+    }
+    const entry = this.resolveEntry(requestedModel);
+    if (!entry) {
+      const t = this.manager.target();
+      return t ? { target: t } : { status: 503, message: `No model matching '${requestedModel}' found. Add one in TurboLLM.` };
+    }
+    {
+      const ms = this.manager.status();
+      if (ms.state === "running" && ms.model && this.keysMatch(ms.model.key, entry)) {
+        this.primaryLastUsed = Date.now();
+        this.manager.touch();
+        return { target: this.manager.target() };
+      }
+    }
+    const slot = this.extraSlots.get(entry.key);
+    if (slot) {
+      const ss = slot.manager.status();
+      if (ss.state === "running") {
+        slot.lastUsedMs = Date.now();
+        slot.manager.touch();
+        return { target: slot.manager.target() };
+      }
+      this.extraSlots.delete(entry.key);
+    }
+    let unlock;
+    const prev = this.swapChain;
+    this.swapChain = new Promise((r) => {
+      unlock = r;
+    });
+    try {
+      await prev;
+      return await this.doLoad(entry);
+    } finally {
+      unlock();
+    }
+  }
+  // ── internal ──────────────────────────────────────────────────────────────
+  async doLoad(entry) {
+    {
+      const ms = this.manager.status();
+      if (ms.state === "running" && ms.model && this.keysMatch(ms.model.key, entry)) {
+        this.primaryLastUsed = Date.now();
+        this.manager.touch();
+        return { target: this.manager.target() };
+      }
+      const slot = this.extraSlots.get(entry.key);
+      if (slot && slot.manager.status().state === "running") {
+        slot.lastUsedMs = Date.now();
+        slot.manager.touch();
+        return { target: slot.manager.target() };
+      }
+    }
+    if (this.comfy?.isBlocked()) {
+      return { status: 503, message: "ComfyUI is rendering \u2014 model swap paused until its queue finishes." };
+    }
+    const active = this.registry.active();
+    if (!active) return { status: 503, message: "No active engine. Set one up in TurboLLM." };
+    if (!engineAcceptsFormat(active.kind, entry.format)) {
+      return { status: 503, message: `Active engine cannot load model format '${entry.format}'.` };
+    }
+    const opts = this.buildOpts(entry, active);
+    if (!opts) return { status: 503, message: "Model is incomplete or unreadable." };
+    const keepN = Math.max(1, this.store.snapshot().gateway.keepN);
+    const needsNewSlot = entry.embedding || this.chatSlotCount() < keepN;
+    const targetManager = needsNewSlot ? this.manager.status().state === "stopped" || this.manager.status().state === "error" ? this.manager : new Manager(this.store) : this.evictChatLru();
+    await targetManager.stopAndWait();
+    await this.comfy?.freeComfyUIBeforeLoad();
+    try {
+      await targetManager.start(opts);
+    } catch (e) {
+      return { status: 503, message: `Engine start failed: ${e.message}` };
+    }
+    const ready = await this.waitReady(targetManager, active.kind);
+    if (!ready) {
+      const s = targetManager.status();
+      return { status: 503, message: s.err?.message ?? "Model failed to become ready." };
+    }
+    const target = targetManager.target();
+    if (!target) return { status: 503, message: "Model loaded but target URL unavailable." };
+    if (targetManager === this.manager) {
+      this.primaryLastUsed = Date.now();
+    } else {
+      this.extraSlots.set(entry.key, { manager: targetManager, modelKey: entry.key, lastUsedMs: Date.now() });
+    }
+    this.store.update((x) => {
+      x.lastLoaded = { modelKey: entry.key, engineId: active.id };
+    });
+    return { target };
+  }
+  /** Count of alive chat (non-embedding) slots. Embedding models don't consume
+   *  a keepN slot so chat models and embedding models can coexist independently. */
+  chatSlotCount() {
+    const isAlive = (s) => s === "running" || s === "starting";
+    const ms = this.manager.status();
+    const primaryAlive = isAlive(ms.state);
+    const primaryEmbed = primaryAlive && !!ms.model && (this.scanner.get(ms.model.key)?.embedding ?? false);
+    const extraChat = [...this.extraSlots.values()].filter(
+      (s) => isAlive(s.manager.status().state) && !(this.scanner.get(s.modelKey)?.embedding ?? false)
+    ).length;
+    return (primaryAlive && !primaryEmbed ? 1 : 0) + extraChat;
+  }
+  /** Evict the least-recently-used chat (non-embedding) slot. Embedding slots are
+   *  skipped; if every alive slot is an embedding model the true LRU is used as
+   *  a fallback so we never deadlock. */
+  evictChatLru() {
+    const isAlive = (s) => s === "running" || s === "starting";
+    const ms = this.manager.status();
+    const primaryAlive = isAlive(ms.state);
+    const primaryEmbed = primaryAlive && !!ms.model && (this.scanner.get(ms.model.key)?.embedding ?? false);
+    let lruManager = this.manager;
+    let lruTime = primaryAlive && !primaryEmbed ? this.primaryLastUsed : Infinity;
+    let lruKey = null;
+    for (const slot of this.extraSlots.values()) {
+      const slotEmbed = this.scanner.get(slot.modelKey)?.embedding ?? false;
+      if (isAlive(slot.manager.status().state) && !slotEmbed && slot.lastUsedMs < lruTime) {
+        lruTime = slot.lastUsedMs;
+        lruManager = slot.manager;
+        lruKey = slot.modelKey;
+      }
+    }
+    if (lruTime === Infinity) {
+      lruTime = primaryAlive ? this.primaryLastUsed : Infinity;
+      lruManager = this.manager;
+      lruKey = null;
+      for (const slot of this.extraSlots.values()) {
+        if (isAlive(slot.manager.status().state) && slot.lastUsedMs < lruTime) {
+          lruTime = slot.lastUsedMs;
+          lruManager = slot.manager;
+          lruKey = slot.modelKey;
+        }
+      }
+    }
+    if (lruKey !== null) this.extraSlots.delete(lruKey);
+    return lruManager;
+  }
+  resolveEntry(requested) {
+    const models = this.scanner.list().models;
+    return models.find((e) => e.key === requested) ?? models.find((e) => e.name === requested) ?? models.find((e) => e.name.toLowerCase() === requested.toLowerCase()) ?? models.find((e) => e.name.toLowerCase().includes(requested.toLowerCase()));
+  }
+  keysMatch(loadedKey, entry) {
+    return loadedKey === entry.key || loadedKey === entry.path;
+  }
+  buildOpts(entry, engine) {
+    if (entry.incomplete || entry.parseError) return null;
+    const cfg2 = this.store.snapshot();
+    const sys = getSysInfo();
+    if (entry.format !== "gguf") {
+      const savedGpu = cfg2.modelProfiles[entry.key]?.gpu;
+      return {
+        engine,
+        model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: entry.nativeCtx, vision: false },
+        modelPath: entry.path,
+        extraArgs: [],
+        tensorParallelSize: savedGpu?.tensorParallelSize
+      };
+    }
+    const saved = cfg2.modelProfiles[entry.key];
+    const profile = resolveProfile(entry, sys, saved, void 0, cfg2.modelDefaults);
+    return {
+      engine,
+      model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: profile.ctx, vision: entry.vision },
+      modelPath: entry.path,
+      extraArgs: profileToArgs(profile, entry, engine.capabilities, sys.cores)
+    };
+  }
+  /** Poll until the manager's engine process becomes ready or fails.
+   *  Mirrors the Manager's internal readiness timeout by engine kind. */
+  async waitReady(manager2, engineKind) {
+    const timeoutMs = engineKind === "vllm" ? 6e5 : 12e4;
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+      const s = manager2.status();
+      if (s.state === "running") return true;
+      if (s.state === "error" || s.state === "stopped") return false;
+      await sleep3(250);
+    }
+    return false;
+  }
+};
+function sleep3(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
 // src/cli-launch.ts
 import { spawn as spawn2 } from "child_process";
 var SUPPORTED = {
@@ -3834,6 +4146,7 @@ function registerApi(app2, d) {
     const engine = {
       id: active?.id ?? "",
       name: active?.name ?? "",
+      kind: active?.kind ?? "",
       state: ms.state,
       port: ms.port,
       pid: ms.pid
@@ -4429,11 +4742,19 @@ function registerApi(app2, d) {
       }
       cuUpdates.url = u;
     }
+    const gwUpdates = {};
+    if (b.gateway?.autoSwap !== void 0) gwUpdates.autoSwap = !!b.gateway.autoSwap;
+    if (b.gateway?.keepN !== void 0) {
+      const v = Number(b.gateway.keepN);
+      if (!Number.isInteger(v) || v < 1 || v > 4) return err(c, 400, "invalid_config_value", "gateway.keepN must be 1\u20134.");
+      gwUpdates.keepN = v;
+    }
     const before = d.store.snapshot().daemon;
     d.store.update((cfg2) => {
       Object.assign(cfg2.daemon, updates);
       Object.assign(cfg2.modelDefaults, mdUpdates);
       Object.assign(cfg2.comfyui, cuUpdates);
+      Object.assign(cfg2.gateway, gwUpdates);
       if (b.autoLoadOnStart !== void 0) cfg2.autoLoadOnStart = !!b.autoLoadOnStart;
       if (telemetryLevel !== void 0) cfg2.telemetry.level = telemetryLevel;
       if (b.hfToken !== void 0) cfg2.hf.token = String(b.hfToken).trim();
@@ -4502,7 +4823,7 @@ function registerApi(app2, d) {
     const q = (c.req.query("q") ?? "").trim();
     if (!q) return c.json({ results: [] });
     try {
-      const results = await d.hf.searchModels(q);
+      const results = await d.hf.searchModels(q, d.registry.active()?.kind);
       const withLocal = results.map((r) => ({ ...r, localCount: localCountFor(d, r.repo) }));
       return c.json({ results: withLocal });
     } catch (e) {
@@ -4698,6 +5019,7 @@ function settingsPayload(d) {
     telemetryLevel,
     modelDefaults: cfg2.modelDefaults,
     comfyui: cfg2.comfyui,
+    gateway: cfg2.gateway,
     // The HF token is write-only over the wire (spec 10 §4): we never echo it back,
     // only whether one is set, so the UI can show "configured" without leaking it.
     hfTokenSet: cfg2.hf.token.length > 0
@@ -5140,7 +5462,9 @@ async function runGeneration(d, stream, ctx) {
     if (!(k in SAMPLING_KEYS) && k !== "stop") samplingOverride[k] = v;
   }
   const reqBody = {
-    model: ms.model.key,
+    // mlx-lm / vLLM serve under a fixed alias and 404 on TurboLLM's internal key;
+    // llama.cpp ignores the field. engineModelAlias() returns the right value per kind.
+    model: engineModelAlias(d.registry.active()?.kind ?? "") ?? ms.model.key,
     messages: engineMessages,
     stream: true,
     stream_options: { include_usage: true },
@@ -5388,7 +5712,7 @@ async function autoTitle(d, convId, prevMessages, assistantReply, target) {
       method: "POST",
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify({
-        model: ms.model?.key,
+        model: engineModelAlias(d.registry.active()?.kind ?? "") ?? ms.model?.key,
         messages: titleMessages,
         stream: false,
         temperature: 0.3,
@@ -5713,13 +6037,6 @@ function cbStop(index) {
 // src/gateway/gateway.ts
 function registerGateway(app2, d) {
   app2.post("/v1/messages", async (c) => {
-    const target = d.manager.target();
-    if (!target) {
-      return c.json(
-        { type: "error", error: { type: "api_error", message: "No model loaded. Load one in TurboLLM." } },
-        503
-      );
-    }
     let req;
     try {
       req = await c.req.json();
@@ -5737,10 +6054,19 @@ function registerGateway(app2, d) {
     }
     const maxLimit = d.store.snapshot().modelDefaults.maxTokens ?? 0;
     req.max_tokens = clampMaxTokens(req.max_tokens, maxLimit) ?? req.max_tokens;
-    d.manager.touch();
+    const routeResult = await d.modelRouter.route(req.model ?? "");
+    if ("status" in routeResult) {
+      return c.json(
+        { type: "error", error: { type: "api_error", message: routeResult.message } },
+        routeResult.status
+      );
+    }
+    const target = routeResult.target;
     const status = d.manager.status();
     const modelName = status.state === "running" ? status.model?.name ?? req.model ?? "local" : req.model ?? "local";
     const oaiBody = mapToOpenAI(req);
+    const oaiAlias = engineModelAlias(d.registry.active()?.kind ?? "");
+    if (oaiAlias) oaiBody.model = oaiAlias;
     d.manager.generationStart();
     let res;
     try {
@@ -5832,46 +6158,44 @@ function registerGateway(app2, d) {
     return c.json({ input_tokens: estimate });
   });
   app2.all("/v1/*", async (c) => {
-    const target = d.manager.target();
-    if (!target) {
-      if (c.req.method === "GET" && c.req.path === "/v1/models") {
+    const url = new URL(c.req.url);
+    const isChat = c.req.method === "POST" && url.pathname === "/v1/chat/completions";
+    let parsedBody = null;
+    if (isChat) {
+      try {
+        parsedBody = await c.req.json();
+      } catch {
+        parsedBody = null;
+      }
+    }
+    const requestedModel = isChat ? parsedBody?.model ?? "" : "";
+    const routeResult = await d.modelRouter.route(requestedModel);
+    if ("status" in routeResult) {
+      if (c.req.method === "GET" && url.pathname === "/v1/models") {
         return c.json({ object: "list", data: [] });
       }
       return c.json(
-        {
-          error: {
-            message: "No model loaded. Load one in TurboLLM.",
-            type: "model_not_loaded",
-            code: "model_not_loaded"
-          }
-        },
+        { error: { message: routeResult.message, type: "model_not_loaded", code: "model_not_loaded" } },
         503
       );
     }
-    d.manager.touch();
-    const url = new URL(c.req.url);
+    const target = routeResult.target;
     const upstream = target + url.pathname + url.search;
     const headers = new Headers(c.req.raw.headers);
     headers.delete("host");
-    const isChat = c.req.method === "POST" && url.pathname === "/v1/chat/completions";
     const maxLimit = d.store.snapshot().modelDefaults.maxTokens ?? 0;
     const init = { method: c.req.method, headers };
     if (c.req.method !== "GET" && c.req.method !== "HEAD") {
-      if (isChat && maxLimit > 0) {
-        let parsed = null;
-        try {
-          parsed = await c.req.json();
-        } catch {
-          parsed = null;
+      if (isChat) {
+        if (parsedBody && maxLimit > 0) {
+          parsedBody.max_tokens = clampMaxTokens(parsedBody.max_tokens, maxLimit);
         }
-        if (parsed) {
-          parsed.max_tokens = clampMaxTokens(parsed.max_tokens, maxLimit);
-          headers.delete("content-length");
-          init.body = JSON.stringify(parsed);
-        } else {
-          init.body = c.req.raw.body;
-          init.duplex = "half";
+        if (parsedBody) {
+          const alias = engineModelAlias(d.registry.active()?.kind ?? "");
+          if (alias) parsedBody.model = alias;
         }
+        headers.delete("content-length");
+        init.body = parsedBody ? JSON.stringify(parsedBody) : "";
       } else {
         init.body = c.req.raw.body;
         init.duplex = "half";
@@ -6155,8 +6479,9 @@ var hf = new HfClient(() => store.snapshot().hf.token, version);
 var downloads = new DownloadManager(store, () => void scanner.rescan(), () => hf.authHeaders());
 var bench = new BenchRunner(manager, store, scanner, registry, version);
 var comfy = new ComfyGuard(store, manager);
+var modelRouter = new ModelRouter(store, registry, manager, scanner, comfy);
 var startedAt = Date.now();
-var deps = { store, registry, manager, scanner, hashes, db, provision, hf, downloads, bench, comfy, version, startedAt };
+var deps = { store, registry, manager, scanner, hashes, db, provision, hf, downloads, bench, modelRouter, comfy, version, startedAt };
 var app = createApp(deps);
 var cfg = store.snapshot();
 var defaultHost = cfg.daemon.lanBind ? "0.0.0.0" : cfg.daemon.host || "127.0.0.1";