turbollm 0.3.0 → 0.6.0

package/dist/cli.js

@@ -87,7 +87,8 @@ function defaultConfig() {
     hf: { token: "" },
     modelDefaults: { ctx: 8192, ngl: 99, imageMaxTokens: 0, maxTokens: 0 },
     featuredOverrideUrl: "",
-    comfyui: { enabled: false, gatePath: "", url: "", reverseGate: false, cachePersist: false }
+    comfyui: { enabled: false, gatePath: "", url: "", reverseGate: false, cachePersist: false },
+    gateway: { autoSwap: true, keepN: 1 }
   };
 }
 var ConfigStore = class _ConfigStore {
@@ -228,6 +229,11 @@ function normalize(c) {
     // Reseated like the other known fields so it isn't dropped on every load.
     cachePersist: !!cu.cachePersist
   };
+  const gw = c.gateway ?? {};
+  c.gateway = {
+    autoSwap: gw.autoSwap !== false,
+    keepN: typeof gw.keepN === "number" && gw.keepN >= 1 ? Math.min(Math.floor(gw.keepN), 4) : 1
+  };
   c.telemetry.level = normalizeTelemetryLevel(c.telemetry.level);
   for (const e of c.engines) {
     e.capabilities ??= { kvTypes: [], flags: [] };
@@ -1966,6 +1972,20 @@ function quantFromName(filename) {
 // src/models/scanner.ts
 var CACHE_VERSION = 2;
 var SPLIT_RE = /^(.*)-(\d{5})-of-(\d{5})\.gguf$/i;
+var EMBED_ARCHS = /* @__PURE__ */ new Set([
+  "bert",
+  "nomic-bert",
+  "jina-bert-v3-base",
+  "jina-bert",
+  "distilbert",
+  "roberta",
+  "xlm-roberta",
+  "electra"
+]);
+var EMBED_FILE_RE = /\b(bge[-_]|nomic[-_]embed|all[-_]minilm|e5[-_]|gte[-_]|stella[-_]embed|jina[-_]embed|mxbai[-_]embed)\b/i;
+function isEmbeddingModel(arch2, name) {
+  return EMBED_ARCHS.has(arch2.toLowerCase()) || EMBED_FILE_RE.test(name);
+}
 var ScannerError = class extends Error {
   constructor(code, message) {
     super(message);
@@ -2111,6 +2131,7 @@ var Scanner = class {
     const quant = meta?.quant || quantFromName(fileName);
     const name = meta?.name || cleanName(fileName);
     const vision = mmprojPath !== null;
+    const arch2 = meta?.arch ?? "unknown";
     return {
       key: `${name.toLowerCase()}|${quant}|${sizeBytes}`,
       name,
@@ -2119,7 +2140,7 @@ var Scanner = class {
       format: "gguf",
       sizeBytes,
       sizeLabel: meta?.sizeLabel ?? "",
-      arch: meta?.arch ?? "unknown",
+      arch: arch2,
       quant,
       nativeCtx: meta?.nativeCtx ?? 0,
       blockCount: meta?.blockCount ?? 0,
@@ -2130,6 +2151,7 @@ var Scanner = class {
       vision,
       mmprojPath: vision ? mmprojPath : null,
       hasChatTemplate: meta?.hasChatTemplate ?? false,
+      embedding: isEmbeddingModel(arch2, fileName),
       incomplete,
       parseError,
       loaded: false,
@@ -2240,6 +2262,7 @@ function mlxEntryFor(dir) {
     vision: false,
     mmprojPath: null,
     hasChatTemplate,
+    embedding: isEmbeddingModel(arch2, basename(dir)),
     incomplete: false,
     parseError,
     loaded: false,
@@ -2342,7 +2365,7 @@ function kvBytesPerElem(t) {
   }
 }
 function defaultSampling() {
-  return { temp: 0.8, topP: 0.95, topK: 40, minP: 0.05, repeatPenalty: 1, presencePenalty: 0 };
+  return { temp: 0.8, topP: 0.95, topK: 40, minP: 0.05, repeatPenalty: 1, presencePenalty: 0, frequencyPenalty: 0, stop: [] };
 }
 function gpuBudgetMb(sys, p) {
   if (sys.gpus.length === 0) return 0;
@@ -2398,7 +2421,13 @@ function deriveDefault(m, sys) {
     mtpHeadPath: "",
     draftModelPath: "",
     sampling: defaultSampling(),
+    contextOverflow: "shift",
+    nKeep: 0,
+    ropeScalingType: "none",
+    ropeFreqBase: 0,
+    ropeFreqScale: 0,
     gpu: defaultGpu(),
+    grammar: "",
     extraArgs: []
   };
   if (m.moe && hasGpu && m.blockCount > 0) {
@@ -2475,6 +2504,21 @@ function profileToArgs(p, m, caps, cores = 0) {
     if (specType) a.push("--spec-type", "draft");
     a.push("--model-draft", p.draftModelPath, "--draft-max", "16", "--draft-min", "1");
   }
+  if (p.sampling.temp !== 0.8 && has("--temp")) a.push("--temp", String(p.sampling.temp));
+  if (p.sampling.topP !== 0.95 && has("--top-p")) a.push("--top-p", String(p.sampling.topP));
+  if (p.sampling.topK !== 40 && has("--top-k")) a.push("--top-k", String(p.sampling.topK));
+  if (p.sampling.minP !== 0.05 && has("--min-p")) a.push("--min-p", String(p.sampling.minP));
+  if (p.sampling.repeatPenalty !== 1 && has("--repeat-penalty")) a.push("--repeat-penalty", String(p.sampling.repeatPenalty));
+  if (p.sampling.presencePenalty !== 0 && has("--presence-penalty")) a.push("--presence-penalty", String(p.sampling.presencePenalty));
+  if (p.sampling.frequencyPenalty !== 0 && has("--frequency-penalty")) a.push("--frequency-penalty", String(p.sampling.frequencyPenalty));
+  if (p.contextOverflow === "keep" && p.nKeep > 0 && has("--n-keep")) a.push("--n-keep", String(p.nKeep));
+  if (p.ropeScalingType !== "none" && has("--rope-scaling")) {
+    a.push("--rope-scaling", p.ropeScalingType);
+    if (p.ropeFreqBase > 0 && has("--rope-freq-base")) a.push("--rope-freq-base", String(p.ropeFreqBase));
+    if (p.ropeFreqScale > 0 && has("--rope-freq-scale")) a.push("--rope-freq-scale", String(p.ropeFreqScale));
+  }
+  if (m.embedding && has("--embeddings")) a.push("--embeddings");
+  if (p.grammar && has("--grammar")) a.push("--grammar", p.grammar);
   a.push(...p.extraArgs);
   return a;
 }
@@ -2685,10 +2729,15 @@ var HfClient = class {
   tokenFn;
   version;
   cache = /* @__PURE__ */ new Map();
-  /** Search GGUF repos (spec 10 §2). Returns up to 30 rows sorted by downloads. */
-  async searchModels(query) {
+  /** Search repos (spec 10 §2). Returns up to 30 rows sorted by downloads.
+   *  Format filter adapts to the active engine kind:
+   *  - llama-server / TurboQuant → filter=gguf
+   *  - mlx                       → filter=mlx (HF library tag)
+   *  - vllm                      → no format filter (searches all HF repos) */
+  async searchModels(query, engineKind) {
     const q = query.trim();
-    const url = `${BASE}/api/models?search=${encodeURIComponent(q)}&filter=gguf&sort=downloads&direction=-1&limit=30&full=false`;
+    const formatFilter = engineKind === "mlx" ? "&filter=mlx" : engineKind === "vllm" ? "" : "&filter=gguf";
+    const url = `${BASE}/api/models?search=${encodeURIComponent(q)}${formatFilter}&sort=downloads&direction=-1&limit=30&full=false`;
     const raw = await this.getJson(url);
     return raw.map((m) => ({
       repo: m.id ?? m.modelId ?? "",
@@ -2704,8 +2753,30 @@ var HfClient = class {
   async getRepo(repo) {
     const info = await this.getJson(`${BASE}/api/models/${repo}`);
     const tree = await this.getJson(`${BASE}/api/models/${repo}/tree/main?recursive=true`);
-    const gguf = tree.filter((e) => e.type === "file" && /\.gguf$/i.test(e.path));
-    const files = groupFiles(repo, gguf);
+    const ggufEntries = tree.filter((e) => e.type === "file" && /\.gguf$/i.test(e.path));
+    const safetensorsEntries = tree.filter((e) => e.type === "file" && /\.safetensors$/i.test(e.path));
+    const isSafetensors = ggufEntries.length === 0 && safetensorsEntries.length > 0;
+    let files;
+    let safetensors;
+    if (isSafetensors) {
+      safetensors = true;
+      const components = tree.filter(
+        (e) => e.type === "file" && (/\.safetensors$/i.test(e.path) || /\.json$/i.test(e.path)) && !e.path.includes("/")
+        // root-level only — no nested model card assets
+      );
+      files = components.map((e) => ({
+        name: e.path,
+        quant: "mlx",
+        sizeBytes: e.lfs?.size ?? e.size ?? 0,
+        parts: 1,
+        mmproj: false,
+        safetensors: true,
+        sha256: e.lfs?.oid,
+        url: this.fileUrl(repo, e.path)
+      }));
+    } else {
+      files = groupFiles(repo, ggufEntries);
+    }
     const gated = info.gated === true || info.gated === "auto" || info.gated === "manual";
     const license = info.cardData?.license ?? (info.tags?.find((t) => t.startsWith("license:"))?.slice("license:".length) || "");
     return {
@@ -2715,7 +2786,8 @@ var HfClient = class {
       downloads: info.downloads ?? 0,
       likes: info.likes ?? 0,
       card: await this.getCard(repo),
-      files
+      files,
+      ...safetensors ? { safetensors } : {}
     };
   }
   /** Fetch the repo README (the model card), strip its YAML frontmatter, and cap the
@@ -2910,27 +2982,30 @@ var DownloadManager = class {
     const dir = this.primaryDir();
     if (!dir) throw new DownloadError("no_model_dir", "Add a model folder in Settings before downloading.");
     let repo = (input.repo ?? "").trim();
+    const subdir = (input.subdir ?? "").trim();
     let url;
     let filename;
     if (input.url) {
       const u = input.url.trim();
       if (!/^https?:\/\//i.test(u)) throw new DownloadError("invalid_url", "URL must start with http:// or https://.");
       const path = safePathname(u);
-      if (!/\.gguf$/i.test(path) && !HF_BLOB_RE.test(u)) {
+      if (!subdir && !/\.gguf$/i.test(path) && !HF_BLOB_RE.test(u)) {
         throw new DownloadError("invalid_url", "URL must point to a .gguf file.");
       }
       filename = basename2(path);
-      if (!/\.gguf$/i.test(filename)) throw new DownloadError("invalid_url", "Could not derive a .gguf filename from that URL.");
+      if (!subdir && !/\.gguf$/i.test(filename)) throw new DownloadError("invalid_url", "Could not derive a .gguf filename from that URL.");
       url = u;
       repo = "";
     } else {
       const rfilename = (input.rfilename ?? "").trim();
       if (!repo || !rfilename) throw new DownloadError("invalid_request", "repo and rfilename are required.");
-      if (!/\.gguf$/i.test(rfilename)) throw new DownloadError("invalid_url", "The file must be a .gguf.");
+      if (!subdir && !/\.gguf$/i.test(rfilename)) throw new DownloadError("invalid_url", "The file must be a .gguf.");
       filename = basename2(rfilename);
       url = `https://huggingface.co/${repo}/resolve/main/${rfilename}`;
     }
     const total = input.size ?? 0;
+    const destDir = subdir ? join10(dir, subdir) : dir;
+    if (subdir) mkdirSync6(destDir, { recursive: true });
     if (total > 0) this.assertDisk(dir, total);
     const id = `dl-${Date.now().toString(36)}-${(this.nextSeq++).toString(36)}`;
     const rec = {
@@ -2938,7 +3013,7 @@ var DownloadManager = class {
       name: filename,
       repo,
       url,
-      dest: join10(dir, filename),
+      dest: join10(destDir, filename),
       total,
       received: 0,
       status: "queued",
@@ -3508,6 +3583,219 @@ function sleep2(ms) {
   return new Promise((r) => setTimeout(r, ms));
 }
 
+// src/gateway/model-router.ts
+var ModelRouter = class {
+  constructor(store2, registry2, manager2, scanner2, comfy2) {
+    this.store = store2;
+    this.registry = registry2;
+    this.manager = manager2;
+    this.scanner = scanner2;
+    this.comfy = comfy2;
+  }
+  store;
+  registry;
+  manager;
+  scanner;
+  comfy;
+  /** Extra pool slots beyond the primary manager. Only populated when keepN > 1. */
+  extraSlots = /* @__PURE__ */ new Map();
+  /** Last-used timestamp for the primary manager slot (for LRU eviction). */
+  primaryLastUsed = 0;
+  /** Promise chain that serialises swap operations so concurrent requests for
+   *  different models queue rather than race. */
+  swapChain = Promise.resolve();
+  /** Route a request to the correct model target URL.
+   *  - If autoSwap is off: returns whatever the primary manager has loaded.
+   *  - If the requested model is already loaded: returns its target immediately.
+   *  - Otherwise: loads the model (swapping / evicting LRU as needed) and waits. */
+  async route(requestedModel) {
+    const cfg2 = this.store.snapshot();
+    if (!cfg2.gateway.autoSwap || !requestedModel.trim()) {
+      const t = this.manager.target();
+      return t ? { target: t } : { status: 503, message: "No model loaded. Load one in TurboLLM." };
+    }
+    const entry = this.resolveEntry(requestedModel);
+    if (!entry) {
+      const t = this.manager.target();
+      return t ? { target: t } : { status: 503, message: `No model matching '${requestedModel}' found. Add one in TurboLLM.` };
+    }
+    {
+      const ms = this.manager.status();
+      if (ms.state === "running" && ms.model && this.keysMatch(ms.model.key, entry)) {
+        this.primaryLastUsed = Date.now();
+        this.manager.touch();
+        return { target: this.manager.target() };
+      }
+    }
+    const slot = this.extraSlots.get(entry.key);
+    if (slot) {
+      const ss = slot.manager.status();
+      if (ss.state === "running") {
+        slot.lastUsedMs = Date.now();
+        slot.manager.touch();
+        return { target: slot.manager.target() };
+      }
+      this.extraSlots.delete(entry.key);
+    }
+    let unlock;
+    const prev = this.swapChain;
+    this.swapChain = new Promise((r) => {
+      unlock = r;
+    });
+    try {
+      await prev;
+      return await this.doLoad(entry);
+    } finally {
+      unlock();
+    }
+  }
+  // ── internal ──────────────────────────────────────────────────────────────
+  async doLoad(entry) {
+    {
+      const ms = this.manager.status();
+      if (ms.state === "running" && ms.model && this.keysMatch(ms.model.key, entry)) {
+        this.primaryLastUsed = Date.now();
+        this.manager.touch();
+        return { target: this.manager.target() };
+      }
+      const slot = this.extraSlots.get(entry.key);
+      if (slot && slot.manager.status().state === "running") {
+        slot.lastUsedMs = Date.now();
+        slot.manager.touch();
+        return { target: slot.manager.target() };
+      }
+    }
+    if (this.comfy?.isBlocked()) {
+      return { status: 503, message: "ComfyUI is rendering \u2014 model swap paused until its queue finishes." };
+    }
+    const active = this.registry.active();
+    if (!active) return { status: 503, message: "No active engine. Set one up in TurboLLM." };
+    if (!engineAcceptsFormat(active.kind, entry.format)) {
+      return { status: 503, message: `Active engine cannot load model format '${entry.format}'.` };
+    }
+    const opts = this.buildOpts(entry, active);
+    if (!opts) return { status: 503, message: "Model is incomplete or unreadable." };
+    const keepN = Math.max(1, this.store.snapshot().gateway.keepN);
+    const needsNewSlot = entry.embedding || this.chatSlotCount() < keepN;
+    const targetManager = needsNewSlot ? this.manager.status().state === "stopped" || this.manager.status().state === "error" ? this.manager : new Manager(this.store) : this.evictChatLru();
+    await targetManager.stopAndWait();
+    await this.comfy?.freeComfyUIBeforeLoad();
+    try {
+      await targetManager.start(opts);
+    } catch (e) {
+      return { status: 503, message: `Engine start failed: ${e.message}` };
+    }
+    const ready = await this.waitReady(targetManager, active.kind);
+    if (!ready) {
+      const s = targetManager.status();
+      return { status: 503, message: s.err?.message ?? "Model failed to become ready." };
+    }
+    const target = targetManager.target();
+    if (!target) return { status: 503, message: "Model loaded but target URL unavailable." };
+    if (targetManager === this.manager) {
+      this.primaryLastUsed = Date.now();
+    } else {
+      this.extraSlots.set(entry.key, { manager: targetManager, modelKey: entry.key, lastUsedMs: Date.now() });
+    }
+    this.store.update((x) => {
+      x.lastLoaded = { modelKey: entry.key, engineId: active.id };
+    });
+    return { target };
+  }
+  /** Count of alive chat (non-embedding) slots. Embedding models don't consume
+   *  a keepN slot so chat models and embedding models can coexist independently. */
+  chatSlotCount() {
+    const isAlive = (s) => s === "running" || s === "starting";
+    const ms = this.manager.status();
+    const primaryAlive = isAlive(ms.state);
+    const primaryEmbed = primaryAlive && !!ms.model && (this.scanner.get(ms.model.key)?.embedding ?? false);
+    const extraChat = [...this.extraSlots.values()].filter(
+      (s) => isAlive(s.manager.status().state) && !(this.scanner.get(s.modelKey)?.embedding ?? false)
+    ).length;
+    return (primaryAlive && !primaryEmbed ? 1 : 0) + extraChat;
+  }
+  /** Evict the least-recently-used chat (non-embedding) slot. Embedding slots are
+   *  skipped; if every alive slot is an embedding model the true LRU is used as
+   *  a fallback so we never deadlock. */
+  evictChatLru() {
+    const isAlive = (s) => s === "running" || s === "starting";
+    const ms = this.manager.status();
+    const primaryAlive = isAlive(ms.state);
+    const primaryEmbed = primaryAlive && !!ms.model && (this.scanner.get(ms.model.key)?.embedding ?? false);
+    let lruManager = this.manager;
+    let lruTime = primaryAlive && !primaryEmbed ? this.primaryLastUsed : Infinity;
+    let lruKey = null;
+    for (const slot of this.extraSlots.values()) {
+      const slotEmbed = this.scanner.get(slot.modelKey)?.embedding ?? false;
+      if (isAlive(slot.manager.status().state) && !slotEmbed && slot.lastUsedMs < lruTime) {
+        lruTime = slot.lastUsedMs;
+        lruManager = slot.manager;
+        lruKey = slot.modelKey;
+      }
+    }
+    if (lruTime === Infinity) {
+      lruTime = primaryAlive ? this.primaryLastUsed : Infinity;
+      lruManager = this.manager;
+      lruKey = null;
+      for (const slot of this.extraSlots.values()) {
+        if (isAlive(slot.manager.status().state) && slot.lastUsedMs < lruTime) {
+          lruTime = slot.lastUsedMs;
+          lruManager = slot.manager;
+          lruKey = slot.modelKey;
+        }
+      }
+    }
+    if (lruKey !== null) this.extraSlots.delete(lruKey);
+    return lruManager;
+  }
+  resolveEntry(requested) {
+    const models = this.scanner.list().models;
+    return models.find((e) => e.key === requested) ?? models.find((e) => e.name === requested) ?? models.find((e) => e.name.toLowerCase() === requested.toLowerCase()) ?? models.find((e) => e.name.toLowerCase().includes(requested.toLowerCase()));
+  }
+  keysMatch(loadedKey, entry) {
+    return loadedKey === entry.key || loadedKey === entry.path;
+  }
+  buildOpts(entry, engine) {
+    if (entry.incomplete || entry.parseError) return null;
+    const cfg2 = this.store.snapshot();
+    const sys = getSysInfo();
+    if (entry.format !== "gguf") {
+      const savedGpu = cfg2.modelProfiles[entry.key]?.gpu;
+      return {
+        engine,
+        model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: entry.nativeCtx, vision: false },
+        modelPath: entry.path,
+        extraArgs: [],
+        tensorParallelSize: savedGpu?.tensorParallelSize
+      };
+    }
+    const saved = cfg2.modelProfiles[entry.key];
+    const profile = resolveProfile(entry, sys, saved, void 0, cfg2.modelDefaults);
+    return {
+      engine,
+      model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: profile.ctx, vision: entry.vision },
+      modelPath: entry.path,
+      extraArgs: profileToArgs(profile, entry, engine.capabilities, sys.cores)
+    };
+  }
+  /** Poll until the manager's engine process becomes ready or fails.
+   *  Mirrors the Manager's internal readiness timeout by engine kind. */
+  async waitReady(manager2, engineKind) {
+    const timeoutMs = engineKind === "vllm" ? 6e5 : 12e4;
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+      const s = manager2.status();
+      if (s.state === "running") return true;
+      if (s.state === "error" || s.state === "stopped") return false;
+      await sleep3(250);
+    }
+    return false;
+  }
+};
+function sleep3(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
 // src/cli-launch.ts
 import { spawn as spawn2 } from "child_process";
 var SUPPORTED = {
@@ -3591,7 +3879,7 @@ import { existsSync as existsSync11, mkdirSync as mkdirSync8, readFileSync as re
 import { basename as basename3, dirname as dirname5, join as join12, resolve, sep } from "path";
 
 // src/comfyui/gate-template.ts
-var GATE_VERSION = 1;
+var GATE_VERSION = 2;
 function gateNodeSource(baseUrl) {
   const base2 = baseUrl.replace(/\/+$/, "");
   const baseLit = JSON.stringify(base2);
@@ -3648,6 +3936,19 @@ def _acquire_blocking() -> None:
     _post("/api/v1/comfyui/acquire", ACQUIRE_TIMEOUT_SEC)
 
 
+def _free_self() -> None:
+    """Unload ComfyUI's own models and clear the CUDA cache so VRAM is available
+    for TurboLLM to reload its model. Called on the release edge (queue drained),
+    before notifying TurboLLM. In-process \u2014 no HTTP roundtrip."""
+    try:
+        import comfy.model_management as mm  # type: ignore[import]
+        mm.unload_all_models()
+        mm.soft_empty_cache()
+        _log("freed ComfyUI model cache.")
+    except Exception as e:  # noqa: BLE001
+        _log(f"could not free ComfyUI models ({e}); continuing.")
+
+
 def _release() -> None:
     _post("/api/v1/comfyui/release", RELEASE_TIMEOUT_SEC)
 
@@ -3697,6 +3998,7 @@ def _install_hooks() -> None:
                     _busy = False
                     do_release = True
             if do_release:
+                _free_self()
                 _release()
         return result
 
@@ -3802,6 +4104,7 @@ function registerApi(app2, d) {
     const engine = {
       id: active?.id ?? "",
       name: active?.name ?? "",
+      kind: active?.kind ?? "",
       state: ms.state,
       port: ms.port,
       pid: ms.pid
@@ -3822,7 +4125,22 @@ function registerApi(app2, d) {
       downloads: { active: d.downloads.activeCount() },
       engineProvision: d.provision.get(),
       // ComfyUI GPU coordination: lets the UI explain a paused/unloaded engine.
-      comfyui: d.comfy?.snapshot() ?? null,
+      // Also expose the installed gate node version so the UI can prompt an upgrade.
+      comfyui: (() => {
+        const snap = d.comfy?.snapshot() ?? null;
+        if (!snap) return null;
+        const gatePath = d.store.snapshot().comfyui.gatePath;
+        let installedVersion = null;
+        if (gatePath) {
+          try {
+            const src = readFileSync6(join12(gatePath, "__init__.py"), "utf-8");
+            const m = src.match(/^GATE_VERSION\s*=\s*(\d+)/m);
+            if (m) installedVersion = Number(m[1]);
+          } catch {
+          }
+        }
+        return { ...snap, installedVersion, currentVersion: GATE_VERSION };
+      })(),
       telemetryLevel: d.store.snapshot().telemetry.level,
       uptimeSec: Math.floor((Date.now() - d.startedAt) / 1e3)
     });
@@ -4382,11 +4700,19 @@ function registerApi(app2, d) {
       }
       cuUpdates.url = u;
     }
+    const gwUpdates = {};
+    if (b.gateway?.autoSwap !== void 0) gwUpdates.autoSwap = !!b.gateway.autoSwap;
+    if (b.gateway?.keepN !== void 0) {
+      const v = Number(b.gateway.keepN);
+      if (!Number.isInteger(v) || v < 1 || v > 4) return err(c, 400, "invalid_config_value", "gateway.keepN must be 1\u20134.");
+      gwUpdates.keepN = v;
+    }
     const before = d.store.snapshot().daemon;
     d.store.update((cfg2) => {
       Object.assign(cfg2.daemon, updates);
       Object.assign(cfg2.modelDefaults, mdUpdates);
       Object.assign(cfg2.comfyui, cuUpdates);
+      Object.assign(cfg2.gateway, gwUpdates);
       if (b.autoLoadOnStart !== void 0) cfg2.autoLoadOnStart = !!b.autoLoadOnStart;
       if (telemetryLevel !== void 0) cfg2.telemetry.level = telemetryLevel;
       if (b.hfToken !== void 0) cfg2.hf.token = String(b.hfToken).trim();
@@ -4455,7 +4781,7 @@ function registerApi(app2, d) {
     const q = (c.req.query("q") ?? "").trim();
     if (!q) return c.json({ results: [] });
     try {
-      const results = await d.hf.searchModels(q);
+      const results = await d.hf.searchModels(q, d.registry.active()?.kind);
       const withLocal = results.map((r) => ({ ...r, localCount: localCountFor(d, r.repo) }));
       return c.json({ results: withLocal });
     } catch (e) {
@@ -4651,6 +4977,7 @@ function settingsPayload(d) {
     telemetryLevel,
     modelDefaults: cfg2.modelDefaults,
     comfyui: cfg2.comfyui,
+    gateway: cfg2.gateway,
     // The HF token is write-only over the wire (spec 10 §4): we never echo it back,
     // only whether one is set, so the UI can show "configured" without leaking it.
     hfTokenSet: cfg2.hf.token.length > 0
@@ -5075,17 +5402,33 @@ ${content}` : b.docContext : content;
 async function runGeneration(d, stream, ctx) {
   const { db: db2 } = d;
   const { convId, conv, engineMessages, assistantMsg, ms, target, ac, disableThinking } = ctx;
-  const modelSampling = {};
-  const convSampling = conv.sampling ?? {};
-  const merged = { ...modelSampling, ...convSampling };
+  const convS = conv.sampling ?? {};
+  const SAMPLING_KEYS = {
+    temp: "temperature",
+    topP: "top_p",
+    topK: "top_k",
+    minP: "min_p",
+    repeatPenalty: "repeat_penalty",
+    presencePenalty: "presence_penalty",
+    frequencyPenalty: "frequency_penalty"
+  };
+  const samplingOverride = {};
+  for (const [camel, snake] of Object.entries(SAMPLING_KEYS)) {
+    if (camel in convS) samplingOverride[snake] = convS[camel];
+  }
+  for (const [k, v] of Object.entries(convS)) {
+    if (!(k in SAMPLING_KEYS) && k !== "stop") samplingOverride[k] = v;
+  }
   const reqBody = {
     model: ms.model.key,
     messages: engineMessages,
     stream: true,
     stream_options: { include_usage: true },
     return_progress: true,
-    ...merged
+    ...samplingOverride
   };
+  const stopStrings = convS.stop;
+  if (stopStrings?.length) reqBody.stop = stopStrings;
   const maxLimit = d.store.snapshot().modelDefaults.maxTokens ?? 0;
   const cappedMax = clampMaxTokens(reqBody.max_tokens, maxLimit);
   if (cappedMax != null) reqBody.max_tokens = cappedMax;
@@ -5123,6 +5466,12 @@ async function runGeneration(d, stream, ctx) {
     const reader = res.body.getReader();
     const decoder = new TextDecoder();
     let buf = "";
+    const cancelReader = () => void reader.cancel();
+    if (ac.signal.aborted) {
+      cancelReader();
+    } else {
+      ac.signal.addEventListener("abort", cancelReader, { once: true });
+    }
     outer: while (true) {
       const { done, value } = await reader.read();
       if (done) break;
@@ -5234,6 +5583,7 @@ async function runGeneration(d, stream, ctx) {
         }
       }
     }
+    ac.signal.removeEventListener("abort", cancelReader);
     if (pendingThinkBuf && inThink) {
       fullReasoning += pendingThinkBuf;
       await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: pendingThinkBuf }) });
@@ -5643,13 +5993,6 @@ function cbStop(index) {
 // src/gateway/gateway.ts
 function registerGateway(app2, d) {
   app2.post("/v1/messages", async (c) => {
-    const target = d.manager.target();
-    if (!target) {
-      return c.json(
-        { type: "error", error: { type: "api_error", message: "No model loaded. Load one in TurboLLM." } },
-        503
-      );
-    }
     let req;
     try {
       req = await c.req.json();
@@ -5667,7 +6010,14 @@ function registerGateway(app2, d) {
     }
     const maxLimit = d.store.snapshot().modelDefaults.maxTokens ?? 0;
     req.max_tokens = clampMaxTokens(req.max_tokens, maxLimit) ?? req.max_tokens;
-    d.manager.touch();
+    const routeResult = await d.modelRouter.route(req.model ?? "");
+    if ("status" in routeResult) {
+      return c.json(
+        { type: "error", error: { type: "api_error", message: routeResult.message } },
+        routeResult.status
+      );
+    }
+    const target = routeResult.target;
     const status = d.manager.status();
     const modelName = status.state === "running" ? status.model?.name ?? req.model ?? "local" : req.model ?? "local";
     const oaiBody = mapToOpenAI(req);
@@ -5762,46 +6112,40 @@ function registerGateway(app2, d) {
     return c.json({ input_tokens: estimate });
   });
   app2.all("/v1/*", async (c) => {
-    const target = d.manager.target();
-    if (!target) {
-      if (c.req.method === "GET" && c.req.path === "/v1/models") {
+    const url = new URL(c.req.url);
+    const isChat = c.req.method === "POST" && url.pathname === "/v1/chat/completions";
+    let parsedBody = null;
+    if (isChat) {
+      try {
+        parsedBody = await c.req.json();
+      } catch {
+        parsedBody = null;
+      }
+    }
+    const requestedModel = isChat ? parsedBody?.model ?? "" : "";
+    const routeResult = await d.modelRouter.route(requestedModel);
+    if ("status" in routeResult) {
+      if (c.req.method === "GET" && url.pathname === "/v1/models") {
         return c.json({ object: "list", data: [] });
       }
       return c.json(
-        {
-          error: {
-            message: "No model loaded. Load one in TurboLLM.",
-            type: "model_not_loaded",
-            code: "model_not_loaded"
-          }
-        },
+        { error: { message: routeResult.message, type: "model_not_loaded", code: "model_not_loaded" } },
         503
       );
     }
-    d.manager.touch();
-    const url = new URL(c.req.url);
+    const target = routeResult.target;
     const upstream = target + url.pathname + url.search;
     const headers = new Headers(c.req.raw.headers);
     headers.delete("host");
-    const isChat = c.req.method === "POST" && url.pathname === "/v1/chat/completions";
     const maxLimit = d.store.snapshot().modelDefaults.maxTokens ?? 0;
     const init = { method: c.req.method, headers };
     if (c.req.method !== "GET" && c.req.method !== "HEAD") {
-      if (isChat && maxLimit > 0) {
-        let parsed = null;
-        try {
-          parsed = await c.req.json();
-        } catch {
-          parsed = null;
-        }
-        if (parsed) {
-          parsed.max_tokens = clampMaxTokens(parsed.max_tokens, maxLimit);
-          headers.delete("content-length");
-          init.body = JSON.stringify(parsed);
-        } else {
-          init.body = c.req.raw.body;
-          init.duplex = "half";
+      if (isChat) {
+        if (parsedBody && maxLimit > 0) {
+          parsedBody.max_tokens = clampMaxTokens(parsedBody.max_tokens, maxLimit);
         }
+        headers.delete("content-length");
+        init.body = parsedBody ? JSON.stringify(parsedBody) : "";
       } else {
         init.body = c.req.raw.body;
         init.duplex = "half";
@@ -6085,8 +6429,9 @@ var hf = new HfClient(() => store.snapshot().hf.token, version);
 var downloads = new DownloadManager(store, () => void scanner.rescan(), () => hf.authHeaders());
 var bench = new BenchRunner(manager, store, scanner, registry, version);
 var comfy = new ComfyGuard(store, manager);
+var modelRouter = new ModelRouter(store, registry, manager, scanner, comfy);
 var startedAt = Date.now();
-var deps = { store, registry, manager, scanner, hashes, db, provision, hf, downloads, bench, comfy, version, startedAt };
+var deps = { store, registry, manager, scanner, hashes, db, provision, hf, downloads, bench, modelRouter, comfy, version, startedAt };
 var app = createApp(deps);
 var cfg = store.snapshot();
 var defaultHost = cfg.daemon.lanBind ? "0.0.0.0" : cfg.daemon.host || "127.0.0.1";