turbollm 0.5.0 → 0.6.0

package/dist/cli.js

@@ -87,7 +87,8 @@ function defaultConfig() {
     hf: { token: "" },
     modelDefaults: { ctx: 8192, ngl: 99, imageMaxTokens: 0, maxTokens: 0 },
     featuredOverrideUrl: "",
-    comfyui: { enabled: false, gatePath: "", url: "", reverseGate: false, cachePersist: false }
+    comfyui: { enabled: false, gatePath: "", url: "", reverseGate: false, cachePersist: false },
+    gateway: { autoSwap: true, keepN: 1 }
   };
 }
 var ConfigStore = class _ConfigStore {
@@ -228,6 +229,11 @@ function normalize(c) {
     // Reseated like the other known fields so it isn't dropped on every load.
     cachePersist: !!cu.cachePersist
   };
+  const gw = c.gateway ?? {};
+  c.gateway = {
+    autoSwap: gw.autoSwap !== false,
+    keepN: typeof gw.keepN === "number" && gw.keepN >= 1 ? Math.min(Math.floor(gw.keepN), 4) : 1
+  };
   c.telemetry.level = normalizeTelemetryLevel(c.telemetry.level);
   for (const e of c.engines) {
     e.capabilities ??= { kvTypes: [], flags: [] };
@@ -1966,6 +1972,20 @@ function quantFromName(filename) {
 // src/models/scanner.ts
 var CACHE_VERSION = 2;
 var SPLIT_RE = /^(.*)-(\d{5})-of-(\d{5})\.gguf$/i;
+var EMBED_ARCHS = /* @__PURE__ */ new Set([
+  "bert",
+  "nomic-bert",
+  "jina-bert-v3-base",
+  "jina-bert",
+  "distilbert",
+  "roberta",
+  "xlm-roberta",
+  "electra"
+]);
+var EMBED_FILE_RE = /\b(bge[-_]|nomic[-_]embed|all[-_]minilm|e5[-_]|gte[-_]|stella[-_]embed|jina[-_]embed|mxbai[-_]embed)\b/i;
+function isEmbeddingModel(arch2, name) {
+  return EMBED_ARCHS.has(arch2.toLowerCase()) || EMBED_FILE_RE.test(name);
+}
 var ScannerError = class extends Error {
   constructor(code, message) {
     super(message);
@@ -2111,6 +2131,7 @@ var Scanner = class {
     const quant = meta?.quant || quantFromName(fileName);
     const name = meta?.name || cleanName(fileName);
     const vision = mmprojPath !== null;
+    const arch2 = meta?.arch ?? "unknown";
     return {
       key: `${name.toLowerCase()}|${quant}|${sizeBytes}`,
       name,
@@ -2119,7 +2140,7 @@ var Scanner = class {
       format: "gguf",
       sizeBytes,
       sizeLabel: meta?.sizeLabel ?? "",
-      arch: meta?.arch ?? "unknown",
+      arch: arch2,
       quant,
       nativeCtx: meta?.nativeCtx ?? 0,
       blockCount: meta?.blockCount ?? 0,
@@ -2130,6 +2151,7 @@ var Scanner = class {
       vision,
       mmprojPath: vision ? mmprojPath : null,
       hasChatTemplate: meta?.hasChatTemplate ?? false,
+      embedding: isEmbeddingModel(arch2, fileName),
       incomplete,
       parseError,
       loaded: false,
@@ -2240,6 +2262,7 @@ function mlxEntryFor(dir) {
     vision: false,
     mmprojPath: null,
     hasChatTemplate,
+    embedding: isEmbeddingModel(arch2, basename(dir)),
     incomplete: false,
     parseError,
     loaded: false,
@@ -2404,6 +2427,7 @@ function deriveDefault(m, sys) {
     ropeFreqBase: 0,
     ropeFreqScale: 0,
     gpu: defaultGpu(),
+    grammar: "",
     extraArgs: []
   };
   if (m.moe && hasGpu && m.blockCount > 0) {
@@ -2493,6 +2517,8 @@ function profileToArgs(p, m, caps, cores = 0) {
     if (p.ropeFreqBase > 0 && has("--rope-freq-base")) a.push("--rope-freq-base", String(p.ropeFreqBase));
     if (p.ropeFreqScale > 0 && has("--rope-freq-scale")) a.push("--rope-freq-scale", String(p.ropeFreqScale));
   }
+  if (m.embedding && has("--embeddings")) a.push("--embeddings");
+  if (p.grammar && has("--grammar")) a.push("--grammar", p.grammar);
   a.push(...p.extraArgs);
   return a;
 }
@@ -2703,10 +2729,15 @@ var HfClient = class {
   tokenFn;
   version;
   cache = /* @__PURE__ */ new Map();
-  /** Search GGUF repos (spec 10 §2). Returns up to 30 rows sorted by downloads. */
-  async searchModels(query) {
+  /** Search repos (spec 10 §2). Returns up to 30 rows sorted by downloads.
+   *  Format filter adapts to the active engine kind:
+   *  - llama-server / TurboQuant → filter=gguf
+   *  - mlx                       → filter=mlx (HF library tag)
+   *  - vllm                      → no format filter (searches all HF repos) */
+  async searchModels(query, engineKind) {
     const q = query.trim();
-    const url = `${BASE}/api/models?search=${encodeURIComponent(q)}&filter=gguf&sort=downloads&direction=-1&limit=30&full=false`;
+    const formatFilter = engineKind === "mlx" ? "&filter=mlx" : engineKind === "vllm" ? "" : "&filter=gguf";
+    const url = `${BASE}/api/models?search=${encodeURIComponent(q)}${formatFilter}&sort=downloads&direction=-1&limit=30&full=false`;
     const raw = await this.getJson(url);
     return raw.map((m) => ({
       repo: m.id ?? m.modelId ?? "",
@@ -2722,8 +2753,30 @@ var HfClient = class {
   async getRepo(repo) {
     const info = await this.getJson(`${BASE}/api/models/${repo}`);
     const tree = await this.getJson(`${BASE}/api/models/${repo}/tree/main?recursive=true`);
-    const gguf = tree.filter((e) => e.type === "file" && /\.gguf$/i.test(e.path));
-    const files = groupFiles(repo, gguf);
+    const ggufEntries = tree.filter((e) => e.type === "file" && /\.gguf$/i.test(e.path));
+    const safetensorsEntries = tree.filter((e) => e.type === "file" && /\.safetensors$/i.test(e.path));
+    const isSafetensors = ggufEntries.length === 0 && safetensorsEntries.length > 0;
+    let files;
+    let safetensors;
+    if (isSafetensors) {
+      safetensors = true;
+      const components = tree.filter(
+        (e) => e.type === "file" && (/\.safetensors$/i.test(e.path) || /\.json$/i.test(e.path)) && !e.path.includes("/")
+        // root-level only — no nested model card assets
+      );
+      files = components.map((e) => ({
+        name: e.path,
+        quant: "mlx",
+        sizeBytes: e.lfs?.size ?? e.size ?? 0,
+        parts: 1,
+        mmproj: false,
+        safetensors: true,
+        sha256: e.lfs?.oid,
+        url: this.fileUrl(repo, e.path)
+      }));
+    } else {
+      files = groupFiles(repo, ggufEntries);
+    }
     const gated = info.gated === true || info.gated === "auto" || info.gated === "manual";
     const license = info.cardData?.license ?? (info.tags?.find((t) => t.startsWith("license:"))?.slice("license:".length) || "");
     return {
@@ -2733,7 +2786,8 @@ var HfClient = class {
       downloads: info.downloads ?? 0,
       likes: info.likes ?? 0,
       card: await this.getCard(repo),
-      files
+      files,
+      ...safetensors ? { safetensors } : {}
     };
   }
   /** Fetch the repo README (the model card), strip its YAML frontmatter, and cap the
@@ -2928,27 +2982,30 @@ var DownloadManager = class {
     const dir = this.primaryDir();
     if (!dir) throw new DownloadError("no_model_dir", "Add a model folder in Settings before downloading.");
     let repo = (input.repo ?? "").trim();
+    const subdir = (input.subdir ?? "").trim();
     let url;
     let filename;
     if (input.url) {
       const u = input.url.trim();
       if (!/^https?:\/\//i.test(u)) throw new DownloadError("invalid_url", "URL must start with http:// or https://.");
       const path = safePathname(u);
-      if (!/\.gguf$/i.test(path) && !HF_BLOB_RE.test(u)) {
+      if (!subdir && !/\.gguf$/i.test(path) && !HF_BLOB_RE.test(u)) {
         throw new DownloadError("invalid_url", "URL must point to a .gguf file.");
       }
       filename = basename2(path);
-      if (!/\.gguf$/i.test(filename)) throw new DownloadError("invalid_url", "Could not derive a .gguf filename from that URL.");
+      if (!subdir && !/\.gguf$/i.test(filename)) throw new DownloadError("invalid_url", "Could not derive a .gguf filename from that URL.");
       url = u;
       repo = "";
     } else {
       const rfilename = (input.rfilename ?? "").trim();
       if (!repo || !rfilename) throw new DownloadError("invalid_request", "repo and rfilename are required.");
-      if (!/\.gguf$/i.test(rfilename)) throw new DownloadError("invalid_url", "The file must be a .gguf.");
+      if (!subdir && !/\.gguf$/i.test(rfilename)) throw new DownloadError("invalid_url", "The file must be a .gguf.");
       filename = basename2(rfilename);
       url = `https://huggingface.co/${repo}/resolve/main/${rfilename}`;
     }
     const total = input.size ?? 0;
+    const destDir = subdir ? join10(dir, subdir) : dir;
+    if (subdir) mkdirSync6(destDir, { recursive: true });
     if (total > 0) this.assertDisk(dir, total);
     const id = `dl-${Date.now().toString(36)}-${(this.nextSeq++).toString(36)}`;
     const rec = {
@@ -2956,7 +3013,7 @@ var DownloadManager = class {
       name: filename,
       repo,
       url,
-      dest: join10(dir, filename),
+      dest: join10(destDir, filename),
       total,
       received: 0,
       status: "queued",
@@ -3526,6 +3583,219 @@ function sleep2(ms) {
   return new Promise((r) => setTimeout(r, ms));
 }
 
+// src/gateway/model-router.ts
+var ModelRouter = class {
+  constructor(store2, registry2, manager2, scanner2, comfy2) {
+    this.store = store2;
+    this.registry = registry2;
+    this.manager = manager2;
+    this.scanner = scanner2;
+    this.comfy = comfy2;
+  }
+  store;
+  registry;
+  manager;
+  scanner;
+  comfy;
+  /** Extra pool slots beyond the primary manager. Only populated when keepN > 1. */
+  extraSlots = /* @__PURE__ */ new Map();
+  /** Last-used timestamp for the primary manager slot (for LRU eviction). */
+  primaryLastUsed = 0;
+  /** Promise chain that serialises swap operations so concurrent requests for
+   *  different models queue rather than race. */
+  swapChain = Promise.resolve();
+  /** Route a request to the correct model target URL.
+   *  - If autoSwap is off: returns whatever the primary manager has loaded.
+   *  - If the requested model is already loaded: returns its target immediately.
+   *  - Otherwise: loads the model (swapping / evicting LRU as needed) and waits. */
+  async route(requestedModel) {
+    const cfg2 = this.store.snapshot();
+    if (!cfg2.gateway.autoSwap || !requestedModel.trim()) {
+      const t = this.manager.target();
+      return t ? { target: t } : { status: 503, message: "No model loaded. Load one in TurboLLM." };
+    }
+    const entry = this.resolveEntry(requestedModel);
+    if (!entry) {
+      const t = this.manager.target();
+      return t ? { target: t } : { status: 503, message: `No model matching '${requestedModel}' found. Add one in TurboLLM.` };
+    }
+    {
+      const ms = this.manager.status();
+      if (ms.state === "running" && ms.model && this.keysMatch(ms.model.key, entry)) {
+        this.primaryLastUsed = Date.now();
+        this.manager.touch();
+        return { target: this.manager.target() };
+      }
+    }
+    const slot = this.extraSlots.get(entry.key);
+    if (slot) {
+      const ss = slot.manager.status();
+      if (ss.state === "running") {
+        slot.lastUsedMs = Date.now();
+        slot.manager.touch();
+        return { target: slot.manager.target() };
+      }
+      this.extraSlots.delete(entry.key);
+    }
+    let unlock;
+    const prev = this.swapChain;
+    this.swapChain = new Promise((r) => {
+      unlock = r;
+    });
+    try {
+      await prev;
+      return await this.doLoad(entry);
+    } finally {
+      unlock();
+    }
+  }
+  // ── internal ──────────────────────────────────────────────────────────────
+  async doLoad(entry) {
+    {
+      const ms = this.manager.status();
+      if (ms.state === "running" && ms.model && this.keysMatch(ms.model.key, entry)) {
+        this.primaryLastUsed = Date.now();
+        this.manager.touch();
+        return { target: this.manager.target() };
+      }
+      const slot = this.extraSlots.get(entry.key);
+      if (slot && slot.manager.status().state === "running") {
+        slot.lastUsedMs = Date.now();
+        slot.manager.touch();
+        return { target: slot.manager.target() };
+      }
+    }
+    if (this.comfy?.isBlocked()) {
+      return { status: 503, message: "ComfyUI is rendering \u2014 model swap paused until its queue finishes." };
+    }
+    const active = this.registry.active();
+    if (!active) return { status: 503, message: "No active engine. Set one up in TurboLLM." };
+    if (!engineAcceptsFormat(active.kind, entry.format)) {
+      return { status: 503, message: `Active engine cannot load model format '${entry.format}'.` };
+    }
+    const opts = this.buildOpts(entry, active);
+    if (!opts) return { status: 503, message: "Model is incomplete or unreadable." };
+    const keepN = Math.max(1, this.store.snapshot().gateway.keepN);
+    const needsNewSlot = entry.embedding || this.chatSlotCount() < keepN;
+    const targetManager = needsNewSlot ? this.manager.status().state === "stopped" || this.manager.status().state === "error" ? this.manager : new Manager(this.store) : this.evictChatLru();
+    await targetManager.stopAndWait();
+    await this.comfy?.freeComfyUIBeforeLoad();
+    try {
+      await targetManager.start(opts);
+    } catch (e) {
+      return { status: 503, message: `Engine start failed: ${e.message}` };
+    }
+    const ready = await this.waitReady(targetManager, active.kind);
+    if (!ready) {
+      const s = targetManager.status();
+      return { status: 503, message: s.err?.message ?? "Model failed to become ready." };
+    }
+    const target = targetManager.target();
+    if (!target) return { status: 503, message: "Model loaded but target URL unavailable." };
+    if (targetManager === this.manager) {
+      this.primaryLastUsed = Date.now();
+    } else {
+      this.extraSlots.set(entry.key, { manager: targetManager, modelKey: entry.key, lastUsedMs: Date.now() });
+    }
+    this.store.update((x) => {
+      x.lastLoaded = { modelKey: entry.key, engineId: active.id };
+    });
+    return { target };
+  }
+  /** Count of alive chat (non-embedding) slots. Embedding models don't consume
+   *  a keepN slot so chat models and embedding models can coexist independently. */
+  chatSlotCount() {
+    const isAlive = (s) => s === "running" || s === "starting";
+    const ms = this.manager.status();
+    const primaryAlive = isAlive(ms.state);
+    const primaryEmbed = primaryAlive && !!ms.model && (this.scanner.get(ms.model.key)?.embedding ?? false);
+    const extraChat = [...this.extraSlots.values()].filter(
+      (s) => isAlive(s.manager.status().state) && !(this.scanner.get(s.modelKey)?.embedding ?? false)
+    ).length;
+    return (primaryAlive && !primaryEmbed ? 1 : 0) + extraChat;
+  }
+  /** Evict the least-recently-used chat (non-embedding) slot. Embedding slots are
+   *  skipped; if every alive slot is an embedding model the true LRU is used as
+   *  a fallback so we never deadlock. */
+  evictChatLru() {
+    const isAlive = (s) => s === "running" || s === "starting";
+    const ms = this.manager.status();
+    const primaryAlive = isAlive(ms.state);
+    const primaryEmbed = primaryAlive && !!ms.model && (this.scanner.get(ms.model.key)?.embedding ?? false);
+    let lruManager = this.manager;
+    let lruTime = primaryAlive && !primaryEmbed ? this.primaryLastUsed : Infinity;
+    let lruKey = null;
+    for (const slot of this.extraSlots.values()) {
+      const slotEmbed = this.scanner.get(slot.modelKey)?.embedding ?? false;
+      if (isAlive(slot.manager.status().state) && !slotEmbed && slot.lastUsedMs < lruTime) {
+        lruTime = slot.lastUsedMs;
+        lruManager = slot.manager;
+        lruKey = slot.modelKey;
+      }
+    }
+    if (lruTime === Infinity) {
+      lruTime = primaryAlive ? this.primaryLastUsed : Infinity;
+      lruManager = this.manager;
+      lruKey = null;
+      for (const slot of this.extraSlots.values()) {
+        if (isAlive(slot.manager.status().state) && slot.lastUsedMs < lruTime) {
+          lruTime = slot.lastUsedMs;
+          lruManager = slot.manager;
+          lruKey = slot.modelKey;
+        }
+      }
+    }
+    if (lruKey !== null) this.extraSlots.delete(lruKey);
+    return lruManager;
+  }
+  resolveEntry(requested) {
+    const models = this.scanner.list().models;
+    return models.find((e) => e.key === requested) ?? models.find((e) => e.name === requested) ?? models.find((e) => e.name.toLowerCase() === requested.toLowerCase()) ?? models.find((e) => e.name.toLowerCase().includes(requested.toLowerCase()));
+  }
+  keysMatch(loadedKey, entry) {
+    return loadedKey === entry.key || loadedKey === entry.path;
+  }
+  buildOpts(entry, engine) {
+    if (entry.incomplete || entry.parseError) return null;
+    const cfg2 = this.store.snapshot();
+    const sys = getSysInfo();
+    if (entry.format !== "gguf") {
+      const savedGpu = cfg2.modelProfiles[entry.key]?.gpu;
+      return {
+        engine,
+        model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: entry.nativeCtx, vision: false },
+        modelPath: entry.path,
+        extraArgs: [],
+        tensorParallelSize: savedGpu?.tensorParallelSize
+      };
+    }
+    const saved = cfg2.modelProfiles[entry.key];
+    const profile = resolveProfile(entry, sys, saved, void 0, cfg2.modelDefaults);
+    return {
+      engine,
+      model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: profile.ctx, vision: entry.vision },
+      modelPath: entry.path,
+      extraArgs: profileToArgs(profile, entry, engine.capabilities, sys.cores)
+    };
+  }
+  /** Poll until the manager's engine process becomes ready or fails.
+   *  Mirrors the Manager's internal readiness timeout by engine kind. */
+  async waitReady(manager2, engineKind) {
+    const timeoutMs = engineKind === "vllm" ? 6e5 : 12e4;
+    const deadline = Date.now() + timeoutMs;
+    while (Date.now() < deadline) {
+      const s = manager2.status();
+      if (s.state === "running") return true;
+      if (s.state === "error" || s.state === "stopped") return false;
+      await sleep3(250);
+    }
+    return false;
+  }
+};
+function sleep3(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
 // src/cli-launch.ts
 import { spawn as spawn2 } from "child_process";
 var SUPPORTED = {
@@ -3834,6 +4104,7 @@ function registerApi(app2, d) {
     const engine = {
       id: active?.id ?? "",
       name: active?.name ?? "",
+      kind: active?.kind ?? "",
       state: ms.state,
       port: ms.port,
       pid: ms.pid
@@ -4429,11 +4700,19 @@ function registerApi(app2, d) {
       }
       cuUpdates.url = u;
     }
+    const gwUpdates = {};
+    if (b.gateway?.autoSwap !== void 0) gwUpdates.autoSwap = !!b.gateway.autoSwap;
+    if (b.gateway?.keepN !== void 0) {
+      const v = Number(b.gateway.keepN);
+      if (!Number.isInteger(v) || v < 1 || v > 4) return err(c, 400, "invalid_config_value", "gateway.keepN must be 1\u20134.");
+      gwUpdates.keepN = v;
+    }
     const before = d.store.snapshot().daemon;
     d.store.update((cfg2) => {
       Object.assign(cfg2.daemon, updates);
       Object.assign(cfg2.modelDefaults, mdUpdates);
       Object.assign(cfg2.comfyui, cuUpdates);
+      Object.assign(cfg2.gateway, gwUpdates);
       if (b.autoLoadOnStart !== void 0) cfg2.autoLoadOnStart = !!b.autoLoadOnStart;
       if (telemetryLevel !== void 0) cfg2.telemetry.level = telemetryLevel;
       if (b.hfToken !== void 0) cfg2.hf.token = String(b.hfToken).trim();
@@ -4502,7 +4781,7 @@ function registerApi(app2, d) {
     const q = (c.req.query("q") ?? "").trim();
     if (!q) return c.json({ results: [] });
     try {
-      const results = await d.hf.searchModels(q);
+      const results = await d.hf.searchModels(q, d.registry.active()?.kind);
       const withLocal = results.map((r) => ({ ...r, localCount: localCountFor(d, r.repo) }));
       return c.json({ results: withLocal });
     } catch (e) {
@@ -4698,6 +4977,7 @@ function settingsPayload(d) {
     telemetryLevel,
     modelDefaults: cfg2.modelDefaults,
     comfyui: cfg2.comfyui,
+    gateway: cfg2.gateway,
     // The HF token is write-only over the wire (spec 10 §4): we never echo it back,
     // only whether one is set, so the UI can show "configured" without leaking it.
     hfTokenSet: cfg2.hf.token.length > 0
@@ -5713,13 +5993,6 @@ function cbStop(index) {
 // src/gateway/gateway.ts
 function registerGateway(app2, d) {
   app2.post("/v1/messages", async (c) => {
-    const target = d.manager.target();
-    if (!target) {
-      return c.json(
-        { type: "error", error: { type: "api_error", message: "No model loaded. Load one in TurboLLM." } },
-        503
-      );
-    }
     let req;
     try {
       req = await c.req.json();
@@ -5737,7 +6010,14 @@ function registerGateway(app2, d) {
     }
     const maxLimit = d.store.snapshot().modelDefaults.maxTokens ?? 0;
     req.max_tokens = clampMaxTokens(req.max_tokens, maxLimit) ?? req.max_tokens;
-    d.manager.touch();
+    const routeResult = await d.modelRouter.route(req.model ?? "");
+    if ("status" in routeResult) {
+      return c.json(
+        { type: "error", error: { type: "api_error", message: routeResult.message } },
+        routeResult.status
+      );
+    }
+    const target = routeResult.target;
     const status = d.manager.status();
     const modelName = status.state === "running" ? status.model?.name ?? req.model ?? "local" : req.model ?? "local";
     const oaiBody = mapToOpenAI(req);
@@ -5832,46 +6112,40 @@ function registerGateway(app2, d) {
     return c.json({ input_tokens: estimate });
   });
   app2.all("/v1/*", async (c) => {
-    const target = d.manager.target();
-    if (!target) {
-      if (c.req.method === "GET" && c.req.path === "/v1/models") {
+    const url = new URL(c.req.url);
+    const isChat = c.req.method === "POST" && url.pathname === "/v1/chat/completions";
+    let parsedBody = null;
+    if (isChat) {
+      try {
+        parsedBody = await c.req.json();
+      } catch {
+        parsedBody = null;
+      }
+    }
+    const requestedModel = isChat ? parsedBody?.model ?? "" : "";
+    const routeResult = await d.modelRouter.route(requestedModel);
+    if ("status" in routeResult) {
+      if (c.req.method === "GET" && url.pathname === "/v1/models") {
         return c.json({ object: "list", data: [] });
       }
       return c.json(
-        {
-          error: {
-            message: "No model loaded. Load one in TurboLLM.",
-            type: "model_not_loaded",
-            code: "model_not_loaded"
-          }
-        },
+        { error: { message: routeResult.message, type: "model_not_loaded", code: "model_not_loaded" } },
         503
       );
     }
-    d.manager.touch();
-    const url = new URL(c.req.url);
+    const target = routeResult.target;
     const upstream = target + url.pathname + url.search;
     const headers = new Headers(c.req.raw.headers);
     headers.delete("host");
-    const isChat = c.req.method === "POST" && url.pathname === "/v1/chat/completions";
     const maxLimit = d.store.snapshot().modelDefaults.maxTokens ?? 0;
     const init = { method: c.req.method, headers };
     if (c.req.method !== "GET" && c.req.method !== "HEAD") {
-      if (isChat && maxLimit > 0) {
-        let parsed = null;
-        try {
-          parsed = await c.req.json();
-        } catch {
-          parsed = null;
-        }
-        if (parsed) {
-          parsed.max_tokens = clampMaxTokens(parsed.max_tokens, maxLimit);
-          headers.delete("content-length");
-          init.body = JSON.stringify(parsed);
-        } else {
-          init.body = c.req.raw.body;
-          init.duplex = "half";
+      if (isChat) {
+        if (parsedBody && maxLimit > 0) {
+          parsedBody.max_tokens = clampMaxTokens(parsedBody.max_tokens, maxLimit);
         }
+        headers.delete("content-length");
+        init.body = parsedBody ? JSON.stringify(parsedBody) : "";
       } else {
         init.body = c.req.raw.body;
         init.duplex = "half";
@@ -6155,8 +6429,9 @@ var hf = new HfClient(() => store.snapshot().hf.token, version);
 var downloads = new DownloadManager(store, () => void scanner.rescan(), () => hf.authHeaders());
 var bench = new BenchRunner(manager, store, scanner, registry, version);
 var comfy = new ComfyGuard(store, manager);
+var modelRouter = new ModelRouter(store, registry, manager, scanner, comfy);
 var startedAt = Date.now();
-var deps = { store, registry, manager, scanner, hashes, db, provision, hf, downloads, bench, comfy, version, startedAt };
+var deps = { store, registry, manager, scanner, hashes, db, provision, hf, downloads, bench, modelRouter, comfy, version, startedAt };
 var app = createApp(deps);
 var cfg = store.snapshot();
 var defaultHost = cfg.daemon.lanBind ? "0.0.0.0" : cfg.daemon.host || "127.0.0.1";