turbollm 0.2.0 → 0.5.0

package/dist/cli.js

@@ -4,7 +4,7 @@
 import { spawn as spawn3 } from "child_process";
 import { openSync as openSync3, readFileSync as readFileSync8 } from "fs";
 import { serve } from "@hono/node-server";
-import { dirname as dirname7, join as join13 } from "path";
+import { dirname as dirname7, join as join14 } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // src/config/config.ts
@@ -87,7 +87,7 @@ function defaultConfig() {
     hf: { token: "" },
     modelDefaults: { ctx: 8192, ngl: 99, imageMaxTokens: 0, maxTokens: 0 },
     featuredOverrideUrl: "",
-    comfyui: { enabled: false, gatePath: "" }
+    comfyui: { enabled: false, gatePath: "", url: "", reverseGate: false, cachePersist: false }
   };
 }
 var ConfigStore = class _ConfigStore {
@@ -217,7 +217,17 @@ function normalize(c) {
   c.autoLoadOnStart ??= false;
   c.featuredOverrideUrl ??= "";
   const cu = c.comfyui ?? {};
-  c.comfyui = { enabled: !!cu.enabled, gatePath: typeof cu.gatePath === "string" ? cu.gatePath : "" };
+  c.comfyui = {
+    enabled: !!cu.enabled,
+    gatePath: typeof cu.gatePath === "string" ? cu.gatePath : "",
+    // Reverse gate (F-011): ComfyUI origin + opt-in toggle. Absent in pre-F-011 configs
+    // → '' / false. Reseated here (like the other known fields) so they aren't dropped.
+    url: typeof cu.url === "string" ? cu.url : "",
+    reverseGate: !!cu.reverseGate,
+    // KV prompt-cache persistence (F-014): opt-in. Absent in pre-F-014 configs → false.
+    // Reseated like the other known fields so it isn't dropped on every load.
+    cachePersist: !!cu.cachePersist
+  };
   c.telemetry.level = normalizeTelemetryLevel(c.telemetry.level);
   for (const e of c.engines) {
     e.capabilities ??= { kvTypes: [], flags: [] };
@@ -244,6 +254,9 @@ function validate(c) {
   if (c.activeEngineId && !c.engines.some((e) => e.id === c.activeEngineId)) {
     throw new ValueError("activeEngineId", "unknown engine id");
   }
+  if (c.comfyui.url && !/^https?:\/\//i.test(c.comfyui.url)) {
+    throw new ValueError("comfyui.url", "must be an http(s):// origin (e.g. http://127.0.0.1:8188)");
+  }
 }
 function isAbsolutePath(p) {
   return /^([a-zA-Z]:[\\/]|[\\/])/.test(p);
@@ -259,9 +272,9 @@ function clampMaxTokens(requested, limit) {
 
 // src/engines/manager.ts
 import { execFile as execFile4, spawn } from "child_process";
-import { createWriteStream as createWriteStream2, existsSync as existsSync5, mkdirSync as mkdirSync4, readFileSync as readFileSync2 } from "fs";
+import { createWriteStream as createWriteStream2, existsSync as existsSync6, mkdirSync as mkdirSync4, readFileSync as readFileSync2 } from "fs";
 import { createServer } from "net";
-import { dirname as dirname2, join as join5 } from "path";
+import { dirname as dirname2, join as join6 } from "path";
 
 // src/engines/mlx.ts
 import { existsSync as existsSync3, mkdirSync as mkdirSync3, rmSync as rmSync2 } from "fs";
@@ -557,21 +570,110 @@ function mlxServerCommand(python, model, port2, host2) {
   };
 }
 
+// src/engines/slot-cache.ts
+import { createHash } from "crypto";
+import { existsSync as existsSync4, readdirSync as readdirSync2, rmSync as rmSync3, statSync } from "fs";
+import { join as join4 } from "path";
+var SAVE_CAP_MS = (() => {
+  const v = Number(process.env.TURBOLLM_SLOT_CACHE_SAVE_CAP_MS);
+  return Number.isFinite(v) && v > 0 ? v : 2500;
+})();
+var TTL_MS = (() => {
+  const v = Number(process.env.TURBOLLM_SLOT_CACHE_TTL_MIN);
+  return Number.isFinite(v) && v > 0 ? v * 6e4 : 60 * 6e4;
+})();
+function slotCacheDir(dataDir) {
+  return join4(dataDir, "slot-cache");
+}
+function slotCacheKey(opts) {
+  const material = `${opts.modelPath}\0${opts.extraArgs.join(" ")}\0${opts.engine.version}`;
+  const hash = createHash("sha256").update(material).digest("hex").slice(0, 16);
+  return `slot-${hash}.bin`;
+}
+function parallelIsOne(extraArgs) {
+  const i = extraArgs.indexOf("--parallel");
+  if (i === -1) return true;
+  return extraArgs[i + 1] === "1";
+}
+function cacheEligible(opts, cfg2) {
+  return cfg2.enabled && cfg2.cachePersist && opts.engine.kind === "llama-server" && !opts.model.vision && parallelIsOne(opts.extraArgs);
+}
+function sweepExpired(dir, ttlMs, now) {
+  if (!existsSync4(dir)) return;
+  let names;
+  try {
+    names = readdirSync2(dir);
+  } catch {
+    return;
+  }
+  for (const name of names) {
+    if (!/^slot-.*\.bin$/.test(name)) continue;
+    const full = join4(dir, name);
+    try {
+      if (now - statSync(full).mtimeMs > ttlMs) rmSync3(full, { force: true });
+    } catch {
+    }
+  }
+}
+async function saveSlot(p) {
+  sweepExpired(p.dir, p.ttlMs, p.now);
+  try {
+    console.log(`[slot-cache] saving the prompt cache before ComfyUI takes the GPU (cap ${p.capMs}ms).`);
+    const res = await p.http(`${p.base}/slots/0?action=save`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ filename: p.filename }),
+      signal: AbortSignal.timeout(p.capMs)
+    });
+    if (!res.ok) {
+      console.warn(`[slot-cache] save returned ${res.status} \u2014 skipping the cache this cycle.`);
+      return false;
+    }
+    return true;
+  } catch (e) {
+    console.warn(`[slot-cache] could not save the prompt cache (${e instanceof Error ? e.message : e}) \u2014 skipping it this cycle.`);
+    return false;
+  }
+}
+async function restoreSlot(p) {
+  try {
+    console.log("[slot-cache] restoring the prompt cache after the ComfyUI reload (skipping the re-prefill).");
+    const res = await p.http(`${p.base}/slots/0?action=restore`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({ filename: p.filename }),
+      signal: AbortSignal.timeout(3e4)
+    });
+    if (!res.ok) {
+      console.warn(`[slot-cache] restore returned ${res.status} \u2014 the prompt will be re-prefilled normally.`);
+      return false;
+    }
+    try {
+      rmSync3(join4(p.dir, p.filename), { force: true });
+    } catch {
+    }
+    return true;
+  } catch (e) {
+    console.warn(`[slot-cache] could not restore the prompt cache (${e instanceof Error ? e.message : e}) \u2014 re-prefilling normally.`);
+    return false;
+  }
+}
+
 // src/engines/vllm.ts
-import { existsSync as existsSync4 } from "fs";
+import { existsSync as existsSync5 } from "fs";
 import { execFile as execFile3 } from "child_process";
-import { join as join4 } from "path";
+import { join as join5 } from "path";
 import { promisify as promisify3 } from "util";
 var execFileP3 = promisify3(execFile3);
 var VLLM_PYTHON = "3.12";
 function venvPython2(envDir) {
-  return process.platform === "win32" ? join4(envDir, "Scripts", "python.exe") : join4(envDir, "bin", "python");
+  return process.platform === "win32" ? join5(envDir, "Scripts", "python.exe") : join5(envDir, "bin", "python");
 }
 async function ensureVllmEnv(root, onProgress) {
   const uv = await ensureUv(root, onProgress);
-  const envDir = join4(root, "vllm", "venv");
+  const envDir = join5(root, "vllm", "venv");
   const py = venvPython2(envDir);
-  if (!existsSync4(py)) {
+  if (!existsSync5(py)) {
     onProgress?.({ phase: "extracting", pct: -1 });
     await execFileP3(uv, ["venv", "--python", VLLM_PYTHON, envDir], { cwd: root });
   }
@@ -591,20 +693,19 @@ async function probeVllm(python) {
   );
   return `vllm ${stdout.trim()}`;
 }
-function vllmServerCommand(python, model, port2, host2) {
-  return {
-    cmd: python,
-    args: [
-      "-m",
-      "vllm.entrypoints.openai.api_server",
-      "--model",
-      model,
-      "--host",
-      host2,
-      "--port",
-      String(port2)
-    ]
-  };
+function vllmServerCommand(python, model, port2, host2, tensorParallelSize = 1) {
+  const args = [
+    "-m",
+    "vllm.entrypoints.openai.api_server",
+    "--model",
+    model,
+    "--host",
+    host2,
+    "--port",
+    String(port2)
+  ];
+  if (tensorParallelSize > 1) args.push("--tensor-parallel-size", String(tensorParallelSize));
+  return { cmd: python, args };
 }
 
 // src/engines/manager.ts
@@ -657,14 +758,23 @@ var Manager = class {
     if (!opts.engine.binPath) throw new Error("no_active_engine");
     if (!opts.modelPath) throw new Error("no_such_model");
     const port2 = await allocPort();
-    const logPath = join5(this.store.dir(), "logs", `engine-${opts.engine.id}.log`);
+    const logPath = join6(this.store.dir(), "logs", `engine-${opts.engine.id}.log`);
     mkdirSync4(dirname2(logPath), { recursive: true });
     const logStream = createWriteStream2(logPath);
     logStream.write(
       `[turbollm] starting engine "${opts.engine.name}" on internal port ${port2} (127.0.0.1 only \u2014 the engine's own port, NOT the TurboLLM app/UI port).
 `
     );
-    const { cmd, args } = engineCommand(opts, port2);
+    const cfg2 = this.store.snapshot();
+    let slotSavePath;
+    if (cfg2.comfyui.enabled && cfg2.comfyui.cachePersist && opts.engine.kind === "llama-server") {
+      const flags = opts.engine.capabilities.flags;
+      if (flags.length === 0 || flags.includes("--slot-save-path")) {
+        slotSavePath = slotCacheDir(this.store.dir());
+        mkdirSync4(slotSavePath, { recursive: true });
+      }
+    }
+    const { cmd, args } = engineCommand(opts, port2, slotSavePath);
     const child = spawn(cmd, args, { cwd: dirname2(cmd), windowsHide: true });
     child.stdout?.pipe(logStream, { end: false });
     child.stderr?.pipe(logStream, { end: false });
@@ -870,23 +980,24 @@ var Manager = class {
     if (idle) this.stop();
   }
 };
-function engineCommand(opts, port2) {
+function engineCommand(opts, port2, slotSavePath) {
   if (opts.engine.kind === "mlx") {
     return mlxServerCommand(opts.engine.binPath, opts.modelPath, port2, "127.0.0.1");
   }
   if (opts.engine.kind === "vllm") {
-    return vllmServerCommand(opts.engine.binPath, opts.modelPath, port2, "127.0.0.1");
+    return vllmServerCommand(opts.engine.binPath, opts.modelPath, port2, "127.0.0.1", opts.tensorParallelSize);
   }
-  return { cmd: opts.engine.binPath, args: buildArgs(opts, port2) };
+  return { cmd: opts.engine.binPath, args: buildArgs(opts, port2, slotSavePath) };
 }
 function readinessTimeoutMs(kind) {
   return kind === "vllm" ? 6e5 : 12e4;
 }
-function buildArgs(opts, port2) {
+function buildArgs(opts, port2, slotSavePath) {
   const args = ["-m", opts.modelPath, "--host", "127.0.0.1", "--port", String(port2)];
   const flags = opts.engine.capabilities.flags;
   if (flags.length === 0 || flags.includes("--metrics")) args.push("--metrics");
   if (flags.includes("--no-webui")) args.push("--no-webui");
+  if (slotSavePath) args.push("--slot-save-path", slotSavePath);
   args.push(...opts.extraArgs);
   return args;
 }
@@ -945,7 +1056,7 @@ function forceKill(child) {
   }
 }
 function readTail(path, n) {
-  if (!path || !existsSync5(path)) return [];
+  if (!path || !existsSync6(path)) return [];
   try {
     const lines = readFileSync2(path, "utf8").replace(/[\r\n]+$/, "").split("\n").map((l) => l.replace(/\r$/, ""));
     return lines.length > n ? lines.slice(-n) : lines;
@@ -958,15 +1069,17 @@ function sleep(ms) {
 }
 
 // src/engines/comfy-guard.ts
+var FREE_TIMEOUT_MS = 1e4;
 var LEASE_MINUTES = (() => {
   const v = Number(process.env.TURBOLLM_COMFY_LEASE_MIN);
   return Number.isFinite(v) && v > 0 ? v : 30;
 })();
 var ComfyGuard = class {
-  constructor(store2, manager2, leaseMinutes = LEASE_MINUTES) {
+  constructor(store2, manager2, leaseMinutes = LEASE_MINUTES, fetchImpl = globalThis.fetch) {
     this.store = store2;
     this.manager = manager2;
     this.leaseMs = Math.max(1, leaseMinutes * 6e4);
+    this.fetchImpl = fetchImpl;
   }
   store;
   manager;
@@ -977,7 +1090,15 @@ var ComfyGuard = class {
   lastSignalAt = 0;
   // Serialize concurrent acquire() calls (rapid-fire enqueues) onto one unload.
   acquiring = null;
+  // Serialize concurrent reverse free-calls (rapid LLM↔image alternation) onto one
+  // POST so we never double-free or race two /free requests at ComfyUI.
+  freeing = null;
+  // KV prompt-cache persistence (F-014): the slot filename we successfully saved THIS
+  // cycle (or null if the save was skipped/failed/ineligible). Gates the restore in
+  // release() — we only restore a cache we actually wrote this acquire.
+  cachedFile = null;
   leaseMs;
+  fetchImpl;
   enabled() {
     return this.store.snapshot().comfyui.enabled;
   }
@@ -997,9 +1118,29 @@ var ComfyGuard = class {
     if (this.acquiring) return this.acquiring;
     this.acquiring = (async () => {
       const st = this.manager.status();
+      this.cachedFile = null;
       if (st.state === "running" || st.state === "starting") {
         const opts = this.manager.currentOpts();
         if (opts && !this.suspended) this.suspended = opts;
+        if (st.state === "running" && opts) {
+          const cfg2 = this.store.snapshot().comfyui;
+          if (cacheEligible(opts, cfg2)) {
+            const base2 = this.manager.target();
+            if (base2) {
+              const filename = slotCacheKey(opts);
+              const ok = await saveSlot({
+                http: this.fetchImpl,
+                base: base2,
+                dir: slotCacheDir(this.store.dir()),
+                filename,
+                capMs: SAVE_CAP_MS,
+                ttlMs: TTL_MS,
+                now: Date.now()
+              });
+              this.cachedFile = ok ? filename : null;
+            }
+          }
+        }
         console.log("[comfy-guard] ComfyUI acquired the GPU \u2014 force-unloading the model.");
         await this.manager.stopAndWait({ force: true });
       }
@@ -1023,10 +1164,74 @@ var ComfyGuard = class {
       console.log("[comfy-guard] ComfyUI released the GPU \u2014 reloading the previous model.");
       try {
         await this.manager.start(opts);
+        if (this.cachedFile) {
+          await this.restoreAfterReady(this.cachedFile);
+        }
       } catch (e) {
         console.warn(`[comfy-guard] reload after ComfyUI failed: ${e instanceof Error ? e.message : e}`);
       }
     }
+    this.cachedFile = null;
+  }
+  /** Wait for the just-reloaded engine to reach 'running', then restore the saved KV cache
+   *  (F-014). Bounded poll: bails on 'error' or after ~130s (just over the manager's 120s
+   *  readiness window). The restore itself is non-fatal — any failure leaves the prefix to
+   *  re-prefill normally. */
+  async restoreAfterReady(filename) {
+    const deadline = Date.now() + 13e4;
+    for (; ; ) {
+      const state = this.manager.status().state;
+      if (state === "running") break;
+      if (state === "error" || Date.now() > deadline) return;
+      await this.sleep(500);
+    }
+    const base2 = this.manager.target();
+    if (!base2) return;
+    try {
+      await restoreSlot({ http: this.fetchImpl, base: base2, dir: slotCacheDir(this.store.dir()), filename });
+    } catch {
+    }
+  }
+  sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+  }
+  /** REVERSE gate (F-011): TurboLLM is about to load a model — ask ComfyUI to drop its
+   *  VRAM first by calling its native `POST {url}/free`. Symmetric counterpart of the
+   *  forward acquire/release gate. Every model-load entry point (the HTTP load route,
+   *  the bench load, the startup auto-load) awaits this before `manager.start(...)`.
+   *
+   *  Fires ONLY when ComfyUI is idle from our side (`!held`): if WE currently hold the
+   *  GPU for ComfyUI a render is in flight, and the forward block already kept this load
+   *  out — never interrupt that render. Any failure (ComfyUI down, connection refused,
+   *  timeout, non-2xx) is NON-FATAL: we log a warning and return so the load proceeds,
+   *  exactly as the gate node treats an unreachable TurboLLM. Concurrent calls (rapid
+   *  LLM↔image alternation) collapse onto one in-flight POST so we never double-free. */
+  async freeComfyUIBeforeLoad() {
+    const cfg2 = this.store.snapshot().comfyui;
+    if (!cfg2.enabled || !cfg2.reverseGate || !cfg2.url || this.held) return;
+    if (this.freeing) return this.freeing;
+    this.freeing = (async () => {
+      const url = `${cfg2.url.replace(/\/+$/, "")}/free`;
+      try {
+        console.log("[comfy-guard] TurboLLM is loading a model \u2014 asking ComfyUI to free its VRAM first.");
+        const res = await this.fetchImpl(url, {
+          method: "POST",
+          headers: { "content-type": "application/json" },
+          body: JSON.stringify({ unload_models: true, free_memory: true }),
+          signal: AbortSignal.timeout(FREE_TIMEOUT_MS)
+        });
+        if (!res.ok) {
+          console.warn(`[comfy-guard] ComfyUI /free returned ${res.status} \u2014 loading anyway.`);
+        }
+      } catch (e) {
+        console.warn(`[comfy-guard] could not reach ComfyUI /free (${e instanceof Error ? e.message : e}) \u2014 loading anyway.`);
+      }
+    })();
+    try {
+      await this.freeing;
+    } finally {
+      this.freeing = null;
+    }
   }
   snapshot() {
     const cfg2 = this.store.snapshot().comfyui;
@@ -1058,12 +1263,12 @@ var ComfyGuard = class {
 };
 
 // src/engines/registry.ts
-import { existsSync as existsSync7 } from "fs";
+import { existsSync as existsSync8 } from "fs";
 import { randomUUID as randomUUID2 } from "crypto";
 
 // src/engines/probe.ts
 import { execFile as execFile5 } from "child_process";
-import { closeSync, existsSync as existsSync6, openSync, readSync, statSync } from "fs";
+import { closeSync, existsSync as existsSync7, openSync, readSync, statSync as statSync2 } from "fs";
 import { dirname as dirname3 } from "path";
 var ProbeError = class extends Error {
   constructor(code, msg) {
@@ -1093,7 +1298,7 @@ function runCaptured(bin, arg) {
   });
 }
 async function probe(bin) {
-  if (!existsSync6(bin) || statSync(bin).isDirectory()) {
+  if (!existsSync7(bin) || statSync2(bin).isDirectory()) {
     throw new ProbeError("binary_not_found", "Binary not found at that path.");
   }
   const fmt = detectFormat(bin);
@@ -1323,7 +1528,7 @@ var Registry = class {
   pruneDeadManagedBuilds() {
     let removed = 0;
     for (const e of this.list().engines) {
-      if (isManagedBuild(e.binPath) && !existsSync7(e.binPath)) {
+      if (isManagedBuild(e.binPath) && !existsSync8(e.binPath)) {
         try {
           this.remove(e.id);
           removed++;
@@ -1506,8 +1711,8 @@ function engineAcceptsFormat(engineKind, format) {
 }
 
 // src/models/scanner.ts
-import { existsSync as existsSync8, lstatSync, readdirSync as readdirSync2, readFileSync as readFileSync3, rmSync as rmSync3, writeFileSync as writeFileSync2 } from "fs";
-import { basename, dirname as dirname4, join as join6 } from "path";
+import { existsSync as existsSync9, lstatSync, readdirSync as readdirSync3, readFileSync as readFileSync3, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
+import { basename, dirname as dirname4, join as join7 } from "path";
 
 // src/gguf/gguf.ts
 import { closeSync as closeSync2, openSync as openSync2, readSync as readSync2 } from "fs";
@@ -1772,7 +1977,7 @@ var ScannerError = class extends Error {
 var Scanner = class {
   constructor(store2) {
     this.store = store2;
-    this.cachePath = join6(store2.dir(), "models-cache.json");
+    this.cachePath = join7(store2.dir(), "models-cache.json");
     this.loadCache();
   }
   store;
@@ -1801,14 +2006,14 @@ var Scanner = class {
     const total = m[3];
     let names;
     try {
-      names = readdirSync2(e.dir);
+      names = readdirSync3(e.dir);
     } catch {
       return [e.path];
     }
     const shards = [];
     for (const name of names) {
       const sm = name.match(SPLIT_RE);
-      if (sm && sm[1] === prefix && sm[3] === total) shards.push(join6(e.dir, name));
+      if (sm && sm[1] === prefix && sm[3] === total) shards.push(join7(e.dir, name));
     }
     return shards.length > 0 ? shards.sort() : [e.path];
   }
@@ -1820,9 +2025,9 @@ var Scanner = class {
     if (!e) throw new ScannerError("no_such_model", "No model with that key.");
     const paths = this.filesFor(key);
     if (e.format === "mlx") {
-      rmSync3(e.path, { recursive: true, force: true });
+      rmSync4(e.path, { recursive: true, force: true });
     } else {
-      for (const p of paths) rmSync3(p, { force: true });
+      for (const p of paths) rmSync4(p, { force: true });
       this.cache.delete(e.path);
     }
     await this.rescan();
@@ -1836,7 +2041,7 @@ var Scanner = class {
       const dirs = this.store.snapshot().modelDirs;
       const scan = { ggufs: [], mlxDirs: [] };
       for (const d of dirs) {
-        if (existsSync8(d)) walk(d, scan);
+        if (existsSync9(d)) walk(d, scan);
         await tick();
       }
       const gguf = await this.build(scan.ggufs);
@@ -1962,7 +2167,7 @@ function isMlxModelDir(names) {
 function walk(dir, out) {
   let names;
   try {
-    names = readdirSync2(dir);
+    names = readdirSync3(dir);
   } catch {
     return;
   }
@@ -1972,7 +2177,7 @@ function walk(dir, out) {
   }
   for (const name of names) {
     if (name === ".git" || name === "node_modules") continue;
-    const full = join6(dir, name);
+    const full = join7(dir, name);
     let st;
     try {
       st = lstatSync(full);
@@ -1990,7 +2195,7 @@ function mlxEntryFor(dir) {
   let cfg2 = {};
   let parseError = null;
   try {
-    const raw = readFileSync3(join6(dir, "config.json"), "utf8").replace(/^/, "");
+    const raw = readFileSync3(join7(dir, "config.json"), "utf8").replace(/^/, "");
     cfg2 = JSON.parse(raw);
   } catch (e) {
     parseError = `Could not read config.json: ${e.message}`;
@@ -1999,16 +2204,16 @@ function mlxEntryFor(dir) {
   let mtimeMs = 0;
   let hasChatTemplate = false;
   try {
-    for (const n of readdirSync2(dir)) {
+    for (const n of readdirSync3(dir)) {
       const lower = n.toLowerCase();
       if (lower.endsWith(".safetensors")) {
-        const st = lstatSync(join6(dir, n));
+        const st = lstatSync(join7(dir, n));
         sizeBytes += st.size;
         mtimeMs = Math.max(mtimeMs, st.mtimeMs);
       }
     }
-    const tc = join6(dir, "tokenizer_config.json");
-    if (existsSync8(tc)) hasChatTemplate = readFileSync3(tc, "utf8").includes("chat_template");
+    const tc = join7(dir, "tokenizer_config.json");
+    if (existsSync9(tc)) hasChatTemplate = readFileSync3(tc, "utf8").includes("chat_template");
   } catch {
   }
   const expertCount = cfg2.num_local_experts ?? cfg2.num_experts ?? 0;
@@ -2051,16 +2256,16 @@ function tick() {
 }
 
 // src/models/hashes.ts
-import { createHash } from "crypto";
+import { createHash as createHash2 } from "crypto";
 import { createReadStream, mkdirSync as mkdirSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync3 } from "fs";
-import { join as join7 } from "path";
+import { join as join8 } from "path";
 var HashStore = class {
   cache = /* @__PURE__ */ new Map();
   pending = /* @__PURE__ */ new Set();
   path;
   constructor(dataDir) {
     mkdirSync5(dataDir, { recursive: true });
-    this.path = join7(dataDir, "model-hashes.json");
+    this.path = join8(dataDir, "model-hashes.json");
     this.load();
   }
   /** Cached sha256 for a file, only when the cached row still matches size+mtime
@@ -2078,7 +2283,7 @@ var HashStore = class {
   }
   compute(path, size, mtime) {
     return new Promise((resolve2) => {
-      const hash = createHash("sha256");
+      const hash = createHash2("sha256");
       const rs = createReadStream(path);
       rs.on("error", () => resolve2());
       rs.on("data", (c) => hash.update(c));
@@ -2110,6 +2315,9 @@ var HashStore = class {
 };
 
 // src/models/profile.ts
+function defaultGpu() {
+  return { splitMode: "layer", tensorSplit: [], mainGpu: -1, tensorParallelSize: 1 };
+}
 var HEAD_DIM = 128;
 function kvBytesPerElem(t) {
   switch (t) {
@@ -2134,10 +2342,18 @@ function kvBytesPerElem(t) {
   }
 }
 function defaultSampling() {
-  return { temp: 0.8, topP: 0.95, topK: 40, minP: 0.05, repeatPenalty: 1, presencePenalty: 0 };
+  return { temp: 0.8, topP: 0.95, topK: 40, minP: 0.05, repeatPenalty: 1, presencePenalty: 0, frequencyPenalty: 0, stop: [] };
+}
+function gpuBudgetMb(sys, p) {
+  if (sys.gpus.length === 0) return 0;
+  if (p?.gpu?.splitMode === "none") {
+    const idx = p.gpu.mainGpu >= 0 ? p.gpu.mainGpu : 0;
+    return sys.gpus[idx]?.vramMb ?? sys.gpus[0]?.vramMb ?? 0;
+  }
+  return sys.gpus.reduce((sum, g) => sum + (g.vramMb || 0), 0);
 }
 function estimateVram(p, m, sys) {
-  const totalVramMb = sys.gpus[0]?.vramMb ?? 0;
+  const totalVramMb = gpuBudgetMb(sys, p);
   if (totalVramMb === 0) return { estMb: 0, totalVramMb: 0, pct: 0, verdict: "cpu" };
   const sizeMb = m.sizeBytes / 1e6;
   const blocks = m.blockCount || 1;
@@ -2182,10 +2398,16 @@ function deriveDefault(m, sys) {
     mtpHeadPath: "",
     draftModelPath: "",
     sampling: defaultSampling(),
+    contextOverflow: "shift",
+    nKeep: 0,
+    ropeScalingType: "none",
+    ropeFreqBase: 0,
+    ropeFreqScale: 0,
+    gpu: defaultGpu(),
     extraArgs: []
   };
   if (m.moe && hasGpu && m.blockCount > 0) {
-    const budget = (sys.gpus[0].vramMb || 0) * 0.85;
+    const budget = gpuBudgetMb(sys, base2) * 0.85;
     base2.nCpuMoe = m.blockCount;
     for (let n = 0; n <= m.blockCount; n += 2) {
       if (estimateVram({ ...base2, nCpuMoe: n }, m, sys).estMb <= budget) {
@@ -2213,13 +2435,24 @@ function resolveProfile(m, sys, saved, overrides, defaults) {
     ...base2,
     ...saved ?? {},
     ...overrides ?? {},
-    sampling: { ...base2.sampling, ...saved?.sampling ?? {}, ...overrides?.sampling ?? {} }
+    sampling: { ...base2.sampling, ...saved?.sampling ?? {}, ...overrides?.sampling ?? {} },
+    // gpu is deep-merged like sampling so a partial override (or an old saved profile
+    // missing some fields) keeps the rest of the defaults instead of going undefined.
+    gpu: { ...base2.gpu, ...saved?.gpu ?? {}, ...overrides?.gpu ?? {} }
   };
 }
 function profileToArgs(p, m, caps, cores = 0) {
   const has = (flag) => caps.flags.length === 0 || caps.flags.includes(flag);
   const a = ["-c", String(p.ctx)];
   if (p.ngl > 0) a.push("-ngl", String(p.ngl));
+  const g = p.gpu;
+  if (g) {
+    if (g.splitMode !== "layer" && has("--split-mode")) a.push("--split-mode", g.splitMode);
+    if (g.splitMode !== "none" && g.tensorSplit.length > 0 && has("--tensor-split")) {
+      a.push("--tensor-split", g.tensorSplit.join(","));
+    }
+    if (g.mainGpu >= 0 && has("--main-gpu")) a.push("--main-gpu", String(g.mainGpu));
+  }
   if (has("--parallel")) a.push("--parallel", String(p.parallel));
   if (p.parallel > 1 && p.kvUnified && has("--kv-unified")) a.push("--kv-unified");
   if (m.moe && p.nCpuMoe > 0 && has("--n-cpu-moe")) a.push("--n-cpu-moe", String(p.nCpuMoe));
@@ -2247,13 +2480,26 @@ function profileToArgs(p, m, caps, cores = 0) {
     if (specType) a.push("--spec-type", "draft");
     a.push("--model-draft", p.draftModelPath, "--draft-max", "16", "--draft-min", "1");
   }
+  if (p.sampling.temp !== 0.8 && has("--temp")) a.push("--temp", String(p.sampling.temp));
+  if (p.sampling.topP !== 0.95 && has("--top-p")) a.push("--top-p", String(p.sampling.topP));
+  if (p.sampling.topK !== 40 && has("--top-k")) a.push("--top-k", String(p.sampling.topK));
+  if (p.sampling.minP !== 0.05 && has("--min-p")) a.push("--min-p", String(p.sampling.minP));
+  if (p.sampling.repeatPenalty !== 1 && has("--repeat-penalty")) a.push("--repeat-penalty", String(p.sampling.repeatPenalty));
+  if (p.sampling.presencePenalty !== 0 && has("--presence-penalty")) a.push("--presence-penalty", String(p.sampling.presencePenalty));
+  if (p.sampling.frequencyPenalty !== 0 && has("--frequency-penalty")) a.push("--frequency-penalty", String(p.sampling.frequencyPenalty));
+  if (p.contextOverflow === "keep" && p.nKeep > 0 && has("--n-keep")) a.push("--n-keep", String(p.nKeep));
+  if (p.ropeScalingType !== "none" && has("--rope-scaling")) {
+    a.push("--rope-scaling", p.ropeScalingType);
+    if (p.ropeFreqBase > 0 && has("--rope-freq-base")) a.push("--rope-freq-base", String(p.ropeFreqBase));
+    if (p.ropeFreqScale > 0 && has("--rope-freq-scale")) a.push("--rope-freq-scale", String(p.ropeFreqScale));
+  }
   a.push(...p.extraArgs);
   return a;
 }
 
 // src/chat/db.ts
 import { DatabaseSync } from "node:sqlite";
-import { join as join8 } from "path";
+import { join as join9 } from "path";
 import { randomUUID as randomUUID3 } from "crypto";
 function safeJson(s) {
   try {
@@ -2271,7 +2517,7 @@ function rowToMsg(r) {
 var ConversationStore = class {
   db;
   constructor(dataDir) {
-    this.db = new DatabaseSync(join8(dataDir, "turbollm.db"));
+    this.db = new DatabaseSync(join9(dataDir, "turbollm.db"));
     this.migrate();
   }
   migrate() {
@@ -2614,19 +2860,19 @@ function base(p) {
 }
 
 // src/downloads/downloads.ts
-import { createHash as createHash2 } from "crypto";
+import { createHash as createHash3 } from "crypto";
 import {
   createWriteStream as createWriteStream3,
-  existsSync as existsSync9,
+  existsSync as existsSync10,
   mkdirSync as mkdirSync6,
   readFileSync as readFileSync5,
   renameSync as renameSync2,
-  rmSync as rmSync4,
+  rmSync as rmSync5,
   statfsSync,
-  statSync as statSync2,
+  statSync as statSync3,
   writeFileSync as writeFileSync4
 } from "fs";
-import { basename as basename2, join as join9 } from "path";
+import { basename as basename2, join as join10 } from "path";
 import { Readable as Readable2 } from "stream";
 import { pipeline as pipeline2 } from "stream/promises";
 var MAX_CONCURRENT = 2;
@@ -2644,10 +2890,10 @@ var DownloadManager = class {
     this.store = store2;
     this.onComplete = onComplete;
     this.authHeaders = authHeaders;
-    const dir = join9(store2.dir(), "downloads");
+    const dir = join10(store2.dir(), "downloads");
     mkdirSync6(dir, { recursive: true });
-    this.manifestPath = join9(dir, "manifest.json");
-    this.provenancePath = join9(dir, "provenance.json");
+    this.manifestPath = join10(dir, "manifest.json");
+    this.provenancePath = join10(dir, "provenance.json");
     this.restore();
     this.loadProvenance();
   }
@@ -2710,7 +2956,7 @@ var DownloadManager = class {
       name: filename,
       repo,
       url,
-      dest: join9(dir, filename),
+      dest: join10(dir, filename),
       total,
       received: 0,
       status: "queued",
@@ -2732,7 +2978,7 @@ var DownloadManager = class {
     this.controllers.get(id)?.abort();
     this.controllers.delete(id);
     if (rec.status !== "done") {
-      rmSync4(`${rec.dest}.part`, { force: true });
+      rmSync5(`${rec.dest}.part`, { force: true });
       rec.status = "cancelled";
       rec.bytesPerSec = 0;
     }
@@ -2746,7 +2992,7 @@ var DownloadManager = class {
     if (!rec) return false;
     this.controllers.get(id)?.abort();
     this.controllers.delete(id);
-    if (rec.status !== "done") rmSync4(`${rec.dest}.part`, { force: true });
+    if (rec.status !== "done") rmSync5(`${rec.dest}.part`, { force: true });
     this.records.delete(id);
     this.persist();
     this.pump();
@@ -2794,9 +3040,9 @@ var DownloadManager = class {
     const part = `${rec.dest}.part`;
     try {
       let startAt = 0;
-      if (existsSync9(part)) {
+      if (existsSync10(part)) {
         try {
-          startAt = statSync2(part).size;
+          startAt = statSync3(part).size;
         } catch {
           startAt = 0;
         }
@@ -2811,13 +3057,13 @@ var DownloadManager = class {
       if (!res.body) throw new DownloadError("download_failed", "Empty response body.");
       const resuming = res.status === 206;
       if (!resuming && startAt > 0) {
-        rmSync4(part, { force: true });
+        rmSync5(part, { force: true });
         startAt = 0;
         rec.received = 0;
       }
       const clen = Number(res.headers.get("content-length") ?? 0);
       if (clen > 0) rec.total = resuming ? startAt + clen : clen;
-      const hash = rec.sha256 ? createHash2("sha256") : null;
+      const hash = rec.sha256 ? createHash3("sha256") : null;
       const verifyHash = hash !== null && startAt === 0;
       let lastTick = Date.now();
       let lastBytes = startAt;
@@ -2836,13 +3082,13 @@ var DownloadManager = class {
       const out = createWriteStream3(part, startAt > 0 ? { flags: "a" } : { flags: "w" });
       await pipeline2(body3, out, { signal: ac.signal });
       if (rec.total > 0 && rec.received !== rec.total) {
-        rmSync4(part, { force: true });
+        rmSync5(part, { force: true });
         throw new DownloadError("size_mismatch", "Download corrupt \u2014 size did not match. Removed the partial file.");
       }
       if (verifyHash && rec.sha256) {
         const got = hash.digest("hex");
         if (got !== rec.sha256) {
-          rmSync4(part, { force: true });
+          rmSync5(part, { force: true });
           throw new DownloadError("checksum_failed", "Checksum failed \u2014 the downloaded file was corrupt.");
         }
       }
@@ -2934,7 +3180,7 @@ var DownloadManager = class {
       let received = 0;
       try {
         const p = `${e.dest}.part`;
-        if (existsSync9(p)) received = statSync2(p).size;
+        if (existsSync10(p)) received = statSync3(p).size;
       } catch {
       }
       this.records.set(e.id, {
@@ -2968,7 +3214,7 @@ function safePathname(u) {
 // src/bench/bench.ts
 import { execFile as execFile6 } from "child_process";
 import { mkdirSync as mkdirSync7, writeFileSync as writeFileSync5 } from "fs";
-import { join as join10 } from "path";
+import { join as join11 } from "path";
 import { randomUUID as randomUUID4 } from "crypto";
 var READY_TIMEOUT_MS = 12e4;
 var TOTAL_BUDGET_MS = 10 * 6e4;
@@ -3236,9 +3482,9 @@ var BenchRunner = class {
           result: { tps: record.tps, ttftMs: record.ttftMs, vramMb: record.vramMb, outcome: "ok" }
         }
       };
-      const queueDir = join10(this.store.dir(), "telemetry", "queue");
+      const queueDir = join11(this.store.dir(), "telemetry", "queue");
       mkdirSync7(queueDir, { recursive: true });
-      writeFileSync5(join10(queueDir, `${randomUUID4()}.json`), JSON.stringify(event));
+      writeFileSync5(join11(queueDir, `${randomUUID4()}.json`), JSON.stringify(event));
     } catch {
     }
   }
@@ -3352,18 +3598,18 @@ Install it:  ${spec.install}
 // src/server.ts
 import { Hono } from "hono";
 import { cors } from "hono/cors";
-import { existsSync as existsSync11, readFileSync as readFileSync7, statSync as statSync4 } from "fs";
+import { existsSync as existsSync12, readFileSync as readFileSync7, statSync as statSync5 } from "fs";
 import { fileURLToPath } from "url";
-import { dirname as dirname6, join as join12, normalize as normalize2 } from "path";
+import { dirname as dirname6, join as join13, normalize as normalize2 } from "path";
 import { Agent, setGlobalDispatcher } from "undici";
 
 // src/api/routes.ts
 import { streamSSE } from "hono/streaming";
-import { existsSync as existsSync10, mkdirSync as mkdirSync8, readFileSync as readFileSync6, readdirSync as readdirSync3, realpathSync, rmSync as rmSync5, statSync as statSync3, writeFileSync as writeFileSync6 } from "fs";
-import { basename as basename3, dirname as dirname5, join as join11, resolve, sep } from "path";
+import { existsSync as existsSync11, mkdirSync as mkdirSync8, readFileSync as readFileSync6, readdirSync as readdirSync4, realpathSync, rmSync as rmSync6, statSync as statSync4, writeFileSync as writeFileSync6 } from "fs";
+import { basename as basename3, dirname as dirname5, join as join12, resolve, sep } from "path";
 
 // src/comfyui/gate-template.ts
-var GATE_VERSION = 1;
+var GATE_VERSION = 2;
 function gateNodeSource(baseUrl) {
   const base2 = baseUrl.replace(/\/+$/, "");
   const baseLit = JSON.stringify(base2);
@@ -3420,6 +3666,19 @@ def _acquire_blocking() -> None:
     _post("/api/v1/comfyui/acquire", ACQUIRE_TIMEOUT_SEC)
 
 
+def _free_self() -> None:
+    """Unload ComfyUI's own models and clear the CUDA cache so VRAM is available
+    for TurboLLM to reload its model. Called on the release edge (queue drained),
+    before notifying TurboLLM. In-process \u2014 no HTTP roundtrip."""
+    try:
+        import comfy.model_management as mm  # type: ignore[import]
+        mm.unload_all_models()
+        mm.soft_empty_cache()
+        _log("freed ComfyUI model cache.")
+    except Exception as e:  # noqa: BLE001
+        _log(f"could not free ComfyUI models ({e}); continuing.")
+
+
 def _release() -> None:
     _post("/api/v1/comfyui/release", RELEASE_TIMEOUT_SEC)
 
@@ -3469,6 +3728,7 @@ def _install_hooks() -> None:
                     _busy = False
                     do_release = True
             if do_release:
+                _free_self()
                 _release()
         return result
 
@@ -3485,7 +3745,7 @@ __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"]
 }
 
 // src/api/routes.ts
-import { createHash as createHash3, randomBytes, randomUUID as randomUUID5 } from "crypto";
+import { createHash as createHash4, randomBytes, randomUUID as randomUUID5 } from "crypto";
 import { homedir as homedir2, networkInterfaces } from "os";
 
 // src/engines/catalog.ts
@@ -3594,7 +3854,22 @@ function registerApi(app2, d) {
       downloads: { active: d.downloads.activeCount() },
       engineProvision: d.provision.get(),
       // ComfyUI GPU coordination: lets the UI explain a paused/unloaded engine.
-      comfyui: d.comfy?.snapshot() ?? null,
+      // Also expose the installed gate node version so the UI can prompt an upgrade.
+      comfyui: (() => {
+        const snap = d.comfy?.snapshot() ?? null;
+        if (!snap) return null;
+        const gatePath = d.store.snapshot().comfyui.gatePath;
+        let installedVersion = null;
+        if (gatePath) {
+          try {
+            const src = readFileSync6(join12(gatePath, "__init__.py"), "utf-8");
+            const m = src.match(/^GATE_VERSION\s*=\s*(\d+)/m);
+            if (m) installedVersion = Number(m[1]);
+          } catch {
+          }
+        }
+        return { ...snap, installedVersion, currentVersion: GATE_VERSION };
+      })(),
       telemetryLevel: d.store.snapshot().telemetry.level,
       uptimeSec: Math.floor((Date.now() - d.startedAt) / 1e3)
     });
@@ -3608,7 +3883,7 @@ function registerApi(app2, d) {
     const sys = getSysInfo();
     const vendor = primaryVendor(sys);
     const recommended = recommendBackendId(vendor, sys.gpus.length > 0);
-    const root = join11(d.store.dir(), "engines");
+    const root = join12(d.store.dir(), "engines");
     const active = d.registry.active();
     const regEngines = d.registry.list().engines;
     const backends = availableBackends().map((b) => {
@@ -3637,7 +3912,7 @@ function registerApi(app2, d) {
     const def = availableBackends().find((x) => x.id === b.backend);
     if (!def) return err(c, 400, "invalid_config_value", "Unknown backend for this platform.");
     if (d.provision.get().active) return err(c, 409, "engine_already_running", "Another engine download is already in progress.");
-    const root = join11(d.store.dir(), "engines");
+    const root = join12(d.store.dir(), "engines");
     const ac = new AbortController();
     provisionAbort = ac;
     void (async () => {
@@ -3673,7 +3948,7 @@ function registerApi(app2, d) {
   app2.delete("/api/v1/engines/backends/:id", async (c) => {
     const def = availableBackends().find((x) => x.id === c.req.param("id"));
     if (!def) return err(c, 400, "invalid_config_value", "Unknown backend for this platform.");
-    const root = join11(d.store.dir(), "engines");
+    const root = join12(d.store.dir(), "engines");
     const bin = installedBackendServer(root, def.id);
     const eng = bin ? d.registry.list().engines.find((e) => e.binPath === bin) : void 0;
     if (eng && d.registry.active()?.id === eng.id) await d.manager.stopAndWait();
@@ -3686,7 +3961,7 @@ function registerApi(app2, d) {
       return err(c, 409, "unsupported_platform", "MLX is only available on macOS (Apple Silicon).");
     }
     if (d.provision.get().active) return err(c, 409, "engine_already_running", "Another engine download is already in progress.");
-    const root = join11(d.store.dir(), "engines");
+    const root = join12(d.store.dir(), "engines");
     void (async () => {
       try {
         d.provision.start("mlx");
@@ -3715,7 +3990,7 @@ function registerApi(app2, d) {
   });
   app2.post("/api/v1/engines/vllm", (c) => {
     if (d.provision.get().active) return err(c, 409, "engine_already_running", "Another engine download is already in progress.");
-    const root = join11(d.store.dir(), "engines");
+    const root = join12(d.store.dir(), "engines");
     void (async () => {
       try {
         d.provision.start("vllm");
@@ -3736,7 +4011,7 @@ function registerApi(app2, d) {
       return err(c, 409, "unsupported_platform", "TurboQuant has no prebuilt binary for this operating system yet.");
     }
     if (d.provision.get().active) return err(c, 409, "engine_already_running", "Another engine download is already in progress.");
-    const root = join11(d.store.dir(), "engines");
+    const root = join12(d.store.dir(), "engines");
     void (async () => {
       try {
         d.provision.start("turboquant");
@@ -3822,16 +4097,16 @@ function registerApi(app2, d) {
     }
     let entries;
     try {
-      entries = readdirSync3(real, { withFileTypes: true }).filter((d2) => !d2.name.startsWith(".")).map((d2) => {
+      entries = readdirSync4(real, { withFileTypes: true }).filter((d2) => !d2.name.startsWith(".")).map((d2) => {
         let isDir = d2.isDirectory();
         if (d2.isSymbolicLink()) {
           try {
-            isDir = statSync3(join11(real, d2.name)).isDirectory();
+            isDir = statSync4(join12(real, d2.name)).isDirectory();
           } catch {
             isDir = false;
           }
         }
-        return { name: d2.name, path: join11(real, d2.name), isDir };
+        return { name: d2.name, path: join12(real, d2.name), isDir };
       }).sort((a, b) => a.isDir === b.isDir ? a.name.localeCompare(b.name) : a.isDir ? -1 : 1);
     } catch {
       return err(c, 400, "fs_read_failed", "Could not read that folder (permission denied or not a directory).");
@@ -3859,11 +4134,13 @@ function registerApi(app2, d) {
       }
       let opts2;
       if (entry.format !== "gguf") {
+        const savedGpu = cfg2.modelProfiles[entry.key]?.gpu;
         opts2 = {
           engine: active,
           model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: entry.nativeCtx, vision: false },
           modelPath: entry.path,
-          extraArgs: []
+          extraArgs: [],
+          tensorParallelSize: savedGpu?.tensorParallelSize
         };
       } else {
         const saved = cfg2.modelProfiles[entry.key];
@@ -3876,6 +4153,7 @@ function registerApi(app2, d) {
         };
       }
       await d.manager.stopAndWait();
+      await d.comfy?.freeComfyUIBeforeLoad();
       try {
         await d.manager.start(opts2);
       } catch (e) {
@@ -3897,6 +4175,7 @@ function registerApi(app2, d) {
     if (!modelPath) return err(c, 409, "no_such_model", "No model specified. Pick one from the Models screen.");
     const opts = { engine: active, model: deriveModel(modelPath, name, extra), modelPath, extraArgs: extra };
     await d.manager.stopAndWait();
+    await d.comfy?.freeComfyUIBeforeLoad();
     try {
       await d.manager.start(opts);
     } catch (e) {
@@ -3927,22 +4206,22 @@ function registerApi(app2, d) {
     const raw = (b.path ?? "").trim();
     if (!raw) return err(c, 400, "invalid_config_value", "Enter the path to your ComfyUI folder.");
     const root = resolve(raw);
-    if (!existsSync10(root) || !statSync3(root).isDirectory()) {
+    if (!existsSync11(root) || !statSync4(root).isDirectory()) {
       return err(c, 400, "invalid_config_value", "That folder does not exist.");
     }
     let customNodes;
     if (basename3(root).toLowerCase() === "custom_nodes") customNodes = root;
-    else if (existsSync10(join11(root, "custom_nodes"))) customNodes = join11(root, "custom_nodes");
+    else if (existsSync11(join12(root, "custom_nodes"))) customNodes = join12(root, "custom_nodes");
     else {
       return err(c, 400, "invalid_config_value", "No 'custom_nodes' folder here. Point me at your ComfyUI folder or its custom_nodes folder.");
     }
     const reqUrl = new URL(c.req.url);
     const port2 = reqUrl.port || (reqUrl.protocol === "https:" ? "443" : "80");
     const base2 = `http://127.0.0.1:${port2}`;
-    const gateDir = join11(customNodes, "turbollm_gate");
+    const gateDir = join12(customNodes, "turbollm_gate");
     try {
       mkdirSync8(gateDir, { recursive: true });
-      writeFileSync6(join11(gateDir, "__init__.py"), gateNodeSource(base2));
+      writeFileSync6(join12(gateDir, "__init__.py"), gateNodeSource(base2));
     } catch (e) {
       return err(c, 500, "fs_write_failed", `Could not write the gate node: ${e instanceof Error ? e.message : e}`);
     }
@@ -3953,9 +4232,9 @@ function registerApi(app2, d) {
   });
   app2.post("/api/v1/comfyui/uninstall", (c) => {
     const dir = d.store.snapshot().comfyui.gatePath;
-    if (dir && existsSync10(dir)) {
+    if (dir && existsSync11(dir)) {
       try {
-        rmSync5(dir, { recursive: true, force: true });
+        rmSync6(dir, { recursive: true, force: true });
       } catch (e) {
         return err(c, 500, "fs_write_failed", `Could not remove the gate node: ${e instanceof Error ? e.message : e}`);
       }
@@ -3988,7 +4267,7 @@ function registerApi(app2, d) {
       });
       while (!aborted) {
         const path = d.manager.logPath();
-        if (path && existsSync10(path)) {
+        if (path && existsSync11(path)) {
           const lines = readFileSync6(path, "utf8").split("\n");
           for (; sent < lines.length - 1; sent++) {
             await stream.writeSSE({ event: "line", data: JSON.stringify({ line: lines[sent].replace(/\r$/, "") }) });
@@ -4004,6 +4283,7 @@ function registerApi(app2, d) {
     const key = (b.modelKey ?? "").trim();
     if (!key) return err(c, 400, "invalid_config_value", "modelKey is required.");
     if (d.comfy?.isBlocked()) return err(c, 409, "comfyui_busy", "ComfyUI is rendering \u2014 benchmarking is paused until its queue finishes.");
+    await d.comfy?.freeComfyUIBeforeLoad();
     try {
       d.bench.start(key, b.base && typeof b.base === "object" ? b.base : void 0);
       return c.json({ accepted: true }, 202);
@@ -4048,7 +4328,7 @@ function registerApi(app2, d) {
     const snap = d.store.snapshot();
     const saved = snap.modelProfiles[e.key];
     const profile = resolveProfile(e, sys, saved, void 0, snap.modelDefaults);
-    return c.json({ ...overlayModel(e, d), profile, vramFit: estimateVram(profile, e, sys), gpu: sys.gpus[0] ?? null, cores: sys.cores });
+    return c.json({ ...overlayModel(e, d), profile, vramFit: estimateVram(profile, e, sys), gpu: sys.gpus[0] ?? null, gpus: sys.gpus, cores: sys.cores });
   });
   app2.put("/api/v1/models/:key/profile", async (c) => {
     const key = decodeURIComponent(c.req.param("key"));
@@ -4058,6 +4338,21 @@ function registerApi(app2, d) {
     if (!p || typeof p.ctx !== "number" || p.ctx < 256) {
       return err(c, 400, "invalid_profile_value", "ctx must be at least 256.");
     }
+    if (p.gpu) {
+      const g = p.gpu;
+      if (!["layer", "row", "none"].includes(g.splitMode)) {
+        return err(c, 400, "invalid_profile_value", "gpu.splitMode must be layer, row, or none.");
+      }
+      if (!Array.isArray(g.tensorSplit) || g.tensorSplit.some((n) => typeof n !== "number" || !(n >= 0))) {
+        return err(c, 400, "invalid_profile_value", "gpu.tensorSplit must be an array of non-negative numbers.");
+      }
+      if (!Number.isInteger(g.mainGpu) || g.mainGpu < -1) {
+        return err(c, 400, "invalid_profile_value", "gpu.mainGpu must be an integer \u2265 -1.");
+      }
+      if (!Number.isInteger(g.tensorParallelSize) || g.tensorParallelSize < 1) {
+        return err(c, 400, "invalid_profile_value", "gpu.tensorParallelSize must be an integer \u2265 1.");
+      }
+    }
     d.store.update((cfg2) => {
       cfg2.modelProfiles[key] = p;
     });
@@ -4126,6 +4421,14 @@ function registerApi(app2, d) {
     }
     const cuUpdates = {};
     if (b.comfyui?.enabled !== void 0) cuUpdates.enabled = !!b.comfyui.enabled;
+    if (b.comfyui?.reverseGate !== void 0) cuUpdates.reverseGate = !!b.comfyui.reverseGate;
+    if (b.comfyui?.url !== void 0) {
+      const u = b.comfyui.url.trim();
+      if (u && !/^https?:\/\//i.test(u)) {
+        return err(c, 400, "invalid_config_value", "comfyui.url must be an http(s):// origin (e.g. http://127.0.0.1:8188).");
+      }
+      cuUpdates.url = u;
+    }
     const before = d.store.snapshot().daemon;
     d.store.update((cfg2) => {
       Object.assign(cfg2.daemon, updates);
@@ -4164,7 +4467,7 @@ function registerApi(app2, d) {
     const b = await body(c);
     const dir = (b.dir ?? "").trim();
     if (!dir || !/^([a-zA-Z]:[\\/]|[\\/])/.test(dir)) return err(c, 400, "invalid_config_value", "Path must be absolute.");
-    if (!existsSync10(dir)) return err(c, 400, "invalid_config_value", "That folder does not exist.");
+    if (!existsSync11(dir)) return err(c, 400, "invalid_config_value", "That folder does not exist.");
     try {
       d.store.update((cfg2) => {
         if (!cfg2.modelDirs.includes(dir)) cfg2.modelDirs.push(dir);
@@ -4498,7 +4801,7 @@ function cleanModelName(p) {
   return basename3(p).replace(/\.gguf$/i, "");
 }
 function readTail2(path, n) {
-  if (!path || !existsSync10(path)) return [];
+  if (!path || !existsSync11(path)) return [];
   try {
     const lines = readFileSync6(path, "utf8").replace(/[\r\n]+$/, "").split("\n").map((l) => l.replace(/\r$/, ""));
     return lines.length > n ? lines.slice(-n) : lines;
@@ -4512,7 +4815,7 @@ function generateApiKey() {
   let key = "";
   for (let i = 0; i < 40; i++) key += charset[buf[i] % 62];
   const full = `tllm-${key}`;
-  const hash = createHash3("sha256").update(full).digest("hex");
+  const hash = createHash4("sha256").update(full).digest("hex");
   return { full, hash, prefix: full.slice(0, 12) };
 }
 function realHome() {
@@ -4819,17 +5122,33 @@ ${content}` : b.docContext : content;
 async function runGeneration(d, stream, ctx) {
   const { db: db2 } = d;
   const { convId, conv, engineMessages, assistantMsg, ms, target, ac, disableThinking } = ctx;
-  const modelSampling = {};
-  const convSampling = conv.sampling ?? {};
-  const merged = { ...modelSampling, ...convSampling };
+  const convS = conv.sampling ?? {};
+  const SAMPLING_KEYS = {
+    temp: "temperature",
+    topP: "top_p",
+    topK: "top_k",
+    minP: "min_p",
+    repeatPenalty: "repeat_penalty",
+    presencePenalty: "presence_penalty",
+    frequencyPenalty: "frequency_penalty"
+  };
+  const samplingOverride = {};
+  for (const [camel, snake] of Object.entries(SAMPLING_KEYS)) {
+    if (camel in convS) samplingOverride[snake] = convS[camel];
+  }
+  for (const [k, v] of Object.entries(convS)) {
+    if (!(k in SAMPLING_KEYS) && k !== "stop") samplingOverride[k] = v;
+  }
   const reqBody = {
     model: ms.model.key,
     messages: engineMessages,
     stream: true,
     stream_options: { include_usage: true },
     return_progress: true,
-    ...merged
+    ...samplingOverride
   };
+  const stopStrings = convS.stop;
+  if (stopStrings?.length) reqBody.stop = stopStrings;
   const maxLimit = d.store.snapshot().modelDefaults.maxTokens ?? 0;
   const cappedMax = clampMaxTokens(reqBody.max_tokens, maxLimit);
   if (cappedMax != null) reqBody.max_tokens = cappedMax;
@@ -4867,6 +5186,12 @@ async function runGeneration(d, stream, ctx) {
     const reader = res.body.getReader();
     const decoder = new TextDecoder();
     let buf = "";
+    const cancelReader = () => void reader.cancel();
+    if (ac.signal.aborted) {
+      cancelReader();
+    } else {
+      ac.signal.addEventListener("abort", cancelReader, { once: true });
+    }
     outer: while (true) {
       const { done, value } = await reader.read();
       if (done) break;
@@ -4978,6 +5303,7 @@ async function runGeneration(d, stream, ctx) {
         }
       }
     }
+    ac.signal.removeEventListener("abort", cancelReader);
     if (pendingThinkBuf && inThink) {
       fullReasoning += pendingThinkBuf;
       await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: pendingThinkBuf }) });
@@ -5629,11 +5955,11 @@ async function recordOpenAiStreamUsage(d, body3) {
 }
 
 // src/auth.ts
-import { createHash as createHash4 } from "crypto";
+import { createHash as createHash5 } from "crypto";
 import { getConnInfo } from "@hono/node-server/conninfo";
 var LOOPBACK = /* @__PURE__ */ new Set(["127.0.0.1", "::1", "::ffff:127.0.0.1"]);
 function hashKey(raw) {
-  return createHash4("sha256").update(raw).digest("hex");
+  return createHash5("sha256").update(raw).digest("hex");
 }
 function presentedKey(c) {
   const direct = c.req.header("X-TurboLLM-Auth") ?? c.req.header("x-api-key");
@@ -5691,7 +6017,7 @@ function lanAuth(d) {
 
 // src/server.ts
 setGlobalDispatcher(new Agent({ keepAliveMaxTimeout: 6e4, connections: 10 }));
-var WEB_ROOT = join12(dirname6(fileURLToPath(import.meta.url)), "webdist");
+var WEB_ROOT = join13(dirname6(fileURLToPath(import.meta.url)), "webdist");
 function createApp(d) {
   const app2 = new Hono();
   app2.use(
@@ -5716,11 +6042,11 @@ function createApp(d) {
     if (path.startsWith("api/") || path.startsWith("v1/")) {
       return c.json({ error: { code: "not_found", message: "Unknown endpoint." } }, 404);
     }
-    let file = normalize2(join12(WEB_ROOT, path || "index.html"));
-    if (!file.startsWith(WEB_ROOT) || !existsSync11(file) || statSync4(file).isDirectory()) {
-      file = join12(WEB_ROOT, "index.html");
+    let file = normalize2(join13(WEB_ROOT, path || "index.html"));
+    if (!file.startsWith(WEB_ROOT) || !existsSync12(file) || statSync5(file).isDirectory()) {
+      file = join13(WEB_ROOT, "index.html");
     }
-    if (!existsSync11(file)) return c.text("web ui not built \u2014 run `npm run build:web`", 500);
+    if (!existsSync12(file)) return c.text("web ui not built \u2014 run `npm run build:web`", 500);
     return new Response(readFileSync7(file), { status: 200, headers: { "Content-Type": contentType(file) } });
   });
   return app2;
@@ -5748,7 +6074,7 @@ function contentType(file) {
 // src/cli.ts
 var version = "0.1.1";
 try {
-  const pkgPath = join13(dirname7(fileURLToPath2(import.meta.url)), "..", "package.json");
+  const pkgPath = join14(dirname7(fileURLToPath2(import.meta.url)), "..", "package.json");
   version = JSON.parse(readFileSync8(pkgPath, "utf8")).version ?? version;
 } catch {
 }
@@ -5818,7 +6144,7 @@ var registry = new Registry(store);
 var pruned = registry.pruneDeadManagedBuilds();
 if (pruned > 0) console.log(`pruned ${pruned} dangling engine build(s)`);
 var provision = new ProvisionState();
-var enginesDir = join13(store.dir(), "engines");
+var enginesDir = join14(store.dir(), "engines");
 void seedDefaultEngines(registry, enginesDir, provision).then(() => registry.ensureProbed());
 var manager = new Manager(store);
 var scanner = new Scanner(store);
@@ -5944,7 +6270,7 @@ var restarting = false;
 function spawnReplacement() {
   let out = "ignore";
   try {
-    out = openSync3(join13(store.dir(), "restart.log"), "a");
+    out = openSync3(join14(store.dir(), "restart.log"), "a");
   } catch {
     out = "ignore";
   }
@@ -6029,7 +6355,10 @@ void (async () => {
       extraArgs: cfg.devModel.extraArgs
     };
   }
-  if (opts) manager.start(opts).catch((e) => console.warn(`auto-load failed: ${e}`));
+  if (opts) {
+    await comfy.freeComfyUIBeforeLoad();
+    manager.start(opts).catch((e) => console.warn(`auto-load failed: ${e}`));
+  }
 })();
 var shuttingDown = false;
 for (const sig of ["SIGINT", "SIGTERM"]) {