turbollm 0.7.1 → 0.7.2

package/dist/cli.js

@@ -288,9 +288,9 @@ function clampMaxTokens(requested, limit) {
 }
 
 // src/engines/manager.ts
-import { execFile as execFile4, spawn } from "child_process";
-import { createWriteStream as createWriteStream2, existsSync as existsSync6, mkdirSync as mkdirSync4, readFileSync as readFileSync2 } from "fs";
-import { createServer } from "net";
+import { execFile as execFile4, spawn, spawnSync } from "child_process";
+import { createWriteStream as createWriteStream2, existsSync as existsSync6, mkdirSync as mkdirSync4, readdirSync as readdirSync3, readFileSync as readFileSync2, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
+import { createConnection, createServer } from "net";
 import { dirname as dirname2, join as join6 } from "path";
 
 // src/engines/mlx.ts
@@ -773,12 +773,35 @@ var BusyError = class extends Error {
     this.name = "BusyError";
   }
 };
-var Manager = class {
+var Manager = class _Manager {
   constructor(store2) {
     this.store = store2;
     setInterval(() => this.watchdogTick(), 6e4).unref();
   }
   store;
+  /** Global single-load lock (rules 1 & 2). Shared across EVERY Manager instance —
+   *  including the gateway keep-N pool's extra slots — so at most ONE model load /
+   *  reload is ever in flight at a time, no matter who requests it. Holding it
+   *  through readiness (not just spawn) guarantees two engines never spin up at
+   *  once and double-allocate VRAM. All load paths funnel through start()/load(),
+   *  the only entry points that touch the engine — nothing spawns an engine without
+   *  passing this gate (rule 3). */
+  static loadGate = Promise.resolve();
+  /** Acquire the global load gate, run `fn` exclusively, then release it. Queued
+   *  callers run in FIFO order; a thrown fn still releases the gate. */
+  static async runExclusive(fn) {
+    const prev = _Manager.loadGate;
+    let release;
+    _Manager.loadGate = new Promise((r) => {
+      release = r;
+    });
+    await prev;
+    try {
+      return await fn();
+    } finally {
+      release();
+    }
+  }
   state = "stopped";
   opts = null;
   port = 0;
@@ -793,7 +816,41 @@ var Manager = class {
   generation = 0;
   liveGen = null;
   session = freshSession();
+  /** Load a model, freeing any currently-loaded one first — the single atomic
+   *  swap entry point (rules 1–3). Runs under the global load gate so the stop +
+   *  optional pre-start hook (e.g. the ComfyUI VRAM free) + spawn + readiness wait
+   *  are one indivisible operation; no other load can interleave. Never throws on a
+   *  model that simply fails to load — callers read status() for the running/error
+   *  outcome. */
+  async load(opts, hooks) {
+    await _Manager.runExclusive(async () => {
+      if (this.state === "running" || this.state === "starting" || this.state === "stopping") {
+        await this.stopAndWait();
+      }
+      if (hooks?.beforeStart) await hooks.beforeStart();
+      await this.startInternal(opts);
+      await this.awaitNotStarting();
+    });
+  }
+  /** Start a model assuming nothing is loaded (throws BusyError otherwise). Held by
+   *  the global load gate through readiness so concurrent loads can't spin up two
+   *  engines at once. Most callers want load() (which stops first); this exists for
+   *  paths that have already ensured the engine is free (bench, ComfyUI reload). */
   async start(opts) {
+    await _Manager.runExclusive(async () => {
+      await this.startInternal(opts);
+      await this.awaitNotStarting();
+    });
+  }
+  /** Wait until the engine leaves the 'starting' state (→ running or error/stopped),
+   *  bounded by the engine kind's readiness window plus a small grace. The internal
+   *  readiness loop flips the state and surfaces errors; this just keeps the load
+   *  gate held until that resolves. */
+  async awaitNotStarting() {
+    const deadline = Date.now() + readinessTimeoutMs(this.opts?.engine.kind ?? "llama-server") + 5e3;
+    while (this.state === "starting" && Date.now() < deadline) await sleep(200);
+  }
+  async startInternal(opts) {
     if (this.state === "starting" || this.state === "running" || this.state === "stopping") {
       throw new BusyError();
     }
@@ -825,6 +882,7 @@ var Manager = class {
     this.opts = opts;
     this.port = port2;
     this.pid = child.pid ?? 0;
+    if (this.pid) writeEnginePid(this.store.dir(), this.pid, port2);
     this.child = child;
     this.startedAt = Date.now();
     this.errInfo = null;
@@ -870,14 +928,8 @@ var Manager = class {
   }
   async restart() {
     const opts = this.opts;
-    const running = this.state === "running" || this.state === "starting";
-    const exited = this.exited;
     if (!opts?.modelPath) throw new Error("no_such_model");
-    if (running) {
-      this.stop();
-      await Promise.race([exited, sleep(1e4)]);
-    }
-    await this.start(opts);
+    await this.load(opts);
   }
   status() {
     const st = { state: this.state, err: this.errInfo, port: this.port, pid: this.pid, model: null, loadElapsedMs: 0 };
@@ -960,6 +1012,7 @@ var Manager = class {
   // ---- internal ----------------------------------------------------------
   onTerminated(child, code, logStream, errMsg) {
     if (this.child !== child) return;
+    if (child.pid) clearEnginePid(this.store.dir(), child.pid);
     const cleanStop = this.state === "stopping" || this.state === "stopped";
     try {
       logStream.write(
@@ -1130,6 +1183,114 @@ function forceKill(child) {
     child.kill("SIGKILL");
   }
 }
+function enginePidDir(dataDir) {
+  return join6(dataDir, "run");
+}
+function writeEnginePid(dataDir, pid, port2) {
+  try {
+    const dir = enginePidDir(dataDir);
+    mkdirSync4(dir, { recursive: true });
+    writeFileSync2(join6(dir, `engine-${pid}.pid`), JSON.stringify({ pid, port: port2, owner: process.pid }));
+  } catch {
+  }
+}
+function clearEnginePid(dataDir, pid) {
+  try {
+    rmSync4(join6(enginePidDir(dataDir), `engine-${pid}.pid`), { force: true });
+  } catch {
+  }
+}
+function pidAlive(pid) {
+  if (!pid) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (e) {
+    return e.code === "EPERM";
+  }
+}
+function readEnginePidFiles(dataDir) {
+  const dir = enginePidDir(dataDir);
+  let names;
+  try {
+    names = readdirSync3(dir).filter((n) => /^engine-\d+\.pid$/.test(n));
+  } catch {
+    return [];
+  }
+  const out = [];
+  for (const name of names) {
+    const file = join6(dir, name);
+    try {
+      const { pid, port: port2, owner } = JSON.parse(readFileSync2(file, "utf8"));
+      if (typeof pid === "number" && pid > 0) {
+        out.push({ pid, port: typeof port2 === "number" ? port2 : 0, owner: typeof owner === "number" ? owner : 0, file });
+      } else rmSync4(file, { force: true });
+    } catch {
+      try {
+        rmSync4(file, { force: true });
+      } catch {
+      }
+    }
+  }
+  return out;
+}
+function portAlive(port2, timeoutMs = 600) {
+  if (!port2) return Promise.resolve(false);
+  return new Promise((resolve2) => {
+    const sock = createConnection({ host: "127.0.0.1", port: port2 });
+    const done = (alive) => {
+      sock.destroy();
+      resolve2(alive);
+    };
+    sock.setTimeout(timeoutMs);
+    sock.once("connect", () => done(true));
+    sock.once("timeout", () => done(false));
+    sock.once("error", () => done(false));
+  });
+}
+async function reapStaleEngines(dataDir) {
+  let killed = 0;
+  for (const { pid, port: port2, owner, file } of readEnginePidFiles(dataDir)) {
+    if (owner && pidAlive(owner)) continue;
+    if (await portAlive(port2)) {
+      killPidTree(pid);
+      killed++;
+    }
+    try {
+      rmSync4(file, { force: true });
+    } catch {
+    }
+  }
+  return killed;
+}
+function killTrackedEnginesSync(dataDir) {
+  for (const { pid, owner, file } of readEnginePidFiles(dataDir)) {
+    if (owner !== process.pid) continue;
+    try {
+      if (process.platform === "win32") {
+        spawnSync("taskkill", ["/PID", String(pid), "/F", "/T"]);
+      } else {
+        process.kill(pid, "SIGKILL");
+      }
+    } catch {
+    }
+    try {
+      rmSync4(file, { force: true });
+    } catch {
+    }
+  }
+}
+function killPidTree(pid) {
+  if (process.platform === "win32") {
+    execFile4("taskkill", ["/PID", String(pid), "/F", "/T"], () => {
+    });
+  } else {
+    try {
+      process.kill(pid, "SIGKILL");
+    } catch {
+    }
+  }
+}
 function readTail(path, n) {
   if (!path || !existsSync6(path)) return [];
   try {
@@ -1790,7 +1951,7 @@ function engineModelAlias(engineKind) {
 }
 
 // src/models/scanner.ts
-import { existsSync as existsSync9, lstatSync, readdirSync as readdirSync3, readFileSync as readFileSync3, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
+import { existsSync as existsSync9, lstatSync, readdirSync as readdirSync4, readFileSync as readFileSync3, rmSync as rmSync5, writeFileSync as writeFileSync3 } from "fs";
 import { basename, dirname as dirname4, join as join7 } from "path";
 
 // src/gguf/gguf.ts
@@ -2099,7 +2260,7 @@ var Scanner = class {
     const total = m[3];
     let names;
     try {
-      names = readdirSync3(e.dir);
+      names = readdirSync4(e.dir);
     } catch {
       return [e.path];
     }
@@ -2118,9 +2279,9 @@ var Scanner = class {
     if (!e) throw new ScannerError("no_such_model", "No model with that key.");
     const paths = this.filesFor(key);
     if (e.format === "mlx") {
-      rmSync4(e.path, { recursive: true, force: true });
+      rmSync5(e.path, { recursive: true, force: true });
     } else {
-      for (const p of paths) rmSync4(p, { force: true });
+      for (const p of paths) rmSync5(p, { force: true });
       this.cache.delete(e.path);
     }
     await this.rescan();
@@ -2247,7 +2408,7 @@ var Scanner = class {
     const entries = {};
     for (const [k, v] of this.cache) entries[k] = v;
     try {
-      writeFileSync2(this.cachePath, JSON.stringify({ version: CACHE_VERSION, entries }));
+      writeFileSync3(this.cachePath, JSON.stringify({ version: CACHE_VERSION, entries }));
     } catch {
     }
   }
@@ -2262,7 +2423,7 @@ function isMlxModelDir(names) {
 function walk(dir, out) {
   let names;
   try {
-    names = readdirSync3(dir);
+    names = readdirSync4(dir);
   } catch {
     return;
   }
@@ -2299,7 +2460,7 @@ function mlxEntryFor(dir) {
   let mtimeMs = 0;
   let hasChatTemplate = false;
   try {
-    for (const n of readdirSync3(dir)) {
+    for (const n of readdirSync4(dir)) {
       const lower = n.toLowerCase();
       if (lower.endsWith(".safetensors")) {
         const st = lstatSync(join7(dir, n));
@@ -2368,7 +2529,7 @@ function tick() {
 
 // src/models/hashes.ts
 import { createHash as createHash2 } from "crypto";
-import { createReadStream, mkdirSync as mkdirSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync3 } from "fs";
+import { createReadStream, mkdirSync as mkdirSync5, readFileSync as readFileSync4, writeFileSync as writeFileSync4 } from "fs";
 import { join as join8 } from "path";
 var HashStore = class {
   cache = /* @__PURE__ */ new Map();
@@ -2419,7 +2580,7 @@ var HashStore = class {
     const entries = {};
     for (const [k, v] of this.cache) entries[k] = v;
     try {
-      writeFileSync3(this.path, JSON.stringify({ version: 1, entries }));
+      writeFileSync4(this.path, JSON.stringify({ version: 1, entries }));
     } catch {
     }
   }
@@ -3026,10 +3187,10 @@ import {
   mkdirSync as mkdirSync6,
   readFileSync as readFileSync5,
   renameSync as renameSync2,
-  rmSync as rmSync5,
+  rmSync as rmSync6,
   statfsSync,
   statSync as statSync3,
-  writeFileSync as writeFileSync4
+  writeFileSync as writeFileSync5
 } from "fs";
 import { basename as basename2, join as join10 } from "path";
 import { Readable as Readable2 } from "stream";
@@ -3140,7 +3301,7 @@ var DownloadManager = class {
     this.controllers.get(id)?.abort();
     this.controllers.delete(id);
     if (rec.status !== "done") {
-      rmSync5(`${rec.dest}.part`, { force: true });
+      rmSync6(`${rec.dest}.part`, { force: true });
       rec.status = "cancelled";
       rec.bytesPerSec = 0;
     }
@@ -3154,7 +3315,7 @@ var DownloadManager = class {
     if (!rec) return false;
     this.controllers.get(id)?.abort();
     this.controllers.delete(id);
-    if (rec.status !== "done") rmSync5(`${rec.dest}.part`, { force: true });
+    if (rec.status !== "done") rmSync6(`${rec.dest}.part`, { force: true });
     this.records.delete(id);
     this.persist();
     this.pump();
@@ -3219,7 +3380,7 @@ var DownloadManager = class {
       if (!res.body) throw new DownloadError("download_failed", "Empty response body.");
       const resuming = res.status === 206;
       if (!resuming && startAt > 0) {
-        rmSync5(part, { force: true });
+        rmSync6(part, { force: true });
         startAt = 0;
         rec.received = 0;
       }
@@ -3244,13 +3405,13 @@ var DownloadManager = class {
       const out = createWriteStream3(part, startAt > 0 ? { flags: "a" } : { flags: "w" });
       await pipeline2(body3, out, { signal: ac.signal });
       if (rec.total > 0 && rec.received !== rec.total) {
-        rmSync5(part, { force: true });
+        rmSync6(part, { force: true });
         throw new DownloadError("size_mismatch", "Download corrupt \u2014 size did not match. Removed the partial file.");
       }
       if (verifyHash && rec.sha256) {
         const got = hash.digest("hex");
         if (got !== rec.sha256) {
-          rmSync5(part, { force: true });
+          rmSync6(part, { force: true });
           throw new DownloadError("checksum_failed", "Checksum failed \u2014 the downloaded file was corrupt.");
         }
       }
@@ -3303,7 +3464,7 @@ var DownloadManager = class {
   }
   saveProvenance() {
     try {
-      writeFileSync4(this.provenancePath, JSON.stringify({ version: 1, entries: this.provenanceList }, null, 2));
+      writeFileSync5(this.provenancePath, JSON.stringify({ version: 1, entries: this.provenanceList }, null, 2));
     } catch {
     }
   }
@@ -3325,7 +3486,7 @@ var DownloadManager = class {
       });
     }
     try {
-      writeFileSync4(this.manifestPath, JSON.stringify({ version: 1, entries }, null, 2));
+      writeFileSync5(this.manifestPath, JSON.stringify({ version: 1, entries }, null, 2));
     } catch {
     }
   }
@@ -3375,7 +3536,7 @@ function safePathname(u) {
 
 // src/bench/bench.ts
 import { execFile as execFile6 } from "child_process";
-import { mkdirSync as mkdirSync7, writeFileSync as writeFileSync5 } from "fs";
+import { mkdirSync as mkdirSync7, writeFileSync as writeFileSync6 } from "fs";
 import { join as join11 } from "path";
 import { randomUUID as randomUUID4 } from "crypto";
 var READY_TIMEOUT_MS = 12e4;
@@ -3646,7 +3807,7 @@ var BenchRunner = class {
       };
       const queueDir = join11(this.store.dir(), "telemetry", "queue");
       mkdirSync7(queueDir, { recursive: true });
-      writeFileSync5(join11(queueDir, `${randomUUID4()}.json`), JSON.stringify(event));
+      writeFileSync6(join11(queueDir, `${randomUUID4()}.json`), JSON.stringify(event));
     } catch {
     }
   }
@@ -3783,16 +3944,15 @@ var ModelRouter = class {
     const keepN = Math.max(1, this.store.snapshot().gateway.keepN);
     const needsNewSlot = entry.embedding || this.chatSlotCount() < keepN;
     const targetManager = needsNewSlot ? this.manager.status().state === "stopped" || this.manager.status().state === "error" ? this.manager : new Manager(this.store) : this.evictChatLru();
-    await targetManager.stopAndWait();
-    await this.comfy?.freeComfyUIBeforeLoad();
     try {
-      await targetManager.start(opts);
+      await targetManager.load(opts, {
+        beforeStart: () => this.comfy?.freeComfyUIBeforeLoad() ?? Promise.resolve()
+      });
     } catch (e) {
       return { status: 503, message: `Engine start failed: ${e.message}` };
     }
-    const ready = await this.waitReady(targetManager, active.kind);
-    if (!ready) {
-      const s = targetManager.status();
+    const s = targetManager.status();
+    if (s.state !== "running") {
       return { status: 503, message: s.err?.message ?? "Model failed to become ready." };
     }
     const target = targetManager.target();
@@ -3884,23 +4044,7 @@ var ModelRouter = class {
       extraArgs: profileToArgs(profile, entry, engine.capabilities, sys.cores)
     };
   }
-  /** Poll until the manager's engine process becomes ready or fails.
-   *  Mirrors the Manager's internal readiness timeout by engine kind. */
-  async waitReady(manager2, engineKind) {
-    const timeoutMs = engineKind === "vllm" ? 6e5 : 12e4;
-    const deadline = Date.now() + timeoutMs;
-    while (Date.now() < deadline) {
-      const s = manager2.status();
-      if (s.state === "running") return true;
-      if (s.state === "error" || s.state === "stopped") return false;
-      await sleep3(250);
-    }
-    return false;
-  }
 };
-function sleep3(ms) {
-  return new Promise((r) => setTimeout(r, ms));
-}
 
 // src/tools/builtin.ts
 import { runInNewContext } from "vm";
@@ -4470,7 +4614,7 @@ import { Agent, setGlobalDispatcher } from "undici";
 
 // src/api/routes.ts
 import { streamSSE } from "hono/streaming";
-import { existsSync as existsSync11, mkdirSync as mkdirSync8, readFileSync as readFileSync6, readdirSync as readdirSync4, realpathSync, rmSync as rmSync6, statSync as statSync4, writeFileSync as writeFileSync6 } from "fs";
+import { existsSync as existsSync11, mkdirSync as mkdirSync8, readFileSync as readFileSync6, readdirSync as readdirSync5, realpathSync, rmSync as rmSync7, statSync as statSync4, writeFileSync as writeFileSync7 } from "fs";
 import { basename as basename3, dirname as dirname5, join as join12, resolve, sep } from "path";
 
 // src/comfyui/gate-template.ts
@@ -4681,6 +4825,23 @@ function catalogEngine(id) {
   return ALL.find((e) => e.id === id);
 }
 
+// src/api/path-utils.ts
+function inferRepoFromPath(filePath, modelDirs) {
+  const norm = (p) => p.replace(/\\/g, "/").replace(/\/+$/, "");
+  const fp = norm(filePath);
+  const seg = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+  for (const dir of modelDirs) {
+    const root = norm(dir);
+    if (!fp.toLowerCase().startsWith(root.toLowerCase() + "/")) continue;
+    const parts = fp.slice(root.length + 1).split("/");
+    if (parts.length >= 2 && seg.test(parts[0]) && seg.test(parts[1])) {
+      return `${parts[0]}/${parts[1]}`;
+    }
+    return null;
+  }
+  return null;
+}
+
 // src/api/routes.ts
 function err(c, status, code, message) {
   return c.json({ error: { code, message } }, status);
@@ -4963,7 +5124,7 @@ function registerApi(app2, d) {
     }
     let entries;
     try {
-      entries = readdirSync4(real, { withFileTypes: true }).filter((d2) => !d2.name.startsWith(".")).map((d2) => {
+      entries = readdirSync5(real, { withFileTypes: true }).filter((d2) => !d2.name.startsWith(".")).map((d2) => {
         let isDir = d2.isDirectory();
         if (d2.isSymbolicLink()) {
           try {
@@ -5020,13 +5181,7 @@ function registerApi(app2, d) {
           extraArgs: profileToArgs(profile, entry, active.capabilities, sys.cores)
         };
       }
-      await d.manager.stopAndWait();
-      await d.comfy?.freeComfyUIBeforeLoad();
-      try {
-        await d.manager.start(opts2);
-      } catch (e) {
-        return startError(c, e);
-      }
+      void d.manager.load(opts2, { beforeStart: () => d.comfy?.freeComfyUIBeforeLoad() ?? Promise.resolve() }).catch((e) => console.warn(`engine load failed: ${e}`));
       d.store.update((x) => {
         x.lastLoaded = { modelKey: entry.key, engineId: active.id };
       });
@@ -5042,13 +5197,7 @@ function registerApi(app2, d) {
     }
     if (!modelPath) return err(c, 409, "no_such_model", "No model specified. Pick one from the Models screen.");
     const opts = { engine: active, model: deriveModel(modelPath, name, extra), modelPath, extraArgs: extra };
-    await d.manager.stopAndWait();
-    await d.comfy?.freeComfyUIBeforeLoad();
-    try {
-      await d.manager.start(opts);
-    } catch (e) {
-      return startError(c, e);
-    }
+    void d.manager.load(opts, { beforeStart: () => d.comfy?.freeComfyUIBeforeLoad() ?? Promise.resolve() }).catch((e) => console.warn(`engine load failed: ${e}`));
     return c.json({ ok: true }, 202);
   });
   app2.post("/api/v1/engine/stop", (c) => {
@@ -5089,7 +5238,7 @@ function registerApi(app2, d) {
     const gateDir = join12(customNodes, "turbollm_gate");
     try {
       mkdirSync8(gateDir, { recursive: true });
-      writeFileSync6(join12(gateDir, "__init__.py"), gateNodeSource(base2));
+      writeFileSync7(join12(gateDir, "__init__.py"), gateNodeSource(base2));
     } catch (e) {
       return err(c, 500, "fs_write_failed", `Could not write the gate node: ${e instanceof Error ? e.message : e}`);
     }
@@ -5102,7 +5251,7 @@ function registerApi(app2, d) {
     const dir = d.store.snapshot().comfyui.gatePath;
     if (dir && existsSync11(dir)) {
       try {
-        rmSync6(dir, { recursive: true, force: true });
+        rmSync7(dir, { recursive: true, force: true });
       } catch (e) {
         return err(c, 500, "fs_write_failed", `Could not remove the gate node: ${e instanceof Error ? e.message : e}`);
       }
@@ -5591,21 +5740,6 @@ function overlayModel(e, d, lastTpsMap) {
   const sourceRepo = provRepo ?? inferRepoFromPath(e.path, snap.modelDirs);
   return { ...e, loaded, hasProfile: e.key in profiles, lastTps, liveTps, benchTps, compatibleWithActiveEngine, sourceRepo };
 }
-function inferRepoFromPath(filePath, modelDirs) {
-  const norm = (p) => p.replace(/\\/g, "/").replace(/\/+$/, "");
-  const fp = norm(filePath);
-  const seg = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
-  for (const dir of modelDirs) {
-    const root = norm(dir);
-    if (!fp.toLowerCase().startsWith(root.toLowerCase() + "/")) continue;
-    const parts = fp.slice(root.length + 1).split("/");
-    if (parts.length >= 2 && seg.test(parts[0]) && seg.test(parts[1])) {
-      return `${parts[0]}/${parts[1]}`;
-    }
-    return null;
-  }
-  return null;
-}
 function formatMismatchMessage(engineKind, format) {
   if (engineKind === "mlx")
     return "The active engine is MLX \u2014 pick a safetensors model, or switch to a llama.cpp engine for GGUF.";
@@ -5725,11 +5859,6 @@ function regErr(c, e) {
   if (e instanceof ValueError) return err(c, 400, "invalid_config_value", e.message);
   return err(c, 500, "internal", e.message);
 }
-function startError(c, e) {
-  if (e instanceof BusyError) return err(c, 409, "engine_already_running", "An engine is already running.");
-  if (e.message === "no_free_port") return err(c, 409, "no_free_port", "No free port for the engine (8081\u20138181 all in use).");
-  return err(c, 500, "engine_start_failed", e.message);
-}
 function deriveModel(modelPath, name, extraArgs) {
   let ctx = 0;
   for (let i = 0; i + 1 < extraArgs.length; i++) {
@@ -5878,12 +6007,95 @@ function buildConnectSnippets(cli, base2, apiKey, modelName) {
 
 // src/chat/chat-routes.ts
 import { streamSSE as streamSSE2 } from "hono/streaming";
-var inflight = /* @__PURE__ */ new Map();
+
+// src/chat/parser.ts
 var THINK_OPEN = "<think>";
 var THINK_CLOSE = "</think>";
 var CHAN_ANALYSIS_OPEN = "<|channel|>analysis<|message|>";
 var CHAN_CLOSE = "<|end|>";
 var CHAN_FINAL_SKIP = "<|start|>assistant<|channel|>final<|message|>";
+function initParseState() {
+  return { phase: "initial", isChannel: false, buf: "" };
+}
+function feedChunk(state, raw) {
+  const events = [];
+  let { phase, isChannel, buf } = state;
+  buf += raw;
+  while (buf.length > 0) {
+    if (phase === "initial") {
+      const thinkIdx = buf.indexOf(THINK_OPEN);
+      const chanIdx = buf.indexOf(CHAN_ANALYSIS_OPEN);
+      const hasThink = thinkIdx >= 0;
+      const hasChan = chanIdx >= 0;
+      const useThink = hasThink && (!hasChan || thinkIdx <= chanIdx);
+      const openIdx = useThink ? thinkIdx : hasChan ? chanIdx : -1;
+      const openTag = useThink ? THINK_OPEN : CHAN_ANALYSIS_OPEN;
+      if (openIdx === 0) {
+        isChannel = !useThink;
+        phase = "reasoning";
+        buf = buf.slice(openTag.length);
+      } else if (openIdx > 0) {
+        events.push({ type: "delta", text: buf.slice(0, openIdx) });
+        isChannel = !useThink;
+        phase = "reasoning";
+        buf = buf.slice(openIdx + openTag.length);
+      } else {
+        const safeLen = buf.length - (CHAN_ANALYSIS_OPEN.length - 1);
+        if (safeLen > 0) {
+          events.push({ type: "delta", text: buf.slice(0, safeLen) });
+          buf = buf.slice(safeLen);
+          phase = "content";
+        } else {
+          break;
+        }
+      }
+    } else if (phase === "reasoning") {
+      const closeTag = isChannel ? CHAN_CLOSE : THINK_CLOSE;
+      const closeIdx = buf.indexOf(closeTag);
+      if (closeIdx >= 0) {
+        if (closeIdx > 0) {
+          events.push({ type: "reasoning", text: buf.slice(0, closeIdx) });
+        }
+        const wasChannel = isChannel;
+        buf = buf.slice(closeIdx + closeTag.length);
+        phase = wasChannel ? "skipFinal" : "content";
+        if (!wasChannel && buf) {
+          events.push({ type: "delta", text: buf });
+          buf = "";
+        }
+      } else if (buf.length >= closeTag.length) {
+        const safe = buf.length - (closeTag.length - 1);
+        events.push({ type: "reasoning", text: buf.slice(0, safe) });
+        buf = buf.slice(safe);
+      } else {
+        break;
+      }
+    } else if (phase === "skipFinal") {
+      if (buf.startsWith(CHAN_FINAL_SKIP)) {
+        buf = buf.slice(CHAN_FINAL_SKIP.length);
+        phase = "content";
+      } else if (CHAN_FINAL_SKIP.startsWith(buf) && buf.length < CHAN_FINAL_SKIP.length) {
+        break;
+      } else {
+        const skipIdx = buf.indexOf(CHAN_FINAL_SKIP);
+        buf = skipIdx >= 0 ? buf.slice(skipIdx + CHAN_FINAL_SKIP.length) : "";
+        phase = "content";
+      }
+    } else {
+      events.push({ type: "delta", text: buf });
+      buf = "";
+      break;
+    }
+  }
+  return { state: { phase, isChannel, buf }, events };
+}
+function flushState(state) {
+  if (!state.buf) return [];
+  return state.phase === "reasoning" ? [{ type: "reasoning", text: state.buf }] : [{ type: "delta", text: state.buf }];
+}
+
+// src/chat/chat-routes.ts
+var inflight = /* @__PURE__ */ new Map();
 var EXPERT_SYSTEM_PROMPT = `You are the TurboLLM in-app expert assistant \u2014 a knowledgeable, friendly guide built into TurboLLM, a local-first desktop app for running large language models on the user's own machine.
 
 Your job is to help the user get the most out of TurboLLM:
@@ -6146,9 +6358,7 @@ async function runGeneration(d, stream, ctx) {
         ac.signal.addEventListener("abort", cancelReader, { once: true });
       }
       let roundContent = "";
-      let parsePhase = "initial";
-      let parseIsChannel = false;
-      let parseBuf = "";
+      let parseState = initParseState();
       let finishReason = "";
       const pendingToolCalls = /* @__PURE__ */ new Map();
       roundLoop: while (true) {
@@ -6202,108 +6412,33 @@ async function runGeneration(d, stream, ctx) {
           }
           const raw_content = delta.content ?? "";
           if (!raw_content) continue;
-          parseBuf += raw_content;
-          while (parseBuf.length > 0) {
-            if (parsePhase === "initial") {
-              const thinkIdx = parseBuf.indexOf(THINK_OPEN);
-              const chanIdx = parseBuf.indexOf(CHAN_ANALYSIS_OPEN);
-              const hasThink = thinkIdx >= 0;
-              const hasChan = chanIdx >= 0;
-              const useThink = hasThink && (!hasChan || thinkIdx <= chanIdx);
-              const openIdx = useThink ? thinkIdx : hasChan ? chanIdx : -1;
-              const openTag = useThink ? THINK_OPEN : CHAN_ANALYSIS_OPEN;
-              if (openIdx === 0) {
-                parseIsChannel = !useThink;
-                parsePhase = "reasoning";
-                if (!thinkStart) thinkStart = Date.now();
-                parseBuf = parseBuf.slice(openTag.length);
-              } else if (openIdx > 0) {
-                const before = parseBuf.slice(0, openIdx);
-                fullContent += before;
-                roundContent += before;
-                if (!ttftMs) ttftMs = Date.now() - requestStart;
-                await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: before }) });
-                parseIsChannel = !useThink;
-                parsePhase = "reasoning";
-                if (!thinkStart) thinkStart = Date.now();
-                parseBuf = parseBuf.slice(openIdx + openTag.length);
-              } else {
-                const safeLen = parseBuf.length - (CHAN_ANALYSIS_OPEN.length - 1);
-                if (safeLen > 0) {
-                  const flush = parseBuf.slice(0, safeLen);
-                  parseBuf = parseBuf.slice(safeLen);
-                  fullContent += flush;
-                  roundContent += flush;
-                  if (!ttftMs) ttftMs = Date.now() - requestStart;
-                  d.manager.setLiveGen({ phase: "gen", pct: 0, outputTokens: ++liveOut });
-                  await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: flush }) });
-                  parsePhase = "content";
-                } else {
-                  break;
-                }
-              }
-            } else if (parsePhase === "reasoning") {
-              const closeTag = parseIsChannel ? CHAN_CLOSE : THINK_CLOSE;
-              const closeIdx = parseBuf.indexOf(closeTag);
-              if (closeIdx >= 0) {
-                if (closeIdx > 0) {
-                  const chunk2 = parseBuf.slice(0, closeIdx);
-                  fullReasoning += chunk2;
-                  await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: chunk2 }) });
-                }
-                thinkEnd = Date.now();
-                const wasChannel = parseIsChannel;
-                parseBuf = parseBuf.slice(closeIdx + closeTag.length);
-                parsePhase = wasChannel ? "skipFinal" : "content";
-                if (!wasChannel && parseBuf) {
-                  fullContent += parseBuf;
-                  roundContent += parseBuf;
-                  if (!ttftMs) ttftMs = Date.now() - requestStart;
-                  await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: parseBuf }) });
-                  parseBuf = "";
-                }
-              } else if (parseBuf.length >= closeTag.length) {
-                const safe = parseBuf.length - (closeTag.length - 1);
-                const chunk2 = parseBuf.slice(0, safe);
-                parseBuf = parseBuf.slice(safe);
-                thinkEnd = Date.now();
-                fullReasoning += chunk2;
-                await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: chunk2 }) });
-              } else {
-                break;
-              }
-            } else if (parsePhase === "skipFinal") {
-              if (parseBuf.startsWith(CHAN_FINAL_SKIP)) {
-                parseBuf = parseBuf.slice(CHAN_FINAL_SKIP.length);
-                parsePhase = "content";
-              } else if (CHAN_FINAL_SKIP.startsWith(parseBuf) && parseBuf.length < CHAN_FINAL_SKIP.length) {
-                break;
-              } else {
-                const skipIdx = parseBuf.indexOf(CHAN_FINAL_SKIP);
-                parseBuf = skipIdx >= 0 ? parseBuf.slice(skipIdx + CHAN_FINAL_SKIP.length) : "";
-                parsePhase = "content";
-              }
+          const { state: nextState, events: parseEvents } = feedChunk(parseState, raw_content);
+          parseState = nextState;
+          for (const ev of parseEvents) {
+            if (ev.type === "reasoning") {
+              if (!thinkStart) thinkStart = Date.now();
+              thinkEnd = Date.now();
+              fullReasoning += ev.text;
+              await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: ev.text }) });
             } else {
+              fullContent += ev.text;
+              roundContent += ev.text;
               if (!ttftMs) ttftMs = Date.now() - requestStart;
-              fullContent += parseBuf;
-              roundContent += parseBuf;
               d.manager.setLiveGen({ phase: "gen", pct: 0, outputTokens: ++liveOut });
-              await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: parseBuf }) });
-              parseBuf = "";
-              break;
+              await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: ev.text }) });
             }
           }
         }
       }
       ac.signal.removeEventListener("abort", cancelReader);
-      if (parseBuf) {
-        if (parsePhase === "reasoning") {
-          fullReasoning += parseBuf;
-          await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: parseBuf }) });
+      for (const ev of flushState(parseState)) {
+        if (ev.type === "reasoning") {
+          fullReasoning += ev.text;
+          await stream.writeSSE({ event: "reasoning", data: JSON.stringify({ delta: ev.text }) });
         } else {
-          fullContent += parseBuf;
-          roundContent += parseBuf;
-          await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: parseBuf }) });
+          fullContent += ev.text;
+          roundContent += ev.text;
+          await stream.writeSSE({ event: "delta", data: JSON.stringify({ delta: ev.text }) });
         }
       }
       if ((finishReason === "tool_calls" || pendingToolCalls.size > 0) && d.tools && toolIter <= MAX_TOOL_ITER) {
@@ -6399,7 +6534,10 @@ async function runGeneration(d, stream, ctx) {
   } catch {
   }
   const finalMsg = db2.getMessage(assistantMsg.id);
-  await stream.writeSSE({ event: "done", data: JSON.stringify({ message: finalMsg }) });
+  try {
+    await stream.writeSSE({ event: "done", data: JSON.stringify({ message: finalMsg }) });
+  } catch {
+  }
   if (!aborted && conv.title === "New chat" && d.store.snapshot().daemon.autoGenerateTitles) {
     setTimeout(() => {
       void autoTitle(d, convId, ctx.engineMessages, fullContent, target);
@@ -6696,7 +6834,8 @@ async function* streamToAnthropic(oaiStream, modelName, msgId, onUsage, onLive)
     failed = true;
     yield sse("error", { error: { type: "api_error", message: "engine stopped" } });
   } finally {
-    reader.releaseLock();
+    await reader.cancel().catch(() => {
+    });
   }
   if (!failed) {
     if (inThinking) {
@@ -6747,6 +6886,15 @@ function cbStop(index) {
 }
 
 // src/gateway/gateway.ts
+function clientAbort(c) {
+  const ac = new AbortController();
+  const sig = c.req.raw.signal;
+  if (sig) {
+    if (sig.aborted) ac.abort();
+    else sig.addEventListener("abort", () => ac.abort(), { once: true });
+  }
+  return ac;
+}
 function registerGateway(app2, d) {
   app2.post("/v1/messages", async (c) => {
     let req;
@@ -6780,12 +6928,14 @@ function registerGateway(app2, d) {
     const oaiAlias = engineModelAlias(d.registry.active()?.kind ?? "");
     if (oaiAlias) oaiBody.model = oaiAlias;
     d.manager.generationStart();
+    const ac = clientAbort(c);
     let res;
     try {
       res = await fetch(`${target}/v1/chat/completions`, {
         method: "POST",
         headers: { "Content-Type": "application/json" },
-        body: JSON.stringify(oaiBody)
+        body: JSON.stringify(oaiBody),
+        signal: ac.signal
       });
     } catch (e) {
       d.manager.generationEnd();
@@ -6824,11 +6974,13 @@ function registerGateway(app2, d) {
         }
       );
       return streamSSE3(c, async (stream) => {
+        stream.onAbort(() => ac.abort());
         try {
           for await (const evt of gen) {
             await stream.writeSSE({ event: evt.event, data: evt.data });
           }
         } finally {
+          ac.abort();
           d.manager.generationEnd();
         }
       });
@@ -6896,7 +7048,8 @@ function registerGateway(app2, d) {
     const headers = new Headers(c.req.raw.headers);
     headers.delete("host");
     const maxLimit = d.store.snapshot().modelDefaults.maxTokens ?? 0;
-    const init = { method: c.req.method, headers };
+    const ac = clientAbort(c);
+    const init = { method: c.req.method, headers, signal: ac.signal };
     if (c.req.method !== "GET" && c.req.method !== "HEAD") {
       if (isChat) {
         if (parsedBody && maxLimit > 0) {
@@ -7124,6 +7277,10 @@ Please upgrade: https://nodejs.org
   );
   process.exit(1);
 }
+process.on("unhandledRejection", (reason) => {
+  if (reason?.name === "AbortError") return;
+  console.warn("unhandledRejection (continuing):", reason);
+});
 var argv = process.argv.slice(2);
 function hasFlag(...names) {
   return names.some((n) => argv.includes(n));
@@ -7176,6 +7333,8 @@ var store = ConfigStore.load(argValue("--config", defaultConfigPath()));
 if (store.brokenBackup()) {
   console.warn(`config was reset; previous file backed up at ${store.brokenBackup()}`);
 }
+var reaped = await reapStaleEngines(store.dir()).catch(() => 0);
+if (reaped > 0) console.log(`reaped ${reaped} orphaned engine process(es) from a previous run`);
 var registry = new Registry(store);
 var pruned = registry.pruneDeadManagedBuilds();
 if (pruned > 0) console.log(`pruned ${pruned} dangling engine build(s)`);
@@ -7398,12 +7557,11 @@ void (async () => {
     };
   }
   if (opts) {
-    await comfy.freeComfyUIBeforeLoad();
-    manager.start(opts).catch((e) => console.warn(`auto-load failed: ${e}`));
+    manager.load(opts, { beforeStart: () => comfy.freeComfyUIBeforeLoad() }).catch((e) => console.warn(`auto-load failed: ${e}`));
   }
 })();
 var shuttingDown = false;
-for (const sig of ["SIGINT", "SIGTERM"]) {
+for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"]) {
   process.on(sig, () => {
     if (shuttingDown) return;
     shuttingDown = true;
@@ -7417,3 +7575,9 @@ for (const sig of ["SIGINT", "SIGTERM"]) {
     setTimeout(() => process.exit(0), 12e3).unref();
   });
 }
+process.on("exit", () => {
+  try {
+    killTrackedEnginesSync(store.dir());
+  } catch {
+  }
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turbollm",
-  "version": "0.7.1",
+  "version": "0.7.2",
   "description": "TurboLLM — local LLM platform: run any inference engine auto-tuned to your GPU, with a web UI and OpenAI/Anthropic-compatible API. Point Claude Code at your own machine in one command.",
   "license": "FSL-1.1-ALv2",
   "author": "Mohit Soni",