turbollm 0.6.0 → 0.6.1

package/dist/cli.js

@@ -705,6 +705,11 @@ function vllmServerCommand(python, model, port2, host2, tensorParallelSize = 1)
     "vllm.entrypoints.openai.api_server",
     "--model",
     model,
+    // Serve under a fixed alias so requests can address the model by a stable name
+    // (TurboLLM's internal key is a display string with spaces). Mirrors mlx-lm's
+    // built-in `default_model` alias; see engineModelAlias() in compat.ts.
+    "--served-model-name",
+    "default_model",
     "--host",
     host2,
     "--port",
@@ -781,7 +786,7 @@ var Manager = class {
       }
     }
     const { cmd, args } = engineCommand(opts, port2, slotSavePath);
-    const child = spawn(cmd, args, { cwd: dirname2(cmd), windowsHide: true });
+    const child = spawn(cmd, args, { cwd: dirname2(cmd), windowsHide: true, env: pyEngineEnv(opts.engine.kind, this.store.dir()) });
     child.stdout?.pipe(logStream, { end: false });
     child.stderr?.pipe(logStream, { end: false });
     this.state = "starting";
@@ -953,10 +958,22 @@ var Manager = class {
     this.resolveExited?.();
   }
   async readiness(child, port2) {
-    const deadline = Date.now() + readinessTimeoutMs(this.opts?.engine.kind ?? "llama-server");
+    const kind = this.opts?.engine.kind ?? "llama-server";
+    const deadline = Date.now() + readinessTimeoutMs(kind);
     for (; ; ) {
       await sleep(500);
       if (this.child !== child || this.state !== "starting") return;
+      if (kind === "mlx" || kind === "vllm") {
+        const loadErr = detectPyLoadFailure(readTail(this.logPathStr, 200));
+        if (loadErr) {
+          if (this.child === child && this.state === "starting") {
+            this.state = "error";
+            this.errInfo = { code: "model_load_failed", message: loadErr, exitCode: -1, logTail: readTail(this.logPathStr, 20) };
+            child.kill("SIGKILL");
+          }
+          return;
+        }
+      }
       if (await probeReady(port2)) {
         if (this.child === child && this.state === "starting") {
           this.state = "running";
@@ -998,6 +1015,27 @@ function engineCommand(opts, port2, slotSavePath) {
 function readinessTimeoutMs(kind) {
   return kind === "vllm" ? 6e5 : 12e4;
 }
+function pyEngineEnv(kind, dataDir) {
+  if (kind !== "mlx" && kind !== "vllm") return void 0;
+  const hfHome = join6(dataDir, "hf-cache");
+  const hubCache = join6(hfHome, "hub");
+  mkdirSync4(hubCache, { recursive: true });
+  return {
+    ...process.env,
+    HF_HUB_OFFLINE: "1",
+    TRANSFORMERS_OFFLINE: "1",
+    HF_HOME: hfHome,
+    HF_HUB_CACHE: hubCache
+  };
+}
+function detectPyLoadFailure(lines) {
+  const text = lines.join("\n");
+  const isLoadCrash = /Exception in thread[^\n]*_generate/.test(text) || /in load_default\b/.test(text) || /in load_model\b/.test(text) || /load_weights/.test(text);
+  if (!isLoadCrash) return null;
+  const errLine = [...lines].reverse().find((l) => /^[A-Za-z_][\w.]*(Error|Exception):/.test(l.trim()));
+  const detail = (errLine ? errLine.trim() : "the model failed to load").slice(0, 200);
+  return `MLX could not load this model \u2014 ${detail} This usually means the installed mlx-lm version does not support this model's architecture or quantization.`;
+}
 function buildArgs(opts, port2, slotSavePath) {
   const args = ["-m", opts.modelPath, "--host", "127.0.0.1", "--port", String(port2)];
   const flags = opts.engine.capabilities.flags;
@@ -1715,6 +1753,10 @@ function engineAcceptsFormat(engineKind, format) {
   if (engineKind === "vllm") return format === "mlx";
   return format === "gguf";
 }
+var ENGINE_MODEL_ALIAS = "default_model";
+function engineModelAlias(engineKind) {
+  return engineKind === "mlx" || engineKind === "vllm" ? ENGINE_MODEL_ALIAS : null;
+}
 
 // src/models/scanner.ts
 import { existsSync as existsSync9, lstatSync, readdirSync as readdirSync3, readFileSync as readFileSync3, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
@@ -5420,7 +5462,9 @@ async function runGeneration(d, stream, ctx) {
     if (!(k in SAMPLING_KEYS) && k !== "stop") samplingOverride[k] = v;
   }
   const reqBody = {
-    model: ms.model.key,
+    // mlx-lm / vLLM serve under a fixed alias and 404 on TurboLLM's internal key;
+    // llama.cpp ignores the field. engineModelAlias() returns the right value per kind.
+    model: engineModelAlias(d.registry.active()?.kind ?? "") ?? ms.model.key,
     messages: engineMessages,
     stream: true,
     stream_options: { include_usage: true },
@@ -5668,7 +5712,7 @@ async function autoTitle(d, convId, prevMessages, assistantReply, target) {
       method: "POST",
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify({
-        model: ms.model?.key,
+        model: engineModelAlias(d.registry.active()?.kind ?? "") ?? ms.model?.key,
         messages: titleMessages,
         stream: false,
         temperature: 0.3,
@@ -6021,6 +6065,8 @@ function registerGateway(app2, d) {
     const status = d.manager.status();
     const modelName = status.state === "running" ? status.model?.name ?? req.model ?? "local" : req.model ?? "local";
     const oaiBody = mapToOpenAI(req);
+    const oaiAlias = engineModelAlias(d.registry.active()?.kind ?? "");
+    if (oaiAlias) oaiBody.model = oaiAlias;
     d.manager.generationStart();
     let res;
     try {
@@ -6144,6 +6190,10 @@ function registerGateway(app2, d) {
         if (parsedBody && maxLimit > 0) {
           parsedBody.max_tokens = clampMaxTokens(parsedBody.max_tokens, maxLimit);
         }
+        if (parsedBody) {
+          const alias = engineModelAlias(d.registry.active()?.kind ?? "");
+          if (alias) parsedBody.model = alias;
+        }
         headers.delete("content-length");
         init.body = parsedBody ? JSON.stringify(parsedBody) : "";
       } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "turbollm",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "TurboLLM — local LLM platform: run any inference engine auto-tuned to your GPU, with a web UI and OpenAI/Anthropic-compatible API. Point Claude Code at your own machine in one command.",
   "license": "FSL-1.1-ALv2",
   "author": "Mohit Soni",