turbollm 0.6.0 → 0.6.2

package/dist/cli.js

@@ -569,12 +569,32 @@ async function probeMlx(python) {
   );
   return `mlx-lm ${stdout.trim()}`;
 }
-function mlxServerCommand(python, model, port2, host2) {
+function mlxServerCommand(python, model, port2, host2, extraArgs = []) {
   return {
     cmd: python,
-    args: ["-m", "mlx_lm", "server", "--model", model, "--host", host2, "--port", String(port2)]
+    args: [
+      "-m",
+      "mlx_lm",
+      "server",
+      "--model",
+      model,
+      "--host",
+      host2,
+      "--port",
+      String(port2),
+      ...extraArgs
+    ]
   };
 }
+function mlxSamplingArgs(s) {
+  if (!s) return [];
+  const a = [];
+  if (typeof s.temp === "number") a.push("--temp", String(s.temp));
+  if (typeof s.topP === "number") a.push("--top-p", String(s.topP));
+  if (typeof s.topK === "number") a.push("--top-k", String(s.topK));
+  if (typeof s.minP === "number") a.push("--min-p", String(s.minP));
+  return a;
+}
 
 // src/engines/slot-cache.ts
 import { createHash } from "crypto";
@@ -705,6 +725,11 @@ function vllmServerCommand(python, model, port2, host2, tensorParallelSize = 1)
     "vllm.entrypoints.openai.api_server",
     "--model",
     model,
+    // Serve under a fixed alias so requests can address the model by a stable name
+    // (TurboLLM's internal key is a display string with spaces). Mirrors mlx-lm's
+    // built-in `default_model` alias; see engineModelAlias() in compat.ts.
+    "--served-model-name",
+    "default_model",
     "--host",
     host2,
     "--port",
@@ -781,7 +806,7 @@ var Manager = class {
       }
     }
     const { cmd, args } = engineCommand(opts, port2, slotSavePath);
-    const child = spawn(cmd, args, { cwd: dirname2(cmd), windowsHide: true });
+    const child = spawn(cmd, args, { cwd: dirname2(cmd), windowsHide: true, env: pyEngineEnv(opts.engine.kind, this.store.dir()) });
     child.stdout?.pipe(logStream, { end: false });
     child.stderr?.pipe(logStream, { end: false });
     this.state = "starting";
@@ -953,10 +978,22 @@ var Manager = class {
     this.resolveExited?.();
   }
   async readiness(child, port2) {
-    const deadline = Date.now() + readinessTimeoutMs(this.opts?.engine.kind ?? "llama-server");
+    const kind = this.opts?.engine.kind ?? "llama-server";
+    const deadline = Date.now() + readinessTimeoutMs(kind);
     for (; ; ) {
       await sleep(500);
       if (this.child !== child || this.state !== "starting") return;
+      if (kind === "mlx" || kind === "vllm") {
+        const loadErr = detectPyLoadFailure(readTail(this.logPathStr, 200));
+        if (loadErr) {
+          if (this.child === child && this.state === "starting") {
+            this.state = "error";
+            this.errInfo = { code: "model_load_failed", message: loadErr, exitCode: -1, logTail: readTail(this.logPathStr, 20) };
+            child.kill("SIGKILL");
+          }
+          return;
+        }
+      }
       if (await probeReady(port2)) {
         if (this.child === child && this.state === "starting") {
           this.state = "running";
@@ -988,7 +1025,7 @@ var Manager = class {
 };
 function engineCommand(opts, port2, slotSavePath) {
   if (opts.engine.kind === "mlx") {
-    return mlxServerCommand(opts.engine.binPath, opts.modelPath, port2, "127.0.0.1");
+    return mlxServerCommand(opts.engine.binPath, opts.modelPath, port2, "127.0.0.1", opts.extraArgs);
   }
   if (opts.engine.kind === "vllm") {
     return vllmServerCommand(opts.engine.binPath, opts.modelPath, port2, "127.0.0.1", opts.tensorParallelSize);
@@ -998,6 +1035,27 @@ function engineCommand(opts, port2, slotSavePath) {
 function readinessTimeoutMs(kind) {
   return kind === "vllm" ? 6e5 : 12e4;
 }
+function pyEngineEnv(kind, dataDir) {
+  if (kind !== "mlx" && kind !== "vllm") return void 0;
+  const hfHome = join6(dataDir, "hf-cache");
+  const hubCache = join6(hfHome, "hub");
+  mkdirSync4(hubCache, { recursive: true });
+  return {
+    ...process.env,
+    HF_HUB_OFFLINE: "1",
+    TRANSFORMERS_OFFLINE: "1",
+    HF_HOME: hfHome,
+    HF_HUB_CACHE: hubCache
+  };
+}
+function detectPyLoadFailure(lines) {
+  const text = lines.join("\n");
+  const isLoadCrash = /Exception in thread[^\n]*_generate/.test(text) || /in load_default\b/.test(text) || /in load_model\b/.test(text) || /load_weights/.test(text);
+  if (!isLoadCrash) return null;
+  const errLine = [...lines].reverse().find((l) => /^[A-Za-z_][\w.]*(Error|Exception):/.test(l.trim()));
+  const detail = (errLine ? errLine.trim() : "the model failed to load").slice(0, 200);
+  return `MLX could not load this model \u2014 ${detail} This usually means the installed mlx-lm version does not support this model's architecture or quantization.`;
+}
 function buildArgs(opts, port2, slotSavePath) {
   const args = ["-m", opts.modelPath, "--host", "127.0.0.1", "--port", String(port2)];
   const flags = opts.engine.capabilities.flags;
@@ -1715,6 +1773,10 @@ function engineAcceptsFormat(engineKind, format) {
   if (engineKind === "vllm") return format === "mlx";
   return format === "gguf";
 }
+var ENGINE_MODEL_ALIAS = "default_model";
+function engineModelAlias(engineKind) {
+  return engineKind === "mlx" || engineKind === "vllm" ? ENGINE_MODEL_ALIAS : null;
+}
 
 // src/models/scanner.ts
 import { existsSync as existsSync9, lstatSync, readdirSync as readdirSync3, readFileSync as readFileSync3, rmSync as rmSync4, writeFileSync as writeFileSync2 } from "fs";
@@ -3760,13 +3822,14 @@ var ModelRouter = class {
     const cfg2 = this.store.snapshot();
     const sys = getSysInfo();
     if (entry.format !== "gguf") {
-      const savedGpu = cfg2.modelProfiles[entry.key]?.gpu;
+      const savedProfile = cfg2.modelProfiles[entry.key];
       return {
         engine,
         model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: entry.nativeCtx, vision: false },
         modelPath: entry.path,
-        extraArgs: [],
-        tensorParallelSize: savedGpu?.tensorParallelSize
+        // MLX honors sampling as launch defaults; vLLM takes no extra flags here.
+        extraArgs: engine.kind === "mlx" ? mlxSamplingArgs(savedProfile?.sampling) : [],
+        tensorParallelSize: savedProfile?.gpu?.tensorParallelSize
       };
     }
     const saved = cfg2.modelProfiles[entry.key];
@@ -4405,13 +4468,15 @@ function registerApi(app2, d) {
       }
       let opts2;
       if (entry.format !== "gguf") {
-        const savedGpu = cfg2.modelProfiles[entry.key]?.gpu;
+        const savedProfile = cfg2.modelProfiles[entry.key];
         opts2 = {
           engine: active,
           model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: entry.nativeCtx, vision: false },
           modelPath: entry.path,
-          extraArgs: [],
-          tensorParallelSize: savedGpu?.tensorParallelSize
+          // MLX honors sampling as launch defaults; vLLM gets no extra flags here
+          // (its multi-GPU shard count maps to --tensor-parallel-size below).
+          extraArgs: active.kind === "mlx" ? mlxSamplingArgs(savedProfile?.sampling) : [],
+          tensorParallelSize: savedProfile?.gpu?.tensorParallelSize
         };
       } else {
         const saved = cfg2.modelProfiles[entry.key];
@@ -5420,7 +5485,9 @@ async function runGeneration(d, stream, ctx) {
     if (!(k in SAMPLING_KEYS) && k !== "stop") samplingOverride[k] = v;
   }
   const reqBody = {
-    model: ms.model.key,
+    // mlx-lm / vLLM serve under a fixed alias and 404 on TurboLLM's internal key;
+    // llama.cpp ignores the field. engineModelAlias() returns the right value per kind.
+    model: engineModelAlias(d.registry.active()?.kind ?? "") ?? ms.model.key,
     messages: engineMessages,
     stream: true,
     stream_options: { include_usage: true },
@@ -5668,7 +5735,7 @@ async function autoTitle(d, convId, prevMessages, assistantReply, target) {
       method: "POST",
       headers: { "Content-Type": "application/json" },
       body: JSON.stringify({
-        model: ms.model?.key,
+        model: engineModelAlias(d.registry.active()?.kind ?? "") ?? ms.model?.key,
         messages: titleMessages,
         stream: false,
         temperature: 0.3,
@@ -6021,6 +6088,8 @@ function registerGateway(app2, d) {
     const status = d.manager.status();
     const modelName = status.state === "running" ? status.model?.name ?? req.model ?? "local" : req.model ?? "local";
     const oaiBody = mapToOpenAI(req);
+    const oaiAlias = engineModelAlias(d.registry.active()?.kind ?? "");
+    if (oaiAlias) oaiBody.model = oaiAlias;
     d.manager.generationStart();
     let res;
     try {
@@ -6144,6 +6213,10 @@ function registerGateway(app2, d) {
         if (parsedBody && maxLimit > 0) {
           parsedBody.max_tokens = clampMaxTokens(parsedBody.max_tokens, maxLimit);
         }
+        if (parsedBody) {
+          const alias = engineModelAlias(d.registry.active()?.kind ?? "");
+          if (alias) parsedBody.model = alias;
+        }
         headers.delete("content-length");
         init.body = parsedBody ? JSON.stringify(parsedBody) : "";
       } else {