turbollm 0.6.1 → 0.6.2

package/dist/cli.js

@@ -569,12 +569,32 @@ async function probeMlx(python) {
   );
   return `mlx-lm ${stdout.trim()}`;
 }
-function mlxServerCommand(python, model, port2, host2) {
+function mlxServerCommand(python, model, port2, host2, extraArgs = []) {
   return {
     cmd: python,
-    args: ["-m", "mlx_lm", "server", "--model", model, "--host", host2, "--port", String(port2)]
+    args: [
+      "-m",
+      "mlx_lm",
+      "server",
+      "--model",
+      model,
+      "--host",
+      host2,
+      "--port",
+      String(port2),
+      ...extraArgs
+    ]
   };
 }
+function mlxSamplingArgs(s) {
+  if (!s) return [];
+  const a = [];
+  if (typeof s.temp === "number") a.push("--temp", String(s.temp));
+  if (typeof s.topP === "number") a.push("--top-p", String(s.topP));
+  if (typeof s.topK === "number") a.push("--top-k", String(s.topK));
+  if (typeof s.minP === "number") a.push("--min-p", String(s.minP));
+  return a;
+}
 
 // src/engines/slot-cache.ts
 import { createHash } from "crypto";
@@ -1005,7 +1025,7 @@ var Manager = class {
 };
 function engineCommand(opts, port2, slotSavePath) {
   if (opts.engine.kind === "mlx") {
-    return mlxServerCommand(opts.engine.binPath, opts.modelPath, port2, "127.0.0.1");
+    return mlxServerCommand(opts.engine.binPath, opts.modelPath, port2, "127.0.0.1", opts.extraArgs);
   }
   if (opts.engine.kind === "vllm") {
     return vllmServerCommand(opts.engine.binPath, opts.modelPath, port2, "127.0.0.1", opts.tensorParallelSize);
@@ -3802,13 +3822,14 @@ var ModelRouter = class {
     const cfg2 = this.store.snapshot();
     const sys = getSysInfo();
     if (entry.format !== "gguf") {
-      const savedGpu = cfg2.modelProfiles[entry.key]?.gpu;
+      const savedProfile = cfg2.modelProfiles[entry.key];
       return {
         engine,
         model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: entry.nativeCtx, vision: false },
         modelPath: entry.path,
-        extraArgs: [],
-        tensorParallelSize: savedGpu?.tensorParallelSize
+        // MLX honors sampling as launch defaults; vLLM takes no extra flags here.
+        extraArgs: engine.kind === "mlx" ? mlxSamplingArgs(savedProfile?.sampling) : [],
+        tensorParallelSize: savedProfile?.gpu?.tensorParallelSize
       };
     }
     const saved = cfg2.modelProfiles[entry.key];
@@ -4447,13 +4468,15 @@ function registerApi(app2, d) {
       }
       let opts2;
       if (entry.format !== "gguf") {
-        const savedGpu = cfg2.modelProfiles[entry.key]?.gpu;
+        const savedProfile = cfg2.modelProfiles[entry.key];
         opts2 = {
           engine: active,
           model: { key: entry.key, name: entry.name, quant: entry.quant, ctx: entry.nativeCtx, vision: false },
           modelPath: entry.path,
-          extraArgs: [],
-          tensorParallelSize: savedGpu?.tensorParallelSize
+          // MLX honors sampling as launch defaults; vLLM gets no extra flags here
+          // (its multi-GPU shard count maps to --tensor-parallel-size below).
+          extraArgs: active.kind === "mlx" ? mlxSamplingArgs(savedProfile?.sampling) : [],
+          tensorParallelSize: savedProfile?.gpu?.tensorParallelSize
         };
       } else {
         const saved = cfg2.modelProfiles[entry.key];