turbollm 1.4.0 → 1.4.2

package/README.md

@@ -1,5 +1,5 @@
 <p align="center">
-  <img src="https://raw.githubusercontent.com/mohitsoni48/Turbo-LLM/main/turbollm/web/public/brand/turbollm-icon-512.jpeg?v=2" width="92" height="92" alt="TurboLLM" />
+  <img src="https://raw.githubusercontent.com/mohitsoni48/TurboLLM/main/turbollm/web/public/brand/turbollm-icon-512.jpeg?v=2" width="92" height="92" alt="TurboLLM" />
 </p>
 
 <h1 align="center">TurboLLM</h1>
@@ -30,7 +30,7 @@ API any tool can talk to. TurboLLM is the **performance & bleeding-edge layer fo
 LLMs** — built for people who today hand-compile forks and hunt forums for the right flags.
 
 <p align="center">
-  <img src="https://raw.githubusercontent.com/mohitsoni48/Turbo-LLM/main/assets/how-it-works.svg?v=2" width="860" alt="How TurboLLM works: clients -> one lightweight daemon -> any engine on your GPU" />
+  <img src="https://raw.githubusercontent.com/mohitsoni48/TurboLLM/main/assets/how-it-works.svg?v=2" width="860" alt="How TurboLLM works: clients -> one lightweight daemon -> any engine on your GPU" />
 </p>
 
 ---
@@ -404,21 +404,27 @@ your machine.
 
 ## How TurboLLM compares
 
-Focused on the differences that matter — all four are good tools.
+Focused on the differences that matter — all four are good tools, and the others move fast.
+Marks reflect mid-2026; verify the moving rows against each tool's current docs.
 
 | | **TurboLLM** | LM Studio | Ollama | Open WebUI |
 |---|:---:|:---:|:---:|:---:|
-| Run **any engine / community forks** | ✅ | ❌ one runtime | ❌ hidden | ❌ |
-| **Auto-tune** launch flags to your GPU | ✅ | ❌ | ❌ | ❌ |
-| **Measured** t/s in the model list | ✅ | ◐ | ◐ | ❌ |
-| **Anthropic** API (tool use) → Claude Code | ✅ | ❌ | ❌ | ❌ |
+| Run **any engine / community forks** | ✅ | ❌ llama.cpp/MLX only | ❌ hidden | ❌ frontend |
+| **Benchmark-based auto-tune** of launch flags | ✅ | ◐ basic offload | ◐ basic offload | ❌ |
+| **Measured** t/s in the model list | ✅ | ◐ per-run | ◐ `--verbose` | ❌ |
+| **Anthropic** API (`/v1/messages`) → Claude Code | ✅ | ✅ 0.4.1+ | ✅ v0.14+ | ❌ |
 | OpenAI-compatible API | ✅ | ✅ | ✅ | ◐ proxy |
-| **Auto-load the requested model** (hot-swap pool) | ✅ | ❌ | ◐ | ❌ |
-| Use existing model folders (no re-download) | ✅ | ◐ | ❌ | ❌ |
-| Speculative decoding (NextN / MTP / draft) | ✅ | ◐ draft | ❌ | ❌ |
+| Auto-load the requested model / multi-model pool | ✅ | ✅ JIT | ✅ | ❌ |
+| Use existing model folders (no re-download) | ✅ | ◐ import | ◐ import | ❌ frontend |
+| Speculative decoding (draft / MTP) | ✅ | ✅ | ◐ env flag | ❌ |
 | Web UI from any LAN device | ✅ | ❌ | ❌ | ✅ |
 | **Lightweight** (no Electron / no Python) | ✅ npm | ❌ Electron | ✅ Go | ❌ Python |
-| Offline-first · no telemetry | ✅ | ◐ | ✅ | ✅ |
+| Offline-first · **no telemetry** | ✅ | ◐ analytics on by default | ✅ | ✅ |
+
+LM Studio and Ollama both added Anthropic `/v1/messages` endpoints in 2026, so the API rows are
+now parity — Claude Code works against any of them. TurboLLM's durable edges are **any engine
+including community forks**, **benchmark-based auto-tuning with a VRAM-fit verdict + measured t/s
+before you commit**, and **zero telemetry**.
 
 Prefer Open WebUI's chat breadth? It works great pointed at TurboLLM's OpenAI endpoint.
 
@@ -463,6 +469,6 @@ shadcn/ui frontend. One TypeScript codebase, shipped as an npm package.
 Source-available under the **Functional Source License 1.1 (Apache-2.0 future grant)** — SPDX
 **`FSL-1.1-ALv2`**. Free for personal use, internal business use, education, and research; the
 only restriction is shipping a competing product. Each release converts to Apache-2.0 two
-years after it's published. Full text: [LICENSE.md](https://github.com/mohitsoni48/Turbo-LLM/blob/main/turbollm/LICENSE.md).
+years after it's published. Full text: [LICENSE.md](https://github.com/mohitsoni48/TurboLLM/blob/main/turbollm/LICENSE.md).
 
 <p align="center"><sub>Built for people who refuse to wait for the mainstream to bless the fast path. ⚡</sub></p>

package/dist/cli.js

@@ -3310,6 +3310,24 @@ function walk(dir, out) {
     }
   }
 }
+function detectSafetensorsQuant(cfg2) {
+  const qc = cfg2.quantization_config;
+  if (qc?.quant_method) {
+    const method = qc.quant_method.toLowerCase();
+    const group0 = qc.config_groups ? Object.values(qc.config_groups)[0] : void 0;
+    const wBits = group0?.weights?.num_bits ?? qc.bits;
+    if (method === "compressed-tensors") {
+      if (wBits) {
+        const aBits = group0?.input_activations?.num_bits ?? 16;
+        return `w${wBits}a${aBits}`;
+      }
+      return "compressed-tensors";
+    }
+    return wBits ? `${method}-${wBits}bit` : method;
+  }
+  const mlxBits = cfg2.quantization?.bits;
+  return mlxBits ? `${mlxBits}bit` : "fp16";
+}
 function mlxEntryFor(dir) {
   let cfg2 = {};
   let parseError = null;
@@ -3350,10 +3368,10 @@ function mlxEntryFor(dir) {
     }
   } catch {
   }
-  const expertCount = cfg2.num_local_experts ?? cfg2.num_experts ?? 0;
-  const bits = cfg2.quantization?.bits;
-  const quant = bits ? `${bits}bit` : "fp16";
-  const arch2 = cfg2.model_type || cfg2.architectures?.[0] || "unknown";
+  const lm = cfg2.text_config ?? {};
+  const expertCount = cfg2.num_local_experts ?? cfg2.num_experts ?? lm.num_local_experts ?? lm.num_experts ?? 0;
+  const quant = detectSafetensorsQuant(cfg2);
+  const arch2 = cfg2.model_type || cfg2.architectures?.[0] || lm.model_type || "unknown";
   const name = basename(dir).replace(/[-_]/g, " ").replace(/\s+/g, " ").trim();
   return {
     key: `${name.toLowerCase()}|mlx-${quant}|${sizeBytes}`,
@@ -3365,9 +3383,9 @@ function mlxEntryFor(dir) {
     sizeLabel: "",
     arch: arch2,
     quant,
-    nativeCtx: cfg2.max_position_embeddings ?? 0,
-    blockCount: cfg2.num_hidden_layers ?? 0,
-    headCountKv: cfg2.num_key_value_heads ?? 0,
+    nativeCtx: cfg2.max_position_embeddings ?? lm.max_position_embeddings ?? 0,
+    blockCount: cfg2.num_hidden_layers ?? lm.num_hidden_layers ?? 0,
+    headCountKv: cfg2.num_key_value_heads ?? lm.num_key_value_heads ?? 0,
     moe: expertCount > 0,
     expertCount,
     nextnLayers: 0,
@@ -7572,8 +7590,9 @@ async function runGeneration(d, stream, ctx) {
         reqBody.reasoning_budget = 0;
         reqBody.chat_template_kwargs = { enable_thinking: false };
       }
-      if (toolDefs.length > 0) reqBody.tools = toolDefs;
-      if (conv.toolPolicy === "force_web_search" && toolIter <= 2 && toolDefs.some((t) => t.function.name === "web_search")) {
+      const toolsSupported = (d.registry.active()?.kind ?? "") !== "vllm" && toolDefs.length > 0;
+      if (toolsSupported) reqBody.tools = toolDefs;
+      if (toolsSupported && conv.toolPolicy === "force_web_search" && toolIter <= 2 && toolDefs.some((t) => t.function.name === "web_search")) {
         reqBody.tool_choice = { type: "function", function: { name: "web_search" } };
       }
       const res = await fetch(`${target}/v1/chat/completions`, {