turbollm 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +291 -79
- package/dist/cli.js +18 -2
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,93 +1,257 @@
|
|
|
1
1
|
<p align="center">
|
|
2
|
-
<img src="https://raw.githubusercontent.com/mohitsoni48/Turbo-LLM/main/turbollm/web/public/brand/turbollm-icon-512.jpeg" width="
|
|
2
|
+
<img src="https://raw.githubusercontent.com/mohitsoni48/Turbo-LLM/main/turbollm/web/public/brand/turbollm-icon-512.jpeg?v=2" width="92" height="92" alt="TurboLLM" />
|
|
3
3
|
</p>
|
|
4
4
|
|
|
5
5
|
<h1 align="center">TurboLLM</h1>
|
|
6
6
|
|
|
7
7
|
<p align="center">
|
|
8
|
-
<strong>Run any local LLM, auto-tuned to your GPU — with a polished web UI
|
|
9
|
-
OpenAI/Anthropic-compatible API.</strong><br/>
|
|
10
|
-
|
|
8
|
+
<strong>Run <em>any</em> local LLM engine, auto-tuned to your GPU — with a polished web UI
|
|
9
|
+
and an OpenAI/Anthropic-compatible API.</strong><br/>
|
|
10
|
+
Bring your own llama.cpp fork. No compiling. No Electron. No Python. Point Claude Code at
|
|
11
|
+
your own machine in one command — fully offline.
|
|
12
|
+
</p>
|
|
13
|
+
|
|
14
|
+
<p align="center">
|
|
15
|
+
<a href="https://www.npmjs.com/package/turbollm"><img src="https://img.shields.io/npm/v/turbollm.svg?color=e2552e" alt="npm version" /></a>
|
|
16
|
+
<a href="https://www.npmjs.com/package/turbollm"><img src="https://img.shields.io/npm/dm/turbollm.svg?color=e2552e" alt="npm downloads" /></a>
|
|
17
|
+
<img src="https://img.shields.io/badge/node-%E2%89%A522-3c873a.svg" alt="node >= 22" />
|
|
18
|
+
<img src="https://img.shields.io/badge/license-FSL--1.1--ALv2-blue.svg" alt="license" />
|
|
19
|
+
<img src="https://img.shields.io/badge/platform-Windows%20%C2%B7%20macOS%20%C2%B7%20Linux-555.svg" alt="platforms" />
|
|
11
20
|
</p>
|
|
12
21
|
|
|
13
22
|
<!-- Brand: shipped app icon web/public/brand/turbollm-icon-512.jpeg · high-res masters web/brand-assets/ (unshipped) · in-app mark web/src/components/Logo.tsx · favicon web/public/favicon.svg -->
|
|
14
23
|
|
|
24
|
+
```bash
|
|
25
|
+
npx turbollm
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
That one command starts a local daemon, opens a browser UI, and serves your models over an
|
|
29
|
+
API any tool can talk to. TurboLLM is the **performance & bleeding-edge layer for local
|
|
30
|
+
LLMs** — built for people who today hand-compile forks and hunt forums for the right flags.
|
|
31
|
+
|
|
32
|
+
<p align="center">
|
|
33
|
+
<img src="https://raw.githubusercontent.com/mohitsoni48/Turbo-LLM/main/assets/how-it-works.svg?v=2" width="860" alt="How TurboLLM works: clients -> one lightweight daemon -> any engine on your GPU" />
|
|
34
|
+
</p>
|
|
35
|
+
|
|
15
36
|
---
|
|
16
37
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
38
|
+
## Contents
|
|
39
|
+
|
|
40
|
+
- [Why TurboLLM](#why-turbollm)
|
|
41
|
+
- [Quick start](#quick-start)
|
|
42
|
+
- [⭐ Bring any engine — the headline feature](#-bring-any-engine--the-headline-feature)
|
|
43
|
+
- [Models — bring your own, or browse Hugging Face](#models)
|
|
44
|
+
- [Auto-tuning & performance](#auto-tuning--performance)
|
|
45
|
+
- [Chat](#chat)
|
|
46
|
+
- [APIs & integrations](#apis--integrations)
|
|
47
|
+
- [Run Claude Code on your own GPU](#run-claude-code-on-your-own-gpu)
|
|
48
|
+
- [Use it from any device on your network](#use-it-from-any-device-on-your-network)
|
|
49
|
+
- [Command-line reference](#command-line-reference)
|
|
50
|
+
- [Configuration & data](#configuration--data)
|
|
51
|
+
- [Requirements](#requirements)
|
|
52
|
+
- [Privacy](#privacy)
|
|
53
|
+
- [How TurboLLM compares](#how-turbollm-compares)
|
|
54
|
+
- [Troubleshooting](#troubleshooting)
|
|
55
|
+
- [Develop from source](#develop-from-source)
|
|
56
|
+
- [License](#license)
|
|
22
57
|
|
|
23
|
-
|
|
24
|
-
the right flags — not as "another easy chat app," but as the **performance & bleeding-edge
|
|
25
|
-
layer for local LLMs**.
|
|
58
|
+
---
|
|
26
59
|
|
|
27
60
|
## Why TurboLLM
|
|
28
61
|
|
|
29
|
-
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
62
|
+
Local-LLM tools make two choices for you, and both cost you performance:
|
|
63
|
+
|
|
64
|
+
1. **They pick the engine.** LM Studio ships one blessed runtime; Ollama hides the engine
|
|
65
|
+
entirely. The fastest community innovations — new quant formats, speculative decoding,
|
|
66
|
+
low-bit KV cache — land in **forks** first, and you can't use them without compiling.
|
|
67
|
+
2. **They don't tell you what speed to expect**, and they don't tune the dozens of launch
|
|
68
|
+
flags (`-c`, `-ngl`, `--n-cpu-moe`, KV type, threads, flash-attn, draft models) that make
|
|
69
|
+
the difference between 20 and 80 tokens/sec on the *same* hardware.
|
|
70
|
+
|
|
71
|
+
TurboLLM does the opposite:
|
|
72
|
+
|
|
73
|
+
- **🔌 Any engine, including forks.** Point it at any `llama-server`-compatible binary — a
|
|
74
|
+
build you compiled, a community fork, or the one it auto-provisions for your GPU. It probes
|
|
75
|
+
the binary's real capabilities and adapts the UI to them. **This is the whole point.**
|
|
76
|
+
- **⚡ Auto-tuned to your hardware.** It benchmarks on load, derives fast defaults, and shows
|
|
77
|
+
a **VRAM-fit verdict before you load** — no more flag guessing.
|
|
78
|
+
- **📊 Real tokens/sec, never faked.** Speed in the model list is *measured on your machine*
|
|
79
|
+
from actual generation — live while you chat, and remembered per model.
|
|
80
|
+
- **🪶 Lightweight.** A ~0.3 MB npm package on Node — **no Electron, no bundled Chromium, no
|
|
81
|
+
Python**. It downloads only the engine your GPU actually needs (Vulkan ≈ 38 MB).
|
|
82
|
+
- **🔌 Drop-in APIs.** OpenAI **and** Anthropic-compatible — so Claude Code and every existing
|
|
83
|
+
tool work unchanged.
|
|
84
|
+
- **🔒 Offline-first & private.** No account, no backend, no internet, **no telemetry.**
|
|
50
85
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
- **Node.js 22 or newer** — the daemon enforces this at startup and exits with a clear
|
|
54
|
-
message if the version is too old. Download: <https://nodejs.org>
|
|
55
|
-
- A GPU is recommended but not required (a CPU build is provisioned as a fallback).
|
|
86
|
+
---
|
|
56
87
|
|
|
57
88
|
## Quick start
|
|
58
89
|
|
|
59
90
|
```bash
|
|
60
|
-
# run
|
|
91
|
+
# run without installing (recommended for first try)
|
|
61
92
|
npx turbollm
|
|
62
93
|
|
|
63
|
-
# or install globally
|
|
94
|
+
# or install globally
|
|
64
95
|
npm install -g turbollm
|
|
65
96
|
turbollm
|
|
66
97
|
```
|
|
67
98
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
99
|
+
**On first run** the daemon:
|
|
100
|
+
|
|
101
|
+
1. Detects your GPU and **downloads a matching `llama-server` build** (CUDA for NVIDIA, ROCm
|
|
102
|
+
for AMD, Metal for Apple, SYCL for Intel, Vulkan otherwise — with a CPU fallback).
|
|
103
|
+
2. Starts on <http://127.0.0.1:6996> and opens your browser.
|
|
104
|
+
3. Drops you on the **Chat** screen, ready to load a model.
|
|
105
|
+
|
|
106
|
+
Then open **Models**, download or pick a GGUF, click **Load**, and start chatting. Stop the
|
|
107
|
+
daemon any time with **Ctrl+C**.
|
|
108
|
+
|
|
109
|
+
<!--
|
|
110
|
+
📸 SCREENSHOTS — drop PNGs into assets/screenshots/ and uncomment. Suggested shots:
|
|
111
|
+
- chat.png : a chat mid-stream showing the live t/s + context meter
|
|
112
|
+
- models.png : the Models › Library with measured t/s per model
|
|
113
|
+
- engines.png : the Engines screen + backend picker (the USP)
|
|
114
|
+
- tuning.png : the model load-params panel (ctx/ngl/NextN/VRAM verdict)
|
|
115
|
+
<p align="center"><img src="https://raw.githubusercontent.com/mohitsoni48/Turbo-LLM/main/assets/screenshots/chat.png" width="860" alt="TurboLLM chat" /></p>
|
|
116
|
+
-->
|
|
117
|
+
|
|
118
|
+
---
|
|
119
|
+
|
|
120
|
+
## ⭐ Bring any engine — the headline feature
|
|
121
|
+
|
|
122
|
+
No other local-LLM app lets you run **whatever inference engine you want**. TurboLLM treats
|
|
123
|
+
the engine as a swappable component.
|
|
124
|
+
|
|
125
|
+
**Add a custom engine** (Engines screen → **Add engine**):
|
|
126
|
+
|
|
127
|
+
1. Compile or download any `llama-server`-compatible binary — stock
|
|
128
|
+
[llama.cpp](https://github.com/ggml-org/llama.cpp), a community fork, or your own build.
|
|
129
|
+
2. Point TurboLLM at the binary. It runs a **capability probe** and learns exactly which
|
|
130
|
+
flags and features that build supports.
|
|
131
|
+
3. Activate it. The load-parameter UI **adapts to that engine** — features the build doesn't
|
|
132
|
+
support are hidden; ones it adds (e.g. low-bit KV cache, NextN) light up.
|
|
133
|
+
|
|
134
|
+
**Auto-provisioned default.** Don't want to fetch anything? On first run TurboLLM downloads
|
|
135
|
+
the right upstream prebuilt for your GPU automatically — and a **backend picker** lets you
|
|
136
|
+
switch between CUDA / ROCm / Metal / SYCL / Vulkan / CPU at any time (it downloads the variant
|
|
137
|
+
you choose, LM Studio-style).
|
|
138
|
+
|
|
139
|
+
**Engine types.** Both **llama.cpp / GGUF** and **MLX** (on macOS) are first-class engine
|
|
140
|
+
kinds — pick the right one per model.
|
|
141
|
+
|
|
142
|
+
**Fully supervised.** Every engine runs under a real state machine: health-gated readiness,
|
|
143
|
+
graceful stop, an **idle auto-stop** watchdog, and **live logs + clear error surfacing** in
|
|
144
|
+
the UI when something fails to load.
|
|
145
|
+
|
|
146
|
+
> Why it matters: fork-exclusive features — **speculative decoding (NextN / MTP / draft)**,
|
|
147
|
+
> low-bit KV cache, new quant formats — are usable on day 0, with **zero compiler knowledge**
|
|
148
|
+
> on your part beyond producing the binary (and often not even that).
|
|
149
|
+
|
|
150
|
+
---
|
|
151
|
+
|
|
152
|
+
## Models
|
|
153
|
+
|
|
154
|
+
- **Use the folders you already have.** Point TurboLLM at any directory of GGUFs — your
|
|
155
|
+
existing LM Studio / Ollama / manual downloads — **no re-downloading.** It parses GGUF
|
|
156
|
+
metadata (arch, params, quant, context, vision) for every file.
|
|
157
|
+
- **Browse & download from Hugging Face**, in-app: search, see the file tree, pick a quant,
|
|
158
|
+
and download with **resume + SHA-256 verification**. Gated models (Llama, Gemma) work via
|
|
159
|
+
your own HF token, which **never leaves your machine**.
|
|
160
|
+
- **Import from any URL** — not just Hugging Face. Paste a direct `.gguf` link (model-author
|
|
161
|
+
sites, mirrors, private servers); it disk-space-checks and downloads through the same
|
|
162
|
+
manager.
|
|
163
|
+
- **Quant recommendation per GPU** and a **VRAM-fit verdict** so you pick a quant that
|
|
164
|
+
actually fits before you commit.
|
|
165
|
+
- **Primary download folder**, real-time **measured t/s per model**, and **delete-from-disk**
|
|
166
|
+
— full library management.
|
|
167
|
+
|
|
168
|
+
---
|
|
169
|
+
|
|
170
|
+
## Auto-tuning & performance
|
|
171
|
+
|
|
172
|
+
- **Auto-benchmark on load** derives fast defaults for your exact GPU.
|
|
173
|
+
- **Real measured tokens/sec** in the model list — **live** while a model is generating,
|
|
174
|
+
**last-session** when it's idle (never a synthetic estimate).
|
|
175
|
+
- **Full load-parameter UI**, a superset of what other tools expose:
|
|
176
|
+
context length, GPU offload (`-ngl`), **MoE CPU-offload (`--n-cpu-moe`)**, parallel slots,
|
|
177
|
+
**KV-cache quant type** (incl. low-bit on supporting forks), CPU threads, flash attention,
|
|
178
|
+
and **speculative decoding (NextN / MTP / draft)**.
|
|
179
|
+
- **Fast by default:** flash attention on, NextN self-speculative decoding on for models that
|
|
180
|
+
carry a draft head, threads auto — best speed out of the box, safely gated to what your
|
|
181
|
+
engine actually accepts.
|
|
182
|
+
- **Saved per-model profiles** — tune once, and it loads that way every time.
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## Chat
|
|
187
|
+
|
|
188
|
+
A genuinely good chat UI, not an afterthought:
|
|
189
|
+
|
|
190
|
+
- **Streaming** with a **stop** button, **live tokens/sec**, **prompt-processing %** and
|
|
191
|
+
**prefill t/s**, **time-to-first-token**, **total time**, exact **token counts**, and a
|
|
192
|
+
**context-usage meter** (filled / max) on every reply.
|
|
193
|
+
- **Thinking control** — toggle reasoning **off** to get a direct answer (saves time and
|
|
194
|
+
tokens), or leave it **on** with collapsible, timed "thought for N s" blocks.
|
|
195
|
+
- **Markdown + syntax-highlighted code** with one-click copy.
|
|
196
|
+
- **Edit, regenerate, delete, copy** any message; **persistent, searchable conversations**
|
|
197
|
+
with rename, delete, and **auto-generated titles**.
|
|
198
|
+
- **Per-chat system prompt** and **per-chat sampling** overrides (temperature, top-p/k, min-p).
|
|
199
|
+
- **Image input** for vision models.
|
|
200
|
+
- **TurboLLM Expert** — a built-in assistant that knows the app and your hardware, for
|
|
201
|
+
onboarding and troubleshooting without leaving the UI.
|
|
202
|
+
|
|
203
|
+
---
|
|
204
|
+
|
|
205
|
+
## APIs & integrations
|
|
206
|
+
|
|
207
|
+
With a model loaded, TurboLLM serves two compatible APIs on the same port:
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
# OpenAI-compatible
|
|
211
|
+
curl http://127.0.0.1:6996/v1/chat/completions \
|
|
212
|
+
-H "Content-Type: application/json" \
|
|
213
|
+
-d '{"model":"local","messages":[{"role":"user","content":"hello"}]}'
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
- **OpenAI-compatible** `/v1/chat/completions`, `/v1/embeddings`, … — point any OpenAI client
|
|
217
|
+
or tool at it.
|
|
218
|
+
- **Anthropic-compatible** `/v1/messages` — including **tool use and streaming** — which is
|
|
219
|
+
what powers Claude Code below. No other local host offers this.
|
|
220
|
+
- **API-key auth** you can require when sharing over a LAN (Settings → Network).
|
|
71
221
|
|
|
72
|
-
|
|
222
|
+
---
|
|
73
223
|
|
|
74
|
-
##
|
|
224
|
+
## Run Claude Code on your own GPU
|
|
75
225
|
|
|
76
|
-
TurboLLM
|
|
77
|
-
|
|
78
|
-
|
|
226
|
+
TurboLLM's Anthropic-compatible endpoint means [Claude
|
|
227
|
+
Code](https://www.npmjs.com/package/@anthropic-ai/claude-code) can run against whatever model
|
|
228
|
+
you've loaded — no cloud key, fully offline. One command wires it up:
|
|
79
229
|
|
|
80
230
|
```bash
|
|
81
231
|
turbollm launch claude # opens Claude Code on your loaded model
|
|
82
232
|
```
|
|
83
233
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
234
|
+
It sets Claude Code's `ANTHROPIC_BASE_URL` / `ANTHROPIC_MODEL` at TurboLLM and execs `claude`;
|
|
235
|
+
extra args are forwarded. If `claude` isn't installed, it tells you how. The in-app
|
|
236
|
+
**Developer** screen also shows copy-paste env snippets for any OpenAI- or Anthropic-compatible
|
|
237
|
+
tool (Open WebUI, Kilo Code, opencode, …).
|
|
238
|
+
|
|
239
|
+
---
|
|
240
|
+
|
|
241
|
+
## Use it from any device on your network
|
|
242
|
+
|
|
243
|
+
The UI runs in the browser, so any phone, tablet, or laptop on your LAN can use the model on
|
|
244
|
+
your GPU box:
|
|
245
|
+
|
|
246
|
+
```bash
|
|
247
|
+
turbollm --addr 0.0.0.0:6996 # bind all interfaces, then open http://<your-ip>:6996
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
Turn on **Require API key** in Settings → Network when you expose it.
|
|
89
251
|
|
|
90
|
-
|
|
252
|
+
---
|
|
253
|
+
|
|
254
|
+
## Command-line reference
|
|
91
255
|
|
|
92
256
|
```bash
|
|
93
257
|
turbollm # start on :6996, open browser
|
|
@@ -99,38 +263,82 @@ turbollm launch claude # start Claude Code against the loaded model
|
|
|
99
263
|
|
|
100
264
|
| Flag | Description |
|
|
101
265
|
|------|-------------|
|
|
102
|
-
| `--port <n>` | Listen on a specific port (default: 6996) |
|
|
266
|
+
| `--port <n>` | Listen on a specific port (default: `6996`) |
|
|
103
267
|
| `--addr <host:port>` | Full host:port override, e.g. `0.0.0.0:6996` for LAN sharing |
|
|
104
268
|
| `--no-open` | Start without opening a browser window |
|
|
105
269
|
| `--config <file>` | Path to a custom config file |
|
|
106
270
|
| `--help`, `-h` | Show usage and exit |
|
|
107
271
|
|
|
108
|
-
|
|
272
|
+
---
|
|
109
273
|
|
|
110
|
-
##
|
|
274
|
+
## Configuration & data
|
|
111
275
|
|
|
112
|
-
|
|
276
|
+
Everything lives under **`~/.turbollm/`** on every OS — `config.json`, the SQLite chat
|
|
277
|
+
database, downloaded engines, models cache, and logs. Back it up or delete it to reset.
|
|
278
|
+
Use `--config <file>` to point at an alternate config (its directory becomes the data dir).
|
|
113
279
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
280
|
+
---
|
|
281
|
+
|
|
282
|
+
## Requirements
|
|
283
|
+
|
|
284
|
+
- **Node.js 22 or newer** — enforced at startup with a clear message. <https://nodejs.org>
|
|
285
|
+
- **Windows, macOS, or Linux.**
|
|
286
|
+
- A GPU is recommended but **not required** — a CPU build is provisioned as a fallback.
|
|
287
|
+
- On Windows, the first time the auto-downloaded `llama-server` runs, SmartScreen/Defender may
|
|
288
|
+
prompt (it's an upstream binary). Allow it once.
|
|
289
|
+
|
|
290
|
+
---
|
|
291
|
+
|
|
292
|
+
## Privacy
|
|
293
|
+
|
|
294
|
+
TurboLLM is **offline-first**: core local use needs no account, no backend, and no internet.
|
|
295
|
+
**No analytics or telemetry are collected.** Your prompts, chats, files, and keys never leave
|
|
296
|
+
your machine.
|
|
297
|
+
|
|
298
|
+
---
|
|
299
|
+
|
|
300
|
+
## How TurboLLM compares
|
|
120
301
|
|
|
121
|
-
|
|
122
|
-
the Claude Code integration above. When sharing over a LAN you can require an API key —
|
|
123
|
-
enable it in **Settings → Network**.
|
|
302
|
+
Focused on the differences that matter — all four are good tools.
|
|
124
303
|
|
|
125
|
-
|
|
304
|
+
| | **TurboLLM** | LM Studio | Ollama | Open WebUI |
|
|
305
|
+
|---|:---:|:---:|:---:|:---:|
|
|
306
|
+
| Run **any engine / community forks** | ✅ | ❌ one runtime | ❌ hidden | ❌ |
|
|
307
|
+
| **Auto-tune** launch flags to your GPU | ✅ | ❌ | ❌ | ❌ |
|
|
308
|
+
| **Measured** t/s in the model list | ✅ | ◐ | ◐ | ❌ |
|
|
309
|
+
| **Anthropic** API (tool use) → Claude Code | ✅ | ❌ | ❌ | ❌ |
|
|
310
|
+
| OpenAI-compatible API | ✅ | ✅ | ✅ | ◐ proxy |
|
|
311
|
+
| Use existing model folders (no re-download) | ✅ | ◐ | ❌ | ❌ |
|
|
312
|
+
| Speculative decoding (NextN / MTP / draft) | ✅ | ◐ draft | ❌ | ❌ |
|
|
313
|
+
| Web UI from any LAN device | ✅ | ❌ | ❌ | ✅ |
|
|
314
|
+
| **Lightweight** (no Electron / no Python) | ✅ npm | ❌ Electron | ✅ Go | ❌ Python |
|
|
315
|
+
| Offline-first · no telemetry | ✅ | ◐ | ✅ | ✅ |
|
|
316
|
+
|
|
317
|
+
Prefer Open WebUI's chat breadth? It works great pointed at TurboLLM's OpenAI endpoint.
|
|
318
|
+
|
|
319
|
+
---
|
|
320
|
+
|
|
321
|
+
## Troubleshooting
|
|
322
|
+
|
|
323
|
+
- **`TurboLLM requires Node.js 22 or newer`** — upgrade Node: <https://nodejs.org>.
|
|
324
|
+
- **Model won't load / OOM** — pick a smaller quant (the VRAM verdict warns you), lower GPU
|
|
325
|
+
offload, or close other GPU apps. Failures surface in the Engines screen with the engine log.
|
|
326
|
+
- **Windows Defender / SmartScreen prompt** — that's the upstream `llama-server` binary on
|
|
327
|
+
first run; allow it once.
|
|
328
|
+
- **Port already in use** — `turbollm --port 9000`.
|
|
329
|
+
- **Slow generation** — open the model's load params; ensure GPU offload is high and flash
|
|
330
|
+
attention / NextN are on for supported models.
|
|
331
|
+
|
|
332
|
+
---
|
|
333
|
+
|
|
334
|
+
## Develop from source
|
|
126
335
|
|
|
127
336
|
```bash
|
|
128
337
|
npm install # daemon deps
|
|
129
338
|
cd web && npm install && cd ..
|
|
130
339
|
|
|
131
340
|
npm run build:web # build the React UI -> src/webdist
|
|
132
|
-
npm run start # run the daemon in dev (hot TS via tsx)
|
|
133
|
-
# open http://127.0.0.1:6996 · curl http://127.0.0.1:6996/api/v1/status
|
|
341
|
+
npm run start # run the daemon in dev (hot TS via tsx) -> :6996
|
|
134
342
|
|
|
135
343
|
npm run build # production bundle -> dist/cli.js (web assets included)
|
|
136
344
|
node dist/cli.js --port 6996
|
|
@@ -139,25 +347,29 @@ node dist/cli.js --port 6996
|
|
|
139
347
|
Frontend hot-reload: `cd web && npm run dev` (proxies `/api` and `/v1` to the daemon on
|
|
140
348
|
:6996).
|
|
141
349
|
|
|
142
|
-
|
|
350
|
+
**Stack:** Node ≥22 · TypeScript · Hono · `node:sqlite` · tsup — and a React 19 + Tailwind v4 +
|
|
351
|
+
shadcn/ui frontend. One TypeScript codebase, shipped as an npm package.
|
|
143
352
|
|
|
144
353
|
```
|
|
145
354
|
turbollm/
|
|
146
|
-
|
|
355
|
+
bin/turbollm.mjs launcher shim (Node guard) -> dist/cli.js
|
|
147
356
|
src/
|
|
148
357
|
cli.ts entrypoint: wiring + graceful shutdown
|
|
149
358
|
server.ts Hono app: CORS, API, gateway, embedded SPA
|
|
150
|
-
config/ config schema + load/save/migrate
|
|
151
359
|
engines/ provisioning, probe, registry, lifecycle state machine
|
|
152
360
|
api/routes.ts /api/v1/* handlers
|
|
153
361
|
gateway/ /v1/* OpenAI + Anthropic gateway
|
|
154
|
-
|
|
155
|
-
web/ React
|
|
362
|
+
models/ · chat/ · hf/ · bench/ · downloads/
|
|
363
|
+
web/ React + TS + Tailwind + shadcn frontend (own package.json)
|
|
156
364
|
```
|
|
157
365
|
|
|
366
|
+
---
|
|
367
|
+
|
|
158
368
|
## License
|
|
159
369
|
|
|
160
|
-
Source-available under the **Functional Source License 1.1 (Apache
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
370
|
+
Source-available under the **Functional Source License 1.1 (Apache-2.0 future grant)** — SPDX
|
|
371
|
+
**`FSL-1.1-ALv2`**. Free for personal use, internal business use, education, and research; the
|
|
372
|
+
only restriction is shipping a competing product. Each release converts to Apache-2.0 two
|
|
373
|
+
years after it's published. Full text: [LICENSE.md](LICENSE.md).
|
|
374
|
+
|
|
375
|
+
<p align="center"><sub>Built for people who refuse to wait for the mainstream to bless the fast path. ⚡</sub></p>
|
package/dist/cli.js
CHANGED
|
@@ -542,6 +542,10 @@ var Manager = class {
|
|
|
542
542
|
const logPath = join4(this.store.dir(), "logs", `engine-${opts.engine.id}.log`);
|
|
543
543
|
mkdirSync4(dirname2(logPath), { recursive: true });
|
|
544
544
|
const logStream = createWriteStream2(logPath);
|
|
545
|
+
logStream.write(
|
|
546
|
+
`[turbollm] starting engine "${opts.engine.name}" on internal port ${port2} (127.0.0.1 only \u2014 the engine's own port, NOT the TurboLLM app/UI port).
|
|
547
|
+
`
|
|
548
|
+
);
|
|
545
549
|
const { cmd, args } = engineCommand(opts, port2);
|
|
546
550
|
const child = spawn(cmd, args, { cwd: dirname2(cmd), windowsHide: true });
|
|
547
551
|
child.stdout?.pipe(logStream, { end: false });
|
|
@@ -671,6 +675,17 @@ var Manager = class {
|
|
|
671
675
|
// ---- internal ----------------------------------------------------------
|
|
672
676
|
onTerminated(child, code, logStream, errMsg) {
|
|
673
677
|
if (this.child !== child) return;
|
|
678
|
+
const cleanStop = this.state === "stopping" || this.state === "stopped";
|
|
679
|
+
try {
|
|
680
|
+
logStream.write(
|
|
681
|
+
cleanStop ? `
|
|
682
|
+
[turbollm] engine stopped \u2014 the model is no longer loaded.
|
|
683
|
+
` : `
|
|
684
|
+
[turbollm] engine process exited unexpectedly (exit ${code})${errMsg ? ` \u2014 ${errMsg}` : ""}. The model did NOT load / is no longer loaded.
|
|
685
|
+
`
|
|
686
|
+
);
|
|
687
|
+
} catch {
|
|
688
|
+
}
|
|
674
689
|
logStream.end();
|
|
675
690
|
if (this.state === "stopping" || this.state === "stopped") {
|
|
676
691
|
this.state = "stopped";
|
|
@@ -752,8 +767,9 @@ async function probeReady(port2) {
|
|
|
752
767
|
try {
|
|
753
768
|
const r = await fetch(`${base2}/health`, { signal: AbortSignal.timeout(1500) });
|
|
754
769
|
if (r.status === 200) return true;
|
|
755
|
-
if (r.status
|
|
770
|
+
if (r.status !== 404 && r.status !== 501) return false;
|
|
756
771
|
} catch {
|
|
772
|
+
return false;
|
|
757
773
|
}
|
|
758
774
|
try {
|
|
759
775
|
const r = await fetch(`${base2}/v1/models`, { signal: AbortSignal.timeout(1500) });
|
|
@@ -4913,7 +4929,7 @@ function contentType(file) {
|
|
|
4913
4929
|
}
|
|
4914
4930
|
|
|
4915
4931
|
// src/cli.ts
|
|
4916
|
-
var version = "0.1.
|
|
4932
|
+
var version = "0.1.1";
|
|
4917
4933
|
try {
|
|
4918
4934
|
const pkgPath = join11(dirname7(fileURLToPath2(import.meta.url)), "..", "package.json");
|
|
4919
4935
|
version = JSON.parse(readFileSync7(pkgPath, "utf8")).version ?? version;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "turbollm",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.1",
|
|
4
4
|
"description": "TurboLLM — local LLM platform: run any inference engine auto-tuned to your GPU, with a web UI and OpenAI/Anthropic-compatible API. Point Claude Code at your own machine in one command.",
|
|
5
5
|
"license": "FSL-1.1-ALv2",
|
|
6
6
|
"author": "Mohit Soni",
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"dev": "tsx watch src/cli.ts",
|
|
41
41
|
"start": "tsx src/cli.ts",
|
|
42
42
|
"typecheck": "tsc --noEmit",
|
|
43
|
-
"test": "
|
|
43
|
+
"test": "tsx --test",
|
|
44
44
|
"build:web": "cd web && npm ci && npm run build",
|
|
45
45
|
"build": "npm run typecheck && tsup && node -e \"const fs=require('fs'),p='dist/cli.js',c=fs.readFileSync(p,'utf8');fs.writeFileSync(p,c.replaceAll('from \\\"sqlite\\\"','from \\\"node:sqlite\\\"'))\" && node -e \"require('fs').cpSync('src/webdist','dist/webdist',{recursive:true,force:true})\"",
|
|
46
46
|
"prepublishOnly": "npm run build:web && npm run build"
|