mlx-stack 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mlx_stack/__init__.py +5 -0
- mlx_stack/_version.py +24 -0
- mlx_stack/cli/__init__.py +5 -0
- mlx_stack/cli/bench.py +221 -0
- mlx_stack/cli/config.py +166 -0
- mlx_stack/cli/down.py +109 -0
- mlx_stack/cli/init.py +180 -0
- mlx_stack/cli/install.py +165 -0
- mlx_stack/cli/logs.py +234 -0
- mlx_stack/cli/main.py +187 -0
- mlx_stack/cli/models.py +304 -0
- mlx_stack/cli/profile.py +65 -0
- mlx_stack/cli/pull.py +134 -0
- mlx_stack/cli/recommend.py +397 -0
- mlx_stack/cli/status.py +111 -0
- mlx_stack/cli/up.py +163 -0
- mlx_stack/cli/watch.py +252 -0
- mlx_stack/core/__init__.py +1 -0
- mlx_stack/core/benchmark.py +1182 -0
- mlx_stack/core/catalog.py +560 -0
- mlx_stack/core/config.py +471 -0
- mlx_stack/core/deps.py +323 -0
- mlx_stack/core/hardware.py +304 -0
- mlx_stack/core/launchd.py +531 -0
- mlx_stack/core/litellm_gen.py +188 -0
- mlx_stack/core/log_rotation.py +231 -0
- mlx_stack/core/log_viewer.py +386 -0
- mlx_stack/core/models.py +639 -0
- mlx_stack/core/paths.py +79 -0
- mlx_stack/core/process.py +887 -0
- mlx_stack/core/pull.py +815 -0
- mlx_stack/core/scoring.py +611 -0
- mlx_stack/core/stack_down.py +317 -0
- mlx_stack/core/stack_init.py +524 -0
- mlx_stack/core/stack_status.py +229 -0
- mlx_stack/core/stack_up.py +856 -0
- mlx_stack/core/watchdog.py +744 -0
- mlx_stack/data/__init__.py +1 -0
- mlx_stack/data/catalog/__init__.py +1 -0
- mlx_stack/data/catalog/deepseek-r1-32b.yaml +46 -0
- mlx_stack/data/catalog/deepseek-r1-8b.yaml +45 -0
- mlx_stack/data/catalog/gemma3-12b.yaml +45 -0
- mlx_stack/data/catalog/gemma3-27b.yaml +45 -0
- mlx_stack/data/catalog/gemma3-4b.yaml +45 -0
- mlx_stack/data/catalog/llama3.3-8b.yaml +44 -0
- mlx_stack/data/catalog/nemotron-49b.yaml +41 -0
- mlx_stack/data/catalog/nemotron-8b.yaml +44 -0
- mlx_stack/data/catalog/qwen3-8b.yaml +45 -0
- mlx_stack/data/catalog/qwen3.5-0.8b.yaml +45 -0
- mlx_stack/data/catalog/qwen3.5-14b.yaml +46 -0
- mlx_stack/data/catalog/qwen3.5-32b.yaml +45 -0
- mlx_stack/data/catalog/qwen3.5-3b.yaml +44 -0
- mlx_stack/data/catalog/qwen3.5-72b.yaml +42 -0
- mlx_stack/data/catalog/qwen3.5-8b.yaml +45 -0
- mlx_stack/py.typed +1 -0
- mlx_stack/utils/__init__.py +1 -0
- mlx_stack-0.1.0.dist-info/METADATA +397 -0
- mlx_stack-0.1.0.dist-info/RECORD +61 -0
- mlx_stack-0.1.0.dist-info/WHEEL +4 -0
- mlx_stack-0.1.0.dist-info/entry_points.txt +2 -0
- mlx_stack-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,397 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: mlx-stack
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: CLI control plane for local LLM infrastructure on Apple Silicon
|
|
5
|
+
Project-URL: Homepage, https://github.com/weklund/mlx-stack
|
|
6
|
+
Project-URL: Repository, https://github.com/weklund/mlx-stack
|
|
7
|
+
Project-URL: Issues, https://github.com/weklund/mlx-stack/issues
|
|
8
|
+
Project-URL: Documentation, https://github.com/weklund/mlx-stack#readme
|
|
9
|
+
Author: mlx-stack contributors
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: apple-silicon,cli,llm,mlx,vllm
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Operating System :: MacOS
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Requires-Python: >=3.13
|
|
22
|
+
Requires-Dist: click>=8.1
|
|
23
|
+
Requires-Dist: httpx>=0.27
|
|
24
|
+
Requires-Dist: huggingface-hub[cli]>=1.8.0
|
|
25
|
+
Requires-Dist: psutil>=5.9
|
|
26
|
+
Requires-Dist: pyyaml>=6.0
|
|
27
|
+
Requires-Dist: rich>=13.0
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# mlx-stack
|
|
31
|
+
|
|
32
|
+
**CLI control plane for local LLM inference infrastructure on Apple Silicon.**
|
|
33
|
+
|
|
34
|
+
[](https://github.com/weklund/mlx-stack/actions/workflows/ci.yml)
|
|
35
|
+
[](https://pypi.org/project/mlx-stack/)
|
|
36
|
+
[](https://www.python.org/downloads/)
|
|
37
|
+
[](LICENSE)
|
|
38
|
+
[](https://support.apple.com/en-us/116943)
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## Table of Contents
|
|
43
|
+
|
|
44
|
+
- [Architecture](#architecture)
|
|
45
|
+
- [Feature Highlights](#feature-highlights)
|
|
46
|
+
- [Installation](#installation)
|
|
47
|
+
- [Quick Start](#quick-start)
|
|
48
|
+
- [CLI Reference](#cli-reference)
|
|
49
|
+
- [Configuration](#configuration)
|
|
50
|
+
- [24/7 Operation](#247-operation)
|
|
51
|
+
- [Model Catalog](#model-catalog)
|
|
52
|
+
- [Architecture Details](#architecture-details)
|
|
53
|
+
- [Development](#development)
|
|
54
|
+
- [Contributing](#contributing)
|
|
55
|
+
- [License](#license)
|
|
56
|
+
|
|
57
|
+
## Architecture
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
┌──────────────────────────────────────────────────┐
|
|
61
|
+
│ mlx-stack CLI │
|
|
62
|
+
│ hardware detection · recommendation · lifecycle │
|
|
63
|
+
└──────────────┬───────────────────────────────────┘
|
|
64
|
+
│
|
|
65
|
+
┌────────────────────────┼────────────────────────┐
|
|
66
|
+
│ │ │
|
|
67
|
+
▼ ▼ ▼
|
|
68
|
+
┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐
|
|
69
|
+
│ vllm-mlx :8000 │ │ vllm-mlx :8001 │ │ vllm-mlx :8002 │
|
|
70
|
+
│ ── standard ── │ │ ── fast ── │ │ ── longctx ── │
|
|
71
|
+
│ Qwen 3.5 14B │ │ Qwen 3.5 3B │ │ DeepSeek R1 8B │
|
|
72
|
+
└────────┬──────────┘ └────────┬──────────┘ └────────┬──────────┘
|
|
73
|
+
│ │ │
|
|
74
|
+
└──────────────────────┼───────────────────────┘
|
|
75
|
+
│
|
|
76
|
+
┌──────────▼──────────┐
|
|
77
|
+
│ LiteLLM Proxy :4000│
|
|
78
|
+
│ routing · fallback │
|
|
79
|
+
│ load balancing │
|
|
80
|
+
└──────────┬──────────┘
|
|
81
|
+
│
|
|
82
|
+
┌──────────▼──────────┐
|
|
83
|
+
│ OpenAI-compatible │
|
|
84
|
+
│ /v1 endpoint │
|
|
85
|
+
│ │
|
|
86
|
+
│ ← Your app / agent │
|
|
87
|
+
└─────────────────────┘
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
mlx-stack orchestrates [vllm-mlx](https://github.com/vllm-project/vllm) model servers and a [LiteLLM](https://github.com/BerriAI/litellm) API gateway to serve large language models locally on Apple Silicon Macs. Each tier runs a dedicated model optimized for a specific workload — quality, speed, or long-context — and LiteLLM routes requests through a single OpenAI-compatible endpoint with automatic fallback.
|
|
91
|
+
|
|
92
|
+
## Feature Highlights
|
|
93
|
+
|
|
94
|
+
- **Hardware-Aware Recommendations** — Detects your Apple Silicon chip (M1–M5, Pro/Max/Ultra), measures memory bandwidth, and recommends an optimal model stack tuned to your exact hardware.
|
|
95
|
+
- **Tiered Model Serving** — Assigns models to `standard`, `fast`, and `longctx` tiers so agents and apps can target the right balance of quality and speed per request.
|
|
96
|
+
- **24/7 Unattended Operation** — Built-in watchdog with auto-restart, flap detection, exponential backoff, and macOS LaunchAgent integration for always-on inference on headless Mac Minis.
|
|
97
|
+
- **One-Command Setup** — `mlx-stack init --accept-defaults` profiles your hardware, picks models, generates configs, and gets you from zero to a running OpenAI-compatible endpoint in minutes.
|
|
98
|
+
- **15-Model Curated Catalog** — Ships with benchmark data for Qwen 3.5, Gemma 3, DeepSeek R1, Nemotron, and Llama 3.3 families — with quality scores, tool-calling metadata, and per-hardware performance data.
|
|
99
|
+
|
|
100
|
+
## Installation
|
|
101
|
+
|
|
102
|
+
The recommended way to install mlx-stack is with [uv](https://docs.astral.sh/uv/):
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
uv tool install mlx-stack
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
This installs `mlx-stack` globally as an isolated tool — no need to manage virtual environments.
|
|
109
|
+
|
|
110
|
+
Alternatively, you can use [pipx](https://pipx.pypa.io/):
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
pipx install mlx-stack
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Or try it without installing:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
uvx mlx-stack profile
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
> **Note:** `uvx` runs in an ephemeral environment, which works great for one-off commands. For the watchdog and LaunchAgent features (`mlx-stack watch`, `mlx-stack install`), use `uv tool install` so the binary has a stable path.
|
|
123
|
+
|
|
124
|
+
## Quick Start
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
# 1. Detect your hardware
|
|
128
|
+
mlx-stack profile
|
|
129
|
+
|
|
130
|
+
# 2. Generate stack configuration
|
|
131
|
+
mlx-stack init --accept-defaults
|
|
132
|
+
|
|
133
|
+
# 3. Download required models
|
|
134
|
+
mlx-stack pull qwen3.5-8b
|
|
135
|
+
|
|
136
|
+
# 4. Start all services
|
|
137
|
+
mlx-stack up
|
|
138
|
+
|
|
139
|
+
# 5. Verify
|
|
140
|
+
mlx-stack status
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
The OpenAI-compatible API is now available at `http://localhost:4000/v1`.
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
# Stop everything when done
|
|
147
|
+
mlx-stack down
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
## CLI Reference
|
|
151
|
+
|
|
152
|
+
### Setup & Configuration
|
|
153
|
+
|
|
154
|
+
| Command | Description |
|
|
155
|
+
|---------|-------------|
|
|
156
|
+
| `mlx-stack profile` | Detect Apple Silicon hardware and save profile to `~/.mlx-stack/profile.json` |
|
|
157
|
+
| `mlx-stack config set <key> <value>` | Set a configuration value |
|
|
158
|
+
| `mlx-stack config get <key>` | Get a configuration value |
|
|
159
|
+
| `mlx-stack config list` | List all configuration values with defaults and sources |
|
|
160
|
+
| `mlx-stack config reset --yes` | Reset all configuration to defaults |
|
|
161
|
+
|
|
162
|
+
### Model Management
|
|
163
|
+
|
|
164
|
+
**`mlx-stack recommend`** — Recommend an optimal model stack based on your hardware profile.
|
|
165
|
+
|
|
166
|
+
| Option | Description |
|
|
167
|
+
|--------|-------------|
|
|
168
|
+
| `--budget <value>` | Memory budget override (e.g., `30gb`). Defaults to 40% of unified memory |
|
|
169
|
+
| `--intent <balanced\|agent-fleet>` | Optimization strategy |
|
|
170
|
+
| `--show-all` | Show all budget-fitting models ranked by score |
|
|
171
|
+
|
|
172
|
+
**`mlx-stack models`** — List locally downloaded models with disk size, quantization, and active stack status.
|
|
173
|
+
|
|
174
|
+
| Option | Description |
|
|
175
|
+
|--------|-------------|
|
|
176
|
+
| `--catalog` | Show all catalog models with hardware-specific benchmark data |
|
|
177
|
+
| `--family <name>` | Filter by model family (e.g., `qwen3.5`) |
|
|
178
|
+
| `--tag <name>` | Filter by tag (e.g., `agent-ready`) |
|
|
179
|
+
| `--tool-calling` | Filter to tool-calling-capable models only |
|
|
180
|
+
|
|
181
|
+
**`mlx-stack pull <model>`** — Download a model from the catalog.
|
|
182
|
+
|
|
183
|
+
| Option | Description |
|
|
184
|
+
|--------|-------------|
|
|
185
|
+
| `--quant <int4\|int8\|bf16>` | Quantization level (default: `int4`) |
|
|
186
|
+
| `--bench` | Run a quick benchmark after download |
|
|
187
|
+
| `--force` | Re-download even if the model already exists |
|
|
188
|
+
|
|
189
|
+
**`mlx-stack init`** — Generate stack definition and LiteLLM proxy configuration.
|
|
190
|
+
|
|
191
|
+
| Option | Description |
|
|
192
|
+
|--------|-------------|
|
|
193
|
+
| `--accept-defaults` | Use defaults without prompting |
|
|
194
|
+
| `--intent <balanced\|agent-fleet>` | Optimization strategy |
|
|
195
|
+
| `--add <model>` | Add a model to the stack (repeatable) |
|
|
196
|
+
| `--remove <tier>` | Remove a tier from the stack (repeatable) |
|
|
197
|
+
| `--force` | Overwrite existing stack configuration |
|
|
198
|
+
|
|
199
|
+
### Stack Lifecycle
|
|
200
|
+
|
|
201
|
+
**`mlx-stack up`** — Start all services: one vllm-mlx process per tier plus the LiteLLM proxy.
|
|
202
|
+
|
|
203
|
+
| Option | Description |
|
|
204
|
+
|--------|-------------|
|
|
205
|
+
| `--dry-run` | Show exact commands without starting anything |
|
|
206
|
+
| `--tier <name>` | Start only the specified tier |
|
|
207
|
+
|
|
208
|
+
**`mlx-stack down`** — Stop all managed services (SIGTERM → 10s grace → SIGKILL).
|
|
209
|
+
|
|
210
|
+
| Option | Description |
|
|
211
|
+
|--------|-------------|
|
|
212
|
+
| `--tier <name>` | Stop only the specified tier |
|
|
213
|
+
|
|
214
|
+
**`mlx-stack status`** — Show health and status of all services (healthy, degraded, down, crashed, stopped).
|
|
215
|
+
|
|
216
|
+
| Option | Description |
|
|
217
|
+
|--------|-------------|
|
|
218
|
+
| `--json` | Output in JSON format |
|
|
219
|
+
|
|
220
|
+
### Diagnostics
|
|
221
|
+
|
|
222
|
+
**`mlx-stack bench <target>`** — Benchmark a running tier or catalog model. Runs 3 iterations and compares against catalog thresholds (PASS/WARN/FAIL).
|
|
223
|
+
|
|
224
|
+
| Option | Description |
|
|
225
|
+
|--------|-------------|
|
|
226
|
+
| `--save` | Persist results for use by `recommend` and `init` scoring |
|
|
227
|
+
|
|
228
|
+
### Ops & Reliability
|
|
229
|
+
|
|
230
|
+
**`mlx-stack logs [service]`** — View and manage service logs. Without arguments, lists all log files.
|
|
231
|
+
|
|
232
|
+
| Option | Description |
|
|
233
|
+
|--------|-------------|
|
|
234
|
+
| `--follow` / `-f` | Follow log output in real-time |
|
|
235
|
+
| `--tail <N>` | Show last N lines (default: 50) |
|
|
236
|
+
| `--service <name>` | Filter to a specific service |
|
|
237
|
+
| `--rotate` | Rotate eligible log files |
|
|
238
|
+
| `--all` | Show archived and current logs chronologically |
|
|
239
|
+
|
|
240
|
+
**`mlx-stack watch`** — Health monitor with auto-restart, flap detection, and log rotation.
|
|
241
|
+
|
|
242
|
+
| Option | Description |
|
|
243
|
+
|--------|-------------|
|
|
244
|
+
| `--interval <seconds>` | Polling interval (default: 30) |
|
|
245
|
+
| `--max-restarts <N>` | Restarts before marking as flapping (default: 5) |
|
|
246
|
+
| `--restart-delay <seconds>` | Base restart delay with exponential backoff (default: 5) |
|
|
247
|
+
| `--daemon` | Run in background as a daemon |
|
|
248
|
+
|
|
249
|
+
**`mlx-stack install`** — Install the watchdog as a macOS LaunchAgent.
|
|
250
|
+
|
|
251
|
+
| Option | Description |
|
|
252
|
+
|--------|-------------|
|
|
253
|
+
| `--status` | Show current LaunchAgent status |
|
|
254
|
+
|
|
255
|
+
**`mlx-stack uninstall`** — Remove the watchdog LaunchAgent. Running services are not affected.
|
|
256
|
+
|
|
257
|
+
## Configuration
|
|
258
|
+
|
|
259
|
+
Configuration is stored in `~/.mlx-stack/config.yaml`. Available keys:
|
|
260
|
+
|
|
261
|
+
| Key | Default | Description |
|
|
262
|
+
|-----|---------|-------------|
|
|
263
|
+
| `openrouter-key` | *(not set)* | OpenRouter API key for cloud fallback |
|
|
264
|
+
| `default-quant` | `int4` | Default quantization level (`int4`, `int8`, `bf16`) |
|
|
265
|
+
| `memory-budget-pct` | `40` | Percentage of unified memory to budget for models (1–100) |
|
|
266
|
+
| `litellm-port` | `4000` | LiteLLM proxy port |
|
|
267
|
+
| `model-dir` | `~/.mlx-stack/models` | Model storage directory |
|
|
268
|
+
| `auto-health-check` | `true` | Run health checks automatically on startup |
|
|
269
|
+
| `log-max-size-mb` | `50` | Maximum log file size in MB before rotation |
|
|
270
|
+
| `log-max-files` | `3` | Number of rotated log files to retain |
|
|
271
|
+
|
|
272
|
+
## 24/7 Operation
|
|
273
|
+
|
|
274
|
+
mlx-stack is designed to run unattended on always-on hardware like a Mac Mini.
|
|
275
|
+
|
|
276
|
+
### Quick setup
|
|
277
|
+
|
|
278
|
+
```bash
|
|
279
|
+
mlx-stack init --accept-defaults
|
|
280
|
+
mlx-stack install
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
This installs a macOS LaunchAgent that starts the watchdog on login. The watchdog:
|
|
284
|
+
|
|
285
|
+
- Monitors service health every 30 seconds
|
|
286
|
+
- Auto-restarts crashed processes with exponential backoff
|
|
287
|
+
- Detects flapping services and stops restart loops
|
|
288
|
+
- Rotates logs automatically to prevent unbounded disk usage
|
|
289
|
+
|
|
290
|
+
### Manual monitoring
|
|
291
|
+
|
|
292
|
+
```bash
|
|
293
|
+
mlx-stack watch # Foreground with Rich status table
|
|
294
|
+
mlx-stack watch --interval 60 # Less frequent polling
|
|
295
|
+
mlx-stack watch --daemon # Background without LaunchAgent
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
### Log management
|
|
299
|
+
|
|
300
|
+
```bash
|
|
301
|
+
mlx-stack logs # List all log files
|
|
302
|
+
mlx-stack logs fast # Last 50 lines of fast tier
|
|
303
|
+
mlx-stack logs fast --follow # Stream in real-time
|
|
304
|
+
mlx-stack logs --rotate # Rotate all eligible logs now
|
|
305
|
+
```
|
|
306
|
+
|
|
307
|
+
### Removing the agent
|
|
308
|
+
|
|
309
|
+
```bash
|
|
310
|
+
mlx-stack uninstall
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
This stops the watchdog and removes the LaunchAgent plist. Running services are not affected.
|
|
314
|
+
|
|
315
|
+
## Model Catalog
|
|
316
|
+
|
|
317
|
+
The built-in catalog includes 15 models across 5 families:
|
|
318
|
+
|
|
319
|
+
| Family | Models | Parameters |
|
|
320
|
+
|--------|--------|------------|
|
|
321
|
+
| Qwen 3.5 | 6 variants | 0.8B, 3B, 8B, 14B, 32B, 72B |
|
|
322
|
+
| Gemma 3 | 3 variants | 4B, 12B, 27B |
|
|
323
|
+
| DeepSeek R1 | 2 variants | 8B, 32B |
|
|
324
|
+
| Nemotron | 2 variants | 8B, 49B |
|
|
325
|
+
| Qwen 3 / Llama 3.3 | 2 variants | 8B each |
|
|
326
|
+
|
|
327
|
+
Each entry includes benchmark data for common Apple Silicon configurations, quality scores, and capability metadata (tool calling, thinking/reasoning, vision).
|
|
328
|
+
|
|
329
|
+
## Architecture Details
|
|
330
|
+
|
|
331
|
+
mlx-stack manages a **tiered local inference stack** with three layers:
|
|
332
|
+
|
|
333
|
+
### Model Servers (vllm-mlx)
|
|
334
|
+
|
|
335
|
+
One [vllm-mlx](https://github.com/vllm-project/vllm) instance per tier, each serving a single model on a dedicated port:
|
|
336
|
+
|
|
337
|
+
- **standard** (port 8000) — Highest-quality model that fits your memory budget. Optimized for accuracy-sensitive tasks.
|
|
338
|
+
- **fast** (port 8001) — Fastest model for latency-sensitive workloads like autocomplete and quick tool calls.
|
|
339
|
+
- **longctx** (port 8002) — Architecturally diverse model (e.g., Mamba2 hybrid) for extended context windows.
|
|
340
|
+
|
|
341
|
+
Each server runs with continuous batching, paged KV cache, and automatic tool-call parsing enabled.
|
|
342
|
+
|
|
343
|
+
### API Gateway (LiteLLM)
|
|
344
|
+
|
|
345
|
+
[LiteLLM](https://github.com/BerriAI/litellm) acts as the unified entry point on port 4000, providing:
|
|
346
|
+
|
|
347
|
+
- **OpenAI-compatible `/v1` API** — Drop-in replacement for `api.openai.com` in any client or agent framework.
|
|
348
|
+
- **Tier-based routing** — Requests target specific tiers by model name, or fall through a configurable chain.
|
|
349
|
+
- **Automatic fallback** — If the primary tier is unavailable, requests cascade to the next healthy tier.
|
|
350
|
+
|
|
351
|
+
### Cloud Fallback (Optional)
|
|
352
|
+
|
|
353
|
+
With an OpenRouter API key configured, a `premium` cloud tier is available as a last-resort fallback, giving you access to frontier models when local capacity is insufficient.
|
|
354
|
+
|
|
355
|
+
### Recommendation Engine
|
|
356
|
+
|
|
357
|
+
The recommendation engine scores all catalog models against your hardware profile:
|
|
358
|
+
|
|
359
|
+
1. **Hardware profiling** — Detects chip variant, GPU cores, unified memory, and memory bandwidth.
|
|
360
|
+
2. **Memory budgeting** — Filters models to those fitting within your configured memory budget (default: 40% of unified memory).
|
|
361
|
+
3. **Composite scoring** — Weights speed, quality, tool-calling capability, and memory efficiency based on your chosen intent (`balanced` or `agent-fleet`).
|
|
362
|
+
4. **Tier assignment** — Assigns top-scoring models to `standard`, `fast`, and `longctx` tiers.
|
|
363
|
+
5. **Local calibration** — Saved benchmark data from `mlx-stack bench --save` overrides catalog estimates for precise scoring.
|
|
364
|
+
|
|
365
|
+
### Process Management
|
|
366
|
+
|
|
367
|
+
- **PID tracking** — Each service writes its PID to `~/.mlx-stack/pids/` for reliable lifecycle management.
|
|
368
|
+
- **Lockfile** — Prevents concurrent `up`/`down` operations via `fcntl.flock`.
|
|
369
|
+
- **Health checks** — HTTP polling with exponential backoff and 120-second timeout per service.
|
|
370
|
+
- **5-state model** — Services are reported as `healthy`, `degraded`, `down`, `crashed`, or `stopped`.
|
|
371
|
+
- **Graceful shutdown** — SIGTERM with 10-second grace period, escalating to SIGKILL.
|
|
372
|
+
|
|
373
|
+
## Development
|
|
374
|
+
|
|
375
|
+
See [DEVELOPING.md](DEVELOPING.md) for the full developer guide, including project architecture, testing strategy, and how to add new models or commands.
|
|
376
|
+
|
|
377
|
+
```bash
|
|
378
|
+
# Install dev dependencies
|
|
379
|
+
uv sync
|
|
380
|
+
|
|
381
|
+
# Run tests
|
|
382
|
+
uv run pytest
|
|
383
|
+
|
|
384
|
+
# Type checking
|
|
385
|
+
uv run python -m pyright
|
|
386
|
+
|
|
387
|
+
# Linting
|
|
388
|
+
uv run ruff check src/ tests/
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
## Contributing
|
|
392
|
+
|
|
393
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on reporting bugs, suggesting features, and submitting pull requests.
|
|
394
|
+
|
|
395
|
+
## License
|
|
396
|
+
|
|
397
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
mlx_stack/__init__.py,sha256=5TQNZPOGwNrNFwmir6ab9zHkTZNPR-cDmKavhzhDXEo,153
|
|
2
|
+
mlx_stack/_version.py,sha256=n_5vdJsPNu7wZ57LGuRL585uvll-hiuvZUBWzdG0RQU,520
|
|
3
|
+
mlx_stack/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
4
|
+
mlx_stack/cli/__init__.py,sha256=ACDxQhPiBLb9vltrJiDW4ZagB_w-tNFYh5yORrMDD0E,94
|
|
5
|
+
mlx_stack/cli/bench.py,sha256=xqi3r9iZFtYyA6-3sG7rLJ2l3F9Hc0mSD5K5KHeJcpY,7615
|
|
6
|
+
mlx_stack/cli/config.py,sha256=DbeOYqDaEO8OCvKbKTKo_96UillHol4MD9rruinE6kQ,5148
|
|
7
|
+
mlx_stack/cli/down.py,sha256=-Cbgi-zIkJz09S8yY_nmOjO7wQBi4mpE_angvWPTgfw,3322
|
|
8
|
+
mlx_stack/cli/init.py,sha256=z8VTvOE5UVlkBy2Kg0veKvoFsDlZy3gSku-2yxFM_YM,5249
|
|
9
|
+
mlx_stack/cli/install.py,sha256=0Lx_KqpeGPGkwjHa04cdu7IqynoPCeszbedqLQklkYA,4727
|
|
10
|
+
mlx_stack/cli/logs.py,sha256=iGmZC5_Ov7uVRRK3N2vPaconS7M2sHqbIL2-2VjuS_8,6671
|
|
11
|
+
mlx_stack/cli/main.py,sha256=mnM05bRRHBF2vo_liQ0PxAQBY2IfCt_-Q58nQKuDHY8,6967
|
|
12
|
+
mlx_stack/cli/models.py,sha256=ADI7WKkiIcohWHtaQrj-kRkvn-i1uEPLdETKTtJBoKQ,9836
|
|
13
|
+
mlx_stack/cli/profile.py,sha256=DnF47wk6-vauG1QlKslr3blCnT64RpyFGoxqE2gOjqs,1924
|
|
14
|
+
mlx_stack/cli/pull.py,sha256=-FNNPTAsb8-AxTkFFpxM3CE02HWexihnD5v1jjGOrOQ,4186
|
|
15
|
+
mlx_stack/cli/recommend.py,sha256=zg5lGE-ME_qcm0yRRkKxwo1Io2QI7XWWBn1gvYWNlhk,12617
|
|
16
|
+
mlx_stack/cli/status.py,sha256=M1ID6RSXu1ZD5EtTEyxJXnM0BCyaHdnSwhrIZR9XbuM,3033
|
|
17
|
+
mlx_stack/cli/up.py,sha256=G65J5NN-9KLie3szD2habuFqvR09AMJMjLLGhT4ekgs,4879
|
|
18
|
+
mlx_stack/cli/watch.py,sha256=-6NJc-etj1H1z-4URBkWjXQtdxwBStC_tt_Kle0OQCA,6922
|
|
19
|
+
mlx_stack/core/__init__.py,sha256=vHZhwkjZ88hSb4Xb1ZgDhY7B0wd76QiCyRDgmaBuqHw,49
|
|
20
|
+
mlx_stack/core/benchmark.py,sha256=KGXWExc3mV3ZtPBimxLH_3tqh1-UoAsiYs3FgEkbtmI,35224
|
|
21
|
+
mlx_stack/core/catalog.py,sha256=E0IL3sYa4IY0MZFe6R234P093L4foHP4Eh0Nhfb-rwE,18044
|
|
22
|
+
mlx_stack/core/config.py,sha256=tM0-8maElb1wcycmU7emBeYBl0DDbiqgWY5mtP79i7Y,13364
|
|
23
|
+
mlx_stack/core/deps.py,sha256=s8ZUKo7aumap1hfuDt3WOrLSvHrhTLPtwjh3c4ROYP8,9886
|
|
24
|
+
mlx_stack/core/hardware.py,sha256=Fyrac4f9P3OZnHyAm-DwW-l31wnCFvjSihPIELbZ0xg,9409
|
|
25
|
+
mlx_stack/core/launchd.py,sha256=pqoiipd1k8Jt_RAaSXkD7jmL1HNGrf9ycWN50xzCow4,15523
|
|
26
|
+
mlx_stack/core/litellm_gen.py,sha256=IBMm-_HZcan3Yq5T4XmSUv2M8z-htE3eTfcTMd1IZ1A,5312
|
|
27
|
+
mlx_stack/core/log_rotation.py,sha256=zdCj1mv_Z9s1eCviCSr7xIWI3V_j4NGu40pTGZYgJc4,7182
|
|
28
|
+
mlx_stack/core/log_viewer.py,sha256=wRsBgErn5X038Z6ej5Mxc8XM9J9gCZDFuwFEX-zh89M,11453
|
|
29
|
+
mlx_stack/core/models.py,sha256=V8YSYitIop19ulTr8pO5LG3Rauq_AyJ3ACo8oaWPvEU,20843
|
|
30
|
+
mlx_stack/core/paths.py,sha256=xxs2I_0-SRuWbDFjdp6GmYnCB3fLhPwcB8WPfWIDvko,2056
|
|
31
|
+
mlx_stack/core/process.py,sha256=DjXMgobAyN1_3Z_TswVDvV85yNIzuQOpAq01X6UdNdg,25674
|
|
32
|
+
mlx_stack/core/pull.py,sha256=r8T3NOIaRcUkBvWsUws6cyltJX5UhjayImgC1E_YzS4,24948
|
|
33
|
+
mlx_stack/core/scoring.py,sha256=Y95MpCiwOVqEbfApwdHUUm6y33aBWkvnVBeokmfa5r0,19724
|
|
34
|
+
mlx_stack/core/stack_down.py,sha256=Zpen8jebSd0IQI-gi2TGQsmBEny4l4tC4FSZZphcekg,9379
|
|
35
|
+
mlx_stack/core/stack_init.py,sha256=sFvOLPYmvWUTIzEyEB_Dzm2m5BuaH40tsmDDH8zUkck,17106
|
|
36
|
+
mlx_stack/core/stack_status.py,sha256=Ue-6qzPQWWIbJtpMcGnFTk_ailrbMMkSPF2JYfPBr6o,6757
|
|
37
|
+
mlx_stack/core/stack_up.py,sha256=7WOk3Y_mqprbvUsQ6d81vrDq5IN-lfHFQ78lPLv1O8w,25981
|
|
38
|
+
mlx_stack/core/watchdog.py,sha256=YCKcLZiezspHWYQwsXrsoy3e4iZNuhARZa6vi4aaZ0Y,21462
|
|
39
|
+
mlx_stack/data/__init__.py,sha256=Ihuwvdj0niAehLr2Yiw_439sFdwZ0p8gqUpGq1ahN-g,39
|
|
40
|
+
mlx_stack/data/catalog/__init__.py,sha256=9WLBBMgdXp6hnPdD3PldIl78RR8QjXiqHmxNyr5FH9g,37
|
|
41
|
+
mlx_stack/data/catalog/deepseek-r1-32b.yaml,sha256=amhzGjxAbHJRv2zrlJwifHLpF6zsd5k8rdhFWnuyOXM,927
|
|
42
|
+
mlx_stack/data/catalog/deepseek-r1-8b.yaml,sha256=pa6BihfFgI1S1yk9vTDk9nKQAlhUEasQd3Ffh3XZTZs,906
|
|
43
|
+
mlx_stack/data/catalog/gemma3-12b.yaml,sha256=VtbNR2_dbClldrNCzhVHNJwwS-4xc88Zt4eiKqZjn2o,844
|
|
44
|
+
mlx_stack/data/catalog/gemma3-27b.yaml,sha256=iHqVRlUQSUpzZ455XyUUb28qAjY7eWmImsAEJf3YDUg,846
|
|
45
|
+
mlx_stack/data/catalog/gemma3-4b.yaml,sha256=ut3oyDhDKCOYYBwBHK952EIbtqB_ah3b9JOB_h-kQmo,846
|
|
46
|
+
mlx_stack/data/catalog/llama3.3-8b.yaml,sha256=gmTSLn9ba52HXr0r8P6TS7pQCIQ1yIQXkSuT_0xQYZQ,870
|
|
47
|
+
mlx_stack/data/catalog/nemotron-49b.yaml,sha256=nWoq7rE2MJi5hnsdgX4Yvn09uXIUZRsc3_I7-CNhqEI,799
|
|
48
|
+
mlx_stack/data/catalog/nemotron-8b.yaml,sha256=68ZabSgwmIdoP7xqn70KNbLFLsN5A40g02NjueNoW9I,843
|
|
49
|
+
mlx_stack/data/catalog/qwen3-8b.yaml,sha256=QPzslOUqENtEHxy1jMGioPzYMLbpjm28zFjhAVq8w_k,823
|
|
50
|
+
mlx_stack/data/catalog/qwen3.5-0.8b.yaml,sha256=CQZsiqAsgjXgY7J6Vc9jaggiJZ7FBxubaef4STJWVS4,858
|
|
51
|
+
mlx_stack/data/catalog/qwen3.5-14b.yaml,sha256=aNdoUAeqDnvc8Xbos2cZN95S3VaMuFUTWr_2lGPz7S4,852
|
|
52
|
+
mlx_stack/data/catalog/qwen3.5-32b.yaml,sha256=PLnHe9gs7m1qAkGxgcvheIQaaXlUX1fwpD7xMcbA2Ak,843
|
|
53
|
+
mlx_stack/data/catalog/qwen3.5-3b.yaml,sha256=AD0sfx0hiDclPABmd7L-pIvcy7DfCyr75m7nN9ApzO4,830
|
|
54
|
+
mlx_stack/data/catalog/qwen3.5-72b.yaml,sha256=IUBvwZ-ORbOIfAzATuDz5Muk2zlF29zPlnaoAQBrUwc,784
|
|
55
|
+
mlx_stack/data/catalog/qwen3.5-8b.yaml,sha256=_ws4BYLKkIb-PLP3Zk4ACORMPLvJDkt7rJUoxYdPcZY,835
|
|
56
|
+
mlx_stack/utils/__init__.py,sha256=A83-xB78mFfkul0y3PLHIHyF61CcTbx4IEzepKabCzc,37
|
|
57
|
+
mlx_stack-0.1.0.dist-info/METADATA,sha256=YvEdPzZVlDSx8eH1tYBZ8dVVoTwFpTT-uOQw_togLNw,16965
|
|
58
|
+
mlx_stack-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
|
|
59
|
+
mlx_stack-0.1.0.dist-info/entry_points.txt,sha256=zZC93ftIoIZGWwMQMNIaL8adv6wvT6HjEAZefDj-iW0,48
|
|
60
|
+
mlx_stack-0.1.0.dist-info/licenses/LICENSE,sha256=KqPG8dgvrddX0b0jUgE_GAlo2QVivpj2h_f-wlKdroI,1079
|
|
61
|
+
mlx_stack-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 mlx-stack contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|