gpu-fit 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gpu_fit-0.1.1/LICENSE +21 -0
- gpu_fit-0.1.1/PKG-INFO +117 -0
- gpu_fit-0.1.1/README.md +102 -0
- gpu_fit-0.1.1/pyproject.toml +28 -0
- gpu_fit-0.1.1/setup.cfg +4 -0
- gpu_fit-0.1.1/src/gpu_fit/__init__.py +3 -0
- gpu_fit-0.1.1/src/gpu_fit/__main__.py +5 -0
- gpu_fit-0.1.1/src/gpu_fit/calc.py +77 -0
- gpu_fit-0.1.1/src/gpu_fit/cli.py +136 -0
- gpu_fit-0.1.1/src/gpu_fit/gpus.py +44 -0
- gpu_fit-0.1.1/src/gpu_fit/model.py +139 -0
- gpu_fit-0.1.1/src/gpu_fit.egg-info/PKG-INFO +117 -0
- gpu_fit-0.1.1/src/gpu_fit.egg-info/SOURCES.txt +15 -0
- gpu_fit-0.1.1/src/gpu_fit.egg-info/dependency_links.txt +1 -0
- gpu_fit-0.1.1/src/gpu_fit.egg-info/entry_points.txt +2 -0
- gpu_fit-0.1.1/src/gpu_fit.egg-info/top_level.txt +1 -0
- gpu_fit-0.1.1/tests/test_calc.py +88 -0
gpu_fit-0.1.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kiran Tallapragada
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
gpu_fit-0.1.1/PKG-INFO
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gpu-fit
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Will this LLM fit on this GPU? vLLM memory maths: weights, KV cache, context and concurrency.
|
|
5
|
+
Author: Kiran Tallapragada
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/KiranTallapragada/gpu-fit
|
|
8
|
+
Keywords: llm,vllm,gpu,kv-cache,inference,vram,self-hosting
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# gpu-fit
|
|
17
|
+
|
|
18
|
+
**Will this LLM fit on this GPU, at this context length, for this many users?**
|
|
19
|
+
|
|
20
|
+
`gpu-fit` answers that before you rent the GPU. It reads the model's config and weight sizes
|
|
21
|
+
from Hugging Face and works out vLLM's memory budget: weights, runtime overhead, KV cache, how
|
|
22
|
+
many requests fit at full context, and the longest single request. Then it prints the
|
|
23
|
+
`vllm serve` command.
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
$ gpu-fit Qwen/Qwen3-32B --context 32k --concurrency 4
|
|
27
|
+
|
|
28
|
+
GPU smallest setup requests at full ctx KV tokens
|
|
29
|
+
L4 8× L4 10.6 347,888
|
|
30
|
+
L40S 4× L40S 11.6 380,656
|
|
31
|
+
A100-80 2× A100-80 9.9 323,484
|
|
32
|
+
H100 2× H100 9.8 320,885
|
|
33
|
+
H200 1× H200 7.9 259,428
|
|
34
|
+
...
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
$ gpu-fit deepseek-ai/DeepSeek-V3 --gpu H200 --tp 8 --context 128k
|
|
39
|
+
|
|
40
|
+
vLLM budget per GPU 126.4 GiB
|
|
41
|
+
weights 80.1 GiB
|
|
42
|
+
runtime overhead (assumed) 2.0 GiB
|
|
43
|
+
left for KV cache 44.2 GiB
|
|
44
|
+
KV cache capacity 675,671 tokens
|
|
45
|
+
Requests at full context 5.2
|
|
46
|
+
Longest single request 163,840 tokens
|
|
47
|
+
|
|
48
|
+
✓ Fits. Suggested command:
|
|
49
|
+
vllm serve deepseek-ai/DeepSeek-V3 --max-model-len 131072 --tensor-parallel-size 8 --max-num-seqs 5
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Install
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
pip install gpu-fit # no dependencies, Python 3.10+
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Why another VRAM calculator?
|
|
59
|
+
|
|
60
|
+
Most calculators multiply parameters by bytes and add a guess. Modern models break that:
|
|
61
|
+
|
|
62
|
+
| Architecture | What changes | gpu-fit |
|
|
63
|
+
|---|---|---|
|
|
64
|
+
| Grouped-query attention | KV per token depends on KV heads, not attention heads | ✓ |
|
|
65
|
+
| Hybrid (linear attention + full attention, e.g. Qwen3-Next family) | Only full-attention layers grow with context; the rest hold fixed state per request | ✓ |
|
|
66
|
+
| MLA (DeepSeek) | Compact latent KV, but **copied to every** tensor-parallel GPU | ✓ |
|
|
67
|
+
| Sliding window (Gemma, Mistral) | Window layers stop growing past the window | ✓ |
|
|
68
|
+
| FP8 / AWQ / GPTQ checkpoints | Exact weight size from the safetensors metadata | ✓ |
|
|
69
|
+
| Tensor parallel beyond KV heads | KV heads get replicated, not split | ✓ |
|
|
70
|
+
| FP8 KV cache | Doubles capacity: `--kv-cache-dtype fp8` | ✓ |
|
|
71
|
+
|
|
72
|
+
## Options
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
gpu-fit MODEL [--gpu L40S | --gpu-mem-gib 48] [--tp 2] [--context 128k]
|
|
76
|
+
[--concurrency 8] [--kv-cache-dtype fp8] [--util 0.9]
|
|
77
|
+
[--overhead-gib 2.0] [--json]
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
- `MODEL`: a Hugging Face id, or a local folder with `config.json` and `*.safetensors`.
|
|
81
|
+
Gated models need `HF_TOKEN` in the environment.
|
|
82
|
+
- Leave out `--gpu` to compare every GPU in the catalog with the smallest tensor-parallel size that works.
|
|
83
|
+
- `--json` for scripts and CI.
|
|
84
|
+
|
|
85
|
+
## How the maths works
|
|
86
|
+
|
|
87
|
+
vLLM takes `gpu_memory_utilization` × GPU memory, loads the weights, profiles a forward pass
|
|
88
|
+
(activations, CUDA graphs), and gives the rest to the KV cache. `gpu-fit` mirrors that:
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
KV pool = memory × utilization − weights / tp − overhead
|
|
92
|
+
KV/token = 2 × full-attention layers × KV heads per GPU × head dim × dtype bytes
|
|
93
|
+
requests = KV pool / (context × KV/token + sliding-window KV + recurrent state)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Accuracy, honestly
|
|
97
|
+
|
|
98
|
+
- Weights and KV per token come straight from the config, so they're exact.
|
|
99
|
+
- **Runtime overhead** (activations, CUDA graphs) is an assumption: 2 GiB per GPU by default.
|
|
100
|
+
It grows with batch size and model width. Override it with `--overhead-gib`.
|
|
101
|
+
- **Hybrid models:** vLLM pads KV pages to hold the recurrent state, so real capacity can be lower than shown.
|
|
102
|
+
- Treat the output as a planning estimate. Confirm on the real GPU before you commit money.
|
|
103
|
+
|
|
104
|
+
**Help make it exact:** share what vLLM printed on your hardware (see [CONTRIBUTING.md](CONTRIBUTING.md)).
|
|
105
|
+
Every measurement improves the defaults for everyone.
|
|
106
|
+
|
|
107
|
+
## Roadmap
|
|
108
|
+
|
|
109
|
+
- [ ] Measured overhead table from community results
|
|
110
|
+
- [ ] Throughput estimate (tokens/s) from memory bandwidth
|
|
111
|
+
- [ ] SGLang and TensorRT-LLM memory models
|
|
112
|
+
- [ ] Web version
|
|
113
|
+
- [ ] Vision encoder and LoRA memory
|
|
114
|
+
|
|
115
|
+
## License
|
|
116
|
+
|
|
117
|
+
MIT
|
gpu_fit-0.1.1/README.md
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# gpu-fit
|
|
2
|
+
|
|
3
|
+
**Will this LLM fit on this GPU, at this context length, for this many users?**
|
|
4
|
+
|
|
5
|
+
`gpu-fit` answers that before you rent the GPU. It reads the model's config and weight sizes
|
|
6
|
+
from Hugging Face and works out vLLM's memory budget: weights, runtime overhead, KV cache, how
|
|
7
|
+
many requests fit at full context, and the longest single request. Then it prints the
|
|
8
|
+
`vllm serve` command.
|
|
9
|
+
|
|
10
|
+
```
|
|
11
|
+
$ gpu-fit Qwen/Qwen3-32B --context 32k --concurrency 4
|
|
12
|
+
|
|
13
|
+
GPU smallest setup requests at full ctx KV tokens
|
|
14
|
+
L4 8× L4 10.6 347,888
|
|
15
|
+
L40S 4× L40S 11.6 380,656
|
|
16
|
+
A100-80 2× A100-80 9.9 323,484
|
|
17
|
+
H100 2× H100 9.8 320,885
|
|
18
|
+
H200 1× H200 7.9 259,428
|
|
19
|
+
...
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
```
|
|
23
|
+
$ gpu-fit deepseek-ai/DeepSeek-V3 --gpu H200 --tp 8 --context 128k
|
|
24
|
+
|
|
25
|
+
vLLM budget per GPU 126.4 GiB
|
|
26
|
+
weights 80.1 GiB
|
|
27
|
+
runtime overhead (assumed) 2.0 GiB
|
|
28
|
+
left for KV cache 44.2 GiB
|
|
29
|
+
KV cache capacity 675,671 tokens
|
|
30
|
+
Requests at full context 5.2
|
|
31
|
+
Longest single request 163,840 tokens
|
|
32
|
+
|
|
33
|
+
✓ Fits. Suggested command:
|
|
34
|
+
vllm serve deepseek-ai/DeepSeek-V3 --max-model-len 131072 --tensor-parallel-size 8 --max-num-seqs 5
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Install
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
pip install gpu-fit # no dependencies, Python 3.10+
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Why another VRAM calculator?
|
|
44
|
+
|
|
45
|
+
Most calculators multiply parameters by bytes and add a guess. Modern models break that:
|
|
46
|
+
|
|
47
|
+
| Architecture | What changes | gpu-fit |
|
|
48
|
+
|---|---|---|
|
|
49
|
+
| Grouped-query attention | KV per token depends on KV heads, not attention heads | ✓ |
|
|
50
|
+
| Hybrid (linear attention + full attention, e.g. Qwen3-Next family) | Only full-attention layers grow with context; the rest hold fixed state per request | ✓ |
|
|
51
|
+
| MLA (DeepSeek) | Compact latent KV, but **copied to every** tensor-parallel GPU | ✓ |
|
|
52
|
+
| Sliding window (Gemma, Mistral) | Window layers stop growing past the window | ✓ |
|
|
53
|
+
| FP8 / AWQ / GPTQ checkpoints | Exact weight size from the safetensors metadata | ✓ |
|
|
54
|
+
| Tensor parallel beyond KV heads | KV heads get replicated, not split | ✓ |
|
|
55
|
+
| FP8 KV cache | Doubles capacity: `--kv-cache-dtype fp8` | ✓ |
|
|
56
|
+
|
|
57
|
+
## Options
|
|
58
|
+
|
|
59
|
+
```
|
|
60
|
+
gpu-fit MODEL [--gpu L40S | --gpu-mem-gib 48] [--tp 2] [--context 128k]
|
|
61
|
+
[--concurrency 8] [--kv-cache-dtype fp8] [--util 0.9]
|
|
62
|
+
[--overhead-gib 2.0] [--json]
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
- `MODEL`: a Hugging Face id, or a local folder with `config.json` and `*.safetensors`.
|
|
66
|
+
Gated models need `HF_TOKEN` in the environment.
|
|
67
|
+
- Leave out `--gpu` to compare every GPU in the catalog with the smallest tensor-parallel size that works.
|
|
68
|
+
- `--json` for scripts and CI.
|
|
69
|
+
|
|
70
|
+
## How the maths works
|
|
71
|
+
|
|
72
|
+
vLLM takes `gpu_memory_utilization` × GPU memory, loads the weights, profiles a forward pass
|
|
73
|
+
(activations, CUDA graphs), and gives the rest to the KV cache. `gpu-fit` mirrors that:
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
KV pool = memory × utilization − weights / tp − overhead
|
|
77
|
+
KV/token = 2 × full-attention layers × KV heads per GPU × head dim × dtype bytes
|
|
78
|
+
requests = KV pool / (context × KV/token + sliding-window KV + recurrent state)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Accuracy, honestly
|
|
82
|
+
|
|
83
|
+
- Weights and KV per token come straight from the config, so they're exact.
|
|
84
|
+
- **Runtime overhead** (activations, CUDA graphs) is an assumption: 2 GiB per GPU by default.
|
|
85
|
+
It grows with batch size and model width. Override it with `--overhead-gib`.
|
|
86
|
+
- **Hybrid models:** vLLM pads KV pages to hold the recurrent state, so real capacity can be lower than shown.
|
|
87
|
+
- Treat the output as a planning estimate. Confirm on the real GPU before you commit money.
|
|
88
|
+
|
|
89
|
+
**Help make it exact:** share what vLLM printed on your hardware (see [CONTRIBUTING.md](CONTRIBUTING.md)).
|
|
90
|
+
Every measurement improves the defaults for everyone.
|
|
91
|
+
|
|
92
|
+
## Roadmap
|
|
93
|
+
|
|
94
|
+
- [ ] Measured overhead table from community results
|
|
95
|
+
- [ ] Throughput estimate (tokens/s) from memory bandwidth
|
|
96
|
+
- [ ] SGLang and TensorRT-LLM memory models
|
|
97
|
+
- [ ] Web version
|
|
98
|
+
- [ ] Vision encoder and LoRA memory
|
|
99
|
+
|
|
100
|
+
## License
|
|
101
|
+
|
|
102
|
+
MIT
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "gpu-fit"
|
|
7
|
+
version = "0.1.1"
|
|
8
|
+
description = "Will this LLM fit on this GPU? vLLM memory maths: weights, KV cache, context and concurrency."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Kiran Tallapragada" }]
|
|
14
|
+
keywords = ["llm", "vllm", "gpu", "kv-cache", "inference", "vram", "self-hosting"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
18
|
+
]
|
|
19
|
+
dependencies = []
|
|
20
|
+
|
|
21
|
+
[project.scripts]
|
|
22
|
+
gpu-fit = "gpu_fit.cli:main"
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/KiranTallapragada/gpu-fit"
|
|
26
|
+
|
|
27
|
+
[tool.setuptools.packages.find]
|
|
28
|
+
where = ["src"]
|
gpu_fit-0.1.1/setup.cfg
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""The vLLM memory model.
|
|
2
|
+
|
|
3
|
+
vLLM takes `gpu_memory_utilization` of the card, loads the weights, profiles a forward
|
|
4
|
+
pass (activations, CUDA graphs, non-torch allocations), and gives everything left to the
|
|
5
|
+
KV cache. We reproduce that budget.
|
|
6
|
+
|
|
7
|
+
Everything is derived from the model config and public vLLM behaviour. The one
|
|
8
|
+
assumption is the per-GPU runtime overhead, which you can override with --overhead-gib.
|
|
9
|
+
Measured results are welcome: see CONTRIBUTING.md.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from typing import Optional
|
|
14
|
+
|
|
15
|
+
from .model import ModelSpec
|
|
16
|
+
|
|
17
|
+
GIB = 1024**3
|
|
18
|
+
|
|
19
|
+
# Activations + CUDA graphs + non-torch memory per GPU. An assumption, not a law:
|
|
20
|
+
# it grows with batch size and hidden size. Override with --overhead-gib.
|
|
21
|
+
DEFAULT_OVERHEAD_GIB = 2.0
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@dataclass
|
|
25
|
+
class Fit:
|
|
26
|
+
gpu_mem_gib: float
|
|
27
|
+
tp: int
|
|
28
|
+
budget_gib: float
|
|
29
|
+
weights_gib: float
|
|
30
|
+
overhead_gib: float
|
|
31
|
+
kv_pool_gib: float
|
|
32
|
+
kv_tokens: int
|
|
33
|
+
context: int
|
|
34
|
+
concurrent: float # full-length requests that fit at once
|
|
35
|
+
max_context_one_request: int
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def fits(self) -> bool:
|
|
39
|
+
return self.kv_pool_gib > 0 and self.concurrent >= 1
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def estimate(
|
|
43
|
+
spec: ModelSpec,
|
|
44
|
+
gpu_mem_gib: float,
|
|
45
|
+
context: int,
|
|
46
|
+
tp: int = 1,
|
|
47
|
+
kv_dtype_bytes: int = 2,
|
|
48
|
+
utilization: float = 0.90,
|
|
49
|
+
overhead_gib: Optional[float] = None,
|
|
50
|
+
) -> Fit:
|
|
51
|
+
overhead = DEFAULT_OVERHEAD_GIB if overhead_gib is None else overhead_gib
|
|
52
|
+
budget = gpu_mem_gib * utilization
|
|
53
|
+
weights = spec.weight_bytes / GIB / tp
|
|
54
|
+
kv_pool = budget - weights - overhead
|
|
55
|
+
|
|
56
|
+
full_tok, swa_tok = spec.kv_bytes_per_token(kv_dtype_bytes, tp)
|
|
57
|
+
usable = max(kv_pool, 0) * GIB
|
|
58
|
+
per_token = full_tok or swa_tok # vLLM reports capacity in full-attention tokens
|
|
59
|
+
|
|
60
|
+
window = spec.sliding_window or context
|
|
61
|
+
state = spec.linear_state_bytes(tp) # fixed per request, independent of length
|
|
62
|
+
per_request = context * full_tok + min(context, window) * swa_tok + state
|
|
63
|
+
concurrent = usable / per_request if per_request else float("inf")
|
|
64
|
+
|
|
65
|
+
if full_tok:
|
|
66
|
+
max_one = int((usable - min(context, window) * swa_tok - state) / full_tok)
|
|
67
|
+
else:
|
|
68
|
+
max_one = spec.max_context or context
|
|
69
|
+
if spec.max_context:
|
|
70
|
+
max_one = min(max_one, spec.max_context)
|
|
71
|
+
|
|
72
|
+
return Fit(
|
|
73
|
+
gpu_mem_gib=gpu_mem_gib, tp=tp, budget_gib=budget, weights_gib=weights,
|
|
74
|
+
overhead_gib=overhead, kv_pool_gib=kv_pool,
|
|
75
|
+
kv_tokens=int(usable / per_token) if per_token else 0,
|
|
76
|
+
context=context, concurrent=concurrent, max_context_one_request=max(max_one, 0),
|
|
77
|
+
)
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""gpu-fit: will this model fit on this GPU, at this context, for this many users (vLLM)?"""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
from dataclasses import asdict
|
|
7
|
+
|
|
8
|
+
from . import __version__
|
|
9
|
+
from .calc import estimate
|
|
10
|
+
from .gpus import GPUS, get_gpu
|
|
11
|
+
from .model import load_model
|
|
12
|
+
|
|
13
|
+
TP_SIZES = (1, 2, 4, 8)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def parse_tokens(text: str) -> int:
|
|
17
|
+
t = text.strip().lower()
|
|
18
|
+
mult = 1
|
|
19
|
+
if t.endswith("k"):
|
|
20
|
+
mult, t = 1024, t[:-1]
|
|
21
|
+
elif t.endswith("m"):
|
|
22
|
+
mult, t = 1024 * 1024, t[:-1]
|
|
23
|
+
return int(float(t) * mult)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _fmt_tokens(n: int) -> str:
|
|
27
|
+
return f"{n:,}"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _command(model: str, fit, args) -> str:
|
|
31
|
+
parts = [f"vllm serve {model}", f"--max-model-len {fit.context}"]
|
|
32
|
+
if fit.tp > 1:
|
|
33
|
+
parts.append(f"--tensor-parallel-size {fit.tp}")
|
|
34
|
+
if args.kv_cache_dtype == "fp8":
|
|
35
|
+
parts.append("--kv-cache-dtype fp8")
|
|
36
|
+
parts.append(f"--max-num-seqs {max(1, int(fit.concurrent)) if args.concurrency is None else args.concurrency}")
|
|
37
|
+
if args.util != 0.9:
|
|
38
|
+
parts.append(f"--gpu-memory-utilization {args.util}")
|
|
39
|
+
return " \\\n ".join(parts)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def report_one(spec, gpu_name, gpu_mem, args) -> int:
|
|
43
|
+
kv_bytes = 1 if args.kv_cache_dtype == "fp8" else 2
|
|
44
|
+
fit = estimate(spec, gpu_mem, args.context, args.tp, kv_bytes, args.util, args.overhead_gib)
|
|
45
|
+
need = args.concurrency or 1
|
|
46
|
+
ok = fit.kv_pool_gib > 0 and fit.concurrent >= need
|
|
47
|
+
|
|
48
|
+
if args.json:
|
|
49
|
+
print(json.dumps({"model": spec.name, "gpu": gpu_name, "fits": ok, **asdict(fit)}, indent=2))
|
|
50
|
+
return 0 if ok else 1
|
|
51
|
+
|
|
52
|
+
print(f"\n{spec.name} on {args.tp}× {gpu_name} ({gpu_mem:.1f} GiB each)")
|
|
53
|
+
print(f"context {_fmt_tokens(args.context)} · KV cache {args.kv_cache_dtype}"
|
|
54
|
+
f" · gpu_memory_utilization {args.util}\n")
|
|
55
|
+
rows = [
|
|
56
|
+
("vLLM budget per GPU", f"{fit.budget_gib:6.1f} GiB"),
|
|
57
|
+
(" weights", f"{fit.weights_gib:6.1f} GiB"),
|
|
58
|
+
(" runtime overhead (assumed)", f"{fit.overhead_gib:6.1f} GiB"),
|
|
59
|
+
(" left for KV cache", f"{fit.kv_pool_gib:6.1f} GiB"),
|
|
60
|
+
("KV cache capacity", f"{_fmt_tokens(fit.kv_tokens)} tokens"),
|
|
61
|
+
("Requests at full context", f"{fit.concurrent:.1f}"),
|
|
62
|
+
("Longest single request", f"{_fmt_tokens(fit.max_context_one_request)} tokens"),
|
|
63
|
+
]
|
|
64
|
+
for label, value in rows:
|
|
65
|
+
print(f" {label:<30} {value}")
|
|
66
|
+
for note in spec.notes:
|
|
67
|
+
print(f" note: {note}")
|
|
68
|
+
|
|
69
|
+
if fit.kv_pool_gib <= 0:
|
|
70
|
+
print("\n✗ Does not fit: the weights alone exceed the budget. Try --tp, a bigger GPU or a quantised model.")
|
|
71
|
+
elif not ok:
|
|
72
|
+
print(f"\n✗ Fits, but only {fit.concurrent:.1f} requests at {_fmt_tokens(args.context)} tokens"
|
|
73
|
+
f" (you asked for {need}). Try --kv-cache-dtype fp8, a shorter context or more GPUs.")
|
|
74
|
+
else:
|
|
75
|
+
print(f"\n✓ Fits. Suggested command:\n\n {_command(spec.name, fit, args)}")
|
|
76
|
+
print()
|
|
77
|
+
return 0 if ok else 1
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def report_scan(spec, args) -> int:
|
|
81
|
+
kv_bytes = 1 if args.kv_cache_dtype == "fp8" else 2
|
|
82
|
+
need = args.concurrency or 1
|
|
83
|
+
results = []
|
|
84
|
+
for gpu in GPUS.values():
|
|
85
|
+
for tp in TP_SIZES:
|
|
86
|
+
fit = estimate(spec, gpu.memory_gib, args.context, tp, kv_bytes, args.util, args.overhead_gib)
|
|
87
|
+
if fit.kv_pool_gib > 0 and fit.concurrent >= need:
|
|
88
|
+
results.append((gpu, fit))
|
|
89
|
+
break
|
|
90
|
+
else:
|
|
91
|
+
results.append((gpu, None))
|
|
92
|
+
|
|
93
|
+
if args.json:
|
|
94
|
+
print(json.dumps([{"gpu": g.name, "fits": f is not None, **(asdict(f) if f else {})}
|
|
95
|
+
for g, f in results], indent=2))
|
|
96
|
+
return 0
|
|
97
|
+
|
|
98
|
+
print(f"\n{spec.name} · context {_fmt_tokens(args.context)} · {need} request(s) at once"
|
|
99
|
+
f" · KV {args.kv_cache_dtype}\n")
|
|
100
|
+
print(f" {'GPU':<13}{'smallest setup':<16}{'requests at full ctx':>22}{'KV tokens':>14}")
|
|
101
|
+
for gpu, fit in results:
|
|
102
|
+
if fit is None:
|
|
103
|
+
print(f" {gpu.name:<13}does not fit on 8 or fewer")
|
|
104
|
+
else:
|
|
105
|
+
print(f" {gpu.name:<13}{f'{fit.tp}× {gpu.name}':<16}{fit.concurrent:>22.1f}{_fmt_tokens(fit.kv_tokens):>14}")
|
|
106
|
+
print("\nDetail for one option: gpu-fit MODEL --gpu <name> --tp <n> ...\n")
|
|
107
|
+
return 0
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def main(argv=None) -> int:
|
|
111
|
+
p = argparse.ArgumentParser(prog="gpu-fit", description=__doc__)
|
|
112
|
+
p.add_argument("model", help="Hugging Face id (org/name) or a local model folder")
|
|
113
|
+
p.add_argument("--gpu", help="GPU name, e.g. L40S, H100, A100-80. Omit to compare all.")
|
|
114
|
+
p.add_argument("--gpu-mem-gib", type=float, help="Custom GPU memory instead of --gpu")
|
|
115
|
+
p.add_argument("--tp", type=int, default=1, help="tensor-parallel GPUs (default 1)")
|
|
116
|
+
p.add_argument("--context", type=parse_tokens, default=parse_tokens("32k"),
|
|
117
|
+
help="tokens per request, e.g. 32k, 128k (default 32k)")
|
|
118
|
+
p.add_argument("--concurrency", type=int, help="requests at full context you need at once")
|
|
119
|
+
p.add_argument("--kv-cache-dtype", choices=["auto", "fp8"], default="auto")
|
|
120
|
+
p.add_argument("--util", type=float, default=0.9, help="gpu_memory_utilization (default 0.9)")
|
|
121
|
+
p.add_argument("--overhead-gib", type=float, help="runtime overhead per GPU (default 2.0)")
|
|
122
|
+
p.add_argument("--json", action="store_true", help="machine-readable output")
|
|
123
|
+
p.add_argument("--version", action="version", version=f"gpu-fit {__version__}")
|
|
124
|
+
args = p.parse_args(argv)
|
|
125
|
+
|
|
126
|
+
spec = load_model(args.model)
|
|
127
|
+
if args.gpu_mem_gib:
|
|
128
|
+
return report_one(spec, "custom", args.gpu_mem_gib, args)
|
|
129
|
+
if args.gpu:
|
|
130
|
+
gpu = get_gpu(args.gpu)
|
|
131
|
+
return report_one(spec, gpu.name, gpu.memory_gib, args)
|
|
132
|
+
return report_scan(spec, args)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
if __name__ == "__main__":
|
|
136
|
+
sys.exit(main())
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""GPU catalog. Memory is what `nvidia-smi --query-gpu=memory.total` reports (MiB),
|
|
2
|
+
which is what vLLM budgets against, not the marketing number."""
|
|
3
|
+
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class Gpu:
|
|
9
|
+
name: str
|
|
10
|
+
memory_mib: int
|
|
11
|
+
fp8: bool # native FP8 (Ada / Hopper / Blackwell)
|
|
12
|
+
note: str = ""
|
|
13
|
+
|
|
14
|
+
@property
|
|
15
|
+
def memory_gib(self) -> float:
|
|
16
|
+
return self.memory_mib / 1024
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
GPUS = {
|
|
20
|
+
g.name.lower(): g
|
|
21
|
+
for g in [
|
|
22
|
+
Gpu("L4", 23034, True),
|
|
23
|
+
Gpu("A10G", 23028, False),
|
|
24
|
+
Gpu("RTX4090", 24564, True),
|
|
25
|
+
Gpu("A100-40", 40960, False),
|
|
26
|
+
Gpu("L40S", 46068, True),
|
|
27
|
+
Gpu("RTX6000-Ada", 49140, True),
|
|
28
|
+
Gpu("A100-80", 81920, False),
|
|
29
|
+
Gpu("H100", 81559, True, "80 GB SXM/PCIe"),
|
|
30
|
+
Gpu("H200", 143771, True),
|
|
31
|
+
Gpu("B200", 183359, True),
|
|
32
|
+
]
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def get_gpu(name: str) -> Gpu:
|
|
37
|
+
key = name.lower().replace("_", "-").replace(" ", "-")
|
|
38
|
+
if key in GPUS:
|
|
39
|
+
return GPUS[key]
|
|
40
|
+
for k, g in GPUS.items():
|
|
41
|
+
if k.replace("-", "") == key.replace("-", ""):
|
|
42
|
+
return g
|
|
43
|
+
known = ", ".join(g.name for g in GPUS.values())
|
|
44
|
+
raise SystemExit(f"Unknown GPU '{name}'. Known: {known}. Or pass --gpu-mem-gib.")
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Read what memory maths needs from a Hugging Face model: its config.json and the
|
|
2
|
+
size of its weights (from the safetensors metadata, so quantised repos are exact)."""
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import urllib.request
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import Optional
|
|
9
|
+
|
|
10
|
+
DTYPE_BYTES = {
|
|
11
|
+
"F64": 8, "F32": 4, "I32": 4, "U32": 4, "BF16": 2, "F16": 2, "I16": 2,
|
|
12
|
+
"F8_E4M3": 1, "F8_E5M2": 1, "I8": 1, "U8": 1, "BOOL": 1,
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass
|
|
17
|
+
class ModelSpec:
|
|
18
|
+
name: str
|
|
19
|
+
weight_bytes: int
|
|
20
|
+
full_attn_layers: int
|
|
21
|
+
kv_heads: int
|
|
22
|
+
head_dim: int
|
|
23
|
+
sliding_layers: int = 0
|
|
24
|
+
sliding_window: Optional[int] = None
|
|
25
|
+
linear_attn_layers: int = 0 # Mamba / linear-attention layers: fixed state, no per-token KV
|
|
26
|
+
linear_state_elems: int = 0 # recurrent state per request per linear layer (elements)
|
|
27
|
+
state_dtype_bytes: int = 2
|
|
28
|
+
mla_dim: Optional[int] = None # DeepSeek-style latent KV: per-layer per-token width
|
|
29
|
+
max_context: Optional[int] = None
|
|
30
|
+
notes: list = field(default_factory=list)
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def hybrid(self) -> bool:
|
|
34
|
+
return self.linear_attn_layers > 0
|
|
35
|
+
|
|
36
|
+
def kv_bytes_per_token(self, kv_dtype_bytes: int, tp: int = 1) -> tuple:
|
|
37
|
+
"""(full-attention bytes/token, sliding-window bytes/token) on ONE GPU."""
|
|
38
|
+
if self.mla_dim: # latent KV is replicated across tensor-parallel ranks
|
|
39
|
+
return self.full_attn_layers * self.mla_dim * kv_dtype_bytes, 0
|
|
40
|
+
heads = max(1, self.kv_heads // tp) if self.kv_heads >= tp else 1
|
|
41
|
+
per_layer = 2 * heads * self.head_dim * kv_dtype_bytes
|
|
42
|
+
return self.full_attn_layers * per_layer, self.sliding_layers * per_layer
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def linear_state_bytes(self, tp: int = 1) -> int:
|
|
46
|
+
"""Recurrent state one request holds across all linear-attention layers, one GPU."""
|
|
47
|
+
return self.linear_attn_layers * self.linear_state_elems * self.state_dtype_bytes // tp
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _linear_state_elems(c: dict) -> int:
|
|
51
|
+
"""Gated-DeltaNet style (Qwen3-Next family): conv state + recurrent matrix per layer."""
|
|
52
|
+
kh, vh = c.get("linear_num_key_heads"), c.get("linear_num_value_heads")
|
|
53
|
+
kd, vd = c.get("linear_key_head_dim"), c.get("linear_value_head_dim")
|
|
54
|
+
if not all((kh, vh, kd, vd)):
|
|
55
|
+
return 0
|
|
56
|
+
conv_dim = 2 * kh * kd + vh * vd
|
|
57
|
+
return conv_dim * (c.get("linear_conv_kernel_dim", 4) - 1) + vh * kd * vd
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _fetch_json(url: str) -> dict:
|
|
61
|
+
headers = {"User-Agent": "gpu-fit"}
|
|
62
|
+
token = os.environ.get("HF_TOKEN")
|
|
63
|
+
if token:
|
|
64
|
+
headers["Authorization"] = f"Bearer {token}"
|
|
65
|
+
with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=30) as r:
|
|
66
|
+
return json.load(r)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def weight_bytes_from_counts(counts: dict) -> int:
|
|
70
|
+
return sum(n * DTYPE_BYTES.get(dt.upper(), 2) for dt, n in counts.items())
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def spec_from_config(name: str, config: dict, weight_bytes: int) -> ModelSpec:
|
|
74
|
+
c = config.get("text_config") or config.get("llm_config") or config
|
|
75
|
+
layers = c.get("num_hidden_layers") or c.get("n_layer") or c.get("num_layers")
|
|
76
|
+
if not layers:
|
|
77
|
+
raise SystemExit(f"{name}: config has no num_hidden_layers; pass a text model config.")
|
|
78
|
+
heads = c.get("num_attention_heads") or c.get("n_head")
|
|
79
|
+
kv_heads = c.get("num_key_value_heads") or heads
|
|
80
|
+
head_dim = c.get("head_dim") or (c["hidden_size"] // heads)
|
|
81
|
+
notes = []
|
|
82
|
+
|
|
83
|
+
full, sliding, linear = layers, 0, 0
|
|
84
|
+
types = c.get("layer_types")
|
|
85
|
+
if types:
|
|
86
|
+
full = sum(t == "full_attention" for t in types)
|
|
87
|
+
sliding = sum(t in ("sliding_attention", "sliding_window") for t in types)
|
|
88
|
+
linear = len(types) - full - sliding
|
|
89
|
+
elif c.get("full_attention_interval"):
|
|
90
|
+
full = layers // c["full_attention_interval"]
|
|
91
|
+
linear = layers - full
|
|
92
|
+
elif c.get("attn_layer_indices") is not None: # Jamba-style
|
|
93
|
+
full = len(c["attn_layer_indices"])
|
|
94
|
+
linear = layers - full
|
|
95
|
+
|
|
96
|
+
window = c.get("sliding_window") if sliding or c.get("use_sliding_window") else None
|
|
97
|
+
if window and not sliding and c.get("use_sliding_window"):
|
|
98
|
+
sliding, full = full, 0
|
|
99
|
+
|
|
100
|
+
mla = None
|
|
101
|
+
if c.get("kv_lora_rank"):
|
|
102
|
+
mla = c["kv_lora_rank"] + c.get("qk_rope_head_dim", 64)
|
|
103
|
+
notes.append("MLA latent KV: compact, but replicated on every tensor-parallel GPU")
|
|
104
|
+
state = _linear_state_elems(c) if linear else 0
|
|
105
|
+
if linear:
|
|
106
|
+
notes.append(f"hybrid: {linear} linear-attention layers carry no per-token KV")
|
|
107
|
+
notes.append("hybrid: vLLM pads KV pages to fit recurrent state, so real capacity can be lower; test before you commit")
|
|
108
|
+
if not state:
|
|
109
|
+
notes.append("recurrent state size unknown for this architecture; not counted")
|
|
110
|
+
|
|
111
|
+
return ModelSpec(
|
|
112
|
+
name=name, weight_bytes=weight_bytes, full_attn_layers=full, kv_heads=kv_heads,
|
|
113
|
+
head_dim=head_dim, sliding_layers=sliding, sliding_window=window,
|
|
114
|
+
linear_attn_layers=linear, linear_state_elems=state, mla_dim=mla,
|
|
115
|
+
max_context=c.get("max_position_embeddings"), notes=notes,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def load_model(model_id: str) -> ModelSpec:
|
|
120
|
+
if os.path.isdir(model_id):
|
|
121
|
+
with open(os.path.join(model_id, "config.json")) as f:
|
|
122
|
+
config = json.load(f)
|
|
123
|
+
size = sum(
|
|
124
|
+
os.path.getsize(os.path.join(model_id, p))
|
|
125
|
+
for p in os.listdir(model_id) if p.endswith(".safetensors")
|
|
126
|
+
)
|
|
127
|
+
return spec_from_config(os.path.basename(model_id.rstrip("/")), config, size)
|
|
128
|
+
|
|
129
|
+
base = "https://huggingface.co"
|
|
130
|
+
try:
|
|
131
|
+
config = _fetch_json(f"{base}/{model_id}/resolve/main/config.json")
|
|
132
|
+
info = _fetch_json(f"{base}/api/models/{model_id}?expand[]=safetensors")
|
|
133
|
+
except Exception as e: # noqa: BLE001
|
|
134
|
+
raise SystemExit(f"Could not read {model_id} from Hugging Face ({e}). "
|
|
135
|
+
"Gated model? Set HF_TOKEN, or pass a local model folder.")
|
|
136
|
+
counts = (info.get("safetensors") or {}).get("parameters")
|
|
137
|
+
if not counts:
|
|
138
|
+
raise SystemExit(f"{model_id}: no safetensors metadata on the Hub; pass a local folder.")
|
|
139
|
+
return spec_from_config(model_id, config, weight_bytes_from_counts(counts))
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gpu-fit
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Will this LLM fit on this GPU? vLLM memory maths: weights, KV cache, context and concurrency.
|
|
5
|
+
Author: Kiran Tallapragada
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/KiranTallapragada/gpu-fit
|
|
8
|
+
Keywords: llm,vllm,gpu,kv-cache,inference,vram,self-hosting
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Dynamic: license-file
|
|
15
|
+
|
|
16
|
+
# gpu-fit
|
|
17
|
+
|
|
18
|
+
**Will this LLM fit on this GPU, at this context length, for this many users?**
|
|
19
|
+
|
|
20
|
+
`gpu-fit` answers that before you rent the GPU. It reads the model's config and weight sizes
|
|
21
|
+
from Hugging Face and works out vLLM's memory budget: weights, runtime overhead, KV cache, how
|
|
22
|
+
many requests fit at full context, and the longest single request. Then it prints the
|
|
23
|
+
`vllm serve` command.
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
$ gpu-fit Qwen/Qwen3-32B --context 32k --concurrency 4
|
|
27
|
+
|
|
28
|
+
GPU smallest setup requests at full ctx KV tokens
|
|
29
|
+
L4 8× L4 10.6 347,888
|
|
30
|
+
L40S 4× L40S 11.6 380,656
|
|
31
|
+
A100-80 2× A100-80 9.9 323,484
|
|
32
|
+
H100 2× H100 9.8 320,885
|
|
33
|
+
H200 1× H200 7.9 259,428
|
|
34
|
+
...
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```
|
|
38
|
+
$ gpu-fit deepseek-ai/DeepSeek-V3 --gpu H200 --tp 8 --context 128k
|
|
39
|
+
|
|
40
|
+
vLLM budget per GPU 126.4 GiB
|
|
41
|
+
weights 80.1 GiB
|
|
42
|
+
runtime overhead (assumed) 2.0 GiB
|
|
43
|
+
left for KV cache 44.2 GiB
|
|
44
|
+
KV cache capacity 675,671 tokens
|
|
45
|
+
Requests at full context 5.2
|
|
46
|
+
Longest single request 163,840 tokens
|
|
47
|
+
|
|
48
|
+
✓ Fits. Suggested command:
|
|
49
|
+
vllm serve deepseek-ai/DeepSeek-V3 --max-model-len 131072 --tensor-parallel-size 8 --max-num-seqs 5
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Install
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
pip install gpu-fit # no dependencies, Python 3.10+
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Why another VRAM calculator?
|
|
59
|
+
|
|
60
|
+
Most calculators multiply parameters by bytes and add a guess. Modern models break that:
|
|
61
|
+
|
|
62
|
+
| Architecture | What changes | gpu-fit |
|
|
63
|
+
|---|---|---|
|
|
64
|
+
| Grouped-query attention | KV per token depends on KV heads, not attention heads | ✓ |
|
|
65
|
+
| Hybrid (linear attention + full attention, e.g. Qwen3-Next family) | Only full-attention layers grow with context; the rest hold fixed state per request | ✓ |
|
|
66
|
+
| MLA (DeepSeek) | Compact latent KV, but **copied to every** tensor-parallel GPU | ✓ |
|
|
67
|
+
| Sliding window (Gemma, Mistral) | Window layers stop growing past the window | ✓ |
|
|
68
|
+
| FP8 / AWQ / GPTQ checkpoints | Exact weight size from the safetensors metadata | ✓ |
|
|
69
|
+
| Tensor parallel beyond KV heads | KV heads get replicated, not split | ✓ |
|
|
70
|
+
| FP8 KV cache | Doubles capacity: `--kv-cache-dtype fp8` | ✓ |
|
|
71
|
+
|
|
72
|
+
## Options
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
gpu-fit MODEL [--gpu L40S | --gpu-mem-gib 48] [--tp 2] [--context 128k]
|
|
76
|
+
[--concurrency 8] [--kv-cache-dtype fp8] [--util 0.9]
|
|
77
|
+
[--overhead-gib 2.0] [--json]
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
- `MODEL`: a Hugging Face id, or a local folder with `config.json` and `*.safetensors`.
|
|
81
|
+
Gated models need `HF_TOKEN` in the environment.
|
|
82
|
+
- Leave out `--gpu` to compare every GPU in the catalog with the smallest tensor-parallel size that works.
|
|
83
|
+
- `--json` for scripts and CI.
|
|
84
|
+
|
|
85
|
+
## How the maths works
|
|
86
|
+
|
|
87
|
+
vLLM takes `gpu_memory_utilization` × GPU memory, loads the weights, profiles a forward pass
|
|
88
|
+
(activations, CUDA graphs), and gives the rest to the KV cache. `gpu-fit` mirrors that:
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
KV pool = memory × utilization − weights / tp − overhead
|
|
92
|
+
KV/token = 2 × full-attention layers × KV heads per GPU × head dim × dtype bytes
|
|
93
|
+
requests = KV pool / (context × KV/token + sliding-window KV + recurrent state)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Accuracy, honestly
|
|
97
|
+
|
|
98
|
+
- Weights and KV per token come straight from the config, so they're exact.
|
|
99
|
+
- **Runtime overhead** (activations, CUDA graphs) is an assumption: 2 GiB per GPU by default.
|
|
100
|
+
It grows with batch size and model width. Override it with `--overhead-gib`.
|
|
101
|
+
- **Hybrid models:** vLLM pads KV pages to hold the recurrent state, so real capacity can be lower than shown.
|
|
102
|
+
- Treat the output as a planning estimate. Confirm on the real GPU before you commit money.
|
|
103
|
+
|
|
104
|
+
**Help make it exact:** share what vLLM printed on your hardware (see [CONTRIBUTING.md](CONTRIBUTING.md)).
|
|
105
|
+
Every measurement improves the defaults for everyone.
|
|
106
|
+
|
|
107
|
+
## Roadmap
|
|
108
|
+
|
|
109
|
+
- [ ] Measured overhead table from community results
|
|
110
|
+
- [ ] Throughput estimate (tokens/s) from memory bandwidth
|
|
111
|
+
- [ ] SGLang and TensorRT-LLM memory models
|
|
112
|
+
- [ ] Web version
|
|
113
|
+
- [ ] Vision encoder and LoRA memory
|
|
114
|
+
|
|
115
|
+
## License
|
|
116
|
+
|
|
117
|
+
MIT
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
src/gpu_fit/__init__.py
|
|
5
|
+
src/gpu_fit/__main__.py
|
|
6
|
+
src/gpu_fit/calc.py
|
|
7
|
+
src/gpu_fit/cli.py
|
|
8
|
+
src/gpu_fit/gpus.py
|
|
9
|
+
src/gpu_fit/model.py
|
|
10
|
+
src/gpu_fit.egg-info/PKG-INFO
|
|
11
|
+
src/gpu_fit.egg-info/SOURCES.txt
|
|
12
|
+
src/gpu_fit.egg-info/dependency_links.txt
|
|
13
|
+
src/gpu_fit.egg-info/entry_points.txt
|
|
14
|
+
src/gpu_fit.egg-info/top_level.txt
|
|
15
|
+
tests/test_calc.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
gpu_fit
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import unittest
|
|
2
|
+
|
|
3
|
+
from gpu_fit.calc import GIB, estimate
|
|
4
|
+
from gpu_fit.cli import parse_tokens
|
|
5
|
+
from gpu_fit.model import spec_from_config, weight_bytes_from_counts
|
|
6
|
+
|
|
7
|
+
DENSE = {"num_hidden_layers": 32, "num_attention_heads": 32, "num_key_value_heads": 8,
|
|
8
|
+
"hidden_size": 4096, "max_position_embeddings": 131072}
|
|
9
|
+
HYBRID = {"text_config": {
|
|
10
|
+
"num_hidden_layers": 64, "num_attention_heads": 24, "num_key_value_heads": 4,
|
|
11
|
+
"head_dim": 256, "hidden_size": 5120, "full_attention_interval": 4,
|
|
12
|
+
"linear_num_key_heads": 16, "linear_num_value_heads": 48,
|
|
13
|
+
"linear_key_head_dim": 128, "linear_value_head_dim": 128, "linear_conv_kernel_dim": 4}}
|
|
14
|
+
MLA = {"num_hidden_layers": 61, "num_attention_heads": 128, "num_key_value_heads": 128,
|
|
15
|
+
"hidden_size": 7168, "kv_lora_rank": 512, "qk_rope_head_dim": 64}
|
|
16
|
+
SWA = {"num_hidden_layers": 4, "num_attention_heads": 8, "num_key_value_heads": 4,
|
|
17
|
+
"head_dim": 128, "hidden_size": 1024, "sliding_window": 1024,
|
|
18
|
+
"layer_types": ["sliding_attention", "sliding_attention", "sliding_attention", "full_attention"]}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class TestModelSpec(unittest.TestCase):
|
|
22
|
+
def test_dense_kv_per_token(self):
|
|
23
|
+
s = spec_from_config("dense", DENSE, 0)
|
|
24
|
+
# 2 (K,V) × 32 layers × 8 heads × 128 dim × 2 bytes
|
|
25
|
+
self.assertEqual(s.kv_bytes_per_token(2), (131072, 0))
|
|
26
|
+
self.assertEqual(s.kv_bytes_per_token(1), (65536, 0))
|
|
27
|
+
|
|
28
|
+
def test_hybrid_counts_only_full_attention_layers(self):
|
|
29
|
+
s = spec_from_config("hybrid", HYBRID, 0)
|
|
30
|
+
self.assertEqual((s.full_attn_layers, s.linear_attn_layers), (16, 48))
|
|
31
|
+
self.assertEqual(s.kv_bytes_per_token(2)[0], 2 * 16 * 4 * 256 * 2)
|
|
32
|
+
conv = (2 * 16 * 128 + 48 * 128) * 3
|
|
33
|
+
self.assertEqual(s.linear_state_elems, conv + 48 * 128 * 128)
|
|
34
|
+
|
|
35
|
+
def test_mla_is_compact_and_replicated_across_tp(self):
|
|
36
|
+
s = spec_from_config("mla", MLA, 0)
|
|
37
|
+
self.assertEqual(s.kv_bytes_per_token(2)[0], 61 * 576 * 2)
|
|
38
|
+
self.assertEqual(s.kv_bytes_per_token(2, tp=8), s.kv_bytes_per_token(2, tp=1))
|
|
39
|
+
|
|
40
|
+
def test_kv_heads_split_then_replicate_beyond_head_count(self):
|
|
41
|
+
s = spec_from_config("dense", DENSE, 0)
|
|
42
|
+
self.assertEqual(s.kv_bytes_per_token(2, tp=2)[0], 131072 // 2)
|
|
43
|
+
self.assertEqual(s.kv_bytes_per_token(2, tp=16)[0], 131072 // 8) # one head each
|
|
44
|
+
|
|
45
|
+
def test_sliding_window_layers(self):
|
|
46
|
+
s = spec_from_config("swa", SWA, 0)
|
|
47
|
+
self.assertEqual((s.full_attn_layers, s.sliding_layers, s.sliding_window), (1, 3, 1024))
|
|
48
|
+
|
|
49
|
+
def test_weight_bytes_by_dtype(self):
|
|
50
|
+
self.assertEqual(weight_bytes_from_counts({"BF16": 10, "F8_E4M3": 100, "F32": 1}), 124)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TestEstimate(unittest.TestCase):
|
|
54
|
+
def test_budget_arithmetic(self):
|
|
55
|
+
s = spec_from_config("dense", DENSE, 16 * GIB)
|
|
56
|
+
f = estimate(s, gpu_mem_gib=80, context=8192, overhead_gib=2.0)
|
|
57
|
+
self.assertAlmostEqual(f.kv_pool_gib, 80 * 0.9 - 16 - 2.0)
|
|
58
|
+
self.assertEqual(f.kv_tokens, int(f.kv_pool_gib * GIB / 131072))
|
|
59
|
+
self.assertAlmostEqual(f.concurrent, f.kv_tokens / 8192, places=2)
|
|
60
|
+
|
|
61
|
+
def test_fp8_kv_doubles_capacity(self):
|
|
62
|
+
s = spec_from_config("dense", DENSE, 16 * GIB)
|
|
63
|
+
a = estimate(s, 80, 8192, kv_dtype_bytes=2)
|
|
64
|
+
b = estimate(s, 80, 8192, kv_dtype_bytes=1)
|
|
65
|
+
self.assertEqual(b.kv_tokens, a.kv_tokens * 2)
|
|
66
|
+
|
|
67
|
+
def test_does_not_fit_when_weights_exceed_budget(self):
|
|
68
|
+
s = spec_from_config("dense", DENSE, 60 * GIB)
|
|
69
|
+
self.assertFalse(estimate(s, 48, 8192).fits)
|
|
70
|
+
self.assertTrue(estimate(s, 48, 8192, tp=2).fits)
|
|
71
|
+
|
|
72
|
+
def test_hybrid_state_reduces_concurrency(self):
|
|
73
|
+
s = spec_from_config("hybrid", HYBRID, 28 * GIB)
|
|
74
|
+
f = estimate(s, 45, 65536)
|
|
75
|
+
no_state = f.kv_pool_gib * GIB / (65536 * s.kv_bytes_per_token(2)[0])
|
|
76
|
+
self.assertLess(f.concurrent, no_state)
|
|
77
|
+
|
|
78
|
+
def test_single_request_capped_by_model_max(self):
|
|
79
|
+
s = spec_from_config("dense", DENSE, 1 * GIB)
|
|
80
|
+
self.assertEqual(estimate(s, 192, 8192).max_context_one_request, 131072)
|
|
81
|
+
|
|
82
|
+
def test_parse_tokens(self):
|
|
83
|
+
self.assertEqual(parse_tokens("128k"), 131072)
|
|
84
|
+
self.assertEqual(parse_tokens("32000"), 32000)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
if __name__ == "__main__":
|
|
88
|
+
unittest.main()
|