localdeploy 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api_server.py +15 -0
- benchmark.py +15 -0
- chat_cli.py +15 -0
- compare_models.py +15 -0
- localdeploy/__init__.py +3 -0
- localdeploy/backends/__init__.py +0 -0
- localdeploy/backends/llamacpp.py +143 -0
- localdeploy/backends/ollama.py +374 -0
- localdeploy/backends/openai_compatible.py +179 -0
- localdeploy/benchmark.py +1996 -0
- localdeploy/chat_cli.py +249 -0
- localdeploy/cli.py +59 -0
- localdeploy/compare_models.py +481 -0
- localdeploy/control/__init__.py +75 -0
- localdeploy/control/_config.py +180 -0
- localdeploy/control/_ollama.py +253 -0
- localdeploy/control/bakeoff.py +216 -0
- localdeploy/control/bench.py +516 -0
- localdeploy/control/calibration.py +191 -0
- localdeploy/control/community.py +161 -0
- localdeploy/control/fit.py +651 -0
- localdeploy/control/hardware.py +602 -0
- localdeploy/control/history.py +125 -0
- localdeploy/control/manifest.py +452 -0
- localdeploy/control/models.py +833 -0
- localdeploy/control/monitor.py +378 -0
- localdeploy/control/profiles.py +161 -0
- localdeploy/control/recommend.py +322 -0
- localdeploy/control/registry.py +956 -0
- localdeploy/control/report.py +407 -0
- localdeploy/control/starter.py +520 -0
- localdeploy/control/updates.py +118 -0
- localdeploy/grader_sandbox.py +268 -0
- localdeploy/server.py +1873 -0
- localdeploy/utils.py +155 -0
- localdeploy/web/favicon.png +0 -0
- localdeploy/web/index.html +891 -0
- localdeploy/web/js/app.js +63 -0
- localdeploy/web/js/benchmark-views.js +73 -0
- localdeploy/web/js/benchmark.js +1691 -0
- localdeploy/web/js/chat.js +779 -0
- localdeploy/web/js/models.js +2758 -0
- localdeploy/web/js/shared.js +454 -0
- localdeploy/web/js/system.js +626 -0
- localdeploy/web/logo.svg +22 -0
- localdeploy/web/styles.css +2032 -0
- localdeploy-0.6.0.dist-info/METADATA +202 -0
- localdeploy-0.6.0.dist-info/RECORD +52 -0
- localdeploy-0.6.0.dist-info/WHEEL +5 -0
- localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
- localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
- localdeploy-0.6.0.dist-info/top_level.txt +5 -0
|
@@ -0,0 +1,651 @@
|
|
|
1
|
+
"""Step 3 - VRAM fit-check.
|
|
2
|
+
|
|
3
|
+
POST /system/fit-check answers "will this model fit?" with a transparent,
|
|
4
|
+
deliberately conservative estimate:
|
|
5
|
+
|
|
6
|
+
required_GB = weights_GB + kv_cache_GB + vision_overhead_GB + overhead_GB
|
|
7
|
+
|
|
8
|
+
The numbers are approximate by design; the breakdown is always returned so the
|
|
9
|
+
verdict is never a black box. The real proof is a short warmup (Step 6).
|
|
10
|
+
|
|
11
|
+
Release R2 adds three things on top of the original formula, all additive and
|
|
12
|
+
never silently changing what "fits" means:
|
|
13
|
+
|
|
14
|
+
- ``confidence``: how much this estimate should be trusted (metadata known +
|
|
15
|
+
calibration samples on file), separate from the fit tier itself.
|
|
16
|
+
- ``calibration``: the measured correction for this exact {GPU, runtime,
|
|
17
|
+
family, quant, context} from ``calibration.py``, shown alongside - never
|
|
18
|
+
substituted into - the raw formula estimate.
|
|
19
|
+
- ``/system/fit-table``: the same estimate swept across common context
|
|
20
|
+
lengths in one call, instead of one context per request.
|
|
21
|
+
"""
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import re
|
|
25
|
+
from typing import Any, Dict, List, Optional
|
|
26
|
+
|
|
27
|
+
from fastapi import APIRouter
|
|
28
|
+
from pydantic import BaseModel, Field
|
|
29
|
+
|
|
30
|
+
from . import calibration
|
|
31
|
+
from ._config import _looks_like_vision
|
|
32
|
+
from .hardware import detect_hardware, estimate_gpu_placement
|
|
33
|
+
|
|
34
|
+
router = APIRouter()
|
|
35
|
+
|
|
36
|
+
# Weight footprint per 1B parameters, in GB, by quantization family.
|
|
37
|
+
_WEIGHT_GB_PER_B = {
|
|
38
|
+
"f16": 2.0,
|
|
39
|
+
"fp16": 2.0,
|
|
40
|
+
"bf16": 2.0,
|
|
41
|
+
"q8": 1.0,
|
|
42
|
+
"q6": 0.75,
|
|
43
|
+
"q5": 0.65,
|
|
44
|
+
"q4": 0.5,
|
|
45
|
+
"q3": 0.42,
|
|
46
|
+
"q2": 0.34,
|
|
47
|
+
}
|
|
48
|
+
_DEFAULT_WEIGHT_GB_PER_B = 0.55 # unknown/default Ollama quant ~= Q4_K_M
|
|
49
|
+
|
|
50
|
+
# KV-cache scaling: per-token cost grows with model size; reduced by KV quant.
|
|
51
|
+
# This is a flat per-parameter coefficient, not an architecture-aware (GQA
|
|
52
|
+
# head count) model - real KV cache varies a lot by attention design. Rather
|
|
53
|
+
# than hardcoding per-family head-count assumptions this repo can't verify,
|
|
54
|
+
# the calibration store (see below) empirically corrects for that drift per
|
|
55
|
+
# {GPU, family, quant, context} from real measurements.
|
|
56
|
+
_KV_MB_PER_TOKEN_PER_B = 0.07 # fp16 baseline, conservative
|
|
57
|
+
_KV_QUANT_FACTOR = {"f16": 1.0, "fp16": 1.0, "q8": 0.5, "q4": 0.25}
|
|
58
|
+
_OVERHEAD_GB = 0.8 # CUDA context + activations
|
|
59
|
+
_VISION_OVERHEAD_GB = 0.6 # approximate vision-tower (encoder + projector) footprint
|
|
60
|
+
_COMFORTABLE_MARGIN_GB = 1.0 # VRAM headroom above which a fit is "comfortable"
|
|
61
|
+
|
|
62
|
+
# Common context tiers for /system/fit-table. Kept in sync with
|
|
63
|
+
# calibration._CONTEXT_BUCKETS so a table row's calibration lookup is exact.
|
|
64
|
+
FIT_TABLE_CONTEXTS = [4096, 8192, 16384, 32768, 65536]
|
|
65
|
+
|
|
66
|
+
_RUNTIME = "ollama" # calibration key runtime component; the estimator is Ollama-centric today
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class FitRequest(BaseModel):
|
|
70
|
+
profile: Optional[str] = None
|
|
71
|
+
model_id: Optional[str] = None
|
|
72
|
+
params_b: Optional[float] = Field(default=None, gt=0, le=10_000)
|
|
73
|
+
quant: Optional[str] = None
|
|
74
|
+
context: Optional[int] = Field(default=None, gt=0, le=10_000_000)
|
|
75
|
+
free_vram_mb: Optional[int] = Field(default=None, ge=0, le=100_000_000)
|
|
76
|
+
size_bytes: Optional[int] = Field(default=None, gt=0, le=1_000_000_000_000_000)
|
|
77
|
+
vision: Optional[bool] = None # None = auto-detect from model_id/profile name
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _parse_params_b(text: Optional[str]) -> Optional[float]:
|
|
81
|
+
"""Pull a parameter count like '4b' / '12b' / '1.5b' out of a model name."""
|
|
82
|
+
if not text:
|
|
83
|
+
return None
|
|
84
|
+
match = re.search(r"(\d+(?:\.\d+)?)\s*b\b", text.lower())
|
|
85
|
+
return float(match.group(1)) if match else None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _weight_gb_per_b(quant: Optional[str]) -> float:
|
|
89
|
+
if not quant:
|
|
90
|
+
return _DEFAULT_WEIGHT_GB_PER_B
|
|
91
|
+
q = quant.lower()
|
|
92
|
+
for key, value in _WEIGHT_GB_PER_B.items():
|
|
93
|
+
if key in q:
|
|
94
|
+
return value
|
|
95
|
+
return _DEFAULT_WEIGHT_GB_PER_B
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _kv_quant_factor(quant: Optional[str]) -> float:
|
|
99
|
+
if not quant:
|
|
100
|
+
return 1.0
|
|
101
|
+
q = quant.lower()
|
|
102
|
+
for key, value in _KV_QUANT_FACTOR.items():
|
|
103
|
+
if key in q:
|
|
104
|
+
return value
|
|
105
|
+
return 1.0
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _parse_quant(text: Optional[str]) -> Optional[str]:
|
|
109
|
+
"""Pull a GGUF-style quant label out of a model or repo name."""
|
|
110
|
+
if not text:
|
|
111
|
+
return None
|
|
112
|
+
lowered = text.lower()
|
|
113
|
+
match = re.search(r"\b(iq[2345]_[a-z0-9_]+|q[234568](?:_[a-z0-9_]+)?)\b", lowered)
|
|
114
|
+
return match.group(1).upper() if match else None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _base_family(model_id: Optional[str]) -> Optional[str]:
|
|
118
|
+
"""'gemma3:12b' -> 'gemma3'; 'hf.co/org/Repo' -> None (not an Ollama library name)."""
|
|
119
|
+
if not model_id:
|
|
120
|
+
return None
|
|
121
|
+
base = model_id.split(":")[0].strip().lower()
|
|
122
|
+
if not base or "/" in base:
|
|
123
|
+
return None
|
|
124
|
+
return base
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _round(value: float) -> float:
|
|
128
|
+
return round(value, 2)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _detected_vram_pool_mb(hw: Dict[str, Any]) -> Optional[int]:
|
|
132
|
+
summary = hw.get("gpu_summary") or {}
|
|
133
|
+
pool = summary.get("best_pool_free_mb")
|
|
134
|
+
if pool is not None:
|
|
135
|
+
return int(pool)
|
|
136
|
+
if hw.get("gpu_available") and hw.get("gpus"):
|
|
137
|
+
gpu = hw["gpus"][0]
|
|
138
|
+
value = gpu.get("vram_free_mb") or gpu.get("vram_total_mb")
|
|
139
|
+
return int(value) if value is not None else None
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _classify(
|
|
144
|
+
required_gb: float, free_vram_gb: Optional[float], ram_available_gb: Optional[float]
|
|
145
|
+
) -> Dict[str, Any]:
|
|
146
|
+
"""Tier the fit into comfortable / tight (soft) / cpu-only (soft) / won't-fit (hard).
|
|
147
|
+
|
|
148
|
+
``verdict`` stays coarse and backward-compatible: FITS when it fits VRAM,
|
|
149
|
+
WONT_FIT when VRAM is known but too small, UNKNOWN when VRAM is unknown.
|
|
150
|
+
``severity`` (ok / soft / hard / unknown) drives the green/yellow/red UI.
|
|
151
|
+
"""
|
|
152
|
+
if free_vram_gb is not None:
|
|
153
|
+
margin = free_vram_gb - required_gb
|
|
154
|
+
if margin >= _COMFORTABLE_MARGIN_GB:
|
|
155
|
+
return {
|
|
156
|
+
"verdict": "FITS", "tier": "comfortable", "severity": "ok",
|
|
157
|
+
"headline": "Fits comfortably on the GPU.", "cpu_deployable": True,
|
|
158
|
+
}
|
|
159
|
+
if margin >= 0:
|
|
160
|
+
return {
|
|
161
|
+
"verdict": "FITS", "tier": "tight", "severity": "soft",
|
|
162
|
+
"headline": f"Fits on the GPU, but headroom is tight (~{_round(margin)} GB).",
|
|
163
|
+
"cpu_deployable": True,
|
|
164
|
+
}
|
|
165
|
+
# Doesn't fit VRAM - can it run on CPU+RAM instead?
|
|
166
|
+
if ram_available_gb is not None:
|
|
167
|
+
if required_gb <= ram_available_gb:
|
|
168
|
+
return {
|
|
169
|
+
"verdict": "WONT_FIT", "tier": "cpu_only", "severity": "soft",
|
|
170
|
+
"headline": "Won't fit the GPU, but can run on CPU (slower).",
|
|
171
|
+
"cpu_deployable": True,
|
|
172
|
+
}
|
|
173
|
+
return {
|
|
174
|
+
"verdict": "WONT_FIT", "tier": "wont_fit", "severity": "hard",
|
|
175
|
+
"headline": "Too large for both GPU VRAM and system RAM.",
|
|
176
|
+
"cpu_deployable": False,
|
|
177
|
+
}
|
|
178
|
+
return {
|
|
179
|
+
"verdict": "WONT_FIT", "tier": "wont_fit_gpu", "severity": "hard",
|
|
180
|
+
"headline": "Won't fit the GPU (system RAM unknown).", "cpu_deployable": None,
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
# No VRAM figure - judge CPU deployability from RAM alone.
|
|
184
|
+
if ram_available_gb is not None:
|
|
185
|
+
if required_gb <= ram_available_gb:
|
|
186
|
+
return {
|
|
187
|
+
"verdict": "UNKNOWN", "tier": "cpu_only", "severity": "soft",
|
|
188
|
+
"headline": "No GPU detected - fits in system RAM, can run on CPU (slower).",
|
|
189
|
+
"cpu_deployable": True,
|
|
190
|
+
}
|
|
191
|
+
return {
|
|
192
|
+
"verdict": "UNKNOWN", "tier": "cpu_too_big", "severity": "hard",
|
|
193
|
+
"headline": "No GPU detected - too large for available system RAM.",
|
|
194
|
+
"cpu_deployable": False,
|
|
195
|
+
}
|
|
196
|
+
return {
|
|
197
|
+
"verdict": "UNKNOWN", "tier": "unknown", "severity": "unknown",
|
|
198
|
+
"headline": "No GPU VRAM detected; pass 'free_vram_mb' to validate against a target.",
|
|
199
|
+
"cpu_deployable": None,
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _resolve_from_profile(req: FitRequest) -> Dict[str, Any]:
|
|
204
|
+
"""Fill missing params/quant/context from a config profile when given."""
|
|
205
|
+
resolved = {
|
|
206
|
+
"model_id": req.model_id,
|
|
207
|
+
"params_b": req.params_b,
|
|
208
|
+
"quant": req.quant,
|
|
209
|
+
"context": req.context,
|
|
210
|
+
"kv_quant": None,
|
|
211
|
+
"size_bytes": req.size_bytes,
|
|
212
|
+
"vision": req.vision,
|
|
213
|
+
}
|
|
214
|
+
if not req.profile:
|
|
215
|
+
return resolved
|
|
216
|
+
# Lazy import: api_server owns config loading and finishes importing first.
|
|
217
|
+
from api_server import load_config
|
|
218
|
+
|
|
219
|
+
profile = load_config().get("profiles", {}).get(req.profile)
|
|
220
|
+
if not profile:
|
|
221
|
+
resolved["error"] = f"Unknown profile '{req.profile}'."
|
|
222
|
+
return resolved
|
|
223
|
+
resolved["model_id"] = resolved["model_id"] or profile.get("model_id")
|
|
224
|
+
if resolved["params_b"] is None:
|
|
225
|
+
resolved["params_b"] = _parse_params_b(profile.get("model_id")) or _parse_params_b(
|
|
226
|
+
profile.get("name")
|
|
227
|
+
)
|
|
228
|
+
resolved["quant"] = resolved["quant"] or profile.get("quantization") or _parse_quant(profile.get("model_id"))
|
|
229
|
+
if resolved["context"] is None:
|
|
230
|
+
resolved["context"] = profile.get("safe_context_limit") or profile.get("context_limit")
|
|
231
|
+
resolved["kv_quant"] = profile.get("kv_cache_type_k") or profile.get("kv_cache_type_v")
|
|
232
|
+
if resolved["vision"] is None:
|
|
233
|
+
resolved["vision"] = bool(profile.get("supports_vision"))
|
|
234
|
+
return resolved
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _confidence(*, size_known: bool, quant_known: bool, calibration_samples: int) -> str:
|
|
238
|
+
"""How much this estimate should be trusted - separate from the fit tier.
|
|
239
|
+
|
|
240
|
+
Starts at metadata completeness (do we know real size vs. a guessed
|
|
241
|
+
params_b, is the quant known vs. assumed default), then rises with
|
|
242
|
+
calibration coverage for this exact configuration.
|
|
243
|
+
"""
|
|
244
|
+
if not size_known:
|
|
245
|
+
return "low"
|
|
246
|
+
base = "medium" if quant_known else "low"
|
|
247
|
+
if calibration_samples >= 10:
|
|
248
|
+
return "high"
|
|
249
|
+
if calibration_samples >= 3:
|
|
250
|
+
return "medium" if base == "low" else "high"
|
|
251
|
+
return base
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _estimate(
|
|
255
|
+
*,
|
|
256
|
+
params_b: Optional[float],
|
|
257
|
+
quant: Optional[str],
|
|
258
|
+
context: int,
|
|
259
|
+
kv_quant: Optional[str],
|
|
260
|
+
size_bytes: Optional[int],
|
|
261
|
+
vision: bool,
|
|
262
|
+
model_id: Optional[str],
|
|
263
|
+
hw: Dict[str, Any],
|
|
264
|
+
free_vram_mb: Optional[int],
|
|
265
|
+
) -> Dict[str, Any]:
|
|
266
|
+
"""Core estimate + classification, shared by fit_check and fit_table.
|
|
267
|
+
|
|
268
|
+
Pure aside from the calibration store read (best-effort, never raises).
|
|
269
|
+
"""
|
|
270
|
+
vision_overhead_gb = _VISION_OVERHEAD_GB if vision else 0.0
|
|
271
|
+
if size_bytes:
|
|
272
|
+
# Installed Ollama/GGUF size is a much better load-footprint anchor than
|
|
273
|
+
# guessing from parameter count. Keep KV cache out of this load estimate;
|
|
274
|
+
# long prompts can still need more memory, which the note makes explicit.
|
|
275
|
+
weights_gb = size_bytes / 1_000_000_000.0
|
|
276
|
+
kv_cache_gb = 0.0
|
|
277
|
+
note = (
|
|
278
|
+
"Estimate uses installed model size plus runtime overhead. Long context "
|
|
279
|
+
"windows can need extra memory during generation."
|
|
280
|
+
)
|
|
281
|
+
size_known = True
|
|
282
|
+
else:
|
|
283
|
+
weights_gb = (params_b or 0.0) * _weight_gb_per_b(quant)
|
|
284
|
+
kv_factor = _kv_quant_factor(kv_quant)
|
|
285
|
+
kv_cache_gb = _KV_MB_PER_TOKEN_PER_B * (params_b or 0.0) * kv_factor * context / 1024.0
|
|
286
|
+
note = "Conservative estimate. The real proof is a short warmup via /models/serve."
|
|
287
|
+
size_known = bool(params_b)
|
|
288
|
+
required_gb = weights_gb + kv_cache_gb + vision_overhead_gb + _OVERHEAD_GB
|
|
289
|
+
|
|
290
|
+
free_vram_mb = free_vram_mb if free_vram_mb is not None else _detected_vram_pool_mb(hw)
|
|
291
|
+
ram_available_mb = (hw.get("system") or {}).get("ram_available_mb")
|
|
292
|
+
free_vram_gb = (free_vram_mb / 1024.0) if free_vram_mb is not None else None
|
|
293
|
+
ram_available_gb = (ram_available_mb / 1024.0) if ram_available_mb is not None else None
|
|
294
|
+
|
|
295
|
+
cls = _classify(required_gb, free_vram_gb, ram_available_gb)
|
|
296
|
+
placement = estimate_gpu_placement(required_gb, hw.get("gpus") or [])
|
|
297
|
+
if placement["mode"] == "multi_gpu_split" and cls["verdict"] == "FITS":
|
|
298
|
+
cls = dict(cls, tier="multi_gpu", headline="Fits only by splitting the model across compatible GPUs.")
|
|
299
|
+
|
|
300
|
+
family = _base_family(model_id)
|
|
301
|
+
corr = calibration.get_correction(
|
|
302
|
+
gpu=calibration.gpu_key(hw), runtime=_RUNTIME, family=family, quant=quant, context=context
|
|
303
|
+
)
|
|
304
|
+
calibrated_required_gb = _round(required_gb * corr["factor"]) if corr["applied"] else None
|
|
305
|
+
|
|
306
|
+
confidence = _confidence(
|
|
307
|
+
size_known=size_known, quant_known=bool(quant), calibration_samples=corr["sample_count"]
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
estimate = {
|
|
311
|
+
"weights": _round(weights_gb),
|
|
312
|
+
"kv_cache": _round(kv_cache_gb),
|
|
313
|
+
"vision_overhead": _round(vision_overhead_gb),
|
|
314
|
+
"overhead": _OVERHEAD_GB,
|
|
315
|
+
"required": _round(required_gb),
|
|
316
|
+
"calibrated_required": calibrated_required_gb,
|
|
317
|
+
}
|
|
318
|
+
if size_bytes:
|
|
319
|
+
estimate["size_gb"] = _round(size_bytes / 1_000_000_000.0)
|
|
320
|
+
|
|
321
|
+
return {
|
|
322
|
+
"verdict": cls["verdict"],
|
|
323
|
+
"tier": cls["tier"],
|
|
324
|
+
"severity": cls["severity"],
|
|
325
|
+
"headline": cls["headline"],
|
|
326
|
+
"cpu_deployable": cls["cpu_deployable"],
|
|
327
|
+
"estimate_gb": estimate,
|
|
328
|
+
"free_vram_gb": _round(free_vram_gb) if free_vram_gb is not None else None,
|
|
329
|
+
"ram_available_gb": _round(ram_available_gb) if ram_available_gb is not None else None,
|
|
330
|
+
"margin_gb": _round(free_vram_gb - required_gb) if free_vram_gb is not None else None,
|
|
331
|
+
"calibrated_margin_gb": (
|
|
332
|
+
_round(free_vram_gb - calibrated_required_gb)
|
|
333
|
+
if free_vram_gb is not None and calibrated_required_gb is not None
|
|
334
|
+
else None
|
|
335
|
+
),
|
|
336
|
+
"placement": placement,
|
|
337
|
+
"confidence": confidence,
|
|
338
|
+
"calibration": corr,
|
|
339
|
+
"note": note,
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
# --- batch fit (one call colors a whole catalog page) -------------------------
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
class FitBatchRequest(BaseModel):
|
|
347
|
+
params_b: List[float] = Field(default_factory=list, max_length=100)
|
|
348
|
+
context: int = Field(default=4096, gt=0)
|
|
349
|
+
free_vram_mb: Optional[int] = Field(default=None, ge=0)
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
@router.post("/system/fit-batch")
|
|
353
|
+
def fit_batch(req: FitBatchRequest) -> Dict[str, Any]:
|
|
354
|
+
"""Fit-classify many parameter counts in one call (single hardware probe),
|
|
355
|
+
so the catalog can badge every size chip without one request per chip."""
|
|
356
|
+
for value in req.params_b:
|
|
357
|
+
if not (0 < value <= 5000):
|
|
358
|
+
return {"success": False, "message": f"params_b values must be in (0, 5000]; got {value}"}
|
|
359
|
+
hw = detect_hardware()
|
|
360
|
+
free_vram_mb = req.free_vram_mb
|
|
361
|
+
if free_vram_mb is None and hw["gpu_available"] and hw["gpus"]:
|
|
362
|
+
free_vram_mb = hw["gpus"][0].get("vram_free_mb")
|
|
363
|
+
free_vram_gb = (free_vram_mb / 1024.0) if free_vram_mb is not None else None
|
|
364
|
+
ram_available_mb = (hw.get("system") or {}).get("ram_available_mb")
|
|
365
|
+
ram_available_gb = (ram_available_mb / 1024.0) if ram_available_mb is not None else None
|
|
366
|
+
|
|
367
|
+
items = []
|
|
368
|
+
for params_b in dict.fromkeys(req.params_b): # dedupe, keep order
|
|
369
|
+
weights_gb = params_b * _DEFAULT_WEIGHT_GB_PER_B
|
|
370
|
+
kv_cache_gb = _KV_MB_PER_TOKEN_PER_B * params_b * req.context / 1024.0
|
|
371
|
+
required_gb = weights_gb + kv_cache_gb + _OVERHEAD_GB
|
|
372
|
+
cls = _classify(required_gb, free_vram_gb, ram_available_gb)
|
|
373
|
+
items.append(
|
|
374
|
+
{
|
|
375
|
+
"params_b": params_b,
|
|
376
|
+
"required_gb": _round(required_gb),
|
|
377
|
+
"verdict": cls["verdict"],
|
|
378
|
+
"tier": cls["tier"],
|
|
379
|
+
"severity": cls["severity"],
|
|
380
|
+
}
|
|
381
|
+
)
|
|
382
|
+
return {
|
|
383
|
+
"success": True,
|
|
384
|
+
"free_vram_gb": _round(free_vram_gb) if free_vram_gb is not None else None,
|
|
385
|
+
"items": items,
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
# --- quantization advisor ----------------------------------------------------
|
|
390
|
+
# One row per common GGUF quant, ordered worst -> best quality. `tag` is the
|
|
391
|
+
# conventional Ollama tag suffix; actual availability varies per family, so the
|
|
392
|
+
# UI links to the family's tags page rather than promising a pullable name.
|
|
393
|
+
_QUANT_LADDER = [
|
|
394
|
+
{"quant": "Q2_K", "quality": "heavy quality loss - last resort"},
|
|
395
|
+
{"quant": "Q3_K_M", "quality": "noticeable quality loss"},
|
|
396
|
+
{"quant": "Q4_K_M", "quality": "good - the usual default pull"},
|
|
397
|
+
{"quant": "Q5_K_M", "quality": "very good - small step up from Q4"},
|
|
398
|
+
{"quant": "Q6_K", "quality": "excellent - near Q8 at less memory"},
|
|
399
|
+
{"quant": "Q8_0", "quality": "near-lossless"},
|
|
400
|
+
{"quant": "F16", "quality": "reference quality, biggest footprint"},
|
|
401
|
+
]
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
class QuantAdviceRequest(BaseModel):
|
|
405
|
+
model_id: Optional[str] = None
|
|
406
|
+
params_b: Optional[float] = Field(default=None, gt=0, le=10_000)
|
|
407
|
+
context: Optional[int] = Field(default=None, gt=0, le=10_000_000)
|
|
408
|
+
free_vram_mb: Optional[int] = Field(default=None, ge=0, le=100_000_000)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
@router.post("/system/quant-advisor")
|
|
412
|
+
def quant_advisor(req: QuantAdviceRequest) -> Dict[str, Any]:
|
|
413
|
+
"""Estimate every common quant of one model family against the budget.
|
|
414
|
+
|
|
415
|
+
Reuses the fit-check formula per quant so the numbers agree with the rest
|
|
416
|
+
of the app, then says which quant is the best quality that still fits.
|
|
417
|
+
"""
|
|
418
|
+
params_b = req.params_b or _parse_params_b(req.model_id)
|
|
419
|
+
if not params_b:
|
|
420
|
+
return {
|
|
421
|
+
"success": False,
|
|
422
|
+
"message": (
|
|
423
|
+
"Could not determine parameter count. Use a model name that "
|
|
424
|
+
"encodes size (e.g. 'gemma3:12b') or pass 'params_b'."
|
|
425
|
+
),
|
|
426
|
+
}
|
|
427
|
+
context = req.context or 4096
|
|
428
|
+
|
|
429
|
+
hw = detect_hardware()
|
|
430
|
+
free_vram_mb = req.free_vram_mb
|
|
431
|
+
if free_vram_mb is None:
|
|
432
|
+
free_vram_mb = _detected_vram_pool_mb(hw)
|
|
433
|
+
free_vram_gb = (free_vram_mb / 1024.0) if free_vram_mb is not None else None
|
|
434
|
+
ram_available_mb = (hw.get("system") or {}).get("ram_available_mb")
|
|
435
|
+
ram_available_gb = (ram_available_mb / 1024.0) if ram_available_mb is not None else None
|
|
436
|
+
|
|
437
|
+
variants = []
|
|
438
|
+
for entry in _QUANT_LADDER:
|
|
439
|
+
weights_gb = params_b * _weight_gb_per_b(entry["quant"])
|
|
440
|
+
kv_cache_gb = _KV_MB_PER_TOKEN_PER_B * params_b * context / 1024.0
|
|
441
|
+
required_gb = weights_gb + kv_cache_gb + _OVERHEAD_GB
|
|
442
|
+
cls = _classify(required_gb, free_vram_gb, ram_available_gb)
|
|
443
|
+
variants.append(
|
|
444
|
+
{
|
|
445
|
+
"quant": entry["quant"],
|
|
446
|
+
"quality": entry["quality"],
|
|
447
|
+
"weights_gb": _round(weights_gb),
|
|
448
|
+
"required_gb": _round(required_gb),
|
|
449
|
+
"margin_gb": _round(free_vram_gb - required_gb) if free_vram_gb is not None else None,
|
|
450
|
+
"verdict": cls["verdict"],
|
|
451
|
+
"tier": cls["tier"],
|
|
452
|
+
"severity": cls["severity"],
|
|
453
|
+
"cpu_deployable": cls["cpu_deployable"],
|
|
454
|
+
"placement": estimate_gpu_placement(required_gb, hw.get("gpus") or []),
|
|
455
|
+
}
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
# Best = highest-quality quant that fits the GPU; note whether that beats
|
|
459
|
+
# the usual Q4 default. Purely derived from the ladder - no special cases.
|
|
460
|
+
gpu_fits = [v for v in variants if v["verdict"] == "FITS"]
|
|
461
|
+
comfortable = [v for v in gpu_fits if v["severity"] == "ok"]
|
|
462
|
+
best = (comfortable or gpu_fits)[-1] if gpu_fits else None
|
|
463
|
+
default_idx = next(i for i, v in enumerate(variants) if v["quant"] == "Q4_K_M")
|
|
464
|
+
if best is None:
|
|
465
|
+
if any(v["cpu_deployable"] for v in variants):
|
|
466
|
+
recommendation = (
|
|
467
|
+
"Nothing fits the GPU budget at this size - the smaller quants can "
|
|
468
|
+
"still run on CPU (slower), or pick a smaller model size."
|
|
469
|
+
)
|
|
470
|
+
else:
|
|
471
|
+
recommendation = "This model size is too large for this machine at any quantization."
|
|
472
|
+
else:
|
|
473
|
+
best_idx = next(i for i, v in enumerate(variants) if v["quant"] == best["quant"])
|
|
474
|
+
if best_idx > default_idx:
|
|
475
|
+
recommendation = (
|
|
476
|
+
f"You have headroom for {best['quant']} (~{best['required_gb']} GB) - "
|
|
477
|
+
f"a quality step up from the usual {variants[default_idx]['quant']} default pull."
|
|
478
|
+
)
|
|
479
|
+
elif best_idx == default_idx:
|
|
480
|
+
recommendation = f"The usual {best['quant']} default is also the best fit for your budget."
|
|
481
|
+
else:
|
|
482
|
+
recommendation = (
|
|
483
|
+
f"The usual Q4_K_M default is too big here - {best['quant']} "
|
|
484
|
+
f"(~{best['required_gb']} GB) is the best quality that fits."
|
|
485
|
+
)
|
|
486
|
+
|
|
487
|
+
family = _base_family(req.model_id)
|
|
488
|
+
return {
|
|
489
|
+
"success": True,
|
|
490
|
+
"model": {"family": family, "params_b": params_b, "context": context},
|
|
491
|
+
"free_vram_gb": _round(free_vram_gb) if free_vram_gb is not None else None,
|
|
492
|
+
"ram_available_gb": _round(ram_available_gb) if ram_available_gb is not None else None,
|
|
493
|
+
"variants": variants,
|
|
494
|
+
"recommendation": recommendation,
|
|
495
|
+
"tags_url": f"https://ollama.com/library/{family}/tags" if family else None,
|
|
496
|
+
"note": (
|
|
497
|
+
"Estimates use the same weights+KV+overhead formula as fit checks. "
|
|
498
|
+
"Exact tag names and availability vary per family - check the tags page. "
|
|
499
|
+
"Some families also ship QAT tags that keep higher-quant quality at Q4 size."
|
|
500
|
+
),
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
@router.post("/system/fit-check")
|
|
505
|
+
def fit_check(req: FitRequest) -> Dict[str, Any]:
|
|
506
|
+
resolved = _resolve_from_profile(req)
|
|
507
|
+
if resolved.get("error"):
|
|
508
|
+
return {"success": False, "verdict": "UNKNOWN", "message": resolved["error"]}
|
|
509
|
+
|
|
510
|
+
params_b = resolved["params_b"] or _parse_params_b(resolved.get("model_id"))
|
|
511
|
+
if not params_b and not resolved.get("size_bytes"):
|
|
512
|
+
return {
|
|
513
|
+
"success": False,
|
|
514
|
+
"verdict": "UNKNOWN",
|
|
515
|
+
"message": (
|
|
516
|
+
"Could not determine parameter count. Pass 'params_b' explicitly "
|
|
517
|
+
"or use a profile/model_id that encodes size (e.g. '12b')."
|
|
518
|
+
),
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
context = resolved["context"] or 4096
|
|
522
|
+
quant = resolved["quant"] or _parse_quant(resolved.get("model_id"))
|
|
523
|
+
vision = resolved["vision"]
|
|
524
|
+
if vision is None:
|
|
525
|
+
vision = _looks_like_vision(resolved.get("model_id") or "")
|
|
526
|
+
|
|
527
|
+
hw = detect_hardware()
|
|
528
|
+
result = _estimate(
|
|
529
|
+
params_b=params_b,
|
|
530
|
+
quant=quant,
|
|
531
|
+
context=context,
|
|
532
|
+
kv_quant=resolved.get("kv_quant"),
|
|
533
|
+
size_bytes=resolved.get("size_bytes"),
|
|
534
|
+
vision=bool(vision),
|
|
535
|
+
model_id=resolved.get("model_id"),
|
|
536
|
+
hw=hw,
|
|
537
|
+
free_vram_mb=req.free_vram_mb,
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
placement = result["placement"]
|
|
541
|
+
if req.free_vram_mb is not None and result["verdict"] == "FITS" and not placement["supported"]:
|
|
542
|
+
placement = {
|
|
543
|
+
"mode": "manual_budget",
|
|
544
|
+
"supported": True,
|
|
545
|
+
"required_mb": int(result["estimate_gb"]["required"] * 1024),
|
|
546
|
+
"available_mb": req.free_vram_mb,
|
|
547
|
+
"gpu_indexes": [],
|
|
548
|
+
"allocations": [],
|
|
549
|
+
"utilization_pct": round(
|
|
550
|
+
result["estimate_gb"]["required"] * 1024 / max(1, req.free_vram_mb) * 100, 1
|
|
551
|
+
),
|
|
552
|
+
"note": "The supplied manual budget fits; automatic GPU placement could not be verified.",
|
|
553
|
+
}
|
|
554
|
+
result["placement"] = placement
|
|
555
|
+
|
|
556
|
+
suggestions = []
|
|
557
|
+
if result["severity"] in ("soft", "hard") and result["tier"] != "tight":
|
|
558
|
+
suggestions = [
|
|
559
|
+
"Use a smaller quantization (e.g. Q4 or Q3).",
|
|
560
|
+
"Lower the context window (try the profile's safe_context_limit).",
|
|
561
|
+
"Deploy to CPU (slower) - pick CPU in the serve panel.",
|
|
562
|
+
]
|
|
563
|
+
|
|
564
|
+
model_info = {"params_b": params_b, "quant": quant or "default (~Q4)", "context": context}
|
|
565
|
+
if result["estimate_gb"].get("size_gb") is not None:
|
|
566
|
+
model_info["size_gb"] = result["estimate_gb"]["size_gb"]
|
|
567
|
+
|
|
568
|
+
return {
|
|
569
|
+
"success": True,
|
|
570
|
+
# verdict stays coarse + backward-compatible (FITS / WONT_FIT / UNKNOWN);
|
|
571
|
+
# tier/severity/headline carry the new soft-vs-hard nuance for the UI.
|
|
572
|
+
"verdict": result["verdict"],
|
|
573
|
+
"tier": result["tier"],
|
|
574
|
+
"severity": result["severity"],
|
|
575
|
+
"headline": result["headline"],
|
|
576
|
+
"cpu_deployable": result["cpu_deployable"],
|
|
577
|
+
"confidence": result["confidence"],
|
|
578
|
+
"calibration": result["calibration"],
|
|
579
|
+
"model": model_info,
|
|
580
|
+
"estimate_gb": result["estimate_gb"],
|
|
581
|
+
"free_vram_gb": result["free_vram_gb"],
|
|
582
|
+
"ram_available_gb": result["ram_available_gb"],
|
|
583
|
+
"margin_gb": result["margin_gb"],
|
|
584
|
+
"calibrated_margin_gb": result["calibrated_margin_gb"],
|
|
585
|
+
"placement": result["placement"],
|
|
586
|
+
"gpu_summary": hw.get("gpu_summary") or {},
|
|
587
|
+
"note": result["note"],
|
|
588
|
+
"suggestions": suggestions,
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
@router.post("/system/fit-table")
|
|
593
|
+
def fit_table(req: FitRequest) -> Dict[str, Any]:
|
|
594
|
+
"""The same estimate as /system/fit-check, swept across common context
|
|
595
|
+
lengths in one call - "will this fit at 8K? 32K? 64K?" without one
|
|
596
|
+
request per tier."""
|
|
597
|
+
resolved = _resolve_from_profile(req)
|
|
598
|
+
if resolved.get("error"):
|
|
599
|
+
return {"success": False, "verdict": "UNKNOWN", "message": resolved["error"]}
|
|
600
|
+
|
|
601
|
+
params_b = resolved["params_b"] or _parse_params_b(resolved.get("model_id"))
|
|
602
|
+
if not params_b and not resolved.get("size_bytes"):
|
|
603
|
+
return {
|
|
604
|
+
"success": False,
|
|
605
|
+
"message": (
|
|
606
|
+
"Could not determine parameter count. Pass 'params_b' explicitly "
|
|
607
|
+
"or use a profile/model_id that encodes size (e.g. '12b')."
|
|
608
|
+
),
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
quant = resolved["quant"] or _parse_quant(resolved.get("model_id"))
|
|
612
|
+
vision = resolved["vision"]
|
|
613
|
+
if vision is None:
|
|
614
|
+
vision = _looks_like_vision(resolved.get("model_id") or "")
|
|
615
|
+
hw = detect_hardware() # one probe, reused for every context tier
|
|
616
|
+
|
|
617
|
+
rows = []
|
|
618
|
+
for context in FIT_TABLE_CONTEXTS:
|
|
619
|
+
result = _estimate(
|
|
620
|
+
params_b=params_b,
|
|
621
|
+
quant=quant,
|
|
622
|
+
context=context,
|
|
623
|
+
kv_quant=resolved.get("kv_quant"),
|
|
624
|
+
size_bytes=resolved.get("size_bytes"),
|
|
625
|
+
vision=bool(vision),
|
|
626
|
+
model_id=resolved.get("model_id"),
|
|
627
|
+
hw=hw,
|
|
628
|
+
free_vram_mb=req.free_vram_mb,
|
|
629
|
+
)
|
|
630
|
+
rows.append(
|
|
631
|
+
{
|
|
632
|
+
"context": context,
|
|
633
|
+
"verdict": result["verdict"],
|
|
634
|
+
"tier": result["tier"],
|
|
635
|
+
"severity": result["severity"],
|
|
636
|
+
"headline": result["headline"],
|
|
637
|
+
"estimate_gb": result["estimate_gb"],
|
|
638
|
+
"margin_gb": result["margin_gb"],
|
|
639
|
+
"calibrated_margin_gb": result["calibrated_margin_gb"],
|
|
640
|
+
"confidence": result["confidence"],
|
|
641
|
+
}
|
|
642
|
+
)
|
|
643
|
+
|
|
644
|
+
free_vram_mb = req.free_vram_mb if req.free_vram_mb is not None else _detected_vram_pool_mb(hw)
|
|
645
|
+
return {
|
|
646
|
+
"success": True,
|
|
647
|
+
"model": {"model_id": resolved.get("model_id"), "params_b": params_b, "quant": quant or "default (~Q4)"},
|
|
648
|
+
"free_vram_gb": _round(free_vram_mb / 1024.0) if free_vram_mb is not None else None,
|
|
649
|
+
"rows": rows,
|
|
650
|
+
"note": "Same weights+KV+overhead formula as /system/fit-check, one row per context length.",
|
|
651
|
+
}
|