localdeploy 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. api_server.py +15 -0
  2. benchmark.py +15 -0
  3. chat_cli.py +15 -0
  4. compare_models.py +15 -0
  5. localdeploy/__init__.py +3 -0
  6. localdeploy/backends/__init__.py +0 -0
  7. localdeploy/backends/llamacpp.py +143 -0
  8. localdeploy/backends/ollama.py +374 -0
  9. localdeploy/backends/openai_compatible.py +179 -0
  10. localdeploy/benchmark.py +1996 -0
  11. localdeploy/chat_cli.py +249 -0
  12. localdeploy/cli.py +59 -0
  13. localdeploy/compare_models.py +481 -0
  14. localdeploy/control/__init__.py +75 -0
  15. localdeploy/control/_config.py +180 -0
  16. localdeploy/control/_ollama.py +253 -0
  17. localdeploy/control/bakeoff.py +216 -0
  18. localdeploy/control/bench.py +516 -0
  19. localdeploy/control/calibration.py +191 -0
  20. localdeploy/control/community.py +161 -0
  21. localdeploy/control/fit.py +651 -0
  22. localdeploy/control/hardware.py +602 -0
  23. localdeploy/control/history.py +125 -0
  24. localdeploy/control/manifest.py +452 -0
  25. localdeploy/control/models.py +833 -0
  26. localdeploy/control/monitor.py +378 -0
  27. localdeploy/control/profiles.py +161 -0
  28. localdeploy/control/recommend.py +322 -0
  29. localdeploy/control/registry.py +956 -0
  30. localdeploy/control/report.py +407 -0
  31. localdeploy/control/starter.py +520 -0
  32. localdeploy/control/updates.py +118 -0
  33. localdeploy/grader_sandbox.py +268 -0
  34. localdeploy/server.py +1873 -0
  35. localdeploy/utils.py +155 -0
  36. localdeploy/web/favicon.png +0 -0
  37. localdeploy/web/index.html +891 -0
  38. localdeploy/web/js/app.js +63 -0
  39. localdeploy/web/js/benchmark-views.js +73 -0
  40. localdeploy/web/js/benchmark.js +1691 -0
  41. localdeploy/web/js/chat.js +779 -0
  42. localdeploy/web/js/models.js +2758 -0
  43. localdeploy/web/js/shared.js +454 -0
  44. localdeploy/web/js/system.js +626 -0
  45. localdeploy/web/logo.svg +22 -0
  46. localdeploy/web/styles.css +2032 -0
  47. localdeploy-0.6.0.dist-info/METADATA +202 -0
  48. localdeploy-0.6.0.dist-info/RECORD +52 -0
  49. localdeploy-0.6.0.dist-info/WHEEL +5 -0
  50. localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
  51. localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
  52. localdeploy-0.6.0.dist-info/top_level.txt +5 -0
@@ -0,0 +1,651 @@
1
+ """Step 3 - VRAM fit-check.
2
+
3
+ POST /system/fit-check answers "will this model fit?" with a transparent,
4
+ deliberately conservative estimate:
5
+
6
+ required_GB = weights_GB + kv_cache_GB + vision_overhead_GB + overhead_GB
7
+
8
+ The numbers are approximate by design; the breakdown is always returned so the
9
+ verdict is never a black box. The real proof is a short warmup (Step 6).
10
+
11
+ Release R2 adds three things on top of the original formula, all additive and
12
+ never silently changing what "fits" means:
13
+
14
+ - ``confidence``: how much this estimate should be trusted (metadata known +
15
+ calibration samples on file), separate from the fit tier itself.
16
+ - ``calibration``: the measured correction for this exact {GPU, runtime,
17
+ family, quant, context} from ``calibration.py``, shown alongside - never
18
+ substituted into - the raw formula estimate.
19
+ - ``/system/fit-table``: the same estimate swept across common context
20
+ lengths in one call, instead of one context per request.
21
+ """
22
+ from __future__ import annotations
23
+
24
+ import re
25
+ from typing import Any, Dict, List, Optional
26
+
27
+ from fastapi import APIRouter
28
+ from pydantic import BaseModel, Field
29
+
30
+ from . import calibration
31
+ from ._config import _looks_like_vision
32
+ from .hardware import detect_hardware, estimate_gpu_placement
33
+
34
+ router = APIRouter()
35
+
36
+ # Weight footprint per 1B parameters, in GB, by quantization family.
37
+ _WEIGHT_GB_PER_B = {
38
+ "f16": 2.0,
39
+ "fp16": 2.0,
40
+ "bf16": 2.0,
41
+ "q8": 1.0,
42
+ "q6": 0.75,
43
+ "q5": 0.65,
44
+ "q4": 0.5,
45
+ "q3": 0.42,
46
+ "q2": 0.34,
47
+ }
48
+ _DEFAULT_WEIGHT_GB_PER_B = 0.55 # unknown/default Ollama quant ~= Q4_K_M
49
+
50
+ # KV-cache scaling: per-token cost grows with model size; reduced by KV quant.
51
+ # This is a flat per-parameter coefficient, not an architecture-aware (GQA
52
+ # head count) model - real KV cache varies a lot by attention design. Rather
53
+ # than hardcoding per-family head-count assumptions this repo can't verify,
54
+ # the calibration store (see below) empirically corrects for that drift per
55
+ # {GPU, family, quant, context} from real measurements.
56
+ _KV_MB_PER_TOKEN_PER_B = 0.07 # fp16 baseline, conservative
57
+ _KV_QUANT_FACTOR = {"f16": 1.0, "fp16": 1.0, "q8": 0.5, "q4": 0.25}
58
+ _OVERHEAD_GB = 0.8 # CUDA context + activations
59
+ _VISION_OVERHEAD_GB = 0.6 # approximate vision-tower (encoder + projector) footprint
60
+ _COMFORTABLE_MARGIN_GB = 1.0 # VRAM headroom above which a fit is "comfortable"
61
+
62
+ # Common context tiers for /system/fit-table. Kept in sync with
63
+ # calibration._CONTEXT_BUCKETS so a table row's calibration lookup is exact.
64
+ FIT_TABLE_CONTEXTS = [4096, 8192, 16384, 32768, 65536]
65
+
66
+ _RUNTIME = "ollama" # calibration key runtime component; the estimator is Ollama-centric today
67
+
68
+
69
+ class FitRequest(BaseModel):
70
+ profile: Optional[str] = None
71
+ model_id: Optional[str] = None
72
+ params_b: Optional[float] = Field(default=None, gt=0, le=10_000)
73
+ quant: Optional[str] = None
74
+ context: Optional[int] = Field(default=None, gt=0, le=10_000_000)
75
+ free_vram_mb: Optional[int] = Field(default=None, ge=0, le=100_000_000)
76
+ size_bytes: Optional[int] = Field(default=None, gt=0, le=1_000_000_000_000_000)
77
+ vision: Optional[bool] = None # None = auto-detect from model_id/profile name
78
+
79
+
80
+ def _parse_params_b(text: Optional[str]) -> Optional[float]:
81
+ """Pull a parameter count like '4b' / '12b' / '1.5b' out of a model name."""
82
+ if not text:
83
+ return None
84
+ match = re.search(r"(\d+(?:\.\d+)?)\s*b\b", text.lower())
85
+ return float(match.group(1)) if match else None
86
+
87
+
88
+ def _weight_gb_per_b(quant: Optional[str]) -> float:
89
+ if not quant:
90
+ return _DEFAULT_WEIGHT_GB_PER_B
91
+ q = quant.lower()
92
+ for key, value in _WEIGHT_GB_PER_B.items():
93
+ if key in q:
94
+ return value
95
+ return _DEFAULT_WEIGHT_GB_PER_B
96
+
97
+
98
+ def _kv_quant_factor(quant: Optional[str]) -> float:
99
+ if not quant:
100
+ return 1.0
101
+ q = quant.lower()
102
+ for key, value in _KV_QUANT_FACTOR.items():
103
+ if key in q:
104
+ return value
105
+ return 1.0
106
+
107
+
108
+ def _parse_quant(text: Optional[str]) -> Optional[str]:
109
+ """Pull a GGUF-style quant label out of a model or repo name."""
110
+ if not text:
111
+ return None
112
+ lowered = text.lower()
113
+ match = re.search(r"\b(iq[2345]_[a-z0-9_]+|q[234568](?:_[a-z0-9_]+)?)\b", lowered)
114
+ return match.group(1).upper() if match else None
115
+
116
+
117
+ def _base_family(model_id: Optional[str]) -> Optional[str]:
118
+ """'gemma3:12b' -> 'gemma3'; 'hf.co/org/Repo' -> None (not an Ollama library name)."""
119
+ if not model_id:
120
+ return None
121
+ base = model_id.split(":")[0].strip().lower()
122
+ if not base or "/" in base:
123
+ return None
124
+ return base
125
+
126
+
127
+ def _round(value: float) -> float:
128
+ return round(value, 2)
129
+
130
+
131
+ def _detected_vram_pool_mb(hw: Dict[str, Any]) -> Optional[int]:
132
+ summary = hw.get("gpu_summary") or {}
133
+ pool = summary.get("best_pool_free_mb")
134
+ if pool is not None:
135
+ return int(pool)
136
+ if hw.get("gpu_available") and hw.get("gpus"):
137
+ gpu = hw["gpus"][0]
138
+ value = gpu.get("vram_free_mb") or gpu.get("vram_total_mb")
139
+ return int(value) if value is not None else None
140
+ return None
141
+
142
+
143
+ def _classify(
144
+ required_gb: float, free_vram_gb: Optional[float], ram_available_gb: Optional[float]
145
+ ) -> Dict[str, Any]:
146
+ """Tier the fit into comfortable / tight (soft) / cpu-only (soft) / won't-fit (hard).
147
+
148
+ ``verdict`` stays coarse and backward-compatible: FITS when it fits VRAM,
149
+ WONT_FIT when VRAM is known but too small, UNKNOWN when VRAM is unknown.
150
+ ``severity`` (ok / soft / hard / unknown) drives the green/yellow/red UI.
151
+ """
152
+ if free_vram_gb is not None:
153
+ margin = free_vram_gb - required_gb
154
+ if margin >= _COMFORTABLE_MARGIN_GB:
155
+ return {
156
+ "verdict": "FITS", "tier": "comfortable", "severity": "ok",
157
+ "headline": "Fits comfortably on the GPU.", "cpu_deployable": True,
158
+ }
159
+ if margin >= 0:
160
+ return {
161
+ "verdict": "FITS", "tier": "tight", "severity": "soft",
162
+ "headline": f"Fits on the GPU, but headroom is tight (~{_round(margin)} GB).",
163
+ "cpu_deployable": True,
164
+ }
165
+ # Doesn't fit VRAM - can it run on CPU+RAM instead?
166
+ if ram_available_gb is not None:
167
+ if required_gb <= ram_available_gb:
168
+ return {
169
+ "verdict": "WONT_FIT", "tier": "cpu_only", "severity": "soft",
170
+ "headline": "Won't fit the GPU, but can run on CPU (slower).",
171
+ "cpu_deployable": True,
172
+ }
173
+ return {
174
+ "verdict": "WONT_FIT", "tier": "wont_fit", "severity": "hard",
175
+ "headline": "Too large for both GPU VRAM and system RAM.",
176
+ "cpu_deployable": False,
177
+ }
178
+ return {
179
+ "verdict": "WONT_FIT", "tier": "wont_fit_gpu", "severity": "hard",
180
+ "headline": "Won't fit the GPU (system RAM unknown).", "cpu_deployable": None,
181
+ }
182
+
183
+ # No VRAM figure - judge CPU deployability from RAM alone.
184
+ if ram_available_gb is not None:
185
+ if required_gb <= ram_available_gb:
186
+ return {
187
+ "verdict": "UNKNOWN", "tier": "cpu_only", "severity": "soft",
188
+ "headline": "No GPU detected - fits in system RAM, can run on CPU (slower).",
189
+ "cpu_deployable": True,
190
+ }
191
+ return {
192
+ "verdict": "UNKNOWN", "tier": "cpu_too_big", "severity": "hard",
193
+ "headline": "No GPU detected - too large for available system RAM.",
194
+ "cpu_deployable": False,
195
+ }
196
+ return {
197
+ "verdict": "UNKNOWN", "tier": "unknown", "severity": "unknown",
198
+ "headline": "No GPU VRAM detected; pass 'free_vram_mb' to validate against a target.",
199
+ "cpu_deployable": None,
200
+ }
201
+
202
+
203
+ def _resolve_from_profile(req: FitRequest) -> Dict[str, Any]:
204
+ """Fill missing params/quant/context from a config profile when given."""
205
+ resolved = {
206
+ "model_id": req.model_id,
207
+ "params_b": req.params_b,
208
+ "quant": req.quant,
209
+ "context": req.context,
210
+ "kv_quant": None,
211
+ "size_bytes": req.size_bytes,
212
+ "vision": req.vision,
213
+ }
214
+ if not req.profile:
215
+ return resolved
216
+ # Lazy import: api_server owns config loading and finishes importing first.
217
+ from api_server import load_config
218
+
219
+ profile = load_config().get("profiles", {}).get(req.profile)
220
+ if not profile:
221
+ resolved["error"] = f"Unknown profile '{req.profile}'."
222
+ return resolved
223
+ resolved["model_id"] = resolved["model_id"] or profile.get("model_id")
224
+ if resolved["params_b"] is None:
225
+ resolved["params_b"] = _parse_params_b(profile.get("model_id")) or _parse_params_b(
226
+ profile.get("name")
227
+ )
228
+ resolved["quant"] = resolved["quant"] or profile.get("quantization") or _parse_quant(profile.get("model_id"))
229
+ if resolved["context"] is None:
230
+ resolved["context"] = profile.get("safe_context_limit") or profile.get("context_limit")
231
+ resolved["kv_quant"] = profile.get("kv_cache_type_k") or profile.get("kv_cache_type_v")
232
+ if resolved["vision"] is None:
233
+ resolved["vision"] = bool(profile.get("supports_vision"))
234
+ return resolved
235
+
236
+
237
+ def _confidence(*, size_known: bool, quant_known: bool, calibration_samples: int) -> str:
238
+ """How much this estimate should be trusted - separate from the fit tier.
239
+
240
+ Starts at metadata completeness (do we know real size vs. a guessed
241
+ params_b, is the quant known vs. assumed default), then rises with
242
+ calibration coverage for this exact configuration.
243
+ """
244
+ if not size_known:
245
+ return "low"
246
+ base = "medium" if quant_known else "low"
247
+ if calibration_samples >= 10:
248
+ return "high"
249
+ if calibration_samples >= 3:
250
+ return "medium" if base == "low" else "high"
251
+ return base
252
+
253
+
254
+ def _estimate(
255
+ *,
256
+ params_b: Optional[float],
257
+ quant: Optional[str],
258
+ context: int,
259
+ kv_quant: Optional[str],
260
+ size_bytes: Optional[int],
261
+ vision: bool,
262
+ model_id: Optional[str],
263
+ hw: Dict[str, Any],
264
+ free_vram_mb: Optional[int],
265
+ ) -> Dict[str, Any]:
266
+ """Core estimate + classification, shared by fit_check and fit_table.
267
+
268
+ Pure aside from the calibration store read (best-effort, never raises).
269
+ """
270
+ vision_overhead_gb = _VISION_OVERHEAD_GB if vision else 0.0
271
+ if size_bytes:
272
+ # Installed Ollama/GGUF size is a much better load-footprint anchor than
273
+ # guessing from parameter count. Keep KV cache out of this load estimate;
274
+ # long prompts can still need more memory, which the note makes explicit.
275
+ weights_gb = size_bytes / 1_000_000_000.0
276
+ kv_cache_gb = 0.0
277
+ note = (
278
+ "Estimate uses installed model size plus runtime overhead. Long context "
279
+ "windows can need extra memory during generation."
280
+ )
281
+ size_known = True
282
+ else:
283
+ weights_gb = (params_b or 0.0) * _weight_gb_per_b(quant)
284
+ kv_factor = _kv_quant_factor(kv_quant)
285
+ kv_cache_gb = _KV_MB_PER_TOKEN_PER_B * (params_b or 0.0) * kv_factor * context / 1024.0
286
+ note = "Conservative estimate. The real proof is a short warmup via /models/serve."
287
+ size_known = bool(params_b)
288
+ required_gb = weights_gb + kv_cache_gb + vision_overhead_gb + _OVERHEAD_GB
289
+
290
+ free_vram_mb = free_vram_mb if free_vram_mb is not None else _detected_vram_pool_mb(hw)
291
+ ram_available_mb = (hw.get("system") or {}).get("ram_available_mb")
292
+ free_vram_gb = (free_vram_mb / 1024.0) if free_vram_mb is not None else None
293
+ ram_available_gb = (ram_available_mb / 1024.0) if ram_available_mb is not None else None
294
+
295
+ cls = _classify(required_gb, free_vram_gb, ram_available_gb)
296
+ placement = estimate_gpu_placement(required_gb, hw.get("gpus") or [])
297
+ if placement["mode"] == "multi_gpu_split" and cls["verdict"] == "FITS":
298
+ cls = dict(cls, tier="multi_gpu", headline="Fits only by splitting the model across compatible GPUs.")
299
+
300
+ family = _base_family(model_id)
301
+ corr = calibration.get_correction(
302
+ gpu=calibration.gpu_key(hw), runtime=_RUNTIME, family=family, quant=quant, context=context
303
+ )
304
+ calibrated_required_gb = _round(required_gb * corr["factor"]) if corr["applied"] else None
305
+
306
+ confidence = _confidence(
307
+ size_known=size_known, quant_known=bool(quant), calibration_samples=corr["sample_count"]
308
+ )
309
+
310
+ estimate = {
311
+ "weights": _round(weights_gb),
312
+ "kv_cache": _round(kv_cache_gb),
313
+ "vision_overhead": _round(vision_overhead_gb),
314
+ "overhead": _OVERHEAD_GB,
315
+ "required": _round(required_gb),
316
+ "calibrated_required": calibrated_required_gb,
317
+ }
318
+ if size_bytes:
319
+ estimate["size_gb"] = _round(size_bytes / 1_000_000_000.0)
320
+
321
+ return {
322
+ "verdict": cls["verdict"],
323
+ "tier": cls["tier"],
324
+ "severity": cls["severity"],
325
+ "headline": cls["headline"],
326
+ "cpu_deployable": cls["cpu_deployable"],
327
+ "estimate_gb": estimate,
328
+ "free_vram_gb": _round(free_vram_gb) if free_vram_gb is not None else None,
329
+ "ram_available_gb": _round(ram_available_gb) if ram_available_gb is not None else None,
330
+ "margin_gb": _round(free_vram_gb - required_gb) if free_vram_gb is not None else None,
331
+ "calibrated_margin_gb": (
332
+ _round(free_vram_gb - calibrated_required_gb)
333
+ if free_vram_gb is not None and calibrated_required_gb is not None
334
+ else None
335
+ ),
336
+ "placement": placement,
337
+ "confidence": confidence,
338
+ "calibration": corr,
339
+ "note": note,
340
+ }
341
+
342
+
343
+ # --- batch fit (one call colors a whole catalog page) -------------------------
344
+
345
+
346
+ class FitBatchRequest(BaseModel):
347
+ params_b: List[float] = Field(default_factory=list, max_length=100)
348
+ context: int = Field(default=4096, gt=0)
349
+ free_vram_mb: Optional[int] = Field(default=None, ge=0)
350
+
351
+
352
+ @router.post("/system/fit-batch")
353
+ def fit_batch(req: FitBatchRequest) -> Dict[str, Any]:
354
+ """Fit-classify many parameter counts in one call (single hardware probe),
355
+ so the catalog can badge every size chip without one request per chip."""
356
+ for value in req.params_b:
357
+ if not (0 < value <= 5000):
358
+ return {"success": False, "message": f"params_b values must be in (0, 5000]; got {value}"}
359
+ hw = detect_hardware()
360
+ free_vram_mb = req.free_vram_mb
361
+ if free_vram_mb is None and hw["gpu_available"] and hw["gpus"]:
362
+ free_vram_mb = hw["gpus"][0].get("vram_free_mb")
363
+ free_vram_gb = (free_vram_mb / 1024.0) if free_vram_mb is not None else None
364
+ ram_available_mb = (hw.get("system") or {}).get("ram_available_mb")
365
+ ram_available_gb = (ram_available_mb / 1024.0) if ram_available_mb is not None else None
366
+
367
+ items = []
368
+ for params_b in dict.fromkeys(req.params_b): # dedupe, keep order
369
+ weights_gb = params_b * _DEFAULT_WEIGHT_GB_PER_B
370
+ kv_cache_gb = _KV_MB_PER_TOKEN_PER_B * params_b * req.context / 1024.0
371
+ required_gb = weights_gb + kv_cache_gb + _OVERHEAD_GB
372
+ cls = _classify(required_gb, free_vram_gb, ram_available_gb)
373
+ items.append(
374
+ {
375
+ "params_b": params_b,
376
+ "required_gb": _round(required_gb),
377
+ "verdict": cls["verdict"],
378
+ "tier": cls["tier"],
379
+ "severity": cls["severity"],
380
+ }
381
+ )
382
+ return {
383
+ "success": True,
384
+ "free_vram_gb": _round(free_vram_gb) if free_vram_gb is not None else None,
385
+ "items": items,
386
+ }
387
+
388
+
389
+ # --- quantization advisor ----------------------------------------------------
390
+ # One row per common GGUF quant, ordered worst -> best quality. `tag` is the
391
+ # conventional Ollama tag suffix; actual availability varies per family, so the
392
+ # UI links to the family's tags page rather than promising a pullable name.
393
+ _QUANT_LADDER = [
394
+ {"quant": "Q2_K", "quality": "heavy quality loss - last resort"},
395
+ {"quant": "Q3_K_M", "quality": "noticeable quality loss"},
396
+ {"quant": "Q4_K_M", "quality": "good - the usual default pull"},
397
+ {"quant": "Q5_K_M", "quality": "very good - small step up from Q4"},
398
+ {"quant": "Q6_K", "quality": "excellent - near Q8 at less memory"},
399
+ {"quant": "Q8_0", "quality": "near-lossless"},
400
+ {"quant": "F16", "quality": "reference quality, biggest footprint"},
401
+ ]
402
+
403
+
404
+ class QuantAdviceRequest(BaseModel):
405
+ model_id: Optional[str] = None
406
+ params_b: Optional[float] = Field(default=None, gt=0, le=10_000)
407
+ context: Optional[int] = Field(default=None, gt=0, le=10_000_000)
408
+ free_vram_mb: Optional[int] = Field(default=None, ge=0, le=100_000_000)
409
+
410
+
411
+ @router.post("/system/quant-advisor")
412
+ def quant_advisor(req: QuantAdviceRequest) -> Dict[str, Any]:
413
+ """Estimate every common quant of one model family against the budget.
414
+
415
+ Reuses the fit-check formula per quant so the numbers agree with the rest
416
+ of the app, then says which quant is the best quality that still fits.
417
+ """
418
+ params_b = req.params_b or _parse_params_b(req.model_id)
419
+ if not params_b:
420
+ return {
421
+ "success": False,
422
+ "message": (
423
+ "Could not determine parameter count. Use a model name that "
424
+ "encodes size (e.g. 'gemma3:12b') or pass 'params_b'."
425
+ ),
426
+ }
427
+ context = req.context or 4096
428
+
429
+ hw = detect_hardware()
430
+ free_vram_mb = req.free_vram_mb
431
+ if free_vram_mb is None:
432
+ free_vram_mb = _detected_vram_pool_mb(hw)
433
+ free_vram_gb = (free_vram_mb / 1024.0) if free_vram_mb is not None else None
434
+ ram_available_mb = (hw.get("system") or {}).get("ram_available_mb")
435
+ ram_available_gb = (ram_available_mb / 1024.0) if ram_available_mb is not None else None
436
+
437
+ variants = []
438
+ for entry in _QUANT_LADDER:
439
+ weights_gb = params_b * _weight_gb_per_b(entry["quant"])
440
+ kv_cache_gb = _KV_MB_PER_TOKEN_PER_B * params_b * context / 1024.0
441
+ required_gb = weights_gb + kv_cache_gb + _OVERHEAD_GB
442
+ cls = _classify(required_gb, free_vram_gb, ram_available_gb)
443
+ variants.append(
444
+ {
445
+ "quant": entry["quant"],
446
+ "quality": entry["quality"],
447
+ "weights_gb": _round(weights_gb),
448
+ "required_gb": _round(required_gb),
449
+ "margin_gb": _round(free_vram_gb - required_gb) if free_vram_gb is not None else None,
450
+ "verdict": cls["verdict"],
451
+ "tier": cls["tier"],
452
+ "severity": cls["severity"],
453
+ "cpu_deployable": cls["cpu_deployable"],
454
+ "placement": estimate_gpu_placement(required_gb, hw.get("gpus") or []),
455
+ }
456
+ )
457
+
458
+ # Best = highest-quality quant that fits the GPU; note whether that beats
459
+ # the usual Q4 default. Purely derived from the ladder - no special cases.
460
+ gpu_fits = [v for v in variants if v["verdict"] == "FITS"]
461
+ comfortable = [v for v in gpu_fits if v["severity"] == "ok"]
462
+ best = (comfortable or gpu_fits)[-1] if gpu_fits else None
463
+ default_idx = next(i for i, v in enumerate(variants) if v["quant"] == "Q4_K_M")
464
+ if best is None:
465
+ if any(v["cpu_deployable"] for v in variants):
466
+ recommendation = (
467
+ "Nothing fits the GPU budget at this size - the smaller quants can "
468
+ "still run on CPU (slower), or pick a smaller model size."
469
+ )
470
+ else:
471
+ recommendation = "This model size is too large for this machine at any quantization."
472
+ else:
473
+ best_idx = next(i for i, v in enumerate(variants) if v["quant"] == best["quant"])
474
+ if best_idx > default_idx:
475
+ recommendation = (
476
+ f"You have headroom for {best['quant']} (~{best['required_gb']} GB) - "
477
+ f"a quality step up from the usual {variants[default_idx]['quant']} default pull."
478
+ )
479
+ elif best_idx == default_idx:
480
+ recommendation = f"The usual {best['quant']} default is also the best fit for your budget."
481
+ else:
482
+ recommendation = (
483
+ f"The usual Q4_K_M default is too big here - {best['quant']} "
484
+ f"(~{best['required_gb']} GB) is the best quality that fits."
485
+ )
486
+
487
+ family = _base_family(req.model_id)
488
+ return {
489
+ "success": True,
490
+ "model": {"family": family, "params_b": params_b, "context": context},
491
+ "free_vram_gb": _round(free_vram_gb) if free_vram_gb is not None else None,
492
+ "ram_available_gb": _round(ram_available_gb) if ram_available_gb is not None else None,
493
+ "variants": variants,
494
+ "recommendation": recommendation,
495
+ "tags_url": f"https://ollama.com/library/{family}/tags" if family else None,
496
+ "note": (
497
+ "Estimates use the same weights+KV+overhead formula as fit checks. "
498
+ "Exact tag names and availability vary per family - check the tags page. "
499
+ "Some families also ship QAT tags that keep higher-quant quality at Q4 size."
500
+ ),
501
+ }
502
+
503
+
504
+ @router.post("/system/fit-check")
505
+ def fit_check(req: FitRequest) -> Dict[str, Any]:
506
+ resolved = _resolve_from_profile(req)
507
+ if resolved.get("error"):
508
+ return {"success": False, "verdict": "UNKNOWN", "message": resolved["error"]}
509
+
510
+ params_b = resolved["params_b"] or _parse_params_b(resolved.get("model_id"))
511
+ if not params_b and not resolved.get("size_bytes"):
512
+ return {
513
+ "success": False,
514
+ "verdict": "UNKNOWN",
515
+ "message": (
516
+ "Could not determine parameter count. Pass 'params_b' explicitly "
517
+ "or use a profile/model_id that encodes size (e.g. '12b')."
518
+ ),
519
+ }
520
+
521
+ context = resolved["context"] or 4096
522
+ quant = resolved["quant"] or _parse_quant(resolved.get("model_id"))
523
+ vision = resolved["vision"]
524
+ if vision is None:
525
+ vision = _looks_like_vision(resolved.get("model_id") or "")
526
+
527
+ hw = detect_hardware()
528
+ result = _estimate(
529
+ params_b=params_b,
530
+ quant=quant,
531
+ context=context,
532
+ kv_quant=resolved.get("kv_quant"),
533
+ size_bytes=resolved.get("size_bytes"),
534
+ vision=bool(vision),
535
+ model_id=resolved.get("model_id"),
536
+ hw=hw,
537
+ free_vram_mb=req.free_vram_mb,
538
+ )
539
+
540
+ placement = result["placement"]
541
+ if req.free_vram_mb is not None and result["verdict"] == "FITS" and not placement["supported"]:
542
+ placement = {
543
+ "mode": "manual_budget",
544
+ "supported": True,
545
+ "required_mb": int(result["estimate_gb"]["required"] * 1024),
546
+ "available_mb": req.free_vram_mb,
547
+ "gpu_indexes": [],
548
+ "allocations": [],
549
+ "utilization_pct": round(
550
+ result["estimate_gb"]["required"] * 1024 / max(1, req.free_vram_mb) * 100, 1
551
+ ),
552
+ "note": "The supplied manual budget fits; automatic GPU placement could not be verified.",
553
+ }
554
+ result["placement"] = placement
555
+
556
+ suggestions = []
557
+ if result["severity"] in ("soft", "hard") and result["tier"] != "tight":
558
+ suggestions = [
559
+ "Use a smaller quantization (e.g. Q4 or Q3).",
560
+ "Lower the context window (try the profile's safe_context_limit).",
561
+ "Deploy to CPU (slower) - pick CPU in the serve panel.",
562
+ ]
563
+
564
+ model_info = {"params_b": params_b, "quant": quant or "default (~Q4)", "context": context}
565
+ if result["estimate_gb"].get("size_gb") is not None:
566
+ model_info["size_gb"] = result["estimate_gb"]["size_gb"]
567
+
568
+ return {
569
+ "success": True,
570
+ # verdict stays coarse + backward-compatible (FITS / WONT_FIT / UNKNOWN);
571
+ # tier/severity/headline carry the new soft-vs-hard nuance for the UI.
572
+ "verdict": result["verdict"],
573
+ "tier": result["tier"],
574
+ "severity": result["severity"],
575
+ "headline": result["headline"],
576
+ "cpu_deployable": result["cpu_deployable"],
577
+ "confidence": result["confidence"],
578
+ "calibration": result["calibration"],
579
+ "model": model_info,
580
+ "estimate_gb": result["estimate_gb"],
581
+ "free_vram_gb": result["free_vram_gb"],
582
+ "ram_available_gb": result["ram_available_gb"],
583
+ "margin_gb": result["margin_gb"],
584
+ "calibrated_margin_gb": result["calibrated_margin_gb"],
585
+ "placement": result["placement"],
586
+ "gpu_summary": hw.get("gpu_summary") or {},
587
+ "note": result["note"],
588
+ "suggestions": suggestions,
589
+ }
590
+
591
+
592
+ @router.post("/system/fit-table")
593
+ def fit_table(req: FitRequest) -> Dict[str, Any]:
594
+ """The same estimate as /system/fit-check, swept across common context
595
+ lengths in one call - "will this fit at 8K? 32K? 64K?" without one
596
+ request per tier."""
597
+ resolved = _resolve_from_profile(req)
598
+ if resolved.get("error"):
599
+ return {"success": False, "verdict": "UNKNOWN", "message": resolved["error"]}
600
+
601
+ params_b = resolved["params_b"] or _parse_params_b(resolved.get("model_id"))
602
+ if not params_b and not resolved.get("size_bytes"):
603
+ return {
604
+ "success": False,
605
+ "message": (
606
+ "Could not determine parameter count. Pass 'params_b' explicitly "
607
+ "or use a profile/model_id that encodes size (e.g. '12b')."
608
+ ),
609
+ }
610
+
611
+ quant = resolved["quant"] or _parse_quant(resolved.get("model_id"))
612
+ vision = resolved["vision"]
613
+ if vision is None:
614
+ vision = _looks_like_vision(resolved.get("model_id") or "")
615
+ hw = detect_hardware() # one probe, reused for every context tier
616
+
617
+ rows = []
618
+ for context in FIT_TABLE_CONTEXTS:
619
+ result = _estimate(
620
+ params_b=params_b,
621
+ quant=quant,
622
+ context=context,
623
+ kv_quant=resolved.get("kv_quant"),
624
+ size_bytes=resolved.get("size_bytes"),
625
+ vision=bool(vision),
626
+ model_id=resolved.get("model_id"),
627
+ hw=hw,
628
+ free_vram_mb=req.free_vram_mb,
629
+ )
630
+ rows.append(
631
+ {
632
+ "context": context,
633
+ "verdict": result["verdict"],
634
+ "tier": result["tier"],
635
+ "severity": result["severity"],
636
+ "headline": result["headline"],
637
+ "estimate_gb": result["estimate_gb"],
638
+ "margin_gb": result["margin_gb"],
639
+ "calibrated_margin_gb": result["calibrated_margin_gb"],
640
+ "confidence": result["confidence"],
641
+ }
642
+ )
643
+
644
+ free_vram_mb = req.free_vram_mb if req.free_vram_mb is not None else _detected_vram_pool_mb(hw)
645
+ return {
646
+ "success": True,
647
+ "model": {"model_id": resolved.get("model_id"), "params_b": params_b, "quant": quant or "default (~Q4)"},
648
+ "free_vram_gb": _round(free_vram_mb / 1024.0) if free_vram_mb is not None else None,
649
+ "rows": rows,
650
+ "note": "Same weights+KV+overhead formula as /system/fit-check, one row per context length.",
651
+ }