localdeploy 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. api_server.py +15 -0
  2. benchmark.py +15 -0
  3. chat_cli.py +15 -0
  4. compare_models.py +15 -0
  5. localdeploy/__init__.py +3 -0
  6. localdeploy/backends/__init__.py +0 -0
  7. localdeploy/backends/llamacpp.py +143 -0
  8. localdeploy/backends/ollama.py +374 -0
  9. localdeploy/backends/openai_compatible.py +179 -0
  10. localdeploy/benchmark.py +1996 -0
  11. localdeploy/chat_cli.py +249 -0
  12. localdeploy/cli.py +59 -0
  13. localdeploy/compare_models.py +481 -0
  14. localdeploy/control/__init__.py +75 -0
  15. localdeploy/control/_config.py +180 -0
  16. localdeploy/control/_ollama.py +253 -0
  17. localdeploy/control/bakeoff.py +216 -0
  18. localdeploy/control/bench.py +516 -0
  19. localdeploy/control/calibration.py +191 -0
  20. localdeploy/control/community.py +161 -0
  21. localdeploy/control/fit.py +651 -0
  22. localdeploy/control/hardware.py +602 -0
  23. localdeploy/control/history.py +125 -0
  24. localdeploy/control/manifest.py +452 -0
  25. localdeploy/control/models.py +833 -0
  26. localdeploy/control/monitor.py +378 -0
  27. localdeploy/control/profiles.py +161 -0
  28. localdeploy/control/recommend.py +322 -0
  29. localdeploy/control/registry.py +956 -0
  30. localdeploy/control/report.py +407 -0
  31. localdeploy/control/starter.py +520 -0
  32. localdeploy/control/updates.py +118 -0
  33. localdeploy/grader_sandbox.py +268 -0
  34. localdeploy/server.py +1873 -0
  35. localdeploy/utils.py +155 -0
  36. localdeploy/web/favicon.png +0 -0
  37. localdeploy/web/index.html +891 -0
  38. localdeploy/web/js/app.js +63 -0
  39. localdeploy/web/js/benchmark-views.js +73 -0
  40. localdeploy/web/js/benchmark.js +1691 -0
  41. localdeploy/web/js/chat.js +779 -0
  42. localdeploy/web/js/models.js +2758 -0
  43. localdeploy/web/js/shared.js +454 -0
  44. localdeploy/web/js/system.js +626 -0
  45. localdeploy/web/logo.svg +22 -0
  46. localdeploy/web/styles.css +2032 -0
  47. localdeploy-0.6.0.dist-info/METADATA +202 -0
  48. localdeploy-0.6.0.dist-info/RECORD +52 -0
  49. localdeploy-0.6.0.dist-info/WHEEL +5 -0
  50. localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
  51. localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
  52. localdeploy-0.6.0.dist-info/top_level.txt +5 -0
@@ -0,0 +1,180 @@
1
+ """Shared config.json read/write helpers for the control plane.
2
+
3
+ `config.json` is the live, gitignored, machine-local file that mirrors what the
4
+ user actually has: pulling a model creates a profile here, deleting one can remove
5
+ it. Several routers (recommend/set-default, profiles CRUD, models/pull auto-create)
6
+ mutate it, so the atomic writer, the example-file guard, and the "build a sensible
7
+ default profile for a pulled model" logic live here in one place.
8
+
9
+ Nothing here imports ``api_server`` at module load (that would be circular); the
10
+ functions that need config loading do a lazy import.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import os
16
+ import re
17
+ import tempfile
18
+ from pathlib import Path
19
+ from typing import Any, Dict, Optional, Tuple
20
+
21
+
22
+ def write_config_atomic(config: Dict[str, Any], path: Path) -> Optional[str]:
23
+ """Atomically write `config` to `path`. Returns an error string, or None on success."""
24
+ try:
25
+ # Atomic write: a concurrent load_config() reader must never observe a
26
+ # half-written file (which would raise JSONDecodeError -> 500).
27
+ path.parent.mkdir(parents=True, exist_ok=True)
28
+ fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), prefix=".config.", suffix=".tmp")
29
+ try:
30
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
31
+ fh.write(json.dumps(config, indent=2))
32
+ os.replace(tmp_path, path)
33
+ finally:
34
+ if os.path.exists(tmp_path):
35
+ os.remove(tmp_path)
36
+ except OSError as exc:
37
+ return f"Could not write {path}: {exc}"
38
+ return None
39
+
40
+
41
+ def refuse_example(path: Path) -> Optional[str]:
42
+ """Guard: never overwrite the shipped, test-fixture config.example.json."""
43
+ if path.name == "config.example.json":
44
+ return (
45
+ "Refusing to overwrite config.example.json. Point CONFIG_PATH at a real "
46
+ "config.json (a missing live config starts empty)."
47
+ )
48
+ return None
49
+
50
+
51
+ # --- profile derivation (auto-create-from-pulled-model) ----------------------
52
+
53
+ # Families whose names signal a vision-language model. Best-effort only; the
54
+ # profile's supports_vision is a hint for the UI, not a hard capability gate.
55
+ _VISION_HINTS = (
56
+ "-vl",
57
+ "vl:",
58
+ "vision",
59
+ "llava",
60
+ "moondream",
61
+ "minicpm-v",
62
+ "-v:",
63
+ # Qwen 3.5+ unified models carry a native vision encoder without `-vl` in
64
+ # the public model name (for example `qwen3.6:27b`).
65
+ "qwen3.5",
66
+ "qwen3.6",
67
+ )
68
+
69
+
70
+ def slugify_profile_name(model_id: str) -> str:
71
+ """Turn a model id into a config-safe profile key.
72
+
73
+ 'gemma3:4b' -> 'gemma3_4b'; 'hf.co/org/Repo-GGUF' -> 'hf_co_org_repo_gguf'.
74
+ """
75
+ slug = re.sub(r"[^a-z0-9]+", "_", (model_id or "").lower()).strip("_")
76
+ return slug or "model"
77
+
78
+
79
+ def _looks_like_vision(model_id: str) -> bool:
80
+ low = (model_id or "").lower()
81
+ return any(hint in low for hint in _VISION_HINTS)
82
+
83
+
84
+ def default_profile_for(
85
+ model_id: str, *, params_b: Optional[float] = None, quant: Optional[str] = None, backend: str = "ollama"
86
+ ) -> Dict[str, Any]:
87
+ """A conservative profile for a freshly pulled/imported model.
88
+
89
+ Mirrors the field shape of the shipped example profiles so the rest of the
90
+ app (fit-check, benchmark, serve) treats it identically to a hand-written one.
91
+ """
92
+ name = slugify_profile_name(model_id)
93
+ supports_vision = _looks_like_vision(model_id)
94
+ try:
95
+ from ..utils import get_backend_base_url
96
+
97
+ base_url = get_backend_base_url({}, backend)
98
+ except Exception:
99
+ base_url = "http://127.0.0.1:11434"
100
+ profile = {
101
+ "name": name,
102
+ "backend": backend,
103
+ "model_id": model_id,
104
+ "base_url": base_url,
105
+ "enabled": True,
106
+ "description": f"Auto-created from pulled model {model_id}.",
107
+ "auto_created": True,
108
+ "supports_vision": supports_vision,
109
+ # Existing hand-authored profiles without this field retain the legacy
110
+ # one-image limit. New vision profiles start conservatively at four and
111
+ # can be raised explicitly after hardware qualification.
112
+ "max_images": 4 if supports_vision else 1,
113
+ "context_limit": 8192,
114
+ "safe_context_limit": 8192,
115
+ "max_prompt_chars": 20000,
116
+ "max_output_tokens": 2048,
117
+ "temperature": 0.2,
118
+ "top_p": 0.9,
119
+ "repeat_penalty": 1.1,
120
+ "timeout_seconds": 240,
121
+ "slow_response_seconds": 60,
122
+ "notes": "Auto-created when this model was pulled. Edit tuning as needed.",
123
+ "quantization": quant,
124
+ "gpu_layers": None,
125
+ "flash_attention": None,
126
+ "kv_cache_type_k": None,
127
+ "kv_cache_type_v": None,
128
+ "batch_size": None,
129
+ "ubatch_size": None,
130
+ "threads": None,
131
+ "mmap": None,
132
+ "mlock": None,
133
+ "params_b": params_b,
134
+ }
135
+ # Ollama thinking models can spend their entire output budget in the
136
+ # separate hidden reasoning field and return empty final content. Fresh
137
+ # profiles default to non-thinking for reliable chat and structured output;
138
+ # users can explicitly enable thinking for long-form reasoning workloads.
139
+ low = (model_id or "").lower()
140
+ if low.startswith("qwen3") or "deepseek-r1" in low:
141
+ profile["think"] = False
142
+ return profile
143
+
144
+
145
+ def ensure_profile_for_model(model_id: str) -> Tuple[Optional[str], bool, Optional[str]]:
146
+ """Ensure the live config has a profile for `model_id`. Best-effort.
147
+
148
+ Returns ``(profile_name, created, error)``. Never raises - auto-creation is a
149
+ convenience that must never turn a successful pull into a failure. When the
150
+ active config is the read-only example fixture, this no-ops (created=False).
151
+ """
152
+ try:
153
+ from api_server import get_config_path, load_config # lazy: avoid circular import
154
+ from .fit import _parse_params_b, _parse_quant
155
+
156
+ path = get_config_path()
157
+ if refuse_example(path):
158
+ return None, False, None # example fixture is read-only; skip silently
159
+ config = load_config()
160
+ profiles = config.setdefault("profiles", {})
161
+ # Already have a profile pointing at this exact model? Leave it untouched.
162
+ for pname, prof in profiles.items():
163
+ if prof.get("model_id") == model_id:
164
+ return pname, False, None
165
+ name = slugify_profile_name(model_id)
166
+ # Avoid clobbering an unrelated profile that happens to share the slug.
167
+ if name in profiles:
168
+ suffix = 2
169
+ while f"{name}_{suffix}" in profiles:
170
+ suffix += 1
171
+ name = f"{name}_{suffix}"
172
+ profiles[name] = default_profile_for(
173
+ model_id, params_b=_parse_params_b(model_id), quant=_parse_quant(model_id)
174
+ )
175
+ if not config.get("default_profile"):
176
+ config["default_profile"] = name
177
+ err = write_config_atomic(config, path)
178
+ return (name, True, None) if not err else (None, False, err)
179
+ except Exception as exc: # pragma: no cover - defensive; pull must still succeed
180
+ return None, False, str(exc)
@@ -0,0 +1,253 @@
1
+ """Shared Ollama control-plane helpers for the web package.
2
+
3
+ One place for the local Ollama HTTP calls used by registry + model lifecycle, so
4
+ those routes never duplicate connection/error handling. Every function either
5
+ returns a clean ``(data, error)`` pair or raises ``BackendCallError`` /
6
+ ``requests.RequestException`` for the caller to translate into a graceful
7
+ response. Nothing here ever talks to a non-loopback host.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import os
13
+ from pathlib import Path
14
+ from typing import Any, Dict, Iterator, List, Optional, Tuple
15
+
16
+ import requests
17
+
18
+ from ..utils import BackendCallError, is_loopback_url, strip_trailing_slash
19
+
20
+
21
+ def base_url() -> str:
22
+ base = strip_trailing_slash(os.getenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434"))
23
+ if not is_loopback_url(base):
24
+ raise BackendCallError(f"Refusing non-local Ollama URL: {base}")
25
+ return base
26
+
27
+
28
+ def list_installed() -> Tuple[List[Dict[str, Any]], Optional[str]]:
29
+ """Installed models via /api/tags. Returns ([], error) when unreachable."""
30
+ try:
31
+ base = base_url()
32
+ except BackendCallError as exc:
33
+ return [], str(exc)
34
+ try:
35
+ response = requests.get(f"{base}/api/tags", timeout=5)
36
+ response.raise_for_status()
37
+ except requests.ConnectionError:
38
+ return [], f"Ollama is not reachable at {base}. Start Ollama and retry."
39
+ except requests.RequestException as exc:
40
+ return [], str(exc)
41
+ out: List[Dict[str, Any]] = []
42
+ for item in response.json().get("models", []):
43
+ out.append(
44
+ {
45
+ "name": item.get("name"),
46
+ "size": item.get("size"),
47
+ "modified_at": item.get("modified_at"),
48
+ "digest": item.get("digest") or None,
49
+ "digest_short": (item.get("digest") or "")[:12] or None,
50
+ "details": item.get("details", {}),
51
+ }
52
+ )
53
+ return out, None
54
+
55
+
56
+ def version() -> Tuple[Optional[str], Optional[str]]:
57
+ """Return the exact Ollama server version when the local daemon is reachable."""
58
+ try:
59
+ base = base_url()
60
+ response = requests.get(f"{base}/api/version", timeout=3)
61
+ response.raise_for_status()
62
+ value = response.json().get("version")
63
+ return (str(value) if value else None), None
64
+ except requests.ConnectionError:
65
+ return None, f"Ollama is not reachable at {base if 'base' in locals() else 'localhost'}."
66
+ except (BackendCallError, requests.RequestException, ValueError) as exc:
67
+ return None, str(exc)
68
+
69
+
70
+ def show_model(model: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
71
+ """Return Ollama's model metadata, including quant and context parameters."""
72
+ try:
73
+ base = base_url()
74
+ response = requests.post(f"{base}/api/show", json={"model": model}, timeout=10)
75
+ response.raise_for_status()
76
+ data = response.json()
77
+ return data if isinstance(data, dict) else None, None
78
+ except requests.ConnectionError:
79
+ return None, f"Ollama is not reachable at {base if 'base' in locals() else 'localhost'}."
80
+ except (BackendCallError, requests.RequestException, ValueError) as exc:
81
+ return None, str(exc)
82
+
83
+
84
+ def list_running() -> Tuple[List[Dict[str, Any]], Optional[str]]:
85
+ """Currently loaded models via /api/ps. Returns ([], error) when unreachable."""
86
+ try:
87
+ base = base_url()
88
+ except BackendCallError as exc:
89
+ return [], str(exc)
90
+ try:
91
+ response = requests.get(f"{base}/api/ps", timeout=5)
92
+ response.raise_for_status()
93
+ except requests.ConnectionError:
94
+ return [], f"Ollama is not reachable at {base}. Start Ollama and retry."
95
+ except requests.RequestException as exc:
96
+ return [], str(exc)
97
+ out: List[Dict[str, Any]] = []
98
+ for item in response.json().get("models", []):
99
+ out.append(
100
+ {
101
+ "name": item.get("name"),
102
+ "size": item.get("size"),
103
+ "size_vram": item.get("size_vram"),
104
+ "expires_at": item.get("expires_at"),
105
+ }
106
+ )
107
+ return out, None
108
+
109
+
110
+ # Warming a model means loading all its weights into memory. On the GPU this is
111
+ # quick; forcing CPU offload (num_gpu=0) for a large model can take minutes, so a
112
+ # flat 120s timeout would spuriously fail those. Both are overridable via env.
113
+ _GPU_LOAD_TIMEOUT = 120
114
+ _CPU_LOAD_TIMEOUT = 600
115
+
116
+
117
+ def _load_timeout(num_gpu: Optional[int]) -> int:
118
+ """Pick a warm-up timeout: longer for CPU offload, overridable via env."""
119
+ override = os.getenv("OLLAMA_LOAD_TIMEOUT")
120
+ if override:
121
+ try:
122
+ return int(override)
123
+ except ValueError:
124
+ pass
125
+ return _CPU_LOAD_TIMEOUT if num_gpu == 0 else _GPU_LOAD_TIMEOUT
126
+
127
+
128
+ def load_model(
129
+ model: str, keep_alive: str = "60m", num_gpu: Optional[int] = None
130
+ ) -> Dict[str, Any]:
131
+ """Warm a model into memory. An empty prompt makes Ollama load without generating.
132
+
133
+ ``num_gpu`` sets how many layers to offload to the GPU (0 = force CPU). When
134
+ None, Ollama decides (auto) - identical to the prior behaviour. The request
135
+ timeout scales with the target device (CPU loads are slower).
136
+ """
137
+ base = base_url()
138
+ payload: Dict[str, Any] = {"model": model, "keep_alive": keep_alive}
139
+ if num_gpu is not None:
140
+ payload["options"] = {"num_gpu": num_gpu}
141
+ response = requests.post(
142
+ f"{base}/api/generate",
143
+ json=payload,
144
+ timeout=_load_timeout(num_gpu),
145
+ )
146
+ response.raise_for_status()
147
+ return response.json()
148
+
149
+
150
+ def unload_model(model: str) -> Dict[str, Any]:
151
+ """Unload a model immediately via keep_alive=0."""
152
+ base = base_url()
153
+ response = requests.post(
154
+ f"{base}/api/generate",
155
+ json={"model": model, "keep_alive": 0},
156
+ timeout=30,
157
+ )
158
+ response.raise_for_status()
159
+ return response.json()
160
+
161
+
162
+ def delete_model(model: str) -> Dict[str, Any]:
163
+ """Delete a model from disk via /api/delete (frees disk space)."""
164
+ base = base_url()
165
+ response = requests.delete(f"{base}/api/delete", json={"name": model}, timeout=30)
166
+ # Ollama returns 200 (often with an empty body) on success, 404 if absent.
167
+ if response.status_code == 404:
168
+ raise BackendCallError(f"Model '{model}' is not installed.")
169
+ response.raise_for_status()
170
+ return {"deleted": model}
171
+
172
+
173
+ def unload_all() -> Tuple[int, Optional[str]]:
174
+ """Unload every currently-loaded model from memory. Returns (count, error)."""
175
+ running, err = list_running()
176
+ if err is not None:
177
+ return 0, err
178
+ unloaded = 0
179
+ for m in running:
180
+ name = m.get("name")
181
+ if not name:
182
+ continue
183
+ try:
184
+ unload_model(name)
185
+ unloaded += 1
186
+ except (BackendCallError, requests.RequestException):
187
+ continue # best-effort; keep unloading the rest
188
+ return unloaded, None
189
+
190
+
191
+ def pull_stream(model: str) -> Iterator[Dict[str, Any]]:
192
+ """Yield progress events from `ollama pull` (JSON lines)."""
193
+ base = base_url()
194
+ with requests.post(
195
+ f"{base}/api/pull",
196
+ json={"name": model, "stream": True},
197
+ stream=True,
198
+ timeout=(10, 120),
199
+ ) as response:
200
+ response.raise_for_status()
201
+ for line in response.iter_lines():
202
+ if not line:
203
+ continue
204
+ try:
205
+ yield json.loads(line)
206
+ except (ValueError, TypeError):
207
+ continue
208
+
209
+
210
+ # --- create-from-GGUF-file (used to import a downloaded file that isn't in any
211
+ # registry: push the blob once, then point /api/create at its digest) --------
212
+
213
+
214
+ def blob_exists(digest: str) -> bool:
215
+ """True if Ollama already has this blob (HEAD /api/blobs/:digest), so a
216
+ retry after a partial import doesn't re-upload the whole file."""
217
+ base = base_url()
218
+ try:
219
+ response = requests.head(f"{base}/api/blobs/sha256:{digest}", timeout=10)
220
+ return response.status_code == 200
221
+ except requests.RequestException:
222
+ return False
223
+
224
+
225
+ def push_blob(file_path: Path, digest: str) -> None:
226
+ """Upload a local file as a content-addressed blob Ollama can reference
227
+ from /api/create. Streams from disk - never loads the whole file into memory."""
228
+ base = base_url()
229
+ with open(file_path, "rb") as fh:
230
+ response = requests.post(f"{base}/api/blobs/sha256:{digest}", data=fh, timeout=(10, 1800))
231
+ # 201 = created, 200 = already present (some Ollama versions no-op instead of 4xx).
232
+ if response.status_code not in (200, 201):
233
+ raise BackendCallError(f"Ollama rejected the uploaded blob (HTTP {response.status_code}): {response.text[:300]}")
234
+
235
+
236
+ def create_stream(model: str, files: Dict[str, str]) -> Iterator[Dict[str, Any]]:
237
+ """Yield progress events from `ollama create` given already-pushed blobs
238
+ (`files` maps a filename to its 'sha256:<digest>')."""
239
+ base = base_url()
240
+ with requests.post(
241
+ f"{base}/api/create",
242
+ json={"model": model, "files": files, "stream": True},
243
+ stream=True,
244
+ timeout=(10, 1800),
245
+ ) as response:
246
+ response.raise_for_status()
247
+ for line in response.iter_lines():
248
+ if not line:
249
+ continue
250
+ try:
251
+ yield json.loads(line)
252
+ except (ValueError, TypeError):
253
+ continue
@@ -0,0 +1,216 @@
1
+ """Automated model bakeoff (Release R6) - "Compare top models for me."
2
+
3
+ POST /system/bakeoff/run streams the strongest end-to-end LocalDeploy workflow
4
+ as one operation: pick fit-safe candidates for a workload within a download
5
+ budget, pull + serve + benchmark each one in turn (sequentially, to avoid
6
+ VRAM contention - same discipline as /benchmark/run), rank them with the
7
+ existing recommend.py scoring formula, then re-serve the winner.
8
+
9
+ This is orchestration only. Every step reuses an existing engine:
10
+ - candidate selection: starter.py's fit-filtered, workload-biased catalog
11
+ - pull: _ollama.pull_stream (same as /models/pull)
12
+ - serve + placement: models._serve_ollama / models._placement
13
+ - benchmark: benchmark.execute_test + bench.py's BENCHMARK_PACKS
14
+ - ranking: recommend.py's rank_candidates (quality x speed x headroom)
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ from typing import Any, Dict, List, Optional
20
+
21
+ from fastapi import APIRouter
22
+ from fastapi.responses import StreamingResponse
23
+ from pydantic import BaseModel, Field
24
+
25
+ from . import _ollama
26
+ from ._config import ensure_profile_for_model
27
+ from .bench import BENCHMARK_PACKS
28
+ from .bench import _bench # memoized, race-safe import of the top-level `benchmark` module
29
+ from .fit import _weight_gb_per_b
30
+ from .models import _matches_model_name, _serve_ollama
31
+ from .recommend import rank_candidates
32
+ from .starter import _fitting_candidates, _rank, _resolve_budget, _with_workload_bias
33
+
34
+ router = APIRouter()
35
+
36
+ # use_case (from the guided-recommend vocabulary) -> default benchmark pack.
37
+ _USE_CASE_PACK = {
38
+ "coding": "coding",
39
+ "structured_extraction": "structured",
40
+ "reasoning": "reasoning",
41
+ "tool_calling": "structured",
42
+ }
43
+ _DEFAULT_PACK = "general"
44
+
45
+
46
+ class BakeoffRequest(BaseModel):
47
+ use_case: Optional[str] = None
48
+ priority: str = Field(default="balanced")
49
+ expected_context: int = Field(default=8192, gt=0, le=1_000_000)
50
+ download_budget_gb: float = Field(default=10.0, gt=0, le=1_000)
51
+ pack: Optional[str] = None
52
+ max_candidates: int = Field(default=3, ge=1, le=5)
53
+ sample_size: int = Field(default=3, ge=1, le=10)
54
+ timeout: int = Field(default=120, ge=1, le=3_600)
55
+ free_vram_mb: Optional[int] = Field(default=None, ge=0, le=100_000_000)
56
+ margin_gb: float = Field(default=2.0, ge=0, le=1_024)
57
+
58
+
59
+ def _select_candidates(
60
+ use_case: Optional[str], priority: str, budget_gb: float, download_budget_gb: float, max_candidates: int
61
+ ) -> List[Dict[str, Any]]:
62
+ """Fit-safe, workload-biased, download-budget-capped picks - the same
63
+ catalog and ranking recommend.py's guided-recommend endpoint uses, just
64
+ greedily packed against a cumulative download-size cap instead of split
65
+ into three labeled buckets."""
66
+ fitting = _fitting_candidates(budget_gb)
67
+ biased = _with_workload_bias(fitting, use_case)
68
+ ranked = _rank(biased, budget_gb, priority)
69
+
70
+ picked: List[Dict[str, Any]] = []
71
+ cumulative_download_gb = 0.0
72
+ for entry in ranked:
73
+ download_gb = round(entry["params_b"] * _weight_gb_per_b("q4"), 1)
74
+ if picked and cumulative_download_gb + download_gb > download_budget_gb:
75
+ continue
76
+ entry = dict(entry)
77
+ entry["download_gb"] = download_gb
78
+ picked.append(entry)
79
+ cumulative_download_gb += download_gb
80
+ if len(picked) >= max_candidates:
81
+ break
82
+ return picked
83
+
84
+
85
+ @router.post("/system/bakeoff/run")
86
+ def bakeoff_run(req: BakeoffRequest):
87
+ def sse(event: Dict[str, Any]) -> str:
88
+ return f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
89
+
90
+ def event_stream():
91
+ raw_budget_gb, budget_gb, source, hw = _resolve_budget(req.free_vram_mb, req.margin_gb)
92
+ if budget_gb is None:
93
+ yield sse({"event": "error", "error": "Could not determine VRAM or system RAM budget."})
94
+ yield "data: [DONE]\n\n"
95
+ return
96
+
97
+ candidates = _select_candidates(req.use_case, req.priority, budget_gb, req.download_budget_gb, req.max_candidates)
98
+ if not candidates:
99
+ yield sse({"event": "error", "error": "No catalog model fits your budget and download cap."})
100
+ yield "data: [DONE]\n\n"
101
+ return
102
+
103
+ bench = _bench()
104
+ pack_id = req.pack or _USE_CASE_PACK.get(req.use_case or "", _DEFAULT_PACK)
105
+ pack_categories = (BENCHMARK_PACKS.get(pack_id) or BENCHMARK_PACKS[_DEFAULT_PACK])["categories"]
106
+ tests = [t for t in bench.builtin_test_cases() if t.category in pack_categories]
107
+ tests = sorted(tests, key=lambda t: t.max_output_tokens)[: max(1, req.sample_size)]
108
+ base_url = bench.api_base_url()
109
+
110
+ yield sse(
111
+ {
112
+ "event": "bakeoff_start",
113
+ "candidates": [c["id"] for c in candidates],
114
+ "pack": pack_id,
115
+ "sample_tests": [t.name for t in tests],
116
+ "budget_source": source,
117
+ "budget_gb": round(budget_gb, 2),
118
+ "download_budget_gb": req.download_budget_gb,
119
+ }
120
+ )
121
+
122
+ scored: List[Dict[str, Any]] = []
123
+ installed, _ = _ollama.list_installed()
124
+ installed_names = {str(m.get("name")) for m in installed if m.get("name")}
125
+
126
+ for candidate in candidates:
127
+ model_id = candidate["id"]
128
+ yield sse({"event": "candidate_start", "model": model_id, "download_gb": candidate["download_gb"]})
129
+
130
+ if not any(_matches_model_name(name, model_id) for name in installed_names):
131
+ yield sse({"event": "pull_start", "model": model_id})
132
+ try:
133
+ for progress in _ollama.pull_stream(model_id):
134
+ yield sse({"event": "pull_progress", "model": model_id, **progress})
135
+ except Exception as exc:
136
+ yield sse({"event": "candidate_failed", "model": model_id, "reason": f"pull failed: {exc}"})
137
+ scored.append({"profile": model_id, "avg_accuracy": 0.0, "avg_latency_s": 0.0, "passed": 0, "tests": 0, "margin_gb": None})
138
+ continue
139
+ yield sse({"event": "pull_end", "model": model_id})
140
+
141
+ profile_name, _created, profile_err = ensure_profile_for_model(model_id)
142
+ if not profile_name:
143
+ yield sse({"event": "candidate_failed", "model": model_id, "reason": profile_err or "could not create a profile"})
144
+ scored.append({"profile": model_id, "avg_accuracy": 0.0, "avg_latency_s": 0.0, "passed": 0, "tests": 0, "margin_gb": None})
145
+ continue
146
+
147
+ yield sse({"event": "deploy_start", "model": model_id})
148
+ served = _serve_ollama(model_id, "10m")
149
+ if not served.get("success"):
150
+ yield sse({"event": "candidate_failed", "model": model_id, "reason": served.get("error") or "deploy failed"})
151
+ scored.append({"profile": profile_name, "avg_accuracy": 0.0, "avg_latency_s": 0.0, "passed": 0, "tests": 0, "margin_gb": candidate.get("margin_gb")})
152
+ continue
153
+ running = served.get("running") or []
154
+ match = next((m for m in running if _matches_model_name(m.get("name"), model_id)), None)
155
+ placement = match.get("placement") if match else None
156
+ yield sse({"event": "deploy_end", "model": model_id, "placement": placement, "warning": served.get("warning")})
157
+
158
+ from api_server import load_config # lazy: api_server owns config loading
159
+
160
+ profile = load_config().get("profiles", {}).get(profile_name, {})
161
+ results = []
162
+ for t in tests:
163
+ yield sse({"event": "test_start", "model": model_id, "name": t.name})
164
+ try:
165
+ item = bench.execute_test(base_url, profile_name, profile, t, req.timeout)
166
+ except Exception:
167
+ item = bench.TestResult(
168
+ name=t.name, category=t.category, success=False, elapsed_seconds=0.0,
169
+ response_length=0, response_preview="", accuracy=0.0, error="execute_test error",
170
+ )
171
+ results.append(item)
172
+ yield sse({"event": "test_result", "model": model_id, "name": item.name, "success": item.success, "accuracy": item.accuracy, "elapsed_seconds": item.elapsed_seconds})
173
+
174
+ accs = [r.accuracy for r in results]
175
+ lats = [r.elapsed_seconds for r in results if r.elapsed_seconds]
176
+ row = {
177
+ "profile": model_id,
178
+ "avg_accuracy": round(sum(accs) / len(accs), 3) if accs else 0.0,
179
+ "avg_latency_s": round(sum(lats) / len(lats), 3) if lats else 0.0,
180
+ "passed": sum(1 for r in results if r.success),
181
+ "tests": len(results),
182
+ "margin_gb": candidate.get("margin_gb"),
183
+ "placement": placement,
184
+ }
185
+ scored.append(row)
186
+ yield sse({"event": "candidate_end", "model": model_id, **row})
187
+
188
+ # Sequential VRAM discipline (same as /benchmark/run): unload before
189
+ # the next candidate. The overall winner gets explicitly re-served below.
190
+ try:
191
+ _ollama.unload_model(model_id)
192
+ except Exception:
193
+ pass
194
+
195
+ ranked = rank_candidates([s for s in scored if s["tests"] > 0]) if any(s["tests"] > 0 for s in scored) else []
196
+ if not ranked:
197
+ yield sse({"event": "error", "error": "Every candidate failed to pull, deploy, or benchmark."})
198
+ yield "data: [DONE]\n\n"
199
+ return
200
+
201
+ winner_model = ranked[0]["profile"]
202
+ yield sse({"event": "deploy_start", "model": winner_model, "final": True})
203
+ winner_served = _serve_ollama(winner_model, "60m")
204
+ yield sse(
205
+ {
206
+ "event": "bakeoff_end",
207
+ "pack": pack_id,
208
+ "ranked": ranked,
209
+ "winner": winner_model,
210
+ "losers": [s["profile"] for s in ranked[1:]],
211
+ "winner_deployed": bool(winner_served.get("success")),
212
+ }
213
+ )
214
+ yield "data: [DONE]\n\n"
215
+
216
+ return StreamingResponse(event_stream(), media_type="text/event-stream")