localdeploy 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api_server.py +15 -0
- benchmark.py +15 -0
- chat_cli.py +15 -0
- compare_models.py +15 -0
- localdeploy/__init__.py +3 -0
- localdeploy/backends/__init__.py +0 -0
- localdeploy/backends/llamacpp.py +143 -0
- localdeploy/backends/ollama.py +374 -0
- localdeploy/backends/openai_compatible.py +179 -0
- localdeploy/benchmark.py +1996 -0
- localdeploy/chat_cli.py +249 -0
- localdeploy/cli.py +59 -0
- localdeploy/compare_models.py +481 -0
- localdeploy/control/__init__.py +75 -0
- localdeploy/control/_config.py +180 -0
- localdeploy/control/_ollama.py +253 -0
- localdeploy/control/bakeoff.py +216 -0
- localdeploy/control/bench.py +516 -0
- localdeploy/control/calibration.py +191 -0
- localdeploy/control/community.py +161 -0
- localdeploy/control/fit.py +651 -0
- localdeploy/control/hardware.py +602 -0
- localdeploy/control/history.py +125 -0
- localdeploy/control/manifest.py +452 -0
- localdeploy/control/models.py +833 -0
- localdeploy/control/monitor.py +378 -0
- localdeploy/control/profiles.py +161 -0
- localdeploy/control/recommend.py +322 -0
- localdeploy/control/registry.py +956 -0
- localdeploy/control/report.py +407 -0
- localdeploy/control/starter.py +520 -0
- localdeploy/control/updates.py +118 -0
- localdeploy/grader_sandbox.py +268 -0
- localdeploy/server.py +1873 -0
- localdeploy/utils.py +155 -0
- localdeploy/web/favicon.png +0 -0
- localdeploy/web/index.html +891 -0
- localdeploy/web/js/app.js +63 -0
- localdeploy/web/js/benchmark-views.js +73 -0
- localdeploy/web/js/benchmark.js +1691 -0
- localdeploy/web/js/chat.js +779 -0
- localdeploy/web/js/models.js +2758 -0
- localdeploy/web/js/shared.js +454 -0
- localdeploy/web/js/system.js +626 -0
- localdeploy/web/logo.svg +22 -0
- localdeploy/web/styles.css +2032 -0
- localdeploy-0.6.0.dist-info/METADATA +202 -0
- localdeploy-0.6.0.dist-info/RECORD +52 -0
- localdeploy-0.6.0.dist-info/WHEEL +5 -0
- localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
- localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
- localdeploy-0.6.0.dist-info/top_level.txt +5 -0
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""Shared config.json read/write helpers for the control plane.
|
|
2
|
+
|
|
3
|
+
`config.json` is the live, gitignored, machine-local file that mirrors what the
|
|
4
|
+
user actually has: pulling a model creates a profile here, deleting one can remove
|
|
5
|
+
it. Several routers (recommend/set-default, profiles CRUD, models/pull auto-create)
|
|
6
|
+
mutate it, so the atomic writer, the example-file guard, and the "build a sensible
|
|
7
|
+
default profile for a pulled model" logic live here in one place.
|
|
8
|
+
|
|
9
|
+
Nothing here imports ``api_server`` at module load (that would be circular); the
|
|
10
|
+
functions that need config loading do a lazy import.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import tempfile
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Dict, Optional, Tuple
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def write_config_atomic(config: Dict[str, Any], path: Path) -> Optional[str]:
|
|
23
|
+
"""Atomically write `config` to `path`. Returns an error string, or None on success."""
|
|
24
|
+
try:
|
|
25
|
+
# Atomic write: a concurrent load_config() reader must never observe a
|
|
26
|
+
# half-written file (which would raise JSONDecodeError -> 500).
|
|
27
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
28
|
+
fd, tmp_path = tempfile.mkstemp(dir=str(path.parent), prefix=".config.", suffix=".tmp")
|
|
29
|
+
try:
|
|
30
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
31
|
+
fh.write(json.dumps(config, indent=2))
|
|
32
|
+
os.replace(tmp_path, path)
|
|
33
|
+
finally:
|
|
34
|
+
if os.path.exists(tmp_path):
|
|
35
|
+
os.remove(tmp_path)
|
|
36
|
+
except OSError as exc:
|
|
37
|
+
return f"Could not write {path}: {exc}"
|
|
38
|
+
return None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def refuse_example(path: Path) -> Optional[str]:
|
|
42
|
+
"""Guard: never overwrite the shipped, test-fixture config.example.json."""
|
|
43
|
+
if path.name == "config.example.json":
|
|
44
|
+
return (
|
|
45
|
+
"Refusing to overwrite config.example.json. Point CONFIG_PATH at a real "
|
|
46
|
+
"config.json (a missing live config starts empty)."
|
|
47
|
+
)
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# --- profile derivation (auto-create-from-pulled-model) ----------------------
|
|
52
|
+
|
|
53
|
+
# Families whose names signal a vision-language model. Best-effort only; the
|
|
54
|
+
# profile's supports_vision is a hint for the UI, not a hard capability gate.
|
|
55
|
+
_VISION_HINTS = (
|
|
56
|
+
"-vl",
|
|
57
|
+
"vl:",
|
|
58
|
+
"vision",
|
|
59
|
+
"llava",
|
|
60
|
+
"moondream",
|
|
61
|
+
"minicpm-v",
|
|
62
|
+
"-v:",
|
|
63
|
+
# Qwen 3.5+ unified models carry a native vision encoder without `-vl` in
|
|
64
|
+
# the public model name (for example `qwen3.6:27b`).
|
|
65
|
+
"qwen3.5",
|
|
66
|
+
"qwen3.6",
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def slugify_profile_name(model_id: str) -> str:
|
|
71
|
+
"""Turn a model id into a config-safe profile key.
|
|
72
|
+
|
|
73
|
+
'gemma3:4b' -> 'gemma3_4b'; 'hf.co/org/Repo-GGUF' -> 'hf_co_org_repo_gguf'.
|
|
74
|
+
"""
|
|
75
|
+
slug = re.sub(r"[^a-z0-9]+", "_", (model_id or "").lower()).strip("_")
|
|
76
|
+
return slug or "model"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _looks_like_vision(model_id: str) -> bool:
|
|
80
|
+
low = (model_id or "").lower()
|
|
81
|
+
return any(hint in low for hint in _VISION_HINTS)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def default_profile_for(
|
|
85
|
+
model_id: str, *, params_b: Optional[float] = None, quant: Optional[str] = None, backend: str = "ollama"
|
|
86
|
+
) -> Dict[str, Any]:
|
|
87
|
+
"""A conservative profile for a freshly pulled/imported model.
|
|
88
|
+
|
|
89
|
+
Mirrors the field shape of the shipped example profiles so the rest of the
|
|
90
|
+
app (fit-check, benchmark, serve) treats it identically to a hand-written one.
|
|
91
|
+
"""
|
|
92
|
+
name = slugify_profile_name(model_id)
|
|
93
|
+
supports_vision = _looks_like_vision(model_id)
|
|
94
|
+
try:
|
|
95
|
+
from ..utils import get_backend_base_url
|
|
96
|
+
|
|
97
|
+
base_url = get_backend_base_url({}, backend)
|
|
98
|
+
except Exception:
|
|
99
|
+
base_url = "http://127.0.0.1:11434"
|
|
100
|
+
profile = {
|
|
101
|
+
"name": name,
|
|
102
|
+
"backend": backend,
|
|
103
|
+
"model_id": model_id,
|
|
104
|
+
"base_url": base_url,
|
|
105
|
+
"enabled": True,
|
|
106
|
+
"description": f"Auto-created from pulled model {model_id}.",
|
|
107
|
+
"auto_created": True,
|
|
108
|
+
"supports_vision": supports_vision,
|
|
109
|
+
# Existing hand-authored profiles without this field retain the legacy
|
|
110
|
+
# one-image limit. New vision profiles start conservatively at four and
|
|
111
|
+
# can be raised explicitly after hardware qualification.
|
|
112
|
+
"max_images": 4 if supports_vision else 1,
|
|
113
|
+
"context_limit": 8192,
|
|
114
|
+
"safe_context_limit": 8192,
|
|
115
|
+
"max_prompt_chars": 20000,
|
|
116
|
+
"max_output_tokens": 2048,
|
|
117
|
+
"temperature": 0.2,
|
|
118
|
+
"top_p": 0.9,
|
|
119
|
+
"repeat_penalty": 1.1,
|
|
120
|
+
"timeout_seconds": 240,
|
|
121
|
+
"slow_response_seconds": 60,
|
|
122
|
+
"notes": "Auto-created when this model was pulled. Edit tuning as needed.",
|
|
123
|
+
"quantization": quant,
|
|
124
|
+
"gpu_layers": None,
|
|
125
|
+
"flash_attention": None,
|
|
126
|
+
"kv_cache_type_k": None,
|
|
127
|
+
"kv_cache_type_v": None,
|
|
128
|
+
"batch_size": None,
|
|
129
|
+
"ubatch_size": None,
|
|
130
|
+
"threads": None,
|
|
131
|
+
"mmap": None,
|
|
132
|
+
"mlock": None,
|
|
133
|
+
"params_b": params_b,
|
|
134
|
+
}
|
|
135
|
+
# Ollama thinking models can spend their entire output budget in the
|
|
136
|
+
# separate hidden reasoning field and return empty final content. Fresh
|
|
137
|
+
# profiles default to non-thinking for reliable chat and structured output;
|
|
138
|
+
# users can explicitly enable thinking for long-form reasoning workloads.
|
|
139
|
+
low = (model_id or "").lower()
|
|
140
|
+
if low.startswith("qwen3") or "deepseek-r1" in low:
|
|
141
|
+
profile["think"] = False
|
|
142
|
+
return profile
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def ensure_profile_for_model(model_id: str) -> Tuple[Optional[str], bool, Optional[str]]:
|
|
146
|
+
"""Ensure the live config has a profile for `model_id`. Best-effort.
|
|
147
|
+
|
|
148
|
+
Returns ``(profile_name, created, error)``. Never raises - auto-creation is a
|
|
149
|
+
convenience that must never turn a successful pull into a failure. When the
|
|
150
|
+
active config is the read-only example fixture, this no-ops (created=False).
|
|
151
|
+
"""
|
|
152
|
+
try:
|
|
153
|
+
from api_server import get_config_path, load_config # lazy: avoid circular import
|
|
154
|
+
from .fit import _parse_params_b, _parse_quant
|
|
155
|
+
|
|
156
|
+
path = get_config_path()
|
|
157
|
+
if refuse_example(path):
|
|
158
|
+
return None, False, None # example fixture is read-only; skip silently
|
|
159
|
+
config = load_config()
|
|
160
|
+
profiles = config.setdefault("profiles", {})
|
|
161
|
+
# Already have a profile pointing at this exact model? Leave it untouched.
|
|
162
|
+
for pname, prof in profiles.items():
|
|
163
|
+
if prof.get("model_id") == model_id:
|
|
164
|
+
return pname, False, None
|
|
165
|
+
name = slugify_profile_name(model_id)
|
|
166
|
+
# Avoid clobbering an unrelated profile that happens to share the slug.
|
|
167
|
+
if name in profiles:
|
|
168
|
+
suffix = 2
|
|
169
|
+
while f"{name}_{suffix}" in profiles:
|
|
170
|
+
suffix += 1
|
|
171
|
+
name = f"{name}_{suffix}"
|
|
172
|
+
profiles[name] = default_profile_for(
|
|
173
|
+
model_id, params_b=_parse_params_b(model_id), quant=_parse_quant(model_id)
|
|
174
|
+
)
|
|
175
|
+
if not config.get("default_profile"):
|
|
176
|
+
config["default_profile"] = name
|
|
177
|
+
err = write_config_atomic(config, path)
|
|
178
|
+
return (name, True, None) if not err else (None, False, err)
|
|
179
|
+
except Exception as exc: # pragma: no cover - defensive; pull must still succeed
|
|
180
|
+
return None, False, str(exc)
|
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
"""Shared Ollama control-plane helpers for the web package.
|
|
2
|
+
|
|
3
|
+
One place for the local Ollama HTTP calls used by registry + model lifecycle, so
|
|
4
|
+
those routes never duplicate connection/error handling. Every function either
|
|
5
|
+
returns a clean ``(data, error)`` pair or raises ``BackendCallError`` /
|
|
6
|
+
``requests.RequestException`` for the caller to translate into a graceful
|
|
7
|
+
response. Nothing here ever talks to a non-loopback host.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
|
15
|
+
|
|
16
|
+
import requests
|
|
17
|
+
|
|
18
|
+
from ..utils import BackendCallError, is_loopback_url, strip_trailing_slash
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def base_url() -> str:
|
|
22
|
+
base = strip_trailing_slash(os.getenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434"))
|
|
23
|
+
if not is_loopback_url(base):
|
|
24
|
+
raise BackendCallError(f"Refusing non-local Ollama URL: {base}")
|
|
25
|
+
return base
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def list_installed() -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
|
29
|
+
"""Installed models via /api/tags. Returns ([], error) when unreachable."""
|
|
30
|
+
try:
|
|
31
|
+
base = base_url()
|
|
32
|
+
except BackendCallError as exc:
|
|
33
|
+
return [], str(exc)
|
|
34
|
+
try:
|
|
35
|
+
response = requests.get(f"{base}/api/tags", timeout=5)
|
|
36
|
+
response.raise_for_status()
|
|
37
|
+
except requests.ConnectionError:
|
|
38
|
+
return [], f"Ollama is not reachable at {base}. Start Ollama and retry."
|
|
39
|
+
except requests.RequestException as exc:
|
|
40
|
+
return [], str(exc)
|
|
41
|
+
out: List[Dict[str, Any]] = []
|
|
42
|
+
for item in response.json().get("models", []):
|
|
43
|
+
out.append(
|
|
44
|
+
{
|
|
45
|
+
"name": item.get("name"),
|
|
46
|
+
"size": item.get("size"),
|
|
47
|
+
"modified_at": item.get("modified_at"),
|
|
48
|
+
"digest": item.get("digest") or None,
|
|
49
|
+
"digest_short": (item.get("digest") or "")[:12] or None,
|
|
50
|
+
"details": item.get("details", {}),
|
|
51
|
+
}
|
|
52
|
+
)
|
|
53
|
+
return out, None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def version() -> Tuple[Optional[str], Optional[str]]:
|
|
57
|
+
"""Return the exact Ollama server version when the local daemon is reachable."""
|
|
58
|
+
try:
|
|
59
|
+
base = base_url()
|
|
60
|
+
response = requests.get(f"{base}/api/version", timeout=3)
|
|
61
|
+
response.raise_for_status()
|
|
62
|
+
value = response.json().get("version")
|
|
63
|
+
return (str(value) if value else None), None
|
|
64
|
+
except requests.ConnectionError:
|
|
65
|
+
return None, f"Ollama is not reachable at {base if 'base' in locals() else 'localhost'}."
|
|
66
|
+
except (BackendCallError, requests.RequestException, ValueError) as exc:
|
|
67
|
+
return None, str(exc)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def show_model(model: str) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
|
|
71
|
+
"""Return Ollama's model metadata, including quant and context parameters."""
|
|
72
|
+
try:
|
|
73
|
+
base = base_url()
|
|
74
|
+
response = requests.post(f"{base}/api/show", json={"model": model}, timeout=10)
|
|
75
|
+
response.raise_for_status()
|
|
76
|
+
data = response.json()
|
|
77
|
+
return data if isinstance(data, dict) else None, None
|
|
78
|
+
except requests.ConnectionError:
|
|
79
|
+
return None, f"Ollama is not reachable at {base if 'base' in locals() else 'localhost'}."
|
|
80
|
+
except (BackendCallError, requests.RequestException, ValueError) as exc:
|
|
81
|
+
return None, str(exc)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def list_running() -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
|
85
|
+
"""Currently loaded models via /api/ps. Returns ([], error) when unreachable."""
|
|
86
|
+
try:
|
|
87
|
+
base = base_url()
|
|
88
|
+
except BackendCallError as exc:
|
|
89
|
+
return [], str(exc)
|
|
90
|
+
try:
|
|
91
|
+
response = requests.get(f"{base}/api/ps", timeout=5)
|
|
92
|
+
response.raise_for_status()
|
|
93
|
+
except requests.ConnectionError:
|
|
94
|
+
return [], f"Ollama is not reachable at {base}. Start Ollama and retry."
|
|
95
|
+
except requests.RequestException as exc:
|
|
96
|
+
return [], str(exc)
|
|
97
|
+
out: List[Dict[str, Any]] = []
|
|
98
|
+
for item in response.json().get("models", []):
|
|
99
|
+
out.append(
|
|
100
|
+
{
|
|
101
|
+
"name": item.get("name"),
|
|
102
|
+
"size": item.get("size"),
|
|
103
|
+
"size_vram": item.get("size_vram"),
|
|
104
|
+
"expires_at": item.get("expires_at"),
|
|
105
|
+
}
|
|
106
|
+
)
|
|
107
|
+
return out, None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
# Warming a model means loading all its weights into memory. On the GPU this is
|
|
111
|
+
# quick; forcing CPU offload (num_gpu=0) for a large model can take minutes, so a
|
|
112
|
+
# flat 120s timeout would spuriously fail those. Both are overridable via env.
|
|
113
|
+
_GPU_LOAD_TIMEOUT = 120
|
|
114
|
+
_CPU_LOAD_TIMEOUT = 600
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _load_timeout(num_gpu: Optional[int]) -> int:
|
|
118
|
+
"""Pick a warm-up timeout: longer for CPU offload, overridable via env."""
|
|
119
|
+
override = os.getenv("OLLAMA_LOAD_TIMEOUT")
|
|
120
|
+
if override:
|
|
121
|
+
try:
|
|
122
|
+
return int(override)
|
|
123
|
+
except ValueError:
|
|
124
|
+
pass
|
|
125
|
+
return _CPU_LOAD_TIMEOUT if num_gpu == 0 else _GPU_LOAD_TIMEOUT
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def load_model(
|
|
129
|
+
model: str, keep_alive: str = "60m", num_gpu: Optional[int] = None
|
|
130
|
+
) -> Dict[str, Any]:
|
|
131
|
+
"""Warm a model into memory. An empty prompt makes Ollama load without generating.
|
|
132
|
+
|
|
133
|
+
``num_gpu`` sets how many layers to offload to the GPU (0 = force CPU). When
|
|
134
|
+
None, Ollama decides (auto) - identical to the prior behaviour. The request
|
|
135
|
+
timeout scales with the target device (CPU loads are slower).
|
|
136
|
+
"""
|
|
137
|
+
base = base_url()
|
|
138
|
+
payload: Dict[str, Any] = {"model": model, "keep_alive": keep_alive}
|
|
139
|
+
if num_gpu is not None:
|
|
140
|
+
payload["options"] = {"num_gpu": num_gpu}
|
|
141
|
+
response = requests.post(
|
|
142
|
+
f"{base}/api/generate",
|
|
143
|
+
json=payload,
|
|
144
|
+
timeout=_load_timeout(num_gpu),
|
|
145
|
+
)
|
|
146
|
+
response.raise_for_status()
|
|
147
|
+
return response.json()
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def unload_model(model: str) -> Dict[str, Any]:
|
|
151
|
+
"""Unload a model immediately via keep_alive=0."""
|
|
152
|
+
base = base_url()
|
|
153
|
+
response = requests.post(
|
|
154
|
+
f"{base}/api/generate",
|
|
155
|
+
json={"model": model, "keep_alive": 0},
|
|
156
|
+
timeout=30,
|
|
157
|
+
)
|
|
158
|
+
response.raise_for_status()
|
|
159
|
+
return response.json()
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def delete_model(model: str) -> Dict[str, Any]:
|
|
163
|
+
"""Delete a model from disk via /api/delete (frees disk space)."""
|
|
164
|
+
base = base_url()
|
|
165
|
+
response = requests.delete(f"{base}/api/delete", json={"name": model}, timeout=30)
|
|
166
|
+
# Ollama returns 200 (often with an empty body) on success, 404 if absent.
|
|
167
|
+
if response.status_code == 404:
|
|
168
|
+
raise BackendCallError(f"Model '{model}' is not installed.")
|
|
169
|
+
response.raise_for_status()
|
|
170
|
+
return {"deleted": model}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def unload_all() -> Tuple[int, Optional[str]]:
|
|
174
|
+
"""Unload every currently-loaded model from memory. Returns (count, error)."""
|
|
175
|
+
running, err = list_running()
|
|
176
|
+
if err is not None:
|
|
177
|
+
return 0, err
|
|
178
|
+
unloaded = 0
|
|
179
|
+
for m in running:
|
|
180
|
+
name = m.get("name")
|
|
181
|
+
if not name:
|
|
182
|
+
continue
|
|
183
|
+
try:
|
|
184
|
+
unload_model(name)
|
|
185
|
+
unloaded += 1
|
|
186
|
+
except (BackendCallError, requests.RequestException):
|
|
187
|
+
continue # best-effort; keep unloading the rest
|
|
188
|
+
return unloaded, None
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def pull_stream(model: str) -> Iterator[Dict[str, Any]]:
|
|
192
|
+
"""Yield progress events from `ollama pull` (JSON lines)."""
|
|
193
|
+
base = base_url()
|
|
194
|
+
with requests.post(
|
|
195
|
+
f"{base}/api/pull",
|
|
196
|
+
json={"name": model, "stream": True},
|
|
197
|
+
stream=True,
|
|
198
|
+
timeout=(10, 120),
|
|
199
|
+
) as response:
|
|
200
|
+
response.raise_for_status()
|
|
201
|
+
for line in response.iter_lines():
|
|
202
|
+
if not line:
|
|
203
|
+
continue
|
|
204
|
+
try:
|
|
205
|
+
yield json.loads(line)
|
|
206
|
+
except (ValueError, TypeError):
|
|
207
|
+
continue
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
# --- create-from-GGUF-file (used to import a downloaded file that isn't in any
|
|
211
|
+
# registry: push the blob once, then point /api/create at its digest) --------
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def blob_exists(digest: str) -> bool:
|
|
215
|
+
"""True if Ollama already has this blob (HEAD /api/blobs/:digest), so a
|
|
216
|
+
retry after a partial import doesn't re-upload the whole file."""
|
|
217
|
+
base = base_url()
|
|
218
|
+
try:
|
|
219
|
+
response = requests.head(f"{base}/api/blobs/sha256:{digest}", timeout=10)
|
|
220
|
+
return response.status_code == 200
|
|
221
|
+
except requests.RequestException:
|
|
222
|
+
return False
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def push_blob(file_path: Path, digest: str) -> None:
|
|
226
|
+
"""Upload a local file as a content-addressed blob Ollama can reference
|
|
227
|
+
from /api/create. Streams from disk - never loads the whole file into memory."""
|
|
228
|
+
base = base_url()
|
|
229
|
+
with open(file_path, "rb") as fh:
|
|
230
|
+
response = requests.post(f"{base}/api/blobs/sha256:{digest}", data=fh, timeout=(10, 1800))
|
|
231
|
+
# 201 = created, 200 = already present (some Ollama versions no-op instead of 4xx).
|
|
232
|
+
if response.status_code not in (200, 201):
|
|
233
|
+
raise BackendCallError(f"Ollama rejected the uploaded blob (HTTP {response.status_code}): {response.text[:300]}")
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def create_stream(model: str, files: Dict[str, str]) -> Iterator[Dict[str, Any]]:
|
|
237
|
+
"""Yield progress events from `ollama create` given already-pushed blobs
|
|
238
|
+
(`files` maps a filename to its 'sha256:<digest>')."""
|
|
239
|
+
base = base_url()
|
|
240
|
+
with requests.post(
|
|
241
|
+
f"{base}/api/create",
|
|
242
|
+
json={"model": model, "files": files, "stream": True},
|
|
243
|
+
stream=True,
|
|
244
|
+
timeout=(10, 1800),
|
|
245
|
+
) as response:
|
|
246
|
+
response.raise_for_status()
|
|
247
|
+
for line in response.iter_lines():
|
|
248
|
+
if not line:
|
|
249
|
+
continue
|
|
250
|
+
try:
|
|
251
|
+
yield json.loads(line)
|
|
252
|
+
except (ValueError, TypeError):
|
|
253
|
+
continue
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Automated model bakeoff (Release R6) - "Compare top models for me."
|
|
2
|
+
|
|
3
|
+
POST /system/bakeoff/run streams the strongest end-to-end LocalDeploy workflow
|
|
4
|
+
as one operation: pick fit-safe candidates for a workload within a download
|
|
5
|
+
budget, pull + serve + benchmark each one in turn (sequentially, to avoid
|
|
6
|
+
VRAM contention - same discipline as /benchmark/run), rank them with the
|
|
7
|
+
existing recommend.py scoring formula, then re-serve the winner.
|
|
8
|
+
|
|
9
|
+
This is orchestration only. Every step reuses an existing engine:
|
|
10
|
+
- candidate selection: starter.py's fit-filtered, workload-biased catalog
|
|
11
|
+
- pull: _ollama.pull_stream (same as /models/pull)
|
|
12
|
+
- serve + placement: models._serve_ollama / models._placement
|
|
13
|
+
- benchmark: benchmark.execute_test + bench.py's BENCHMARK_PACKS
|
|
14
|
+
- ranking: recommend.py's rank_candidates (quality x speed x headroom)
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
from typing import Any, Dict, List, Optional
|
|
20
|
+
|
|
21
|
+
from fastapi import APIRouter
|
|
22
|
+
from fastapi.responses import StreamingResponse
|
|
23
|
+
from pydantic import BaseModel, Field
|
|
24
|
+
|
|
25
|
+
from . import _ollama
|
|
26
|
+
from ._config import ensure_profile_for_model
|
|
27
|
+
from .bench import BENCHMARK_PACKS
|
|
28
|
+
from .bench import _bench # memoized, race-safe import of the top-level `benchmark` module
|
|
29
|
+
from .fit import _weight_gb_per_b
|
|
30
|
+
from .models import _matches_model_name, _serve_ollama
|
|
31
|
+
from .recommend import rank_candidates
|
|
32
|
+
from .starter import _fitting_candidates, _rank, _resolve_budget, _with_workload_bias
|
|
33
|
+
|
|
34
|
+
router = APIRouter()
|
|
35
|
+
|
|
36
|
+
# use_case (from the guided-recommend vocabulary) -> default benchmark pack.
|
|
37
|
+
_USE_CASE_PACK = {
|
|
38
|
+
"coding": "coding",
|
|
39
|
+
"structured_extraction": "structured",
|
|
40
|
+
"reasoning": "reasoning",
|
|
41
|
+
"tool_calling": "structured",
|
|
42
|
+
}
|
|
43
|
+
_DEFAULT_PACK = "general"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class BakeoffRequest(BaseModel):
|
|
47
|
+
use_case: Optional[str] = None
|
|
48
|
+
priority: str = Field(default="balanced")
|
|
49
|
+
expected_context: int = Field(default=8192, gt=0, le=1_000_000)
|
|
50
|
+
download_budget_gb: float = Field(default=10.0, gt=0, le=1_000)
|
|
51
|
+
pack: Optional[str] = None
|
|
52
|
+
max_candidates: int = Field(default=3, ge=1, le=5)
|
|
53
|
+
sample_size: int = Field(default=3, ge=1, le=10)
|
|
54
|
+
timeout: int = Field(default=120, ge=1, le=3_600)
|
|
55
|
+
free_vram_mb: Optional[int] = Field(default=None, ge=0, le=100_000_000)
|
|
56
|
+
margin_gb: float = Field(default=2.0, ge=0, le=1_024)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _select_candidates(
|
|
60
|
+
use_case: Optional[str], priority: str, budget_gb: float, download_budget_gb: float, max_candidates: int
|
|
61
|
+
) -> List[Dict[str, Any]]:
|
|
62
|
+
"""Fit-safe, workload-biased, download-budget-capped picks - the same
|
|
63
|
+
catalog and ranking recommend.py's guided-recommend endpoint uses, just
|
|
64
|
+
greedily packed against a cumulative download-size cap instead of split
|
|
65
|
+
into three labeled buckets."""
|
|
66
|
+
fitting = _fitting_candidates(budget_gb)
|
|
67
|
+
biased = _with_workload_bias(fitting, use_case)
|
|
68
|
+
ranked = _rank(biased, budget_gb, priority)
|
|
69
|
+
|
|
70
|
+
picked: List[Dict[str, Any]] = []
|
|
71
|
+
cumulative_download_gb = 0.0
|
|
72
|
+
for entry in ranked:
|
|
73
|
+
download_gb = round(entry["params_b"] * _weight_gb_per_b("q4"), 1)
|
|
74
|
+
if picked and cumulative_download_gb + download_gb > download_budget_gb:
|
|
75
|
+
continue
|
|
76
|
+
entry = dict(entry)
|
|
77
|
+
entry["download_gb"] = download_gb
|
|
78
|
+
picked.append(entry)
|
|
79
|
+
cumulative_download_gb += download_gb
|
|
80
|
+
if len(picked) >= max_candidates:
|
|
81
|
+
break
|
|
82
|
+
return picked
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@router.post("/system/bakeoff/run")
|
|
86
|
+
def bakeoff_run(req: BakeoffRequest):
|
|
87
|
+
def sse(event: Dict[str, Any]) -> str:
|
|
88
|
+
return f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
|
|
89
|
+
|
|
90
|
+
def event_stream():
|
|
91
|
+
raw_budget_gb, budget_gb, source, hw = _resolve_budget(req.free_vram_mb, req.margin_gb)
|
|
92
|
+
if budget_gb is None:
|
|
93
|
+
yield sse({"event": "error", "error": "Could not determine VRAM or system RAM budget."})
|
|
94
|
+
yield "data: [DONE]\n\n"
|
|
95
|
+
return
|
|
96
|
+
|
|
97
|
+
candidates = _select_candidates(req.use_case, req.priority, budget_gb, req.download_budget_gb, req.max_candidates)
|
|
98
|
+
if not candidates:
|
|
99
|
+
yield sse({"event": "error", "error": "No catalog model fits your budget and download cap."})
|
|
100
|
+
yield "data: [DONE]\n\n"
|
|
101
|
+
return
|
|
102
|
+
|
|
103
|
+
bench = _bench()
|
|
104
|
+
pack_id = req.pack or _USE_CASE_PACK.get(req.use_case or "", _DEFAULT_PACK)
|
|
105
|
+
pack_categories = (BENCHMARK_PACKS.get(pack_id) or BENCHMARK_PACKS[_DEFAULT_PACK])["categories"]
|
|
106
|
+
tests = [t for t in bench.builtin_test_cases() if t.category in pack_categories]
|
|
107
|
+
tests = sorted(tests, key=lambda t: t.max_output_tokens)[: max(1, req.sample_size)]
|
|
108
|
+
base_url = bench.api_base_url()
|
|
109
|
+
|
|
110
|
+
yield sse(
|
|
111
|
+
{
|
|
112
|
+
"event": "bakeoff_start",
|
|
113
|
+
"candidates": [c["id"] for c in candidates],
|
|
114
|
+
"pack": pack_id,
|
|
115
|
+
"sample_tests": [t.name for t in tests],
|
|
116
|
+
"budget_source": source,
|
|
117
|
+
"budget_gb": round(budget_gb, 2),
|
|
118
|
+
"download_budget_gb": req.download_budget_gb,
|
|
119
|
+
}
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
scored: List[Dict[str, Any]] = []
|
|
123
|
+
installed, _ = _ollama.list_installed()
|
|
124
|
+
installed_names = {str(m.get("name")) for m in installed if m.get("name")}
|
|
125
|
+
|
|
126
|
+
for candidate in candidates:
|
|
127
|
+
model_id = candidate["id"]
|
|
128
|
+
yield sse({"event": "candidate_start", "model": model_id, "download_gb": candidate["download_gb"]})
|
|
129
|
+
|
|
130
|
+
if not any(_matches_model_name(name, model_id) for name in installed_names):
|
|
131
|
+
yield sse({"event": "pull_start", "model": model_id})
|
|
132
|
+
try:
|
|
133
|
+
for progress in _ollama.pull_stream(model_id):
|
|
134
|
+
yield sse({"event": "pull_progress", "model": model_id, **progress})
|
|
135
|
+
except Exception as exc:
|
|
136
|
+
yield sse({"event": "candidate_failed", "model": model_id, "reason": f"pull failed: {exc}"})
|
|
137
|
+
scored.append({"profile": model_id, "avg_accuracy": 0.0, "avg_latency_s": 0.0, "passed": 0, "tests": 0, "margin_gb": None})
|
|
138
|
+
continue
|
|
139
|
+
yield sse({"event": "pull_end", "model": model_id})
|
|
140
|
+
|
|
141
|
+
profile_name, _created, profile_err = ensure_profile_for_model(model_id)
|
|
142
|
+
if not profile_name:
|
|
143
|
+
yield sse({"event": "candidate_failed", "model": model_id, "reason": profile_err or "could not create a profile"})
|
|
144
|
+
scored.append({"profile": model_id, "avg_accuracy": 0.0, "avg_latency_s": 0.0, "passed": 0, "tests": 0, "margin_gb": None})
|
|
145
|
+
continue
|
|
146
|
+
|
|
147
|
+
yield sse({"event": "deploy_start", "model": model_id})
|
|
148
|
+
served = _serve_ollama(model_id, "10m")
|
|
149
|
+
if not served.get("success"):
|
|
150
|
+
yield sse({"event": "candidate_failed", "model": model_id, "reason": served.get("error") or "deploy failed"})
|
|
151
|
+
scored.append({"profile": profile_name, "avg_accuracy": 0.0, "avg_latency_s": 0.0, "passed": 0, "tests": 0, "margin_gb": candidate.get("margin_gb")})
|
|
152
|
+
continue
|
|
153
|
+
running = served.get("running") or []
|
|
154
|
+
match = next((m for m in running if _matches_model_name(m.get("name"), model_id)), None)
|
|
155
|
+
placement = match.get("placement") if match else None
|
|
156
|
+
yield sse({"event": "deploy_end", "model": model_id, "placement": placement, "warning": served.get("warning")})
|
|
157
|
+
|
|
158
|
+
from api_server import load_config # lazy: api_server owns config loading
|
|
159
|
+
|
|
160
|
+
profile = load_config().get("profiles", {}).get(profile_name, {})
|
|
161
|
+
results = []
|
|
162
|
+
for t in tests:
|
|
163
|
+
yield sse({"event": "test_start", "model": model_id, "name": t.name})
|
|
164
|
+
try:
|
|
165
|
+
item = bench.execute_test(base_url, profile_name, profile, t, req.timeout)
|
|
166
|
+
except Exception:
|
|
167
|
+
item = bench.TestResult(
|
|
168
|
+
name=t.name, category=t.category, success=False, elapsed_seconds=0.0,
|
|
169
|
+
response_length=0, response_preview="", accuracy=0.0, error="execute_test error",
|
|
170
|
+
)
|
|
171
|
+
results.append(item)
|
|
172
|
+
yield sse({"event": "test_result", "model": model_id, "name": item.name, "success": item.success, "accuracy": item.accuracy, "elapsed_seconds": item.elapsed_seconds})
|
|
173
|
+
|
|
174
|
+
accs = [r.accuracy for r in results]
|
|
175
|
+
lats = [r.elapsed_seconds for r in results if r.elapsed_seconds]
|
|
176
|
+
row = {
|
|
177
|
+
"profile": model_id,
|
|
178
|
+
"avg_accuracy": round(sum(accs) / len(accs), 3) if accs else 0.0,
|
|
179
|
+
"avg_latency_s": round(sum(lats) / len(lats), 3) if lats else 0.0,
|
|
180
|
+
"passed": sum(1 for r in results if r.success),
|
|
181
|
+
"tests": len(results),
|
|
182
|
+
"margin_gb": candidate.get("margin_gb"),
|
|
183
|
+
"placement": placement,
|
|
184
|
+
}
|
|
185
|
+
scored.append(row)
|
|
186
|
+
yield sse({"event": "candidate_end", "model": model_id, **row})
|
|
187
|
+
|
|
188
|
+
# Sequential VRAM discipline (same as /benchmark/run): unload before
|
|
189
|
+
# the next candidate. The overall winner gets explicitly re-served below.
|
|
190
|
+
try:
|
|
191
|
+
_ollama.unload_model(model_id)
|
|
192
|
+
except Exception:
|
|
193
|
+
pass
|
|
194
|
+
|
|
195
|
+
ranked = rank_candidates([s for s in scored if s["tests"] > 0]) if any(s["tests"] > 0 for s in scored) else []
|
|
196
|
+
if not ranked:
|
|
197
|
+
yield sse({"event": "error", "error": "Every candidate failed to pull, deploy, or benchmark."})
|
|
198
|
+
yield "data: [DONE]\n\n"
|
|
199
|
+
return
|
|
200
|
+
|
|
201
|
+
winner_model = ranked[0]["profile"]
|
|
202
|
+
yield sse({"event": "deploy_start", "model": winner_model, "final": True})
|
|
203
|
+
winner_served = _serve_ollama(winner_model, "60m")
|
|
204
|
+
yield sse(
|
|
205
|
+
{
|
|
206
|
+
"event": "bakeoff_end",
|
|
207
|
+
"pack": pack_id,
|
|
208
|
+
"ranked": ranked,
|
|
209
|
+
"winner": winner_model,
|
|
210
|
+
"losers": [s["profile"] for s in ranked[1:]],
|
|
211
|
+
"winner_deployed": bool(winner_served.get("success")),
|
|
212
|
+
}
|
|
213
|
+
)
|
|
214
|
+
yield "data: [DONE]\n\n"
|
|
215
|
+
|
|
216
|
+
return StreamingResponse(event_stream(), media_type="text/event-stream")
|