localdeploy 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api_server.py +15 -0
- benchmark.py +15 -0
- chat_cli.py +15 -0
- compare_models.py +15 -0
- localdeploy/__init__.py +3 -0
- localdeploy/backends/__init__.py +0 -0
- localdeploy/backends/llamacpp.py +143 -0
- localdeploy/backends/ollama.py +374 -0
- localdeploy/backends/openai_compatible.py +179 -0
- localdeploy/benchmark.py +1996 -0
- localdeploy/chat_cli.py +249 -0
- localdeploy/cli.py +59 -0
- localdeploy/compare_models.py +481 -0
- localdeploy/control/__init__.py +75 -0
- localdeploy/control/_config.py +180 -0
- localdeploy/control/_ollama.py +253 -0
- localdeploy/control/bakeoff.py +216 -0
- localdeploy/control/bench.py +516 -0
- localdeploy/control/calibration.py +191 -0
- localdeploy/control/community.py +161 -0
- localdeploy/control/fit.py +651 -0
- localdeploy/control/hardware.py +602 -0
- localdeploy/control/history.py +125 -0
- localdeploy/control/manifest.py +452 -0
- localdeploy/control/models.py +833 -0
- localdeploy/control/monitor.py +378 -0
- localdeploy/control/profiles.py +161 -0
- localdeploy/control/recommend.py +322 -0
- localdeploy/control/registry.py +956 -0
- localdeploy/control/report.py +407 -0
- localdeploy/control/starter.py +520 -0
- localdeploy/control/updates.py +118 -0
- localdeploy/grader_sandbox.py +268 -0
- localdeploy/server.py +1873 -0
- localdeploy/utils.py +155 -0
- localdeploy/web/favicon.png +0 -0
- localdeploy/web/index.html +891 -0
- localdeploy/web/js/app.js +63 -0
- localdeploy/web/js/benchmark-views.js +73 -0
- localdeploy/web/js/benchmark.js +1691 -0
- localdeploy/web/js/chat.js +779 -0
- localdeploy/web/js/models.js +2758 -0
- localdeploy/web/js/shared.js +454 -0
- localdeploy/web/js/system.js +626 -0
- localdeploy/web/logo.svg +22 -0
- localdeploy/web/styles.css +2032 -0
- localdeploy-0.6.0.dist-info/METADATA +202 -0
- localdeploy-0.6.0.dist-info/RECORD +52 -0
- localdeploy-0.6.0.dist-info/WHEEL +5 -0
- localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
- localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
- localdeploy-0.6.0.dist-info/top_level.txt +5 -0
|
@@ -0,0 +1,516 @@
|
|
|
1
|
+
"""Steps 7-8 - benchmark over the web.
|
|
2
|
+
|
|
3
|
+
GET /benchmark/example -> the canonical question-set example
|
|
4
|
+
POST /benchmark/validate -> validate an uploaded question set (no run)
|
|
5
|
+
POST /benchmark/run -> run the benchmark, streaming per-test results
|
|
6
|
+
|
|
7
|
+
All heavy lifting (grading, the run generator, the schema) lives in the
|
|
8
|
+
top-level `benchmark.py` so the CLI and this endpoint share one implementation.
|
|
9
|
+
`benchmark` is imported lazily so the server boot stays light and free of a
|
|
10
|
+
hard import-order coupling.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import threading
|
|
16
|
+
from datetime import datetime, timezone
|
|
17
|
+
from typing import Any, Dict, List, Optional
|
|
18
|
+
|
|
19
|
+
from fastapi import APIRouter, Request
|
|
20
|
+
from fastapi.responses import StreamingResponse
|
|
21
|
+
from pydantic import BaseModel, Field
|
|
22
|
+
|
|
23
|
+
router = APIRouter()
|
|
24
|
+
|
|
25
|
+
_bench_lock = threading.Lock()
|
|
26
|
+
_bench_module = None
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _bench():
|
|
30
|
+
"""Lazy, memoized import of the top-level `benchmark` module.
|
|
31
|
+
|
|
32
|
+
FastAPI runs each request handler in a threadpool worker, so the first
|
|
33
|
+
few requests after startup (e.g. a page load firing several /benchmark/*
|
|
34
|
+
calls at once) can call this concurrently from different threads before
|
|
35
|
+
`benchmark` has ever been imported. A bare `import benchmark` in that
|
|
36
|
+
window is not reliably safe against concurrent first-import from
|
|
37
|
+
multiple threads; double-checked locking here makes the import happen
|
|
38
|
+
exactly once, with every other caller blocking on the lock rather than
|
|
39
|
+
racing it.
|
|
40
|
+
"""
|
|
41
|
+
global _bench_module
|
|
42
|
+
if _bench_module is None:
|
|
43
|
+
with _bench_lock:
|
|
44
|
+
if _bench_module is None:
|
|
45
|
+
import benchmark # avoids importing the benchmark engine at boot
|
|
46
|
+
|
|
47
|
+
_bench_module = benchmark
|
|
48
|
+
return _bench_module
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@router.get("/benchmark/example")
|
|
52
|
+
def benchmark_example() -> Dict[str, Any]:
|
|
53
|
+
return _bench().EXAMPLE_QUESTION_SET
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@router.get("/benchmark/test-bench")
|
|
57
|
+
def benchmark_test_bench() -> Dict[str, Any]:
|
|
58
|
+
bench = _bench()
|
|
59
|
+
tests = bench.builtin_test_cases()
|
|
60
|
+
categories: Dict[str, int] = {}
|
|
61
|
+
for test in tests:
|
|
62
|
+
categories[test.category] = categories.get(test.category, 0) + 1
|
|
63
|
+
return {
|
|
64
|
+
"success": True,
|
|
65
|
+
"test_count": len(tests),
|
|
66
|
+
"categories": categories,
|
|
67
|
+
"question_set": bench.BUILTIN_QUESTION_SET,
|
|
68
|
+
"tests": [{"name": t.name, "category": t.category, "max_output_tokens": t.max_output_tokens} for t in tests],
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@router.post("/benchmark/validate")
|
|
73
|
+
async def benchmark_validate(request: Request) -> Dict[str, Any]:
|
|
74
|
+
try:
|
|
75
|
+
payload = await request.json()
|
|
76
|
+
except Exception as exc:
|
|
77
|
+
return {
|
|
78
|
+
"success": True,
|
|
79
|
+
"valid": False,
|
|
80
|
+
"question_count": 0,
|
|
81
|
+
"errors": [{"index": -1, "error": f"invalid JSON body: {exc}"}],
|
|
82
|
+
"grader_types": _bench().GRADER_TYPES,
|
|
83
|
+
}
|
|
84
|
+
return _bench().validate_question_set(payload)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
# --- benchmark packs (Release R4) --------------------------------------------
|
|
88
|
+
# A pack is a named, reusable preset over /benchmark/run's existing
|
|
89
|
+
# `include_categories` filter - not a new engine. "all" (the default UI
|
|
90
|
+
# behaviour today) intentionally has no entry here; omit include_categories
|
|
91
|
+
# for that.
|
|
92
|
+
BENCHMARK_PACKS: Dict[str, Dict[str, Any]] = {
|
|
93
|
+
"general": {
|
|
94
|
+
"label": "General assistant",
|
|
95
|
+
"categories": ["planning", "classification"],
|
|
96
|
+
"description": "Planning and classification - everyday assistant tasks.",
|
|
97
|
+
},
|
|
98
|
+
"coding": {
|
|
99
|
+
"label": "Coding",
|
|
100
|
+
"categories": ["code"],
|
|
101
|
+
"description": "Non-trivial Python/SQL, graded by AST parsing and unit tests.",
|
|
102
|
+
},
|
|
103
|
+
"structured": {
|
|
104
|
+
"label": "JSON and structured extraction",
|
|
105
|
+
"categories": ["structured", "structured_hard"],
|
|
106
|
+
"description": "Schema-shaped JSON output, including nested/conditional schemas.",
|
|
107
|
+
},
|
|
108
|
+
"reasoning": {
|
|
109
|
+
"label": "Reasoning",
|
|
110
|
+
"categories": ["math", "planning"],
|
|
111
|
+
"description": "Multi-step math and planning tasks that reward chain-of-thought.",
|
|
112
|
+
},
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
@router.get("/benchmark/packs")
|
|
117
|
+
def benchmark_packs() -> Dict[str, Any]:
|
|
118
|
+
bench = _bench()
|
|
119
|
+
tests = bench.builtin_test_cases()
|
|
120
|
+
known_categories = {t.category for t in tests}
|
|
121
|
+
packs = []
|
|
122
|
+
for key, pack in BENCHMARK_PACKS.items():
|
|
123
|
+
cats = [c for c in pack["categories"] if c in known_categories]
|
|
124
|
+
packs.append(
|
|
125
|
+
{
|
|
126
|
+
"id": key,
|
|
127
|
+
"label": pack["label"],
|
|
128
|
+
"description": pack["description"],
|
|
129
|
+
"categories": cats,
|
|
130
|
+
"test_count": sum(1 for t in tests if t.category in cats),
|
|
131
|
+
}
|
|
132
|
+
)
|
|
133
|
+
return {"success": True, "packs": packs}
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class RunRequest(BaseModel):
|
|
137
|
+
profiles: Optional[List[str]] = None
|
|
138
|
+
questions: Optional[Dict[str, Any]] = None
|
|
139
|
+
device: Optional[str] = None
|
|
140
|
+
max_output_tokens: Optional[int] = None
|
|
141
|
+
timeout: int = 240
|
|
142
|
+
skip_categories: Optional[List[str]] = None
|
|
143
|
+
include_categories: Optional[List[str]] = None
|
|
144
|
+
pack: Optional[str] = None # convenience: expands to include_categories via BENCHMARK_PACKS
|
|
145
|
+
repetitions: int = Field(default=1, ge=1, le=10)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@router.post("/benchmark/run")
|
|
149
|
+
def benchmark_run(req: RunRequest):
|
|
150
|
+
bench = _bench()
|
|
151
|
+
from api_server import load_config # lazy: api_server owns config loading
|
|
152
|
+
|
|
153
|
+
profiles_map = load_config().get("profiles", {})
|
|
154
|
+
selected = req.profiles or [n for n, p in profiles_map.items() if p.get("enabled", False)]
|
|
155
|
+
selected = [n for n in selected if n in profiles_map]
|
|
156
|
+
if not selected:
|
|
157
|
+
return {"success": False, "error": "No profiles selected or enabled in config."}
|
|
158
|
+
|
|
159
|
+
# Build a fresh test list so request-specific limits and filtering cannot
|
|
160
|
+
# mutate the built-in definitions used by another request.
|
|
161
|
+
if req.questions is not None:
|
|
162
|
+
report = bench.validate_question_set(req.questions)
|
|
163
|
+
if not report["valid"]:
|
|
164
|
+
return {"success": False, "error": "invalid question set", "validation": report}
|
|
165
|
+
tests = bench.build_test_cases(req.questions)
|
|
166
|
+
else:
|
|
167
|
+
skip = set(req.skip_categories or [])
|
|
168
|
+
include = set(req.include_categories or [])
|
|
169
|
+
if not include and req.pack and req.pack in BENCHMARK_PACKS:
|
|
170
|
+
include = set(BENCHMARK_PACKS[req.pack]["categories"])
|
|
171
|
+
tests = [
|
|
172
|
+
t
|
|
173
|
+
for t in bench.builtin_test_cases()
|
|
174
|
+
if t.category not in skip and (not include or t.category in include)
|
|
175
|
+
]
|
|
176
|
+
if not tests:
|
|
177
|
+
return {"success": False, "error": "No tests selected."}
|
|
178
|
+
|
|
179
|
+
if req.max_output_tokens:
|
|
180
|
+
for test in tests:
|
|
181
|
+
if test.category != "classification":
|
|
182
|
+
test.max_output_tokens = max(test.max_output_tokens, req.max_output_tokens)
|
|
183
|
+
|
|
184
|
+
base_url = bench.api_base_url()
|
|
185
|
+
|
|
186
|
+
def event_stream():
|
|
187
|
+
def sse(event: Dict[str, Any]) -> str:
|
|
188
|
+
return f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
|
|
189
|
+
|
|
190
|
+
model_routes = None
|
|
191
|
+
pre_existing: set[str] = set()
|
|
192
|
+
unloaded_profiles: set[str] = set()
|
|
193
|
+
requested_device = (req.device or "auto").strip().lower()
|
|
194
|
+
|
|
195
|
+
def ensure_model_routes():
|
|
196
|
+
nonlocal model_routes
|
|
197
|
+
if model_routes is None:
|
|
198
|
+
from . import models as routes # lazy: avoid extra startup coupling
|
|
199
|
+
|
|
200
|
+
model_routes = routes
|
|
201
|
+
return model_routes
|
|
202
|
+
|
|
203
|
+
def model_for_profile(profile_name: str) -> Optional[str]:
|
|
204
|
+
profile = profiles_map.get(profile_name) or {}
|
|
205
|
+
if str(profile.get("backend", "ollama")).lower() != "ollama":
|
|
206
|
+
return None
|
|
207
|
+
return str(profile.get("model_id") or profile_name)
|
|
208
|
+
|
|
209
|
+
def matches_running(model_id: str, running_name: str) -> bool:
|
|
210
|
+
routes = ensure_model_routes()
|
|
211
|
+
return routes._matches_model_name(running_name, model_id)
|
|
212
|
+
|
|
213
|
+
def was_pre_existing(model_id: str) -> bool:
|
|
214
|
+
return any(matches_running(model_id, name) for name in pre_existing)
|
|
215
|
+
|
|
216
|
+
def current_placement(profile_name: str) -> Optional[str]:
|
|
217
|
+
model_id = model_for_profile(profile_name)
|
|
218
|
+
if not model_id:
|
|
219
|
+
return None
|
|
220
|
+
try:
|
|
221
|
+
routes = ensure_model_routes()
|
|
222
|
+
running, _ = routes._ollama.list_running()
|
|
223
|
+
for item in running:
|
|
224
|
+
item.update(routes._placement(item.get("size"), item.get("size_vram")))
|
|
225
|
+
if matches_running(model_id, str(item.get("name") or "")):
|
|
226
|
+
placement = item.get("placement")
|
|
227
|
+
return str(placement).lower() if placement else None
|
|
228
|
+
except Exception:
|
|
229
|
+
return None
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
def should_unload(profile_name: str) -> bool:
|
|
233
|
+
model_id = model_for_profile(profile_name)
|
|
234
|
+
if not model_id:
|
|
235
|
+
return False
|
|
236
|
+
if requested_device in {"cpu", "gpu"}:
|
|
237
|
+
return True
|
|
238
|
+
return not was_pre_existing(model_id)
|
|
239
|
+
|
|
240
|
+
def unload_profile(profile_name: str) -> Optional[Dict[str, Any]]:
|
|
241
|
+
if profile_name in unloaded_profiles or not should_unload(profile_name):
|
|
242
|
+
return None
|
|
243
|
+
model_id = model_for_profile(profile_name)
|
|
244
|
+
if not model_id:
|
|
245
|
+
return None
|
|
246
|
+
unloaded_profiles.add(profile_name)
|
|
247
|
+
try:
|
|
248
|
+
ensure_model_routes()._ollama.unload_model(model_id)
|
|
249
|
+
except Exception as exc:
|
|
250
|
+
return {
|
|
251
|
+
"event": "benchmark_unload_error",
|
|
252
|
+
"profile": profile_name,
|
|
253
|
+
"model_id": model_id,
|
|
254
|
+
"error": str(exc),
|
|
255
|
+
}
|
|
256
|
+
return {
|
|
257
|
+
"event": "benchmark_unload_end",
|
|
258
|
+
"profile": profile_name,
|
|
259
|
+
"model_id": model_id,
|
|
260
|
+
"message": f"Unloaded temporary benchmark model '{model_id}'.",
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
try:
|
|
264
|
+
try:
|
|
265
|
+
routes = ensure_model_routes()
|
|
266
|
+
running, _ = routes._ollama.list_running()
|
|
267
|
+
pre_existing = {str(item.get("name")) for item in running if item.get("name")}
|
|
268
|
+
except Exception:
|
|
269
|
+
pre_existing = set()
|
|
270
|
+
|
|
271
|
+
from localdeploy import __version__
|
|
272
|
+
from . import _ollama
|
|
273
|
+
from .hardware import detect_hardware
|
|
274
|
+
|
|
275
|
+
installed, _ = _ollama.list_installed()
|
|
276
|
+
installed_by_name = {str(item.get("name") or ""): item for item in installed}
|
|
277
|
+
ollama_version, _ = _ollama.version()
|
|
278
|
+
profile_provenance: Dict[str, Any] = {}
|
|
279
|
+
for profile_name in selected:
|
|
280
|
+
profile = profiles_map[profile_name]
|
|
281
|
+
backend = str(profile.get("backend") or "ollama").lower()
|
|
282
|
+
model_id = str(profile.get("model_id") or profile_name)
|
|
283
|
+
installed_item = next(
|
|
284
|
+
(
|
|
285
|
+
item
|
|
286
|
+
for installed_name, item in installed_by_name.items()
|
|
287
|
+
if backend == "ollama" and matches_running(model_id, installed_name)
|
|
288
|
+
),
|
|
289
|
+
{},
|
|
290
|
+
)
|
|
291
|
+
details = installed_item.get("details") or {}
|
|
292
|
+
shown: Dict[str, Any] = {}
|
|
293
|
+
if backend == "ollama":
|
|
294
|
+
shown_result, _ = _ollama.show_model(model_id)
|
|
295
|
+
shown = shown_result or {}
|
|
296
|
+
shown_details = shown.get("details") if isinstance(shown.get("details"), dict) else {}
|
|
297
|
+
profile_provenance[profile_name] = {
|
|
298
|
+
"backend": backend,
|
|
299
|
+
"backend_version": ollama_version if backend == "ollama" else None,
|
|
300
|
+
"model": model_id,
|
|
301
|
+
"model_digest": installed_item.get("digest"),
|
|
302
|
+
"quant": details.get("quantization_level")
|
|
303
|
+
or shown_details.get("quantization_level")
|
|
304
|
+
or profile.get("quantization"),
|
|
305
|
+
"context": profile.get("context_limit"),
|
|
306
|
+
"warm_state": "warm"
|
|
307
|
+
if backend == "ollama" and any(matches_running(model_id, name) for name in pre_existing)
|
|
308
|
+
else "cold",
|
|
309
|
+
}
|
|
310
|
+
provenance = {
|
|
311
|
+
"recorded_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
312
|
+
"localdeploy_version": __version__,
|
|
313
|
+
"hardware": detect_hardware(),
|
|
314
|
+
"providers": {"ollama": {"version": ollama_version}},
|
|
315
|
+
"profiles": profile_provenance,
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
# For a forced CPU/GPU run, pin the same num_gpu on every inference
|
|
319
|
+
# call (not just the warm-up) so the benchmark measures the requested
|
|
320
|
+
# device end-to-end instead of whatever Ollama re-places it onto.
|
|
321
|
+
forced_num_gpu: Optional[int] = None
|
|
322
|
+
force_deploy = requested_device in {"cpu", "gpu"}
|
|
323
|
+
if force_deploy:
|
|
324
|
+
routes = ensure_model_routes()
|
|
325
|
+
forced_num_gpu = routes._resolve_num_gpu(requested_device, None)
|
|
326
|
+
# Fail fast, before deploying anything, if any selected profile
|
|
327
|
+
# can't be device-forced.
|
|
328
|
+
for profile_name in selected:
|
|
329
|
+
backend = str(profiles_map[profile_name].get("backend", "ollama")).lower()
|
|
330
|
+
if backend != "ollama":
|
|
331
|
+
yield sse(
|
|
332
|
+
{
|
|
333
|
+
"event": "error",
|
|
334
|
+
"error": (
|
|
335
|
+
"Benchmark device forcing is only supported for Ollama profiles; "
|
|
336
|
+
f"{profile_name} uses {backend}"
|
|
337
|
+
),
|
|
338
|
+
}
|
|
339
|
+
)
|
|
340
|
+
return
|
|
341
|
+
|
|
342
|
+
peak_vram_by_profile: Dict[str, int] = {}
|
|
343
|
+
|
|
344
|
+
def _track_peak_vram(profile_name: str) -> None:
|
|
345
|
+
# Best-effort (nvidia-smi only, same as the CLI's VRAM sampling);
|
|
346
|
+
# a missing/failed reading just means no peak_vram_mb is reported.
|
|
347
|
+
used = bench.nvidia_smi_used_mb()
|
|
348
|
+
if used is not None:
|
|
349
|
+
peak_vram_by_profile[profile_name] = max(peak_vram_by_profile.get(profile_name, 0), used)
|
|
350
|
+
|
|
351
|
+
for event in bench.iter_run(
|
|
352
|
+
base_url,
|
|
353
|
+
profiles_map,
|
|
354
|
+
selected,
|
|
355
|
+
tests,
|
|
356
|
+
req.timeout,
|
|
357
|
+
num_gpu=forced_num_gpu,
|
|
358
|
+
repetitions=req.repetitions,
|
|
359
|
+
provenance=provenance,
|
|
360
|
+
):
|
|
361
|
+
if force_deploy and event.get("event") == "profile_start":
|
|
362
|
+
# Deploy just this profile's model right before its tests run
|
|
363
|
+
# (instead of deploying every selected profile up front) so a
|
|
364
|
+
# multi-profile forced-device comparison never stacks more
|
|
365
|
+
# than one model in VRAM/RAM at a time.
|
|
366
|
+
profile_name = str(event.get("profile") or "")
|
|
367
|
+
model_id = str(event.get("model_id") or profile_name)
|
|
368
|
+
yield sse(
|
|
369
|
+
{
|
|
370
|
+
"event": "deploy_start",
|
|
371
|
+
"profile": profile_name,
|
|
372
|
+
"model_id": model_id,
|
|
373
|
+
"device": requested_device,
|
|
374
|
+
}
|
|
375
|
+
)
|
|
376
|
+
served = model_routes._serve_ollama(model_id, "60m", forced_num_gpu)
|
|
377
|
+
if not served.get("success"):
|
|
378
|
+
yield sse(
|
|
379
|
+
{
|
|
380
|
+
"event": "error",
|
|
381
|
+
"error": served.get("error") or served.get("message") or "model deploy failed",
|
|
382
|
+
"profile": profile_name,
|
|
383
|
+
"device": requested_device,
|
|
384
|
+
}
|
|
385
|
+
)
|
|
386
|
+
unload_event = unload_profile(profile_name)
|
|
387
|
+
if unload_event:
|
|
388
|
+
yield sse(unload_event)
|
|
389
|
+
break
|
|
390
|
+
deploy_end = {
|
|
391
|
+
"event": "deploy_end",
|
|
392
|
+
"profile": profile_name,
|
|
393
|
+
"model_id": model_id,
|
|
394
|
+
"device": requested_device,
|
|
395
|
+
}
|
|
396
|
+
if served.get("warning"):
|
|
397
|
+
deploy_end["warning"] = served["warning"]
|
|
398
|
+
yield sse(deploy_end)
|
|
399
|
+
|
|
400
|
+
if event.get("event") == "test_result":
|
|
401
|
+
_track_peak_vram(str(event.get("profile") or ""))
|
|
402
|
+
if event.get("event") == "profile_end":
|
|
403
|
+
profile_name = str(event.get("profile") or "")
|
|
404
|
+
actual = current_placement(profile_name)
|
|
405
|
+
if actual:
|
|
406
|
+
event["actual_device"] = actual
|
|
407
|
+
if profile_name in peak_vram_by_profile and isinstance(event.get("summary"), dict):
|
|
408
|
+
event["summary"]["peak_vram_mb"] = peak_vram_by_profile[profile_name]
|
|
409
|
+
yield sse(event)
|
|
410
|
+
if event.get("event") in {"profile_end", "profile_aborted"}:
|
|
411
|
+
unload_event = unload_profile(str(event.get("profile") or ""))
|
|
412
|
+
if unload_event:
|
|
413
|
+
yield sse(unload_event)
|
|
414
|
+
except Exception as exc: # never break the SSE contract
|
|
415
|
+
yield sse({"event": "error", "error": str(exc)})
|
|
416
|
+
finally:
|
|
417
|
+
# Best-effort cleanup for client aborts and unexpected stream exits.
|
|
418
|
+
for profile_name in selected:
|
|
419
|
+
if profile_name in unloaded_profiles or not should_unload(profile_name):
|
|
420
|
+
continue
|
|
421
|
+
model_id = model_for_profile(profile_name)
|
|
422
|
+
if not model_id:
|
|
423
|
+
continue
|
|
424
|
+
unloaded_profiles.add(profile_name)
|
|
425
|
+
try:
|
|
426
|
+
ensure_model_routes()._ollama.unload_model(model_id)
|
|
427
|
+
except Exception:
|
|
428
|
+
pass
|
|
429
|
+
yield "data: [DONE]\n\n"
|
|
430
|
+
|
|
431
|
+
return StreamingResponse(event_stream(), media_type="text/event-stream")
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
# --- context-scaling sweep (Release R4) --------------------------------------
|
|
435
|
+
# "How does memory, prompt-processing speed, first-token latency, and
|
|
436
|
+
# generation speed change with context?" - the same profile, the same short
|
|
437
|
+
# test subset used by /system/recommend, run once per context tier via the
|
|
438
|
+
# /chat endpoint's existing context_limit override. No separate estimation or
|
|
439
|
+
# grading logic: this reuses execute_test, so numbers agree with every other
|
|
440
|
+
# benchmark surface in the app.
|
|
441
|
+
DEFAULT_SWEEP_CONTEXTS = [4096, 8192, 16384, 32768, 65536]
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
class ContextSweepRequest(BaseModel):
|
|
445
|
+
profile: str
|
|
446
|
+
contexts: List[int] = Field(default_factory=lambda: list(DEFAULT_SWEEP_CONTEXTS))
|
|
447
|
+
sample_size: int = Field(default=3, ge=1, le=10)
|
|
448
|
+
timeout: int = Field(default=120, ge=1, le=3_600)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
@router.post("/benchmark/context-sweep")
|
|
452
|
+
def benchmark_context_sweep(req: ContextSweepRequest):
|
|
453
|
+
bench = _bench()
|
|
454
|
+
from api_server import load_config # lazy: api_server owns config loading
|
|
455
|
+
|
|
456
|
+
profiles_map = load_config().get("profiles", {})
|
|
457
|
+
|
|
458
|
+
def event_stream():
|
|
459
|
+
def sse(event: Dict[str, Any]) -> str:
|
|
460
|
+
return f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
|
|
461
|
+
|
|
462
|
+
if req.profile not in profiles_map:
|
|
463
|
+
yield sse({"event": "error", "error": f"Unknown profile '{req.profile}'."})
|
|
464
|
+
yield "data: [DONE]\n\n"
|
|
465
|
+
return
|
|
466
|
+
|
|
467
|
+
profile = profiles_map[req.profile]
|
|
468
|
+
tests = sorted(bench.builtin_test_cases(), key=lambda t: t.max_output_tokens)[: max(1, req.sample_size)]
|
|
469
|
+
base_url = bench.api_base_url()
|
|
470
|
+
contexts = sorted(dict.fromkeys(c for c in req.contexts if c > 0))
|
|
471
|
+
yield sse(
|
|
472
|
+
{
|
|
473
|
+
"event": "sweep_start", "profile": req.profile, "contexts": contexts,
|
|
474
|
+
"sample_tests": [t.name for t in tests],
|
|
475
|
+
}
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
for context in contexts:
|
|
479
|
+
yield sse({"event": "context_start", "context": context})
|
|
480
|
+
results = []
|
|
481
|
+
for t in tests:
|
|
482
|
+
try:
|
|
483
|
+
item = bench.execute_test(base_url, req.profile, profile, t, req.timeout, context_override=context)
|
|
484
|
+
except Exception:
|
|
485
|
+
item = bench.TestResult(
|
|
486
|
+
name=t.name, category=t.category, success=False, elapsed_seconds=0.0,
|
|
487
|
+
response_length=0, response_preview="", accuracy=0.0, error="execute_test error",
|
|
488
|
+
)
|
|
489
|
+
results.append(item)
|
|
490
|
+
yield sse(
|
|
491
|
+
{
|
|
492
|
+
"event": "test_result", "context": context, "name": item.name,
|
|
493
|
+
"success": item.success, "elapsed_seconds": item.elapsed_seconds,
|
|
494
|
+
"context_limit_used": item.context_limit_used, "metrics": item.metrics,
|
|
495
|
+
}
|
|
496
|
+
)
|
|
497
|
+
successes = [r for r in results if r.success]
|
|
498
|
+
rates = [r.approx_tokens_per_second for r in successes if r.approx_tokens_per_second is not None]
|
|
499
|
+
ttft_values = [r.metrics.get("ttft_ms") for r in successes if r.metrics.get("ttft_ms") is not None]
|
|
500
|
+
actual_contexts = {r.context_limit_used for r in successes if r.context_limit_used}
|
|
501
|
+
yield sse(
|
|
502
|
+
{
|
|
503
|
+
"event": "context_end",
|
|
504
|
+
"context": context,
|
|
505
|
+
"actual_context_used": next(iter(actual_contexts), None) if len(actual_contexts) == 1 else None,
|
|
506
|
+
"passed": len(successes),
|
|
507
|
+
"tests": len(results),
|
|
508
|
+
"mean_tokens_per_second": round(sum(rates) / len(rates), 2) if rates else None,
|
|
509
|
+
"mean_ttft_ms": round(sum(ttft_values) / len(ttft_values), 1) if ttft_values else None,
|
|
510
|
+
"vram_used_mb": bench.nvidia_smi_used_mb(),
|
|
511
|
+
}
|
|
512
|
+
)
|
|
513
|
+
yield sse({"event": "sweep_end", "profile": req.profile})
|
|
514
|
+
yield "data: [DONE]\n\n"
|
|
515
|
+
|
|
516
|
+
return StreamingResponse(event_stream(), media_type="text/event-stream")
|