localdeploy 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. api_server.py +15 -0
  2. benchmark.py +15 -0
  3. chat_cli.py +15 -0
  4. compare_models.py +15 -0
  5. localdeploy/__init__.py +3 -0
  6. localdeploy/backends/__init__.py +0 -0
  7. localdeploy/backends/llamacpp.py +143 -0
  8. localdeploy/backends/ollama.py +374 -0
  9. localdeploy/backends/openai_compatible.py +179 -0
  10. localdeploy/benchmark.py +1996 -0
  11. localdeploy/chat_cli.py +249 -0
  12. localdeploy/cli.py +59 -0
  13. localdeploy/compare_models.py +481 -0
  14. localdeploy/control/__init__.py +75 -0
  15. localdeploy/control/_config.py +180 -0
  16. localdeploy/control/_ollama.py +253 -0
  17. localdeploy/control/bakeoff.py +216 -0
  18. localdeploy/control/bench.py +516 -0
  19. localdeploy/control/calibration.py +191 -0
  20. localdeploy/control/community.py +161 -0
  21. localdeploy/control/fit.py +651 -0
  22. localdeploy/control/hardware.py +602 -0
  23. localdeploy/control/history.py +125 -0
  24. localdeploy/control/manifest.py +452 -0
  25. localdeploy/control/models.py +833 -0
  26. localdeploy/control/monitor.py +378 -0
  27. localdeploy/control/profiles.py +161 -0
  28. localdeploy/control/recommend.py +322 -0
  29. localdeploy/control/registry.py +956 -0
  30. localdeploy/control/report.py +407 -0
  31. localdeploy/control/starter.py +520 -0
  32. localdeploy/control/updates.py +118 -0
  33. localdeploy/grader_sandbox.py +268 -0
  34. localdeploy/server.py +1873 -0
  35. localdeploy/utils.py +155 -0
  36. localdeploy/web/favicon.png +0 -0
  37. localdeploy/web/index.html +891 -0
  38. localdeploy/web/js/app.js +63 -0
  39. localdeploy/web/js/benchmark-views.js +73 -0
  40. localdeploy/web/js/benchmark.js +1691 -0
  41. localdeploy/web/js/chat.js +779 -0
  42. localdeploy/web/js/models.js +2758 -0
  43. localdeploy/web/js/shared.js +454 -0
  44. localdeploy/web/js/system.js +626 -0
  45. localdeploy/web/logo.svg +22 -0
  46. localdeploy/web/styles.css +2032 -0
  47. localdeploy-0.6.0.dist-info/METADATA +202 -0
  48. localdeploy-0.6.0.dist-info/RECORD +52 -0
  49. localdeploy-0.6.0.dist-info/WHEEL +5 -0
  50. localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
  51. localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
  52. localdeploy-0.6.0.dist-info/top_level.txt +5 -0
@@ -0,0 +1,516 @@
1
+ """Steps 7-8 - benchmark over the web.
2
+
3
+ GET /benchmark/example -> the canonical question-set example
4
+ POST /benchmark/validate -> validate an uploaded question set (no run)
5
+ POST /benchmark/run -> run the benchmark, streaming per-test results
6
+
7
+ All heavy lifting (grading, the run generator, the schema) lives in the
8
+ top-level `benchmark.py` so the CLI and this endpoint share one implementation.
9
+ `benchmark` is imported lazily so the server boot stays light and free of a
10
+ hard import-order coupling.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import threading
16
+ from datetime import datetime, timezone
17
+ from typing import Any, Dict, List, Optional
18
+
19
+ from fastapi import APIRouter, Request
20
+ from fastapi.responses import StreamingResponse
21
+ from pydantic import BaseModel, Field
22
+
23
+ router = APIRouter()
24
+
25
+ _bench_lock = threading.Lock()
26
+ _bench_module = None
27
+
28
+
29
+ def _bench():
30
+ """Lazy, memoized import of the top-level `benchmark` module.
31
+
32
+ FastAPI runs each request handler in a threadpool worker, so the first
33
+ few requests after startup (e.g. a page load firing several /benchmark/*
34
+ calls at once) can call this concurrently from different threads before
35
+ `benchmark` has ever been imported. A bare `import benchmark` in that
36
+ window is not reliably safe against concurrent first-import from
37
+ multiple threads; double-checked locking here makes the import happen
38
+ exactly once, with every other caller blocking on the lock rather than
39
+ racing it.
40
+ """
41
+ global _bench_module
42
+ if _bench_module is None:
43
+ with _bench_lock:
44
+ if _bench_module is None:
45
+ import benchmark # avoids importing the benchmark engine at boot
46
+
47
+ _bench_module = benchmark
48
+ return _bench_module
49
+
50
+
51
+ @router.get("/benchmark/example")
52
+ def benchmark_example() -> Dict[str, Any]:
53
+ return _bench().EXAMPLE_QUESTION_SET
54
+
55
+
56
+ @router.get("/benchmark/test-bench")
57
+ def benchmark_test_bench() -> Dict[str, Any]:
58
+ bench = _bench()
59
+ tests = bench.builtin_test_cases()
60
+ categories: Dict[str, int] = {}
61
+ for test in tests:
62
+ categories[test.category] = categories.get(test.category, 0) + 1
63
+ return {
64
+ "success": True,
65
+ "test_count": len(tests),
66
+ "categories": categories,
67
+ "question_set": bench.BUILTIN_QUESTION_SET,
68
+ "tests": [{"name": t.name, "category": t.category, "max_output_tokens": t.max_output_tokens} for t in tests],
69
+ }
70
+
71
+
72
+ @router.post("/benchmark/validate")
73
+ async def benchmark_validate(request: Request) -> Dict[str, Any]:
74
+ try:
75
+ payload = await request.json()
76
+ except Exception as exc:
77
+ return {
78
+ "success": True,
79
+ "valid": False,
80
+ "question_count": 0,
81
+ "errors": [{"index": -1, "error": f"invalid JSON body: {exc}"}],
82
+ "grader_types": _bench().GRADER_TYPES,
83
+ }
84
+ return _bench().validate_question_set(payload)
85
+
86
+
87
+ # --- benchmark packs (Release R4) --------------------------------------------
88
+ # A pack is a named, reusable preset over /benchmark/run's existing
89
+ # `include_categories` filter - not a new engine. "all" (the default UI
90
+ # behaviour today) intentionally has no entry here; omit include_categories
91
+ # for that.
92
+ BENCHMARK_PACKS: Dict[str, Dict[str, Any]] = {
93
+ "general": {
94
+ "label": "General assistant",
95
+ "categories": ["planning", "classification"],
96
+ "description": "Planning and classification - everyday assistant tasks.",
97
+ },
98
+ "coding": {
99
+ "label": "Coding",
100
+ "categories": ["code"],
101
+ "description": "Non-trivial Python/SQL, graded by AST parsing and unit tests.",
102
+ },
103
+ "structured": {
104
+ "label": "JSON and structured extraction",
105
+ "categories": ["structured", "structured_hard"],
106
+ "description": "Schema-shaped JSON output, including nested/conditional schemas.",
107
+ },
108
+ "reasoning": {
109
+ "label": "Reasoning",
110
+ "categories": ["math", "planning"],
111
+ "description": "Multi-step math and planning tasks that reward chain-of-thought.",
112
+ },
113
+ }
114
+
115
+
116
+ @router.get("/benchmark/packs")
117
+ def benchmark_packs() -> Dict[str, Any]:
118
+ bench = _bench()
119
+ tests = bench.builtin_test_cases()
120
+ known_categories = {t.category for t in tests}
121
+ packs = []
122
+ for key, pack in BENCHMARK_PACKS.items():
123
+ cats = [c for c in pack["categories"] if c in known_categories]
124
+ packs.append(
125
+ {
126
+ "id": key,
127
+ "label": pack["label"],
128
+ "description": pack["description"],
129
+ "categories": cats,
130
+ "test_count": sum(1 for t in tests if t.category in cats),
131
+ }
132
+ )
133
+ return {"success": True, "packs": packs}
134
+
135
+
136
+ class RunRequest(BaseModel):
137
+ profiles: Optional[List[str]] = None
138
+ questions: Optional[Dict[str, Any]] = None
139
+ device: Optional[str] = None
140
+ max_output_tokens: Optional[int] = None
141
+ timeout: int = 240
142
+ skip_categories: Optional[List[str]] = None
143
+ include_categories: Optional[List[str]] = None
144
+ pack: Optional[str] = None # convenience: expands to include_categories via BENCHMARK_PACKS
145
+ repetitions: int = Field(default=1, ge=1, le=10)
146
+
147
+
148
+ @router.post("/benchmark/run")
149
+ def benchmark_run(req: RunRequest):
150
+ bench = _bench()
151
+ from api_server import load_config # lazy: api_server owns config loading
152
+
153
+ profiles_map = load_config().get("profiles", {})
154
+ selected = req.profiles or [n for n, p in profiles_map.items() if p.get("enabled", False)]
155
+ selected = [n for n in selected if n in profiles_map]
156
+ if not selected:
157
+ return {"success": False, "error": "No profiles selected or enabled in config."}
158
+
159
+ # Build a fresh test list so request-specific limits and filtering cannot
160
+ # mutate the built-in definitions used by another request.
161
+ if req.questions is not None:
162
+ report = bench.validate_question_set(req.questions)
163
+ if not report["valid"]:
164
+ return {"success": False, "error": "invalid question set", "validation": report}
165
+ tests = bench.build_test_cases(req.questions)
166
+ else:
167
+ skip = set(req.skip_categories or [])
168
+ include = set(req.include_categories or [])
169
+ if not include and req.pack and req.pack in BENCHMARK_PACKS:
170
+ include = set(BENCHMARK_PACKS[req.pack]["categories"])
171
+ tests = [
172
+ t
173
+ for t in bench.builtin_test_cases()
174
+ if t.category not in skip and (not include or t.category in include)
175
+ ]
176
+ if not tests:
177
+ return {"success": False, "error": "No tests selected."}
178
+
179
+ if req.max_output_tokens:
180
+ for test in tests:
181
+ if test.category != "classification":
182
+ test.max_output_tokens = max(test.max_output_tokens, req.max_output_tokens)
183
+
184
+ base_url = bench.api_base_url()
185
+
186
+ def event_stream():
187
+ def sse(event: Dict[str, Any]) -> str:
188
+ return f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
189
+
190
+ model_routes = None
191
+ pre_existing: set[str] = set()
192
+ unloaded_profiles: set[str] = set()
193
+ requested_device = (req.device or "auto").strip().lower()
194
+
195
+ def ensure_model_routes():
196
+ nonlocal model_routes
197
+ if model_routes is None:
198
+ from . import models as routes # lazy: avoid extra startup coupling
199
+
200
+ model_routes = routes
201
+ return model_routes
202
+
203
+ def model_for_profile(profile_name: str) -> Optional[str]:
204
+ profile = profiles_map.get(profile_name) or {}
205
+ if str(profile.get("backend", "ollama")).lower() != "ollama":
206
+ return None
207
+ return str(profile.get("model_id") or profile_name)
208
+
209
+ def matches_running(model_id: str, running_name: str) -> bool:
210
+ routes = ensure_model_routes()
211
+ return routes._matches_model_name(running_name, model_id)
212
+
213
+ def was_pre_existing(model_id: str) -> bool:
214
+ return any(matches_running(model_id, name) for name in pre_existing)
215
+
216
+ def current_placement(profile_name: str) -> Optional[str]:
217
+ model_id = model_for_profile(profile_name)
218
+ if not model_id:
219
+ return None
220
+ try:
221
+ routes = ensure_model_routes()
222
+ running, _ = routes._ollama.list_running()
223
+ for item in running:
224
+ item.update(routes._placement(item.get("size"), item.get("size_vram")))
225
+ if matches_running(model_id, str(item.get("name") or "")):
226
+ placement = item.get("placement")
227
+ return str(placement).lower() if placement else None
228
+ except Exception:
229
+ return None
230
+ return None
231
+
232
+ def should_unload(profile_name: str) -> bool:
233
+ model_id = model_for_profile(profile_name)
234
+ if not model_id:
235
+ return False
236
+ if requested_device in {"cpu", "gpu"}:
237
+ return True
238
+ return not was_pre_existing(model_id)
239
+
240
+ def unload_profile(profile_name: str) -> Optional[Dict[str, Any]]:
241
+ if profile_name in unloaded_profiles or not should_unload(profile_name):
242
+ return None
243
+ model_id = model_for_profile(profile_name)
244
+ if not model_id:
245
+ return None
246
+ unloaded_profiles.add(profile_name)
247
+ try:
248
+ ensure_model_routes()._ollama.unload_model(model_id)
249
+ except Exception as exc:
250
+ return {
251
+ "event": "benchmark_unload_error",
252
+ "profile": profile_name,
253
+ "model_id": model_id,
254
+ "error": str(exc),
255
+ }
256
+ return {
257
+ "event": "benchmark_unload_end",
258
+ "profile": profile_name,
259
+ "model_id": model_id,
260
+ "message": f"Unloaded temporary benchmark model '{model_id}'.",
261
+ }
262
+
263
+ try:
264
+ try:
265
+ routes = ensure_model_routes()
266
+ running, _ = routes._ollama.list_running()
267
+ pre_existing = {str(item.get("name")) for item in running if item.get("name")}
268
+ except Exception:
269
+ pre_existing = set()
270
+
271
+ from localdeploy import __version__
272
+ from . import _ollama
273
+ from .hardware import detect_hardware
274
+
275
+ installed, _ = _ollama.list_installed()
276
+ installed_by_name = {str(item.get("name") or ""): item for item in installed}
277
+ ollama_version, _ = _ollama.version()
278
+ profile_provenance: Dict[str, Any] = {}
279
+ for profile_name in selected:
280
+ profile = profiles_map[profile_name]
281
+ backend = str(profile.get("backend") or "ollama").lower()
282
+ model_id = str(profile.get("model_id") or profile_name)
283
+ installed_item = next(
284
+ (
285
+ item
286
+ for installed_name, item in installed_by_name.items()
287
+ if backend == "ollama" and matches_running(model_id, installed_name)
288
+ ),
289
+ {},
290
+ )
291
+ details = installed_item.get("details") or {}
292
+ shown: Dict[str, Any] = {}
293
+ if backend == "ollama":
294
+ shown_result, _ = _ollama.show_model(model_id)
295
+ shown = shown_result or {}
296
+ shown_details = shown.get("details") if isinstance(shown.get("details"), dict) else {}
297
+ profile_provenance[profile_name] = {
298
+ "backend": backend,
299
+ "backend_version": ollama_version if backend == "ollama" else None,
300
+ "model": model_id,
301
+ "model_digest": installed_item.get("digest"),
302
+ "quant": details.get("quantization_level")
303
+ or shown_details.get("quantization_level")
304
+ or profile.get("quantization"),
305
+ "context": profile.get("context_limit"),
306
+ "warm_state": "warm"
307
+ if backend == "ollama" and any(matches_running(model_id, name) for name in pre_existing)
308
+ else "cold",
309
+ }
310
+ provenance = {
311
+ "recorded_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
312
+ "localdeploy_version": __version__,
313
+ "hardware": detect_hardware(),
314
+ "providers": {"ollama": {"version": ollama_version}},
315
+ "profiles": profile_provenance,
316
+ }
317
+
318
+ # For a forced CPU/GPU run, pin the same num_gpu on every inference
319
+ # call (not just the warm-up) so the benchmark measures the requested
320
+ # device end-to-end instead of whatever Ollama re-places it onto.
321
+ forced_num_gpu: Optional[int] = None
322
+ force_deploy = requested_device in {"cpu", "gpu"}
323
+ if force_deploy:
324
+ routes = ensure_model_routes()
325
+ forced_num_gpu = routes._resolve_num_gpu(requested_device, None)
326
+ # Fail fast, before deploying anything, if any selected profile
327
+ # can't be device-forced.
328
+ for profile_name in selected:
329
+ backend = str(profiles_map[profile_name].get("backend", "ollama")).lower()
330
+ if backend != "ollama":
331
+ yield sse(
332
+ {
333
+ "event": "error",
334
+ "error": (
335
+ "Benchmark device forcing is only supported for Ollama profiles; "
336
+ f"{profile_name} uses {backend}"
337
+ ),
338
+ }
339
+ )
340
+ return
341
+
342
+ peak_vram_by_profile: Dict[str, int] = {}
343
+
344
+ def _track_peak_vram(profile_name: str) -> None:
345
+ # Best-effort (nvidia-smi only, same as the CLI's VRAM sampling);
346
+ # a missing/failed reading just means no peak_vram_mb is reported.
347
+ used = bench.nvidia_smi_used_mb()
348
+ if used is not None:
349
+ peak_vram_by_profile[profile_name] = max(peak_vram_by_profile.get(profile_name, 0), used)
350
+
351
+ for event in bench.iter_run(
352
+ base_url,
353
+ profiles_map,
354
+ selected,
355
+ tests,
356
+ req.timeout,
357
+ num_gpu=forced_num_gpu,
358
+ repetitions=req.repetitions,
359
+ provenance=provenance,
360
+ ):
361
+ if force_deploy and event.get("event") == "profile_start":
362
+ # Deploy just this profile's model right before its tests run
363
+ # (instead of deploying every selected profile up front) so a
364
+ # multi-profile forced-device comparison never stacks more
365
+ # than one model in VRAM/RAM at a time.
366
+ profile_name = str(event.get("profile") or "")
367
+ model_id = str(event.get("model_id") or profile_name)
368
+ yield sse(
369
+ {
370
+ "event": "deploy_start",
371
+ "profile": profile_name,
372
+ "model_id": model_id,
373
+ "device": requested_device,
374
+ }
375
+ )
376
+ served = model_routes._serve_ollama(model_id, "60m", forced_num_gpu)
377
+ if not served.get("success"):
378
+ yield sse(
379
+ {
380
+ "event": "error",
381
+ "error": served.get("error") or served.get("message") or "model deploy failed",
382
+ "profile": profile_name,
383
+ "device": requested_device,
384
+ }
385
+ )
386
+ unload_event = unload_profile(profile_name)
387
+ if unload_event:
388
+ yield sse(unload_event)
389
+ break
390
+ deploy_end = {
391
+ "event": "deploy_end",
392
+ "profile": profile_name,
393
+ "model_id": model_id,
394
+ "device": requested_device,
395
+ }
396
+ if served.get("warning"):
397
+ deploy_end["warning"] = served["warning"]
398
+ yield sse(deploy_end)
399
+
400
+ if event.get("event") == "test_result":
401
+ _track_peak_vram(str(event.get("profile") or ""))
402
+ if event.get("event") == "profile_end":
403
+ profile_name = str(event.get("profile") or "")
404
+ actual = current_placement(profile_name)
405
+ if actual:
406
+ event["actual_device"] = actual
407
+ if profile_name in peak_vram_by_profile and isinstance(event.get("summary"), dict):
408
+ event["summary"]["peak_vram_mb"] = peak_vram_by_profile[profile_name]
409
+ yield sse(event)
410
+ if event.get("event") in {"profile_end", "profile_aborted"}:
411
+ unload_event = unload_profile(str(event.get("profile") or ""))
412
+ if unload_event:
413
+ yield sse(unload_event)
414
+ except Exception as exc: # never break the SSE contract
415
+ yield sse({"event": "error", "error": str(exc)})
416
+ finally:
417
+ # Best-effort cleanup for client aborts and unexpected stream exits.
418
+ for profile_name in selected:
419
+ if profile_name in unloaded_profiles or not should_unload(profile_name):
420
+ continue
421
+ model_id = model_for_profile(profile_name)
422
+ if not model_id:
423
+ continue
424
+ unloaded_profiles.add(profile_name)
425
+ try:
426
+ ensure_model_routes()._ollama.unload_model(model_id)
427
+ except Exception:
428
+ pass
429
+ yield "data: [DONE]\n\n"
430
+
431
+ return StreamingResponse(event_stream(), media_type="text/event-stream")
432
+
433
+
434
+ # --- context-scaling sweep (Release R4) --------------------------------------
435
+ # "How does memory, prompt-processing speed, first-token latency, and
436
+ # generation speed change with context?" - the same profile, the same short
437
+ # test subset used by /system/recommend, run once per context tier via the
438
+ # /chat endpoint's existing context_limit override. No separate estimation or
439
+ # grading logic: this reuses execute_test, so numbers agree with every other
440
+ # benchmark surface in the app.
441
+ DEFAULT_SWEEP_CONTEXTS = [4096, 8192, 16384, 32768, 65536]
442
+
443
+
444
+ class ContextSweepRequest(BaseModel):
445
+ profile: str
446
+ contexts: List[int] = Field(default_factory=lambda: list(DEFAULT_SWEEP_CONTEXTS))
447
+ sample_size: int = Field(default=3, ge=1, le=10)
448
+ timeout: int = Field(default=120, ge=1, le=3_600)
449
+
450
+
451
+ @router.post("/benchmark/context-sweep")
452
+ def benchmark_context_sweep(req: ContextSweepRequest):
453
+ bench = _bench()
454
+ from api_server import load_config # lazy: api_server owns config loading
455
+
456
+ profiles_map = load_config().get("profiles", {})
457
+
458
+ def event_stream():
459
+ def sse(event: Dict[str, Any]) -> str:
460
+ return f"data: {json.dumps(event, ensure_ascii=False)}\n\n"
461
+
462
+ if req.profile not in profiles_map:
463
+ yield sse({"event": "error", "error": f"Unknown profile '{req.profile}'."})
464
+ yield "data: [DONE]\n\n"
465
+ return
466
+
467
+ profile = profiles_map[req.profile]
468
+ tests = sorted(bench.builtin_test_cases(), key=lambda t: t.max_output_tokens)[: max(1, req.sample_size)]
469
+ base_url = bench.api_base_url()
470
+ contexts = sorted(dict.fromkeys(c for c in req.contexts if c > 0))
471
+ yield sse(
472
+ {
473
+ "event": "sweep_start", "profile": req.profile, "contexts": contexts,
474
+ "sample_tests": [t.name for t in tests],
475
+ }
476
+ )
477
+
478
+ for context in contexts:
479
+ yield sse({"event": "context_start", "context": context})
480
+ results = []
481
+ for t in tests:
482
+ try:
483
+ item = bench.execute_test(base_url, req.profile, profile, t, req.timeout, context_override=context)
484
+ except Exception:
485
+ item = bench.TestResult(
486
+ name=t.name, category=t.category, success=False, elapsed_seconds=0.0,
487
+ response_length=0, response_preview="", accuracy=0.0, error="execute_test error",
488
+ )
489
+ results.append(item)
490
+ yield sse(
491
+ {
492
+ "event": "test_result", "context": context, "name": item.name,
493
+ "success": item.success, "elapsed_seconds": item.elapsed_seconds,
494
+ "context_limit_used": item.context_limit_used, "metrics": item.metrics,
495
+ }
496
+ )
497
+ successes = [r for r in results if r.success]
498
+ rates = [r.approx_tokens_per_second for r in successes if r.approx_tokens_per_second is not None]
499
+ ttft_values = [r.metrics.get("ttft_ms") for r in successes if r.metrics.get("ttft_ms") is not None]
500
+ actual_contexts = {r.context_limit_used for r in successes if r.context_limit_used}
501
+ yield sse(
502
+ {
503
+ "event": "context_end",
504
+ "context": context,
505
+ "actual_context_used": next(iter(actual_contexts), None) if len(actual_contexts) == 1 else None,
506
+ "passed": len(successes),
507
+ "tests": len(results),
508
+ "mean_tokens_per_second": round(sum(rates) / len(rates), 2) if rates else None,
509
+ "mean_ttft_ms": round(sum(ttft_values) / len(ttft_values), 1) if ttft_values else None,
510
+ "vram_used_mb": bench.nvidia_smi_used_mb(),
511
+ }
512
+ )
513
+ yield sse({"event": "sweep_end", "profile": req.profile})
514
+ yield "data: [DONE]\n\n"
515
+
516
+ return StreamingResponse(event_stream(), media_type="text/event-stream")