localdeploy 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. api_server.py +15 -0
  2. benchmark.py +15 -0
  3. chat_cli.py +15 -0
  4. compare_models.py +15 -0
  5. localdeploy/__init__.py +3 -0
  6. localdeploy/backends/__init__.py +0 -0
  7. localdeploy/backends/llamacpp.py +143 -0
  8. localdeploy/backends/ollama.py +374 -0
  9. localdeploy/backends/openai_compatible.py +179 -0
  10. localdeploy/benchmark.py +1996 -0
  11. localdeploy/chat_cli.py +249 -0
  12. localdeploy/cli.py +59 -0
  13. localdeploy/compare_models.py +481 -0
  14. localdeploy/control/__init__.py +75 -0
  15. localdeploy/control/_config.py +180 -0
  16. localdeploy/control/_ollama.py +253 -0
  17. localdeploy/control/bakeoff.py +216 -0
  18. localdeploy/control/bench.py +516 -0
  19. localdeploy/control/calibration.py +191 -0
  20. localdeploy/control/community.py +161 -0
  21. localdeploy/control/fit.py +651 -0
  22. localdeploy/control/hardware.py +602 -0
  23. localdeploy/control/history.py +125 -0
  24. localdeploy/control/manifest.py +452 -0
  25. localdeploy/control/models.py +833 -0
  26. localdeploy/control/monitor.py +378 -0
  27. localdeploy/control/profiles.py +161 -0
  28. localdeploy/control/recommend.py +322 -0
  29. localdeploy/control/registry.py +956 -0
  30. localdeploy/control/report.py +407 -0
  31. localdeploy/control/starter.py +520 -0
  32. localdeploy/control/updates.py +118 -0
  33. localdeploy/grader_sandbox.py +268 -0
  34. localdeploy/server.py +1873 -0
  35. localdeploy/utils.py +155 -0
  36. localdeploy/web/favicon.png +0 -0
  37. localdeploy/web/index.html +891 -0
  38. localdeploy/web/js/app.js +63 -0
  39. localdeploy/web/js/benchmark-views.js +73 -0
  40. localdeploy/web/js/benchmark.js +1691 -0
  41. localdeploy/web/js/chat.js +779 -0
  42. localdeploy/web/js/models.js +2758 -0
  43. localdeploy/web/js/shared.js +454 -0
  44. localdeploy/web/js/system.js +626 -0
  45. localdeploy/web/logo.svg +22 -0
  46. localdeploy/web/styles.css +2032 -0
  47. localdeploy-0.6.0.dist-info/METADATA +202 -0
  48. localdeploy-0.6.0.dist-info/RECORD +52 -0
  49. localdeploy-0.6.0.dist-info/WHEEL +5 -0
  50. localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
  51. localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
  52. localdeploy-0.6.0.dist-info/top_level.txt +5 -0
@@ -0,0 +1,191 @@
1
+ """Estimated-vs-observed VRAM calibration (Release R2).
2
+
3
+ Every successful Ollama warm-up (see ``models._serve_ollama``) knows two
4
+ numbers for the same deployment: what ``fit.py`` predicted before load, and
5
+ what Ollama actually placed on the GPU (``size_vram`` from ``/api/ps``). This
6
+ module stores that pair, keyed by the configuration it applies to, and turns
7
+ repeated samples into a correction factor future fit checks can apply.
8
+
9
+ Storage: one JSON file at ``app_home()/calibration.json`` (schema_version 1),
10
+ gitignored and machine-local like config.json/history. Writes are atomic
11
+ (reuses ``_config.write_config_atomic``) so a crash mid-write can't corrupt it.
12
+
13
+ Nothing here is silent: ``get_correction`` always reports ``sample_count`` and
14
+ ``factor`` so a caller can show "calibrated +7.7% from 5 samples on this GPU"
15
+ instead of quietly nudging a number.
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import re
20
+ import statistics
21
+ import threading
22
+ import time
23
+ from pathlib import Path
24
+ from typing import Any, Dict, List, Optional
25
+
26
+ from ._config import write_config_atomic
27
+ from .hardware import detect_hardware
28
+
29
+ _SCHEMA_VERSION = 1
30
+ _MAX_SAMPLES_PER_KEY = 50 # oldest samples drop off; recent hardware/driver state matters most
31
+ # Guards the load-mutate-save sequence in record_sample: two concurrent serves
32
+ # (e.g. two models warming up close together) would otherwise both read the
33
+ # same on-disk state and the second write would silently clobber the first.
34
+ _write_lock = threading.Lock()
35
+
36
+ # Context is bucketed (not stored exact) so "4096 vs 4100" doesn't fragment
37
+ # calibration into useless singleton buckets. Buckets mirror fit-table tiers.
38
+ _CONTEXT_BUCKETS = [4096, 8192, 16384, 32768, 65536, 131072]
39
+
40
+
41
+ def _context_bucket(context: Optional[int]) -> int:
42
+ context = int(context or 4096)
43
+ for bucket in _CONTEXT_BUCKETS:
44
+ if context <= bucket:
45
+ return bucket
46
+ return _CONTEXT_BUCKETS[-1]
47
+
48
+
49
+ def gpu_key(hw: Optional[Dict[str, Any]] = None) -> str:
50
+ """A stable-ish identifier for 'the GPU pool fit checks are budgeting against'.
51
+
52
+ Uses the best compatible pool's vendor+backend+name (not free/total VRAM,
53
+ which changes sample to sample). Falls back to 'cpu' when no GPU was
54
+ detected, so CPU-only calibration still groups sensibly.
55
+ """
56
+ hw = hw or detect_hardware()
57
+ gpus = hw.get("gpus") or []
58
+ if not gpus:
59
+ return "cpu"
60
+ gpu = gpus[0]
61
+ name = re.sub(r"[^a-z0-9]+", "-", str(gpu.get("name") or "gpu").lower()).strip("-")
62
+ return f"{gpu.get('vendor') or '?'}:{gpu.get('backend') or '?'}:{name}"
63
+
64
+
65
+ def sample_key(gpu: str, runtime: str, family: Optional[str], quant: Optional[str], context: Optional[int]) -> str:
66
+ fam = (family or "unknown").lower()
67
+ q = (quant or "default").lower()
68
+ return f"{gpu}|{runtime}|{fam}|{q}|{_context_bucket(context)}"
69
+
70
+
71
+ def _store_path() -> Path:
72
+ from ..utils import app_home
73
+
74
+ return app_home() / "calibration.json"
75
+
76
+
77
+ def _load() -> Dict[str, Any]:
78
+ path = _store_path()
79
+ if not path.is_file():
80
+ return {"schema_version": _SCHEMA_VERSION, "samples": {}}
81
+ try:
82
+ import json
83
+
84
+ data = json.loads(path.read_text(encoding="utf-8"))
85
+ except (OSError, ValueError):
86
+ return {"schema_version": _SCHEMA_VERSION, "samples": {}}
87
+ if not isinstance(data, dict) or not isinstance(data.get("samples"), dict):
88
+ return {"schema_version": _SCHEMA_VERSION, "samples": {}}
89
+ return data
90
+
91
+
92
+ def _save(data: Dict[str, Any]) -> Optional[str]:
93
+ data["schema_version"] = _SCHEMA_VERSION
94
+ return write_config_atomic(data, _store_path())
95
+
96
+
97
+ def record_sample(
98
+ *,
99
+ gpu: str,
100
+ runtime: str,
101
+ family: Optional[str],
102
+ quant: Optional[str],
103
+ context: Optional[int],
104
+ estimated_gb: float,
105
+ observed_gb: float,
106
+ ) -> Dict[str, Any]:
107
+ """Append one (estimated, observed) pair. Best-effort: never raises.
108
+
109
+ Returns the sample recorded plus the key it landed under, or an
110
+ ``error`` string when the store couldn't be written.
111
+ """
112
+ key = sample_key(gpu, runtime, family, quant, context)
113
+ try:
114
+ with _write_lock:
115
+ data = _load()
116
+ samples: List[Dict[str, Any]] = data["samples"].setdefault(key, [])
117
+ samples.append(
118
+ {
119
+ "ts": time.time(),
120
+ "estimated_gb": round(float(estimated_gb), 3),
121
+ "observed_gb": round(float(observed_gb), 3),
122
+ }
123
+ )
124
+ del samples[:-_MAX_SAMPLES_PER_KEY]
125
+ err = _save(data)
126
+ if err:
127
+ return {"key": key, "error": err}
128
+ return {"key": key, "sample_count": len(samples)}
129
+ except Exception as exc: # calibration is a bonus signal, never a hard dependency
130
+ return {"key": key, "error": str(exc)}
131
+
132
+
133
+ def _confidence_for(sample_count: int) -> str:
134
+ if sample_count >= 10:
135
+ return "high"
136
+ if sample_count >= 3:
137
+ return "medium"
138
+ if sample_count >= 1:
139
+ return "low"
140
+ return "none"
141
+
142
+
143
+ def get_correction(
144
+ *, gpu: str, runtime: str, family: Optional[str], quant: Optional[str], context: Optional[int]
145
+ ) -> Dict[str, Any]:
146
+ """Median observed/estimated ratio for this exact key. No cross-key blending -
147
+ materially different configurations are never averaged together."""
148
+ key = sample_key(gpu, runtime, family, quant, context)
149
+ data = _load()
150
+ samples = data["samples"].get(key) or []
151
+ if not samples:
152
+ return {"key": key, "applied": False, "factor": 1.0, "sample_count": 0, "confidence": "none"}
153
+ ratios = [
154
+ s["observed_gb"] / s["estimated_gb"]
155
+ for s in samples
156
+ if isinstance(s.get("estimated_gb"), (int, float)) and s.get("estimated_gb", 0) > 0
157
+ ]
158
+ if not ratios:
159
+ return {"key": key, "applied": False, "factor": 1.0, "sample_count": 0, "confidence": "none"}
160
+ factor = statistics.median(ratios)
161
+ return {
162
+ "key": key,
163
+ "applied": True,
164
+ "factor": round(factor, 4),
165
+ "sample_count": len(ratios),
166
+ "confidence": _confidence_for(len(ratios)),
167
+ "last_sample_estimated_gb": samples[-1]["estimated_gb"],
168
+ "last_sample_observed_gb": samples[-1]["observed_gb"],
169
+ }
170
+
171
+
172
+ def stats() -> Dict[str, Any]:
173
+ """Overall calibration coverage: how many keys have data, and the mean
174
+ absolute prediction error across every recorded sample (the estimator
175
+ self-grading metric)."""
176
+ data = _load()
177
+ all_samples: List[Dict[str, Any]] = []
178
+ for samples in data["samples"].values():
179
+ all_samples.extend(samples)
180
+ if not all_samples:
181
+ return {"keys": 0, "samples": 0, "mean_abs_error_pct": None}
182
+ errors_pct = [
183
+ abs(s["observed_gb"] - s["estimated_gb"]) / s["estimated_gb"] * 100
184
+ for s in all_samples
185
+ if s.get("estimated_gb")
186
+ ]
187
+ return {
188
+ "keys": len(data["samples"]),
189
+ "samples": len(all_samples),
190
+ "mean_abs_error_pct": round(statistics.mean(errors_pct), 2) if errors_pct else None,
191
+ }
@@ -0,0 +1,161 @@
1
+ """Community benchmark sharing - local-only groundwork (Release R8).
2
+
3
+ LocalDeploy's whole premise is "no telemetry" (see README Privacy & Security).
4
+ A real community dataset needs a hosted server, a moderation/abuse story, and
5
+ a data-retention policy - an infrastructure and product decision this repo
6
+ has not made, not something to wire up silently inside a feature PR. Both
7
+ endpoints below are honest about that: they compute and preview exactly what
8
+ *would* be shared, and "export" saves that anonymized snapshot to a local
9
+ file - there is no submission call anywhere in this module, because there is
10
+ nowhere to submit to yet.
11
+
12
+ POST /system/community/preview -> the anonymized payload, never persisted
13
+ POST /system/community/export -> the same payload, saved to
14
+ reports/community-contributions/ locally
15
+ """
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ import platform
20
+ import time
21
+ from pathlib import Path
22
+ from typing import Any, Dict, Optional
23
+
24
+ from fastapi import APIRouter
25
+ from pydantic import BaseModel
26
+
27
+ from .. import __version__
28
+
29
+ router = APIRouter()
30
+
31
+ SCHEMA_VERSION = 1
32
+
33
+ # Fields explicitly never copied into the shared payload, no matter what the
34
+ # input card contains - listed here (not just in a docstring) so it doubles
35
+ # as documentation of the promise and is easy to audit.
36
+ EXCLUDED_FIELDS = [
37
+ "username", "computer name", "IP address",
38
+ "model prompts", "model responses", "response previews",
39
+ "filesystem paths", "API keys", "local profile name",
40
+ ]
41
+
42
+ _TEST_NUMERIC_FIELDS = ("name", "category", "success", "accuracy", "elapsed_seconds", "approx_tokens_per_second")
43
+
44
+
45
+ def _os_category() -> str:
46
+ return {"Windows": "Windows", "Darwin": "macOS", "Linux": "Linux"}.get(platform.system(), platform.system())
47
+
48
+
49
+ def _round_or_none(value: Any, digits: int = 1) -> Optional[float]:
50
+ try:
51
+ return round(float(value), digits) if value is not None else None
52
+ except (TypeError, ValueError):
53
+ return None
54
+
55
+
56
+ def _anonymize_test(test: Dict[str, Any]) -> Dict[str, Any]:
57
+ """Numeric result fields only - never response_preview or error text,
58
+ which can contain fragments of the user's actual prompts/outputs."""
59
+ out = {k: test.get(k) for k in _TEST_NUMERIC_FIELDS}
60
+ metrics = test.get("metrics") or {}
61
+ numeric_metrics = {
62
+ k: metrics.get(k)
63
+ for k in ("ttft_ms", "tokens_per_second", "prompt_tokens_per_second")
64
+ if metrics.get(k) is not None
65
+ }
66
+ if numeric_metrics:
67
+ out["metrics"] = numeric_metrics
68
+ return out
69
+
70
+
71
+ def anonymize_card(card: Dict[str, Any]) -> Dict[str, Any]:
72
+ """Whitelist-copy a report-card-shaped payload (report.py's build_card
73
+ output, or an equivalent run record) down to what the community-sharing
74
+ plan defines as safe to share. Everything not explicitly copied here is
75
+ dropped - this is a whitelist, not a blocklist, by design."""
76
+ hardware = card.get("hardware") or {}
77
+ provenance = card.get("provenance") or {}
78
+ prov_hardware = provenance.get("hardware") or {}
79
+ gpus = prov_hardware.get("gpus") or hardware.get("gpus") or []
80
+ best_gpu = gpus[0] if gpus else {}
81
+ system = prov_hardware.get("system") or {}
82
+
83
+ profile_name = card.get("profile")
84
+ profile_provenance = (provenance.get("profiles") or {}).get(profile_name, {}) if profile_name else {}
85
+
86
+ summary = card.get("summary") or {}
87
+ tests = [_anonymize_test(t) for t in (card.get("tests") or [])]
88
+
89
+ return {
90
+ "schema_version": SCHEMA_VERSION,
91
+ "hardware": {
92
+ "gpu": best_gpu.get("name") or hardware.get("gpu"),
93
+ "vram_gb": _round_or_none((best_gpu.get("vram_total_mb") or hardware.get("vram_total_mb") or 0) / 1024.0) or None,
94
+ "cpu": system.get("cpu_model"),
95
+ "ram_gb": _round_or_none((system.get("ram_total_mb") or 0) / 1024.0) or None,
96
+ "os_category": _os_category(),
97
+ },
98
+ "runtime": {
99
+ "provider": profile_provenance.get("backend") or "ollama",
100
+ "version": profile_provenance.get("backend_version"),
101
+ },
102
+ "model": {
103
+ "id": card.get("model_id"),
104
+ "digest": profile_provenance.get("model_digest"),
105
+ "quantization": profile_provenance.get("quant"),
106
+ "context": profile_provenance.get("context"),
107
+ },
108
+ "device": card.get("device"),
109
+ "performance": {
110
+ "avg_accuracy": summary.get("avg_accuracy"),
111
+ "avg_latency_s": summary.get("avg_latency_s"),
112
+ "avg_tokens_per_second": summary.get("avg_tokens_per_second"),
113
+ "avg_ttft_ms": summary.get("avg_ttft_ms"),
114
+ "peak_vram_mb": card.get("peak_vram_mb"),
115
+ },
116
+ "tests": tests,
117
+ "repetitions": card.get("repetitions") or 1,
118
+ "localdeploy_version": __version__,
119
+ }
120
+
121
+
122
+ class CommunityShareRequest(BaseModel):
123
+ card: Dict[str, Any]
124
+
125
+
126
+ @router.post("/system/community/preview")
127
+ def community_preview(req: CommunityShareRequest) -> Dict[str, Any]:
128
+ return {
129
+ "success": True,
130
+ "would_share": anonymize_card(req.card),
131
+ "excluded_fields": EXCLUDED_FIELDS,
132
+ "note": (
133
+ "Preview only - nothing is sent anywhere. LocalDeploy does not have a "
134
+ "community server yet, so there is currently no submit action; "
135
+ "'Save locally' keeps this anonymized snapshot on disk for later."
136
+ ),
137
+ }
138
+
139
+
140
+ def _contributions_dir() -> Path:
141
+ from ..utils import app_home
142
+
143
+ return app_home() / "reports" / "community-contributions"
144
+
145
+
146
+ @router.post("/system/community/export")
147
+ def community_export(req: CommunityShareRequest) -> Dict[str, Any]:
148
+ payload = anonymize_card(req.card)
149
+ try:
150
+ directory = _contributions_dir()
151
+ directory.mkdir(parents=True, exist_ok=True)
152
+ path = directory / f"contribution-{int(time.time() * 1000)}.json"
153
+ path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
154
+ except OSError as exc:
155
+ return {"success": False, "error": f"Could not save locally: {exc}"}
156
+ return {
157
+ "success": True,
158
+ "path": str(path),
159
+ "would_share": payload,
160
+ "note": "Saved locally only - not transmitted anywhere. LocalDeploy has no community server yet.",
161
+ }