localdeploy 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api_server.py +15 -0
- benchmark.py +15 -0
- chat_cli.py +15 -0
- compare_models.py +15 -0
- localdeploy/__init__.py +3 -0
- localdeploy/backends/__init__.py +0 -0
- localdeploy/backends/llamacpp.py +143 -0
- localdeploy/backends/ollama.py +374 -0
- localdeploy/backends/openai_compatible.py +179 -0
- localdeploy/benchmark.py +1996 -0
- localdeploy/chat_cli.py +249 -0
- localdeploy/cli.py +59 -0
- localdeploy/compare_models.py +481 -0
- localdeploy/control/__init__.py +75 -0
- localdeploy/control/_config.py +180 -0
- localdeploy/control/_ollama.py +253 -0
- localdeploy/control/bakeoff.py +216 -0
- localdeploy/control/bench.py +516 -0
- localdeploy/control/calibration.py +191 -0
- localdeploy/control/community.py +161 -0
- localdeploy/control/fit.py +651 -0
- localdeploy/control/hardware.py +602 -0
- localdeploy/control/history.py +125 -0
- localdeploy/control/manifest.py +452 -0
- localdeploy/control/models.py +833 -0
- localdeploy/control/monitor.py +378 -0
- localdeploy/control/profiles.py +161 -0
- localdeploy/control/recommend.py +322 -0
- localdeploy/control/registry.py +956 -0
- localdeploy/control/report.py +407 -0
- localdeploy/control/starter.py +520 -0
- localdeploy/control/updates.py +118 -0
- localdeploy/grader_sandbox.py +268 -0
- localdeploy/server.py +1873 -0
- localdeploy/utils.py +155 -0
- localdeploy/web/favicon.png +0 -0
- localdeploy/web/index.html +891 -0
- localdeploy/web/js/app.js +63 -0
- localdeploy/web/js/benchmark-views.js +73 -0
- localdeploy/web/js/benchmark.js +1691 -0
- localdeploy/web/js/chat.js +779 -0
- localdeploy/web/js/models.js +2758 -0
- localdeploy/web/js/shared.js +454 -0
- localdeploy/web/js/system.js +626 -0
- localdeploy/web/logo.svg +22 -0
- localdeploy/web/styles.css +2032 -0
- localdeploy-0.6.0.dist-info/METADATA +202 -0
- localdeploy-0.6.0.dist-info/RECORD +52 -0
- localdeploy-0.6.0.dist-info/WHEEL +5 -0
- localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
- localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
- localdeploy-0.6.0.dist-info/top_level.txt +5 -0
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
"""Estimated-vs-observed VRAM calibration (Release R2).
|
|
2
|
+
|
|
3
|
+
Every successful Ollama warm-up (see ``models._serve_ollama``) knows two
|
|
4
|
+
numbers for the same deployment: what ``fit.py`` predicted before load, and
|
|
5
|
+
what Ollama actually placed on the GPU (``size_vram`` from ``/api/ps``). This
|
|
6
|
+
module stores that pair, keyed by the configuration it applies to, and turns
|
|
7
|
+
repeated samples into a correction factor future fit checks can apply.
|
|
8
|
+
|
|
9
|
+
Storage: one JSON file at ``app_home()/calibration.json`` (schema_version 1),
|
|
10
|
+
gitignored and machine-local like config.json/history. Writes are atomic
|
|
11
|
+
(reuses ``_config.write_config_atomic``) so a crash mid-write can't corrupt it.
|
|
12
|
+
|
|
13
|
+
Nothing here is silent: ``get_correction`` always reports ``sample_count`` and
|
|
14
|
+
``factor`` so a caller can show "calibrated +7.7% from 5 samples on this GPU"
|
|
15
|
+
instead of quietly nudging a number.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import re
|
|
20
|
+
import statistics
|
|
21
|
+
import threading
|
|
22
|
+
import time
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
from typing import Any, Dict, List, Optional
|
|
25
|
+
|
|
26
|
+
from ._config import write_config_atomic
|
|
27
|
+
from .hardware import detect_hardware
|
|
28
|
+
|
|
29
|
+
_SCHEMA_VERSION = 1
|
|
30
|
+
_MAX_SAMPLES_PER_KEY = 50 # oldest samples drop off; recent hardware/driver state matters most
|
|
31
|
+
# Guards the load-mutate-save sequence in record_sample: two concurrent serves
|
|
32
|
+
# (e.g. two models warming up close together) would otherwise both read the
|
|
33
|
+
# same on-disk state and the second write would silently clobber the first.
|
|
34
|
+
_write_lock = threading.Lock()
|
|
35
|
+
|
|
36
|
+
# Context is bucketed (not stored exact) so "4096 vs 4100" doesn't fragment
|
|
37
|
+
# calibration into useless singleton buckets. Buckets mirror fit-table tiers.
|
|
38
|
+
_CONTEXT_BUCKETS = [4096, 8192, 16384, 32768, 65536, 131072]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _context_bucket(context: Optional[int]) -> int:
|
|
42
|
+
context = int(context or 4096)
|
|
43
|
+
for bucket in _CONTEXT_BUCKETS:
|
|
44
|
+
if context <= bucket:
|
|
45
|
+
return bucket
|
|
46
|
+
return _CONTEXT_BUCKETS[-1]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def gpu_key(hw: Optional[Dict[str, Any]] = None) -> str:
|
|
50
|
+
"""A stable-ish identifier for 'the GPU pool fit checks are budgeting against'.
|
|
51
|
+
|
|
52
|
+
Uses the best compatible pool's vendor+backend+name (not free/total VRAM,
|
|
53
|
+
which changes sample to sample). Falls back to 'cpu' when no GPU was
|
|
54
|
+
detected, so CPU-only calibration still groups sensibly.
|
|
55
|
+
"""
|
|
56
|
+
hw = hw or detect_hardware()
|
|
57
|
+
gpus = hw.get("gpus") or []
|
|
58
|
+
if not gpus:
|
|
59
|
+
return "cpu"
|
|
60
|
+
gpu = gpus[0]
|
|
61
|
+
name = re.sub(r"[^a-z0-9]+", "-", str(gpu.get("name") or "gpu").lower()).strip("-")
|
|
62
|
+
return f"{gpu.get('vendor') or '?'}:{gpu.get('backend') or '?'}:{name}"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def sample_key(gpu: str, runtime: str, family: Optional[str], quant: Optional[str], context: Optional[int]) -> str:
|
|
66
|
+
fam = (family or "unknown").lower()
|
|
67
|
+
q = (quant or "default").lower()
|
|
68
|
+
return f"{gpu}|{runtime}|{fam}|{q}|{_context_bucket(context)}"
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _store_path() -> Path:
|
|
72
|
+
from ..utils import app_home
|
|
73
|
+
|
|
74
|
+
return app_home() / "calibration.json"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _load() -> Dict[str, Any]:
|
|
78
|
+
path = _store_path()
|
|
79
|
+
if not path.is_file():
|
|
80
|
+
return {"schema_version": _SCHEMA_VERSION, "samples": {}}
|
|
81
|
+
try:
|
|
82
|
+
import json
|
|
83
|
+
|
|
84
|
+
data = json.loads(path.read_text(encoding="utf-8"))
|
|
85
|
+
except (OSError, ValueError):
|
|
86
|
+
return {"schema_version": _SCHEMA_VERSION, "samples": {}}
|
|
87
|
+
if not isinstance(data, dict) or not isinstance(data.get("samples"), dict):
|
|
88
|
+
return {"schema_version": _SCHEMA_VERSION, "samples": {}}
|
|
89
|
+
return data
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _save(data: Dict[str, Any]) -> Optional[str]:
|
|
93
|
+
data["schema_version"] = _SCHEMA_VERSION
|
|
94
|
+
return write_config_atomic(data, _store_path())
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def record_sample(
|
|
98
|
+
*,
|
|
99
|
+
gpu: str,
|
|
100
|
+
runtime: str,
|
|
101
|
+
family: Optional[str],
|
|
102
|
+
quant: Optional[str],
|
|
103
|
+
context: Optional[int],
|
|
104
|
+
estimated_gb: float,
|
|
105
|
+
observed_gb: float,
|
|
106
|
+
) -> Dict[str, Any]:
|
|
107
|
+
"""Append one (estimated, observed) pair. Best-effort: never raises.
|
|
108
|
+
|
|
109
|
+
Returns the sample recorded plus the key it landed under, or an
|
|
110
|
+
``error`` string when the store couldn't be written.
|
|
111
|
+
"""
|
|
112
|
+
key = sample_key(gpu, runtime, family, quant, context)
|
|
113
|
+
try:
|
|
114
|
+
with _write_lock:
|
|
115
|
+
data = _load()
|
|
116
|
+
samples: List[Dict[str, Any]] = data["samples"].setdefault(key, [])
|
|
117
|
+
samples.append(
|
|
118
|
+
{
|
|
119
|
+
"ts": time.time(),
|
|
120
|
+
"estimated_gb": round(float(estimated_gb), 3),
|
|
121
|
+
"observed_gb": round(float(observed_gb), 3),
|
|
122
|
+
}
|
|
123
|
+
)
|
|
124
|
+
del samples[:-_MAX_SAMPLES_PER_KEY]
|
|
125
|
+
err = _save(data)
|
|
126
|
+
if err:
|
|
127
|
+
return {"key": key, "error": err}
|
|
128
|
+
return {"key": key, "sample_count": len(samples)}
|
|
129
|
+
except Exception as exc: # calibration is a bonus signal, never a hard dependency
|
|
130
|
+
return {"key": key, "error": str(exc)}
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _confidence_for(sample_count: int) -> str:
|
|
134
|
+
if sample_count >= 10:
|
|
135
|
+
return "high"
|
|
136
|
+
if sample_count >= 3:
|
|
137
|
+
return "medium"
|
|
138
|
+
if sample_count >= 1:
|
|
139
|
+
return "low"
|
|
140
|
+
return "none"
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def get_correction(
|
|
144
|
+
*, gpu: str, runtime: str, family: Optional[str], quant: Optional[str], context: Optional[int]
|
|
145
|
+
) -> Dict[str, Any]:
|
|
146
|
+
"""Median observed/estimated ratio for this exact key. No cross-key blending -
|
|
147
|
+
materially different configurations are never averaged together."""
|
|
148
|
+
key = sample_key(gpu, runtime, family, quant, context)
|
|
149
|
+
data = _load()
|
|
150
|
+
samples = data["samples"].get(key) or []
|
|
151
|
+
if not samples:
|
|
152
|
+
return {"key": key, "applied": False, "factor": 1.0, "sample_count": 0, "confidence": "none"}
|
|
153
|
+
ratios = [
|
|
154
|
+
s["observed_gb"] / s["estimated_gb"]
|
|
155
|
+
for s in samples
|
|
156
|
+
if isinstance(s.get("estimated_gb"), (int, float)) and s.get("estimated_gb", 0) > 0
|
|
157
|
+
]
|
|
158
|
+
if not ratios:
|
|
159
|
+
return {"key": key, "applied": False, "factor": 1.0, "sample_count": 0, "confidence": "none"}
|
|
160
|
+
factor = statistics.median(ratios)
|
|
161
|
+
return {
|
|
162
|
+
"key": key,
|
|
163
|
+
"applied": True,
|
|
164
|
+
"factor": round(factor, 4),
|
|
165
|
+
"sample_count": len(ratios),
|
|
166
|
+
"confidence": _confidence_for(len(ratios)),
|
|
167
|
+
"last_sample_estimated_gb": samples[-1]["estimated_gb"],
|
|
168
|
+
"last_sample_observed_gb": samples[-1]["observed_gb"],
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def stats() -> Dict[str, Any]:
|
|
173
|
+
"""Overall calibration coverage: how many keys have data, and the mean
|
|
174
|
+
absolute prediction error across every recorded sample (the estimator
|
|
175
|
+
self-grading metric)."""
|
|
176
|
+
data = _load()
|
|
177
|
+
all_samples: List[Dict[str, Any]] = []
|
|
178
|
+
for samples in data["samples"].values():
|
|
179
|
+
all_samples.extend(samples)
|
|
180
|
+
if not all_samples:
|
|
181
|
+
return {"keys": 0, "samples": 0, "mean_abs_error_pct": None}
|
|
182
|
+
errors_pct = [
|
|
183
|
+
abs(s["observed_gb"] - s["estimated_gb"]) / s["estimated_gb"] * 100
|
|
184
|
+
for s in all_samples
|
|
185
|
+
if s.get("estimated_gb")
|
|
186
|
+
]
|
|
187
|
+
return {
|
|
188
|
+
"keys": len(data["samples"]),
|
|
189
|
+
"samples": len(all_samples),
|
|
190
|
+
"mean_abs_error_pct": round(statistics.mean(errors_pct), 2) if errors_pct else None,
|
|
191
|
+
}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
"""Community benchmark sharing - local-only groundwork (Release R8).
|
|
2
|
+
|
|
3
|
+
LocalDeploy's whole premise is "no telemetry" (see README Privacy & Security).
|
|
4
|
+
A real community dataset needs a hosted server, a moderation/abuse story, and
|
|
5
|
+
a data-retention policy - an infrastructure and product decision this repo
|
|
6
|
+
has not made, not something to wire up silently inside a feature PR. Both
|
|
7
|
+
endpoints below are honest about that: they compute and preview exactly what
|
|
8
|
+
*would* be shared, and "export" saves that anonymized snapshot to a local
|
|
9
|
+
file - there is no submission call anywhere in this module, because there is
|
|
10
|
+
nowhere to submit to yet.
|
|
11
|
+
|
|
12
|
+
POST /system/community/preview -> the anonymized payload, never persisted
|
|
13
|
+
POST /system/community/export -> the same payload, saved to
|
|
14
|
+
reports/community-contributions/ locally
|
|
15
|
+
"""
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
import platform
|
|
20
|
+
import time
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, Dict, Optional
|
|
23
|
+
|
|
24
|
+
from fastapi import APIRouter
|
|
25
|
+
from pydantic import BaseModel
|
|
26
|
+
|
|
27
|
+
from .. import __version__
|
|
28
|
+
|
|
29
|
+
router = APIRouter()
|
|
30
|
+
|
|
31
|
+
SCHEMA_VERSION = 1
|
|
32
|
+
|
|
33
|
+
# Fields explicitly never copied into the shared payload, no matter what the
|
|
34
|
+
# input card contains - listed here (not just in a docstring) so it doubles
|
|
35
|
+
# as documentation of the promise and is easy to audit.
|
|
36
|
+
EXCLUDED_FIELDS = [
|
|
37
|
+
"username", "computer name", "IP address",
|
|
38
|
+
"model prompts", "model responses", "response previews",
|
|
39
|
+
"filesystem paths", "API keys", "local profile name",
|
|
40
|
+
]
|
|
41
|
+
|
|
42
|
+
_TEST_NUMERIC_FIELDS = ("name", "category", "success", "accuracy", "elapsed_seconds", "approx_tokens_per_second")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _os_category() -> str:
|
|
46
|
+
return {"Windows": "Windows", "Darwin": "macOS", "Linux": "Linux"}.get(platform.system(), platform.system())
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _round_or_none(value: Any, digits: int = 1) -> Optional[float]:
|
|
50
|
+
try:
|
|
51
|
+
return round(float(value), digits) if value is not None else None
|
|
52
|
+
except (TypeError, ValueError):
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _anonymize_test(test: Dict[str, Any]) -> Dict[str, Any]:
|
|
57
|
+
"""Numeric result fields only - never response_preview or error text,
|
|
58
|
+
which can contain fragments of the user's actual prompts/outputs."""
|
|
59
|
+
out = {k: test.get(k) for k in _TEST_NUMERIC_FIELDS}
|
|
60
|
+
metrics = test.get("metrics") or {}
|
|
61
|
+
numeric_metrics = {
|
|
62
|
+
k: metrics.get(k)
|
|
63
|
+
for k in ("ttft_ms", "tokens_per_second", "prompt_tokens_per_second")
|
|
64
|
+
if metrics.get(k) is not None
|
|
65
|
+
}
|
|
66
|
+
if numeric_metrics:
|
|
67
|
+
out["metrics"] = numeric_metrics
|
|
68
|
+
return out
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def anonymize_card(card: Dict[str, Any]) -> Dict[str, Any]:
|
|
72
|
+
"""Whitelist-copy a report-card-shaped payload (report.py's build_card
|
|
73
|
+
output, or an equivalent run record) down to what the community-sharing
|
|
74
|
+
plan defines as safe to share. Everything not explicitly copied here is
|
|
75
|
+
dropped - this is a whitelist, not a blocklist, by design."""
|
|
76
|
+
hardware = card.get("hardware") or {}
|
|
77
|
+
provenance = card.get("provenance") or {}
|
|
78
|
+
prov_hardware = provenance.get("hardware") or {}
|
|
79
|
+
gpus = prov_hardware.get("gpus") or hardware.get("gpus") or []
|
|
80
|
+
best_gpu = gpus[0] if gpus else {}
|
|
81
|
+
system = prov_hardware.get("system") or {}
|
|
82
|
+
|
|
83
|
+
profile_name = card.get("profile")
|
|
84
|
+
profile_provenance = (provenance.get("profiles") or {}).get(profile_name, {}) if profile_name else {}
|
|
85
|
+
|
|
86
|
+
summary = card.get("summary") or {}
|
|
87
|
+
tests = [_anonymize_test(t) for t in (card.get("tests") or [])]
|
|
88
|
+
|
|
89
|
+
return {
|
|
90
|
+
"schema_version": SCHEMA_VERSION,
|
|
91
|
+
"hardware": {
|
|
92
|
+
"gpu": best_gpu.get("name") or hardware.get("gpu"),
|
|
93
|
+
"vram_gb": _round_or_none((best_gpu.get("vram_total_mb") or hardware.get("vram_total_mb") or 0) / 1024.0) or None,
|
|
94
|
+
"cpu": system.get("cpu_model"),
|
|
95
|
+
"ram_gb": _round_or_none((system.get("ram_total_mb") or 0) / 1024.0) or None,
|
|
96
|
+
"os_category": _os_category(),
|
|
97
|
+
},
|
|
98
|
+
"runtime": {
|
|
99
|
+
"provider": profile_provenance.get("backend") or "ollama",
|
|
100
|
+
"version": profile_provenance.get("backend_version"),
|
|
101
|
+
},
|
|
102
|
+
"model": {
|
|
103
|
+
"id": card.get("model_id"),
|
|
104
|
+
"digest": profile_provenance.get("model_digest"),
|
|
105
|
+
"quantization": profile_provenance.get("quant"),
|
|
106
|
+
"context": profile_provenance.get("context"),
|
|
107
|
+
},
|
|
108
|
+
"device": card.get("device"),
|
|
109
|
+
"performance": {
|
|
110
|
+
"avg_accuracy": summary.get("avg_accuracy"),
|
|
111
|
+
"avg_latency_s": summary.get("avg_latency_s"),
|
|
112
|
+
"avg_tokens_per_second": summary.get("avg_tokens_per_second"),
|
|
113
|
+
"avg_ttft_ms": summary.get("avg_ttft_ms"),
|
|
114
|
+
"peak_vram_mb": card.get("peak_vram_mb"),
|
|
115
|
+
},
|
|
116
|
+
"tests": tests,
|
|
117
|
+
"repetitions": card.get("repetitions") or 1,
|
|
118
|
+
"localdeploy_version": __version__,
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class CommunityShareRequest(BaseModel):
|
|
123
|
+
card: Dict[str, Any]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@router.post("/system/community/preview")
|
|
127
|
+
def community_preview(req: CommunityShareRequest) -> Dict[str, Any]:
|
|
128
|
+
return {
|
|
129
|
+
"success": True,
|
|
130
|
+
"would_share": anonymize_card(req.card),
|
|
131
|
+
"excluded_fields": EXCLUDED_FIELDS,
|
|
132
|
+
"note": (
|
|
133
|
+
"Preview only - nothing is sent anywhere. LocalDeploy does not have a "
|
|
134
|
+
"community server yet, so there is currently no submit action; "
|
|
135
|
+
"'Save locally' keeps this anonymized snapshot on disk for later."
|
|
136
|
+
),
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _contributions_dir() -> Path:
|
|
141
|
+
from ..utils import app_home
|
|
142
|
+
|
|
143
|
+
return app_home() / "reports" / "community-contributions"
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@router.post("/system/community/export")
|
|
147
|
+
def community_export(req: CommunityShareRequest) -> Dict[str, Any]:
|
|
148
|
+
payload = anonymize_card(req.card)
|
|
149
|
+
try:
|
|
150
|
+
directory = _contributions_dir()
|
|
151
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
152
|
+
path = directory / f"contribution-{int(time.time() * 1000)}.json"
|
|
153
|
+
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
154
|
+
except OSError as exc:
|
|
155
|
+
return {"success": False, "error": f"Could not save locally: {exc}"}
|
|
156
|
+
return {
|
|
157
|
+
"success": True,
|
|
158
|
+
"path": str(path),
|
|
159
|
+
"would_share": payload,
|
|
160
|
+
"note": "Saved locally only - not transmitted anywhere. LocalDeploy has no community server yet.",
|
|
161
|
+
}
|