localdeploy 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api_server.py +15 -0
- benchmark.py +15 -0
- chat_cli.py +15 -0
- compare_models.py +15 -0
- localdeploy/__init__.py +3 -0
- localdeploy/backends/__init__.py +0 -0
- localdeploy/backends/llamacpp.py +143 -0
- localdeploy/backends/ollama.py +374 -0
- localdeploy/backends/openai_compatible.py +179 -0
- localdeploy/benchmark.py +1996 -0
- localdeploy/chat_cli.py +249 -0
- localdeploy/cli.py +59 -0
- localdeploy/compare_models.py +481 -0
- localdeploy/control/__init__.py +75 -0
- localdeploy/control/_config.py +180 -0
- localdeploy/control/_ollama.py +253 -0
- localdeploy/control/bakeoff.py +216 -0
- localdeploy/control/bench.py +516 -0
- localdeploy/control/calibration.py +191 -0
- localdeploy/control/community.py +161 -0
- localdeploy/control/fit.py +651 -0
- localdeploy/control/hardware.py +602 -0
- localdeploy/control/history.py +125 -0
- localdeploy/control/manifest.py +452 -0
- localdeploy/control/models.py +833 -0
- localdeploy/control/monitor.py +378 -0
- localdeploy/control/profiles.py +161 -0
- localdeploy/control/recommend.py +322 -0
- localdeploy/control/registry.py +956 -0
- localdeploy/control/report.py +407 -0
- localdeploy/control/starter.py +520 -0
- localdeploy/control/updates.py +118 -0
- localdeploy/grader_sandbox.py +268 -0
- localdeploy/server.py +1873 -0
- localdeploy/utils.py +155 -0
- localdeploy/web/favicon.png +0 -0
- localdeploy/web/index.html +891 -0
- localdeploy/web/js/app.js +63 -0
- localdeploy/web/js/benchmark-views.js +73 -0
- localdeploy/web/js/benchmark.js +1691 -0
- localdeploy/web/js/chat.js +779 -0
- localdeploy/web/js/models.js +2758 -0
- localdeploy/web/js/shared.js +454 -0
- localdeploy/web/js/system.js +626 -0
- localdeploy/web/logo.svg +22 -0
- localdeploy/web/styles.css +2032 -0
- localdeploy-0.6.0.dist-info/METADATA +202 -0
- localdeploy-0.6.0.dist-info/RECORD +52 -0
- localdeploy-0.6.0.dist-info/WHEEL +5 -0
- localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
- localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
- localdeploy-0.6.0.dist-info/top_level.txt +5 -0
|
@@ -0,0 +1,481 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import base64
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import statistics
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
12
|
+
|
|
13
|
+
import requests
|
|
14
|
+
from dotenv import load_dotenv
|
|
15
|
+
|
|
16
|
+
from api_server import get_global_limits, load_config, run_local_request
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
from localdeploy.utils import api_auth_headers, api_client_base_url, app_home
|
|
20
|
+
|
|
21
|
+
APP_DIR = Path(__file__).resolve().parents[1] # repo root in a checkout
|
|
22
|
+
APP_HOME = app_home()
|
|
23
|
+
load_dotenv(APP_HOME / ".env")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
BASIC_PROMPT = "Explain what this local LLM server is doing in 3 bullet points."
|
|
27
|
+
REASONING_PROMPT = (
|
|
28
|
+
"A laptop can process 18 images per minute. How long will it take to process 153 images? "
|
|
29
|
+
"Show your reasoning briefly."
|
|
30
|
+
)
|
|
31
|
+
CODING_PROMPT = (
|
|
32
|
+
"Write a Python function that validates whether a string is valid JSON and returns a tuple "
|
|
33
|
+
"of success and error message."
|
|
34
|
+
)
|
|
35
|
+
JSON_PROMPT = "Return only valid JSON with keys: model_capability, strengths, weaknesses, recommended_use. No markdown."
|
|
36
|
+
LOCAL_API_PROMPT = (
|
|
37
|
+
"You are being used inside a local desktop app. Give a short, practical answer explaining how the app should call your API."
|
|
38
|
+
)
|
|
39
|
+
VISION_PROMPT = "Describe the image accurately. Then extract any visible text. Then list any uncertainty."
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def parse_bool(value: str) -> bool:
|
|
43
|
+
lowered = str(value).strip().lower()
|
|
44
|
+
if lowered in {"true", "1", "yes", "y", "on"}:
|
|
45
|
+
return True
|
|
46
|
+
if lowered in {"false", "0", "no", "n", "off"}:
|
|
47
|
+
return False
|
|
48
|
+
raise argparse.ArgumentTypeError("Expected true or false.")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def server_base_url() -> str:
|
|
52
|
+
return api_client_base_url()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def is_server_running() -> bool:
|
|
56
|
+
try:
|
|
57
|
+
response = requests.get(f"{server_base_url()}/health", headers=api_auth_headers(), timeout=10)
|
|
58
|
+
return response.ok
|
|
59
|
+
except requests.RequestException:
|
|
60
|
+
return False
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def enabled_profiles(config: Dict[str, Any]) -> List[str]:
|
|
64
|
+
return [name for name, profile in config.get("profiles", {}).items() if profile.get("enabled", False)]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def prompt_limit_for_profile(profile: Dict[str, Any]) -> int:
|
|
68
|
+
global_limit = int(get_global_limits()["max_prompt_chars"])
|
|
69
|
+
profile_limit = int(profile.get("max_prompt_chars") or global_limit)
|
|
70
|
+
return min(global_limit, profile_limit)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def make_long_context_prompt(profile: Dict[str, Any]) -> str:
|
|
74
|
+
limit = prompt_limit_for_profile(profile)
|
|
75
|
+
prefix = "Summarize this synthetic local deployment test document in 5 concise bullets.\n\n"
|
|
76
|
+
suffix = "\n\nFocus on the deployment purpose, safety limits, model profiles, and benchmark strategy."
|
|
77
|
+
section = (
|
|
78
|
+
"Section {n}: This Windows local LLM deployment runs only against localhost. "
|
|
79
|
+
"It uses Ollama as the default backend, keeps Gemma 3 4B as the stable profile, "
|
|
80
|
+
"tests Gemma 3 12B with conservative context, and supports optional llama.cpp GGUF profiles "
|
|
81
|
+
"for quantization, GPU layer, KV cache, and context experiments.\n"
|
|
82
|
+
)
|
|
83
|
+
budget = max(500, min(limit - len(prefix) - len(suffix) - 50, 9000))
|
|
84
|
+
document = ""
|
|
85
|
+
n = 1
|
|
86
|
+
while len(document) < budget:
|
|
87
|
+
document += section.format(n=n)
|
|
88
|
+
n += 1
|
|
89
|
+
document = document[:budget]
|
|
90
|
+
return f"{prefix}{document}{suffix}"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_prompts_for_profile(profile: Dict[str, Any]) -> List[Tuple[str, str]]:
|
|
94
|
+
return [
|
|
95
|
+
("basic", BASIC_PROMPT),
|
|
96
|
+
("reasoning", REASONING_PROMPT),
|
|
97
|
+
("coding", CODING_PROMPT),
|
|
98
|
+
("long_context", make_long_context_prompt(profile)),
|
|
99
|
+
("json_compliance", JSON_PROMPT),
|
|
100
|
+
("local_api_use", LOCAL_API_PROMPT),
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def image_to_base64(path: Path) -> Tuple[Optional[str], Optional[str]]:
|
|
105
|
+
if not path.exists():
|
|
106
|
+
return None, f"Invalid image path: {path}"
|
|
107
|
+
if not path.is_file():
|
|
108
|
+
return None, f"Image path is not a file: {path}"
|
|
109
|
+
try:
|
|
110
|
+
return base64.b64encode(path.read_bytes()).decode("ascii"), None
|
|
111
|
+
except OSError as exc:
|
|
112
|
+
return None, f"Could not read image file '{path}': {exc}"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def call_server(endpoint: str, payload: Dict[str, Any], timeout: int) -> Dict[str, Any]:
|
|
116
|
+
url = f"{server_base_url()}/{endpoint.lstrip('/')}"
|
|
117
|
+
try:
|
|
118
|
+
response = requests.post(url, json=payload, headers=api_auth_headers(), timeout=timeout + 10)
|
|
119
|
+
except requests.Timeout:
|
|
120
|
+
return {
|
|
121
|
+
"success": False,
|
|
122
|
+
"elapsed_seconds": timeout,
|
|
123
|
+
"error": f"Local API server request timed out after {timeout} seconds.",
|
|
124
|
+
}
|
|
125
|
+
except requests.ConnectionError:
|
|
126
|
+
return {
|
|
127
|
+
"success": False,
|
|
128
|
+
"elapsed_seconds": 0,
|
|
129
|
+
"error": f"Local API server is not reachable at {server_base_url()}.",
|
|
130
|
+
}
|
|
131
|
+
try:
|
|
132
|
+
data = response.json()
|
|
133
|
+
except ValueError:
|
|
134
|
+
return {
|
|
135
|
+
"success": False,
|
|
136
|
+
"elapsed_seconds": 0,
|
|
137
|
+
"error": f"Server returned non-JSON HTTP {response.status_code}: {response.text[:500]}",
|
|
138
|
+
}
|
|
139
|
+
if not response.ok:
|
|
140
|
+
return {
|
|
141
|
+
"success": False,
|
|
142
|
+
"elapsed_seconds": data.get("elapsed_seconds", 0),
|
|
143
|
+
"error": data.get("detail") or data.get("error") or f"HTTP {response.status_code}",
|
|
144
|
+
}
|
|
145
|
+
return data
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def call_model(use_server: bool, endpoint: str, payload: Dict[str, Any], timeout: int) -> Dict[str, Any]:
|
|
149
|
+
if use_server:
|
|
150
|
+
return call_server(endpoint, payload, timeout)
|
|
151
|
+
return run_local_request("vision" if endpoint == "vision" else "chat", payload)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def response_text(result: Dict[str, Any]) -> str:
|
|
155
|
+
value = result.get("response")
|
|
156
|
+
if value is None:
|
|
157
|
+
return ""
|
|
158
|
+
if isinstance(value, str):
|
|
159
|
+
return value
|
|
160
|
+
return json.dumps(value, ensure_ascii=False)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def strict_json_ok(text: str) -> bool:
|
|
164
|
+
try:
|
|
165
|
+
json.loads(text.strip())
|
|
166
|
+
return True
|
|
167
|
+
except Exception:
|
|
168
|
+
return False
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def quality_score(test_name: str, text: str) -> float:
|
|
172
|
+
lowered = text.lower()
|
|
173
|
+
if not text.strip():
|
|
174
|
+
return 0.0
|
|
175
|
+
if test_name == "json_compliance":
|
|
176
|
+
return 1.0 if strict_json_ok(text) else 0.2
|
|
177
|
+
if test_name == "reasoning":
|
|
178
|
+
if "8.5" in lowered or ("8" in lowered and "30" in lowered):
|
|
179
|
+
return 1.0
|
|
180
|
+
return 0.5 if "153" in lowered and "18" in lowered else 0.2
|
|
181
|
+
if test_name == "coding":
|
|
182
|
+
score = 0.0
|
|
183
|
+
score += 0.35 if "json.loads" in lowered else 0.0
|
|
184
|
+
score += 0.25 if "except" in lowered else 0.0
|
|
185
|
+
score += 0.2 if "tuple" in lowered or "return true" in lowered else 0.0
|
|
186
|
+
score += 0.2 if "error" in lowered else 0.0
|
|
187
|
+
return min(1.0, score)
|
|
188
|
+
if test_name == "local_api_use":
|
|
189
|
+
score = 0.0
|
|
190
|
+
score += 0.35 if "post" in lowered else 0.0
|
|
191
|
+
score += 0.35 if "/chat" in lowered or "/vision" in lowered or "http" in lowered else 0.0
|
|
192
|
+
score += 0.3 if "json" in lowered else 0.0
|
|
193
|
+
return min(1.0, score)
|
|
194
|
+
if test_name == "vision":
|
|
195
|
+
return 1.0 if len(text) > 80 else 0.5
|
|
196
|
+
return 1.0 if len(text) > 80 else 0.6
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def classify_error(error: str) -> str:
|
|
200
|
+
lowered = error.lower()
|
|
201
|
+
if "timed out" in lowered or "timeout" in lowered:
|
|
202
|
+
return "timeout"
|
|
203
|
+
if "too large" in lowered or "exceeds configured limit" in lowered or "limit" in lowered:
|
|
204
|
+
return "exceeded limits"
|
|
205
|
+
if "not running" in lowered or "unreachable" in lowered or "not reachable" in lowered:
|
|
206
|
+
return "backend unavailable"
|
|
207
|
+
if "not available" in lowered or "model not found" in lowered or "pull" in lowered:
|
|
208
|
+
return "model not pulled"
|
|
209
|
+
if "gguf file path not found" in lowered:
|
|
210
|
+
return "gguf path missing"
|
|
211
|
+
if "memory" in lowered or "cuda" in lowered or "out of memory" in lowered:
|
|
212
|
+
return "memory failure"
|
|
213
|
+
return "failed"
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def suitability_score(value: Any) -> float:
|
|
217
|
+
if value is True:
|
|
218
|
+
return 1.0
|
|
219
|
+
if value == "fallback":
|
|
220
|
+
return 0.55
|
|
221
|
+
if value == "experimental":
|
|
222
|
+
return 0.45
|
|
223
|
+
if value is False:
|
|
224
|
+
return 0.35
|
|
225
|
+
return 0.5
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def summarize_profile(profile_name: str, profile: Dict[str, Any], tests: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
229
|
+
total = len(tests)
|
|
230
|
+
successes = [item for item in tests if item["success"]]
|
|
231
|
+
failures = [item for item in tests if not item["success"]]
|
|
232
|
+
reliability = len(successes) / total if total else 0.0
|
|
233
|
+
elapsed_values = [item["elapsed_seconds"] for item in successes if item.get("elapsed_seconds") is not None]
|
|
234
|
+
avg_elapsed = statistics.mean(elapsed_values) if elapsed_values else None
|
|
235
|
+
slow_threshold = int(profile.get("slow_response_seconds") or os.getenv("SLOW_RESPONSE_SECONDS", "60"))
|
|
236
|
+
speed = 0.0
|
|
237
|
+
if avg_elapsed is not None:
|
|
238
|
+
speed = min(1.0, slow_threshold / max(avg_elapsed, 0.001))
|
|
239
|
+
quality = statistics.mean([item["quality_score"] for item in successes]) if successes else 0.0
|
|
240
|
+
json_tests = [item for item in tests if item["name"] == "json_compliance"]
|
|
241
|
+
json_compliance = 1.0 if json_tests and json_tests[0].get("json_ok") else 0.0
|
|
242
|
+
vision_tests = [item for item in tests if item["name"] == "vision"]
|
|
243
|
+
vision_score = 1.0 if vision_tests and vision_tests[0]["success"] else (0.5 if not vision_tests else 0.0)
|
|
244
|
+
suitability = suitability_score(profile.get("recommended_for_8gb_vram"))
|
|
245
|
+
score = (
|
|
246
|
+
reliability * 0.35
|
|
247
|
+
+ speed * 0.20
|
|
248
|
+
+ quality * 0.20
|
|
249
|
+
+ json_compliance * 0.10
|
|
250
|
+
+ vision_score * 0.05
|
|
251
|
+
+ suitability * 0.10
|
|
252
|
+
)
|
|
253
|
+
failure_types = sorted({item["classification"] for item in failures})
|
|
254
|
+
status = "works" if reliability >= 0.8 else "unstable" if reliability > 0 else "fails"
|
|
255
|
+
if avg_elapsed is not None and avg_elapsed >= slow_threshold:
|
|
256
|
+
status = f"{status}, too slow"
|
|
257
|
+
if any(item["classification"] == "exceeded limits" for item in failures):
|
|
258
|
+
status = f"{status}, exceeded limits"
|
|
259
|
+
return {
|
|
260
|
+
"profile": profile_name,
|
|
261
|
+
"status": status,
|
|
262
|
+
"score": score,
|
|
263
|
+
"reliability": reliability,
|
|
264
|
+
"average_elapsed_seconds": avg_elapsed,
|
|
265
|
+
"average_tokens_per_second": statistics.mean(
|
|
266
|
+
[item["approx_tokens_per_second"] for item in successes if item["approx_tokens_per_second"] is not None]
|
|
267
|
+
)
|
|
268
|
+
if successes
|
|
269
|
+
else None,
|
|
270
|
+
"quality": quality,
|
|
271
|
+
"json_compliance": json_compliance,
|
|
272
|
+
"vision_score": vision_score,
|
|
273
|
+
"suitability": suitability,
|
|
274
|
+
"failure_types": failure_types,
|
|
275
|
+
"recommended_for_8gb_vram": profile.get("recommended_for_8gb_vram"),
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def recommend_profile(summaries: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
|
280
|
+
viable = [item for item in summaries if item.get("reliability", 0.0) > 0.0]
|
|
281
|
+
if not viable:
|
|
282
|
+
return None
|
|
283
|
+
ranked = sorted(viable, key=lambda item: item["score"], reverse=True)
|
|
284
|
+
best = ranked[0]
|
|
285
|
+
four_b = next((item for item in viable if item["profile"] == "gemma3_4b_ollama_safe"), None)
|
|
286
|
+
if four_b and four_b["reliability"] >= 0.8:
|
|
287
|
+
best_elapsed = best.get("average_elapsed_seconds")
|
|
288
|
+
four_elapsed = four_b.get("average_elapsed_seconds")
|
|
289
|
+
best_is_much_slower = (
|
|
290
|
+
best_elapsed is not None
|
|
291
|
+
and four_elapsed is not None
|
|
292
|
+
and best_elapsed > max(four_elapsed * 2.0, four_elapsed + 20)
|
|
293
|
+
)
|
|
294
|
+
if best["profile"] != four_b["profile"] and (best["score"] < four_b["score"] + 0.15 or best_is_much_slower):
|
|
295
|
+
return four_b
|
|
296
|
+
return best
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def print_test_line(profile_name: str, item: Dict[str, Any]) -> None:
|
|
300
|
+
if item["success"]:
|
|
301
|
+
tps = item["approx_tokens_per_second"]
|
|
302
|
+
tps_text = f", ~{tps:.2f} tok/s" if tps is not None else ""
|
|
303
|
+
print(f" PASS {item['name']}: {item['elapsed_seconds']:.2f}s, {item['response_length']} chars{tps_text}")
|
|
304
|
+
else:
|
|
305
|
+
print(f" FAIL {item['name']}: {item['classification']} - {item['error']}")
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def build_payload(
|
|
309
|
+
profile_name: str,
|
|
310
|
+
prompt: str,
|
|
311
|
+
args: argparse.Namespace,
|
|
312
|
+
images_base64: Optional[List[str]] = None,
|
|
313
|
+
) -> Dict[str, Any]:
|
|
314
|
+
payload: Dict[str, Any] = {
|
|
315
|
+
"profile": profile_name,
|
|
316
|
+
"prompt": prompt,
|
|
317
|
+
"safe_mode": args.safe_mode,
|
|
318
|
+
"allow_clamp": False,
|
|
319
|
+
"stream": False,
|
|
320
|
+
}
|
|
321
|
+
if args.max_output_tokens is not None:
|
|
322
|
+
payload["max_output_tokens"] = args.max_output_tokens
|
|
323
|
+
if args.context_limit is not None:
|
|
324
|
+
payload["context_limit"] = args.context_limit
|
|
325
|
+
if args.timeout is not None:
|
|
326
|
+
payload["timeout_seconds"] = args.timeout
|
|
327
|
+
if images_base64:
|
|
328
|
+
payload["images_base64"] = images_base64
|
|
329
|
+
return payload
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def run_profile_tests(
|
|
333
|
+
profile_name: str,
|
|
334
|
+
profile: Dict[str, Any],
|
|
335
|
+
args: argparse.Namespace,
|
|
336
|
+
use_server: bool,
|
|
337
|
+
image_base64: Optional[str],
|
|
338
|
+
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
|
|
339
|
+
print(f"\nProfile: {profile_name}")
|
|
340
|
+
print(f" backend={profile.get('backend')} model={profile.get('model_id')} recommended_8gb={profile.get('recommended_for_8gb_vram')}")
|
|
341
|
+
tests: List[Dict[str, Any]] = []
|
|
342
|
+
timeout = args.timeout or int(profile.get("timeout_seconds") or os.getenv("REQUEST_TIMEOUT_SECONDS", "180"))
|
|
343
|
+
|
|
344
|
+
prompt_tests = test_prompts_for_profile(profile)
|
|
345
|
+
for test_name, prompt in prompt_tests:
|
|
346
|
+
endpoint = "chat"
|
|
347
|
+
payload = build_payload(profile_name, prompt, args)
|
|
348
|
+
started = time.perf_counter()
|
|
349
|
+
result = call_model(use_server, endpoint, payload, timeout)
|
|
350
|
+
elapsed = float(result.get("elapsed_seconds") or (time.perf_counter() - started))
|
|
351
|
+
text = response_text(result)
|
|
352
|
+
approx_tps = ((len(text) / 4) / elapsed) if result.get("success") and elapsed > 0 else None
|
|
353
|
+
item = {
|
|
354
|
+
"name": test_name,
|
|
355
|
+
"success": bool(result.get("success")),
|
|
356
|
+
"elapsed_seconds": elapsed,
|
|
357
|
+
"response_length": len(text),
|
|
358
|
+
"approx_tokens_per_second": approx_tps,
|
|
359
|
+
"quality_score": quality_score(test_name, text) if result.get("success") else 0.0,
|
|
360
|
+
"json_ok": strict_json_ok(text) if test_name == "json_compliance" and result.get("success") else False,
|
|
361
|
+
"classification": "ok" if result.get("success") else classify_error(str(result.get("error") or "")),
|
|
362
|
+
"error": result.get("error"),
|
|
363
|
+
"warning": result.get("warning"),
|
|
364
|
+
}
|
|
365
|
+
tests.append(item)
|
|
366
|
+
print_test_line(profile_name, item)
|
|
367
|
+
|
|
368
|
+
if image_base64:
|
|
369
|
+
payload = build_payload(profile_name, VISION_PROMPT, args, [image_base64])
|
|
370
|
+
started = time.perf_counter()
|
|
371
|
+
result = call_model(use_server, "vision", payload, timeout)
|
|
372
|
+
elapsed = float(result.get("elapsed_seconds") or (time.perf_counter() - started))
|
|
373
|
+
text = response_text(result)
|
|
374
|
+
approx_tps = ((len(text) / 4) / elapsed) if result.get("success") and elapsed > 0 else None
|
|
375
|
+
item = {
|
|
376
|
+
"name": "vision",
|
|
377
|
+
"success": bool(result.get("success")),
|
|
378
|
+
"elapsed_seconds": elapsed,
|
|
379
|
+
"response_length": len(text),
|
|
380
|
+
"approx_tokens_per_second": approx_tps,
|
|
381
|
+
"quality_score": quality_score("vision", text) if result.get("success") else 0.0,
|
|
382
|
+
"json_ok": False,
|
|
383
|
+
"classification": "ok" if result.get("success") else classify_error(str(result.get("error") or "")),
|
|
384
|
+
"error": result.get("error"),
|
|
385
|
+
"warning": result.get("warning"),
|
|
386
|
+
}
|
|
387
|
+
tests.append(item)
|
|
388
|
+
print_test_line(profile_name, item)
|
|
389
|
+
|
|
390
|
+
summary = summarize_profile(profile_name, profile, tests)
|
|
391
|
+
print(
|
|
392
|
+
f" Summary: {summary['status']} | score={summary['score']:.3f} "
|
|
393
|
+
f"reliability={summary['reliability']:.0%} quality={summary['quality']:.2f}"
|
|
394
|
+
)
|
|
395
|
+
return tests, summary
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
def main() -> int:
|
|
399
|
+
parser = argparse.ArgumentParser(description="Benchmark local Ollama and optional llama.cpp model profiles.")
|
|
400
|
+
parser.add_argument("--profile", help="Profile name from config.json.")
|
|
401
|
+
parser.add_argument("--all", action="store_true", help="Test all enabled profiles.")
|
|
402
|
+
parser.add_argument("--image", help="Optional image path for a vision test.")
|
|
403
|
+
parser.add_argument("--safe-mode", type=parse_bool, default=True, help="true/false. Defaults to true.")
|
|
404
|
+
parser.add_argument("--max-output-tokens", type=int, help="Requested output token cap.")
|
|
405
|
+
parser.add_argument("--context-limit", type=int, help="Requested context limit.")
|
|
406
|
+
parser.add_argument("--timeout", type=int, help="Request timeout seconds.")
|
|
407
|
+
args = parser.parse_args()
|
|
408
|
+
|
|
409
|
+
try:
|
|
410
|
+
config = load_config()
|
|
411
|
+
except Exception as exc:
|
|
412
|
+
print(f"Could not load config.json: {exc}")
|
|
413
|
+
return 2
|
|
414
|
+
|
|
415
|
+
profiles = config.get("profiles", {})
|
|
416
|
+
if args.all:
|
|
417
|
+
profile_names = enabled_profiles(config)
|
|
418
|
+
else:
|
|
419
|
+
selected = args.profile or os.getenv("DEFAULT_MODEL_PROFILE") or config.get("default_profile")
|
|
420
|
+
profile_names = [selected] if selected else []
|
|
421
|
+
|
|
422
|
+
if not profile_names:
|
|
423
|
+
print("No profiles selected. Pass --profile, set DEFAULT_MODEL_PROFILE, set config.json's "
|
|
424
|
+
"default_profile, or use --all to test every enabled profile.")
|
|
425
|
+
return 2
|
|
426
|
+
|
|
427
|
+
missing = [name for name in profile_names if name not in profiles]
|
|
428
|
+
if missing:
|
|
429
|
+
print(f"Unknown profile(s): {', '.join(missing)}")
|
|
430
|
+
return 2
|
|
431
|
+
|
|
432
|
+
image_base64: Optional[str] = None
|
|
433
|
+
if args.image:
|
|
434
|
+
image_base64, image_error = image_to_base64(Path(args.image))
|
|
435
|
+
if image_error:
|
|
436
|
+
print(image_error)
|
|
437
|
+
return 2
|
|
438
|
+
|
|
439
|
+
use_server = is_server_running()
|
|
440
|
+
mode = f"local API server at {server_base_url()}" if use_server else "direct backend calls"
|
|
441
|
+
print(f"Test mode: {mode}")
|
|
442
|
+
print(f"Safe mode: {args.safe_mode}")
|
|
443
|
+
if args.max_output_tokens:
|
|
444
|
+
print(f"Requested max_output_tokens: {args.max_output_tokens}")
|
|
445
|
+
if args.context_limit:
|
|
446
|
+
print(f"Requested context_limit: {args.context_limit}")
|
|
447
|
+
|
|
448
|
+
all_summaries: List[Dict[str, Any]] = []
|
|
449
|
+
for profile_name in profile_names:
|
|
450
|
+
_, summary = run_profile_tests(profile_name, profiles[profile_name], args, use_server, image_base64)
|
|
451
|
+
all_summaries.append(summary)
|
|
452
|
+
|
|
453
|
+
print("\nScoring summary")
|
|
454
|
+
for summary in sorted(all_summaries, key=lambda item: item["score"], reverse=True):
|
|
455
|
+
avg = summary["average_elapsed_seconds"]
|
|
456
|
+
avg_text = f"{avg:.2f}s" if avg is not None else "n/a"
|
|
457
|
+
tps = summary["average_tokens_per_second"]
|
|
458
|
+
tps_text = f"{tps:.2f}" if tps is not None else "n/a"
|
|
459
|
+
failures = ", ".join(summary["failure_types"]) if summary["failure_types"] else "none"
|
|
460
|
+
print(
|
|
461
|
+
f"- {summary['profile']}: score={summary['score']:.3f}, status={summary['status']}, "
|
|
462
|
+
f"avg={avg_text}, tok/s={tps_text}, failures={failures}, "
|
|
463
|
+
f"8GB={summary['recommended_for_8gb_vram']}"
|
|
464
|
+
)
|
|
465
|
+
|
|
466
|
+
recommended = recommend_profile(all_summaries)
|
|
467
|
+
if recommended:
|
|
468
|
+
print(f"\nRecommended profile: {recommended['profile']}")
|
|
469
|
+
if recommended["profile"] == "gemma3_4b_ollama_safe":
|
|
470
|
+
print("Reason: it is the safest default when speed and reliability matter on 8 GB VRAM.")
|
|
471
|
+
elif "12b" in recommended["profile"]:
|
|
472
|
+
print("Reason: 12B only won because it ran reliably enough for the measured workload.")
|
|
473
|
+
else:
|
|
474
|
+
print("\nRecommended profile: none")
|
|
475
|
+
print("Reason: no tested profile completed successfully. Check Ollama, pulled models, or enabled backends.")
|
|
476
|
+
|
|
477
|
+
return 0
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
if __name__ == "__main__":
|
|
481
|
+
sys.exit(main())
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Additive web UI and control-plane endpoints for LocalDeploy.
|
|
2
|
+
|
|
3
|
+
This package is mounted only when ``ENABLE_WEB_UI`` is true (the default). It
|
|
4
|
+
never changes the behavior of the existing API: every route lives under a new
|
|
5
|
+
prefix (``/system/*``, ``/registry/*``, ``/models/*``) so it cannot collide
|
|
6
|
+
with the original endpoints in ``api_server.py``.
|
|
7
|
+
|
|
8
|
+
Feature routers are included here as each implementation step lands.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from fastapi import APIRouter
|
|
13
|
+
|
|
14
|
+
router = APIRouter()
|
|
15
|
+
|
|
16
|
+
# --- feature routers (registered as steps land) ------------------------------
|
|
17
|
+
from . import hardware # noqa: E402 (import after router exists)
|
|
18
|
+
|
|
19
|
+
router.include_router(hardware.router)
|
|
20
|
+
|
|
21
|
+
from . import fit # noqa: E402
|
|
22
|
+
|
|
23
|
+
router.include_router(fit.router)
|
|
24
|
+
|
|
25
|
+
from . import registry # noqa: E402
|
|
26
|
+
|
|
27
|
+
router.include_router(registry.router)
|
|
28
|
+
|
|
29
|
+
from . import models # noqa: E402
|
|
30
|
+
|
|
31
|
+
router.include_router(models.router)
|
|
32
|
+
|
|
33
|
+
from . import bench # noqa: E402
|
|
34
|
+
|
|
35
|
+
router.include_router(bench.router)
|
|
36
|
+
|
|
37
|
+
from . import report # noqa: E402
|
|
38
|
+
|
|
39
|
+
router.include_router(report.router)
|
|
40
|
+
|
|
41
|
+
from . import recommend # noqa: E402
|
|
42
|
+
|
|
43
|
+
router.include_router(recommend.router)
|
|
44
|
+
|
|
45
|
+
from . import starter # noqa: E402
|
|
46
|
+
|
|
47
|
+
router.include_router(starter.router)
|
|
48
|
+
|
|
49
|
+
from . import profiles # noqa: E402
|
|
50
|
+
|
|
51
|
+
router.include_router(profiles.router)
|
|
52
|
+
|
|
53
|
+
from . import history # noqa: E402
|
|
54
|
+
|
|
55
|
+
router.include_router(history.router)
|
|
56
|
+
|
|
57
|
+
from . import monitor # noqa: E402
|
|
58
|
+
|
|
59
|
+
router.include_router(monitor.router)
|
|
60
|
+
|
|
61
|
+
from . import manifest # noqa: E402
|
|
62
|
+
|
|
63
|
+
router.include_router(manifest.router)
|
|
64
|
+
|
|
65
|
+
from . import bakeoff # noqa: E402
|
|
66
|
+
|
|
67
|
+
router.include_router(bakeoff.router)
|
|
68
|
+
|
|
69
|
+
from . import updates # noqa: E402
|
|
70
|
+
|
|
71
|
+
router.include_router(updates.router)
|
|
72
|
+
|
|
73
|
+
from . import community # noqa: E402
|
|
74
|
+
|
|
75
|
+
router.include_router(community.router)
|