localdeploy 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api_server.py +15 -0
- benchmark.py +15 -0
- chat_cli.py +15 -0
- compare_models.py +15 -0
- localdeploy/__init__.py +3 -0
- localdeploy/backends/__init__.py +0 -0
- localdeploy/backends/llamacpp.py +143 -0
- localdeploy/backends/ollama.py +374 -0
- localdeploy/backends/openai_compatible.py +179 -0
- localdeploy/benchmark.py +1996 -0
- localdeploy/chat_cli.py +249 -0
- localdeploy/cli.py +59 -0
- localdeploy/compare_models.py +481 -0
- localdeploy/control/__init__.py +75 -0
- localdeploy/control/_config.py +180 -0
- localdeploy/control/_ollama.py +253 -0
- localdeploy/control/bakeoff.py +216 -0
- localdeploy/control/bench.py +516 -0
- localdeploy/control/calibration.py +191 -0
- localdeploy/control/community.py +161 -0
- localdeploy/control/fit.py +651 -0
- localdeploy/control/hardware.py +602 -0
- localdeploy/control/history.py +125 -0
- localdeploy/control/manifest.py +452 -0
- localdeploy/control/models.py +833 -0
- localdeploy/control/monitor.py +378 -0
- localdeploy/control/profiles.py +161 -0
- localdeploy/control/recommend.py +322 -0
- localdeploy/control/registry.py +956 -0
- localdeploy/control/report.py +407 -0
- localdeploy/control/starter.py +520 -0
- localdeploy/control/updates.py +118 -0
- localdeploy/grader_sandbox.py +268 -0
- localdeploy/server.py +1873 -0
- localdeploy/utils.py +155 -0
- localdeploy/web/favicon.png +0 -0
- localdeploy/web/index.html +891 -0
- localdeploy/web/js/app.js +63 -0
- localdeploy/web/js/benchmark-views.js +73 -0
- localdeploy/web/js/benchmark.js +1691 -0
- localdeploy/web/js/chat.js +779 -0
- localdeploy/web/js/models.js +2758 -0
- localdeploy/web/js/shared.js +454 -0
- localdeploy/web/js/system.js +626 -0
- localdeploy/web/logo.svg +22 -0
- localdeploy/web/styles.css +2032 -0
- localdeploy-0.6.0.dist-info/METADATA +202 -0
- localdeploy-0.6.0.dist-info/RECORD +52 -0
- localdeploy-0.6.0.dist-info/WHEEL +5 -0
- localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
- localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
- localdeploy-0.6.0.dist-info/top_level.txt +5 -0
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
|
|
9
|
+
from ..utils import BackendCallError, get_backend_base_url
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _headers(profile: Dict[str, Any]) -> Dict[str, str]:
|
|
13
|
+
token = str(profile.get("api_key") or "").strip()
|
|
14
|
+
return {"Authorization": f"Bearer {token}"} if token else {}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _messages(prepared: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
18
|
+
supplied = prepared.get("messages") or []
|
|
19
|
+
if supplied:
|
|
20
|
+
return [dict(item) for item in supplied if isinstance(item, dict)]
|
|
21
|
+
messages: List[Dict[str, Any]] = []
|
|
22
|
+
if prepared.get("system_prompt"):
|
|
23
|
+
messages.append({"role": "system", "content": prepared["system_prompt"]})
|
|
24
|
+
messages.append({"role": "user", "content": prepared.get("prompt") or ""})
|
|
25
|
+
return messages
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _payload(prepared: Dict[str, Any], stream: bool) -> Dict[str, Any]:
|
|
29
|
+
model = str(prepared["model"])
|
|
30
|
+
if prepared.get("backend") == "llamacpp" and model.lower().endswith(".gguf"):
|
|
31
|
+
model = Path(model).stem
|
|
32
|
+
payload: Dict[str, Any] = {
|
|
33
|
+
"model": model,
|
|
34
|
+
"messages": _messages(prepared),
|
|
35
|
+
"stream": stream,
|
|
36
|
+
"max_tokens": prepared["max_output_tokens_used"],
|
|
37
|
+
}
|
|
38
|
+
for name in ("temperature", "top_p", "repeat_penalty", "response_format"):
|
|
39
|
+
if prepared.get(name) is not None:
|
|
40
|
+
payload[name] = prepared[name]
|
|
41
|
+
tools = prepared.get("tools") or []
|
|
42
|
+
if tools and prepared.get("tool_choice") != "none":
|
|
43
|
+
payload["tools"] = tools
|
|
44
|
+
if prepared.get("tool_choice") is not None:
|
|
45
|
+
payload["tool_choice"] = prepared["tool_choice"]
|
|
46
|
+
return payload
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _error(response: requests.Response, provider: str) -> str:
|
|
50
|
+
body = response.text.strip()
|
|
51
|
+
try:
|
|
52
|
+
data = response.json()
|
|
53
|
+
error = data.get("error")
|
|
54
|
+
if isinstance(error, dict):
|
|
55
|
+
body = str(error.get("message") or body)
|
|
56
|
+
elif error:
|
|
57
|
+
body = str(error)
|
|
58
|
+
except Exception:
|
|
59
|
+
pass
|
|
60
|
+
return f"{provider} request failed with HTTP {response.status_code}. Details: {body[:1000]}"
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def call_openai_compatible(prepared: Dict[str, Any]) -> Dict[str, Any]:
|
|
64
|
+
profile = prepared["profile"]
|
|
65
|
+
provider = str(prepared["backend"])
|
|
66
|
+
base_url = get_backend_base_url(profile, provider)
|
|
67
|
+
try:
|
|
68
|
+
response = requests.post(
|
|
69
|
+
f"{base_url}/v1/chat/completions",
|
|
70
|
+
json=_payload(prepared, False),
|
|
71
|
+
headers=_headers(profile),
|
|
72
|
+
timeout=prepared["timeout_seconds"],
|
|
73
|
+
)
|
|
74
|
+
except requests.Timeout as exc:
|
|
75
|
+
raise BackendCallError(f"{provider} timed out after {prepared['timeout_seconds']} seconds.") from exc
|
|
76
|
+
except requests.ConnectionError as exc:
|
|
77
|
+
raise BackendCallError(f"{provider} is not reachable at {base_url}.") from exc
|
|
78
|
+
if not response.ok:
|
|
79
|
+
raise BackendCallError(_error(response, provider))
|
|
80
|
+
try:
|
|
81
|
+
data = response.json()
|
|
82
|
+
choice = data["choices"][0]
|
|
83
|
+
message = choice.get("message") or {}
|
|
84
|
+
except Exception as exc:
|
|
85
|
+
raise BackendCallError(f"{provider} returned an invalid chat response: {response.text[:500]}") from exc
|
|
86
|
+
usage = data.get("usage") if isinstance(data.get("usage"), dict) else {}
|
|
87
|
+
details = usage.get("completion_tokens_details") if isinstance(usage.get("completion_tokens_details"), dict) else {}
|
|
88
|
+
return {
|
|
89
|
+
"content": str(message.get("content") or ""),
|
|
90
|
+
"tool_calls": message.get("tool_calls") or [],
|
|
91
|
+
"done_reason": choice.get("finish_reason"),
|
|
92
|
+
"metrics": {
|
|
93
|
+
"prompt_eval_count": usage.get("prompt_tokens"),
|
|
94
|
+
"eval_count": usage.get("completion_tokens"),
|
|
95
|
+
"tokens_per_second": details.get("tokens_per_second") or usage.get("tokens_per_second"),
|
|
96
|
+
},
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def stream_openai_compatible(prepared: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
|
|
101
|
+
profile = prepared["profile"]
|
|
102
|
+
provider = str(prepared["backend"])
|
|
103
|
+
base_url = get_backend_base_url(profile, provider)
|
|
104
|
+
try:
|
|
105
|
+
response = requests.post(
|
|
106
|
+
f"{base_url}/v1/chat/completions",
|
|
107
|
+
json=_payload(prepared, True),
|
|
108
|
+
headers=_headers(profile),
|
|
109
|
+
timeout=prepared["timeout_seconds"],
|
|
110
|
+
stream=True,
|
|
111
|
+
)
|
|
112
|
+
except requests.Timeout as exc:
|
|
113
|
+
raise BackendCallError(f"{provider} timed out after {prepared['timeout_seconds']} seconds.") from exc
|
|
114
|
+
except requests.ConnectionError as exc:
|
|
115
|
+
raise BackendCallError(f"{provider} is not reachable at {base_url}.") from exc
|
|
116
|
+
with response:
|
|
117
|
+
if not response.ok:
|
|
118
|
+
raise BackendCallError(_error(response, provider))
|
|
119
|
+
for raw in response.iter_lines():
|
|
120
|
+
if not raw:
|
|
121
|
+
continue
|
|
122
|
+
line = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw)
|
|
123
|
+
if not line.startswith("data:"):
|
|
124
|
+
continue
|
|
125
|
+
value = line[5:].strip()
|
|
126
|
+
if value == "[DONE]":
|
|
127
|
+
break
|
|
128
|
+
try:
|
|
129
|
+
data = json.loads(value)
|
|
130
|
+
choice = data["choices"][0]
|
|
131
|
+
delta = choice.get("delta") or {}
|
|
132
|
+
except Exception:
|
|
133
|
+
continue
|
|
134
|
+
yield {
|
|
135
|
+
"content": str(delta.get("content") or ""),
|
|
136
|
+
"tool_calls": delta.get("tool_calls") or [],
|
|
137
|
+
"done": bool(choice.get("finish_reason")),
|
|
138
|
+
"done_reason": choice.get("finish_reason"),
|
|
139
|
+
"metrics": data.get("usage"),
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def embed_openai_compatible(
|
|
144
|
+
profile: Dict[str, Any], backend: str, model: str, inputs: List[str], timeout: int
|
|
145
|
+
) -> Dict[str, Any]:
|
|
146
|
+
base_url = get_backend_base_url(profile, backend)
|
|
147
|
+
if backend == "llamacpp" and model.lower().endswith(".gguf"):
|
|
148
|
+
model = Path(model).stem
|
|
149
|
+
try:
|
|
150
|
+
response = requests.post(
|
|
151
|
+
f"{base_url}/v1/embeddings",
|
|
152
|
+
json={"model": model, "input": inputs, "encoding_format": "float"},
|
|
153
|
+
headers=_headers(profile),
|
|
154
|
+
timeout=timeout,
|
|
155
|
+
)
|
|
156
|
+
except requests.Timeout as exc:
|
|
157
|
+
raise BackendCallError(f"{backend} embedding request timed out after {timeout} seconds.") from exc
|
|
158
|
+
except requests.ConnectionError as exc:
|
|
159
|
+
raise BackendCallError(f"{backend} is not reachable at {base_url}.") from exc
|
|
160
|
+
if not response.ok:
|
|
161
|
+
raise BackendCallError(_error(response, backend))
|
|
162
|
+
data = response.json()
|
|
163
|
+
rows = sorted(data.get("data") or [], key=lambda item: int(item.get("index", 0)))
|
|
164
|
+
return {"embeddings": [item.get("embedding") or [] for item in rows], "usage": data.get("usage") or {}}
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def list_openai_models(
|
|
168
|
+
base_url: str, provider: str, headers: Optional[Dict[str, str]] = None
|
|
169
|
+
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
|
|
170
|
+
try:
|
|
171
|
+
response = requests.get(f"{base_url.rstrip('/')}/v1/models", headers=headers or {}, timeout=3)
|
|
172
|
+
except (requests.ConnectionError, requests.Timeout):
|
|
173
|
+
return [], f"{provider} is not reachable at {base_url}."
|
|
174
|
+
if not response.ok:
|
|
175
|
+
return [], _error(response, provider)
|
|
176
|
+
try:
|
|
177
|
+
return [item for item in response.json().get("data", []) if item.get("id")], None
|
|
178
|
+
except Exception as exc:
|
|
179
|
+
return [], f"{provider} returned invalid model inventory: {exc}"
|