localdeploy 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. api_server.py +15 -0
  2. benchmark.py +15 -0
  3. chat_cli.py +15 -0
  4. compare_models.py +15 -0
  5. localdeploy/__init__.py +3 -0
  6. localdeploy/backends/__init__.py +0 -0
  7. localdeploy/backends/llamacpp.py +143 -0
  8. localdeploy/backends/ollama.py +374 -0
  9. localdeploy/backends/openai_compatible.py +179 -0
  10. localdeploy/benchmark.py +1996 -0
  11. localdeploy/chat_cli.py +249 -0
  12. localdeploy/cli.py +59 -0
  13. localdeploy/compare_models.py +481 -0
  14. localdeploy/control/__init__.py +75 -0
  15. localdeploy/control/_config.py +180 -0
  16. localdeploy/control/_ollama.py +253 -0
  17. localdeploy/control/bakeoff.py +216 -0
  18. localdeploy/control/bench.py +516 -0
  19. localdeploy/control/calibration.py +191 -0
  20. localdeploy/control/community.py +161 -0
  21. localdeploy/control/fit.py +651 -0
  22. localdeploy/control/hardware.py +602 -0
  23. localdeploy/control/history.py +125 -0
  24. localdeploy/control/manifest.py +452 -0
  25. localdeploy/control/models.py +833 -0
  26. localdeploy/control/monitor.py +378 -0
  27. localdeploy/control/profiles.py +161 -0
  28. localdeploy/control/recommend.py +322 -0
  29. localdeploy/control/registry.py +956 -0
  30. localdeploy/control/report.py +407 -0
  31. localdeploy/control/starter.py +520 -0
  32. localdeploy/control/updates.py +118 -0
  33. localdeploy/grader_sandbox.py +268 -0
  34. localdeploy/server.py +1873 -0
  35. localdeploy/utils.py +155 -0
  36. localdeploy/web/favicon.png +0 -0
  37. localdeploy/web/index.html +891 -0
  38. localdeploy/web/js/app.js +63 -0
  39. localdeploy/web/js/benchmark-views.js +73 -0
  40. localdeploy/web/js/benchmark.js +1691 -0
  41. localdeploy/web/js/chat.js +779 -0
  42. localdeploy/web/js/models.js +2758 -0
  43. localdeploy/web/js/shared.js +454 -0
  44. localdeploy/web/js/system.js +626 -0
  45. localdeploy/web/logo.svg +22 -0
  46. localdeploy/web/styles.css +2032 -0
  47. localdeploy-0.6.0.dist-info/METADATA +202 -0
  48. localdeploy-0.6.0.dist-info/RECORD +52 -0
  49. localdeploy-0.6.0.dist-info/WHEEL +5 -0
  50. localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
  51. localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
  52. localdeploy-0.6.0.dist-info/top_level.txt +5 -0
@@ -0,0 +1,179 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from pathlib import Path
5
+ from typing import Any, Dict, Iterator, List, Optional, Tuple
6
+
7
+ import requests
8
+
9
+ from ..utils import BackendCallError, get_backend_base_url
10
+
11
+
12
+ def _headers(profile: Dict[str, Any]) -> Dict[str, str]:
13
+ token = str(profile.get("api_key") or "").strip()
14
+ return {"Authorization": f"Bearer {token}"} if token else {}
15
+
16
+
17
+ def _messages(prepared: Dict[str, Any]) -> List[Dict[str, Any]]:
18
+ supplied = prepared.get("messages") or []
19
+ if supplied:
20
+ return [dict(item) for item in supplied if isinstance(item, dict)]
21
+ messages: List[Dict[str, Any]] = []
22
+ if prepared.get("system_prompt"):
23
+ messages.append({"role": "system", "content": prepared["system_prompt"]})
24
+ messages.append({"role": "user", "content": prepared.get("prompt") or ""})
25
+ return messages
26
+
27
+
28
+ def _payload(prepared: Dict[str, Any], stream: bool) -> Dict[str, Any]:
29
+ model = str(prepared["model"])
30
+ if prepared.get("backend") == "llamacpp" and model.lower().endswith(".gguf"):
31
+ model = Path(model).stem
32
+ payload: Dict[str, Any] = {
33
+ "model": model,
34
+ "messages": _messages(prepared),
35
+ "stream": stream,
36
+ "max_tokens": prepared["max_output_tokens_used"],
37
+ }
38
+ for name in ("temperature", "top_p", "repeat_penalty", "response_format"):
39
+ if prepared.get(name) is not None:
40
+ payload[name] = prepared[name]
41
+ tools = prepared.get("tools") or []
42
+ if tools and prepared.get("tool_choice") != "none":
43
+ payload["tools"] = tools
44
+ if prepared.get("tool_choice") is not None:
45
+ payload["tool_choice"] = prepared["tool_choice"]
46
+ return payload
47
+
48
+
49
+ def _error(response: requests.Response, provider: str) -> str:
50
+ body = response.text.strip()
51
+ try:
52
+ data = response.json()
53
+ error = data.get("error")
54
+ if isinstance(error, dict):
55
+ body = str(error.get("message") or body)
56
+ elif error:
57
+ body = str(error)
58
+ except Exception:
59
+ pass
60
+ return f"{provider} request failed with HTTP {response.status_code}. Details: {body[:1000]}"
61
+
62
+
63
+ def call_openai_compatible(prepared: Dict[str, Any]) -> Dict[str, Any]:
64
+ profile = prepared["profile"]
65
+ provider = str(prepared["backend"])
66
+ base_url = get_backend_base_url(profile, provider)
67
+ try:
68
+ response = requests.post(
69
+ f"{base_url}/v1/chat/completions",
70
+ json=_payload(prepared, False),
71
+ headers=_headers(profile),
72
+ timeout=prepared["timeout_seconds"],
73
+ )
74
+ except requests.Timeout as exc:
75
+ raise BackendCallError(f"{provider} timed out after {prepared['timeout_seconds']} seconds.") from exc
76
+ except requests.ConnectionError as exc:
77
+ raise BackendCallError(f"{provider} is not reachable at {base_url}.") from exc
78
+ if not response.ok:
79
+ raise BackendCallError(_error(response, provider))
80
+ try:
81
+ data = response.json()
82
+ choice = data["choices"][0]
83
+ message = choice.get("message") or {}
84
+ except Exception as exc:
85
+ raise BackendCallError(f"{provider} returned an invalid chat response: {response.text[:500]}") from exc
86
+ usage = data.get("usage") if isinstance(data.get("usage"), dict) else {}
87
+ details = usage.get("completion_tokens_details") if isinstance(usage.get("completion_tokens_details"), dict) else {}
88
+ return {
89
+ "content": str(message.get("content") or ""),
90
+ "tool_calls": message.get("tool_calls") or [],
91
+ "done_reason": choice.get("finish_reason"),
92
+ "metrics": {
93
+ "prompt_eval_count": usage.get("prompt_tokens"),
94
+ "eval_count": usage.get("completion_tokens"),
95
+ "tokens_per_second": details.get("tokens_per_second") or usage.get("tokens_per_second"),
96
+ },
97
+ }
98
+
99
+
100
+ def stream_openai_compatible(prepared: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
101
+ profile = prepared["profile"]
102
+ provider = str(prepared["backend"])
103
+ base_url = get_backend_base_url(profile, provider)
104
+ try:
105
+ response = requests.post(
106
+ f"{base_url}/v1/chat/completions",
107
+ json=_payload(prepared, True),
108
+ headers=_headers(profile),
109
+ timeout=prepared["timeout_seconds"],
110
+ stream=True,
111
+ )
112
+ except requests.Timeout as exc:
113
+ raise BackendCallError(f"{provider} timed out after {prepared['timeout_seconds']} seconds.") from exc
114
+ except requests.ConnectionError as exc:
115
+ raise BackendCallError(f"{provider} is not reachable at {base_url}.") from exc
116
+ with response:
117
+ if not response.ok:
118
+ raise BackendCallError(_error(response, provider))
119
+ for raw in response.iter_lines():
120
+ if not raw:
121
+ continue
122
+ line = raw.decode("utf-8", errors="replace") if isinstance(raw, bytes) else str(raw)
123
+ if not line.startswith("data:"):
124
+ continue
125
+ value = line[5:].strip()
126
+ if value == "[DONE]":
127
+ break
128
+ try:
129
+ data = json.loads(value)
130
+ choice = data["choices"][0]
131
+ delta = choice.get("delta") or {}
132
+ except Exception:
133
+ continue
134
+ yield {
135
+ "content": str(delta.get("content") or ""),
136
+ "tool_calls": delta.get("tool_calls") or [],
137
+ "done": bool(choice.get("finish_reason")),
138
+ "done_reason": choice.get("finish_reason"),
139
+ "metrics": data.get("usage"),
140
+ }
141
+
142
+
143
+ def embed_openai_compatible(
144
+ profile: Dict[str, Any], backend: str, model: str, inputs: List[str], timeout: int
145
+ ) -> Dict[str, Any]:
146
+ base_url = get_backend_base_url(profile, backend)
147
+ if backend == "llamacpp" and model.lower().endswith(".gguf"):
148
+ model = Path(model).stem
149
+ try:
150
+ response = requests.post(
151
+ f"{base_url}/v1/embeddings",
152
+ json={"model": model, "input": inputs, "encoding_format": "float"},
153
+ headers=_headers(profile),
154
+ timeout=timeout,
155
+ )
156
+ except requests.Timeout as exc:
157
+ raise BackendCallError(f"{backend} embedding request timed out after {timeout} seconds.") from exc
158
+ except requests.ConnectionError as exc:
159
+ raise BackendCallError(f"{backend} is not reachable at {base_url}.") from exc
160
+ if not response.ok:
161
+ raise BackendCallError(_error(response, backend))
162
+ data = response.json()
163
+ rows = sorted(data.get("data") or [], key=lambda item: int(item.get("index", 0)))
164
+ return {"embeddings": [item.get("embedding") or [] for item in rows], "usage": data.get("usage") or {}}
165
+
166
+
167
+ def list_openai_models(
168
+ base_url: str, provider: str, headers: Optional[Dict[str, str]] = None
169
+ ) -> Tuple[List[Dict[str, Any]], Optional[str]]:
170
+ try:
171
+ response = requests.get(f"{base_url.rstrip('/')}/v1/models", headers=headers or {}, timeout=3)
172
+ except (requests.ConnectionError, requests.Timeout):
173
+ return [], f"{provider} is not reachable at {base_url}."
174
+ if not response.ok:
175
+ return [], _error(response, provider)
176
+ try:
177
+ return [item for item in response.json().get("data", []) if item.get("id")], None
178
+ except Exception as exc:
179
+ return [], f"{provider} returned invalid model inventory: {exc}"