localdeploy 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. api_server.py +15 -0
  2. benchmark.py +15 -0
  3. chat_cli.py +15 -0
  4. compare_models.py +15 -0
  5. localdeploy/__init__.py +3 -0
  6. localdeploy/backends/__init__.py +0 -0
  7. localdeploy/backends/llamacpp.py +143 -0
  8. localdeploy/backends/ollama.py +374 -0
  9. localdeploy/backends/openai_compatible.py +179 -0
  10. localdeploy/benchmark.py +1996 -0
  11. localdeploy/chat_cli.py +249 -0
  12. localdeploy/cli.py +59 -0
  13. localdeploy/compare_models.py +481 -0
  14. localdeploy/control/__init__.py +75 -0
  15. localdeploy/control/_config.py +180 -0
  16. localdeploy/control/_ollama.py +253 -0
  17. localdeploy/control/bakeoff.py +216 -0
  18. localdeploy/control/bench.py +516 -0
  19. localdeploy/control/calibration.py +191 -0
  20. localdeploy/control/community.py +161 -0
  21. localdeploy/control/fit.py +651 -0
  22. localdeploy/control/hardware.py +602 -0
  23. localdeploy/control/history.py +125 -0
  24. localdeploy/control/manifest.py +452 -0
  25. localdeploy/control/models.py +833 -0
  26. localdeploy/control/monitor.py +378 -0
  27. localdeploy/control/profiles.py +161 -0
  28. localdeploy/control/recommend.py +322 -0
  29. localdeploy/control/registry.py +956 -0
  30. localdeploy/control/report.py +407 -0
  31. localdeploy/control/starter.py +520 -0
  32. localdeploy/control/updates.py +118 -0
  33. localdeploy/grader_sandbox.py +268 -0
  34. localdeploy/server.py +1873 -0
  35. localdeploy/utils.py +155 -0
  36. localdeploy/web/favicon.png +0 -0
  37. localdeploy/web/index.html +891 -0
  38. localdeploy/web/js/app.js +63 -0
  39. localdeploy/web/js/benchmark-views.js +73 -0
  40. localdeploy/web/js/benchmark.js +1691 -0
  41. localdeploy/web/js/chat.js +779 -0
  42. localdeploy/web/js/models.js +2758 -0
  43. localdeploy/web/js/shared.js +454 -0
  44. localdeploy/web/js/system.js +626 -0
  45. localdeploy/web/logo.svg +22 -0
  46. localdeploy/web/styles.css +2032 -0
  47. localdeploy-0.6.0.dist-info/METADATA +202 -0
  48. localdeploy-0.6.0.dist-info/RECORD +52 -0
  49. localdeploy-0.6.0.dist-info/WHEEL +5 -0
  50. localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
  51. localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
  52. localdeploy-0.6.0.dist-info/top_level.txt +5 -0
api_server.py ADDED
@@ -0,0 +1,15 @@
1
+ """Compatibility shim - the implementation lives in ``localdeploy.server``.
2
+
3
+ ``sys.modules`` aliasing makes ``import api_server`` yield the *same* module object
4
+ as ``localdeploy.server``, so attribute access, monkeypatching, and ``from
5
+ api_server import X`` all behave exactly as before the module moved into the
6
+ package. Run directly (``python api_server.py``) to serve the API.
7
+ """
8
+ import sys
9
+
10
+ from localdeploy import server as _mod
11
+
12
+ sys.modules[__name__] = _mod
13
+
14
+ if __name__ == "__main__":
15
+ _mod.serve()
benchmark.py ADDED
@@ -0,0 +1,15 @@
1
+ """Compatibility shim - the implementation lives in ``localdeploy.benchmark``.
2
+
3
+ ``sys.modules`` aliasing makes ``import benchmark`` yield the *same* module object
4
+ as ``localdeploy.benchmark``, so attribute access, monkeypatching, and ``from
5
+ benchmark import X`` all behave exactly as before the module moved into the
6
+ package.
7
+ """
8
+ import sys
9
+
10
+ from localdeploy import benchmark as _mod
11
+
12
+ sys.modules[__name__] = _mod
13
+
14
+ if __name__ == "__main__":
15
+ raise SystemExit(_mod.main())
chat_cli.py ADDED
@@ -0,0 +1,15 @@
1
+ """Compatibility shim - the implementation lives in ``localdeploy.chat_cli``.
2
+
3
+ ``sys.modules`` aliasing makes ``import chat_cli`` yield the *same* module object
4
+ as ``localdeploy.chat_cli``, so attribute access, monkeypatching, and ``from
5
+ chat_cli import X`` all behave exactly as before the module moved into the
6
+ package.
7
+ """
8
+ import sys
9
+
10
+ from localdeploy import chat_cli as _mod
11
+
12
+ sys.modules[__name__] = _mod
13
+
14
+ if __name__ == "__main__":
15
+ raise SystemExit(_mod.main())
compare_models.py ADDED
@@ -0,0 +1,15 @@
1
+ """Compatibility shim - the implementation lives in ``localdeploy.compare_models``.
2
+
3
+ ``sys.modules`` aliasing makes ``import compare_models`` yield the *same* module object
4
+ as ``localdeploy.compare_models``, so attribute access, monkeypatching, and ``from
5
+ compare_models import X`` all behave exactly as before the module moved into the
6
+ package.
7
+ """
8
+ import sys
9
+
10
+ from localdeploy import compare_models as _mod
11
+
12
+ sys.modules[__name__] = _mod
13
+
14
+ if __name__ == "__main__":
15
+ raise SystemExit(_mod.main())
@@ -0,0 +1,3 @@
1
+ """LocalDeploy manages and benchmarks AI models on local runtimes."""
2
+
3
+ __version__ = "0.6.0"
File without changes
@@ -0,0 +1,143 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from pathlib import Path
5
+ from typing import Any, Dict, List
6
+
7
+ import requests
8
+
9
+ from ..utils import (
10
+ BackendCallError,
11
+ get_backend_base_url,
12
+ is_loopback_url,
13
+ strip_trailing_slash,
14
+ )
15
+
16
+
17
+ def gguf_path_exists(model_id: str) -> bool:
18
+ if not model_id.lower().endswith(".gguf"):
19
+ return True
20
+ return Path(model_id).expanduser().exists()
21
+
22
+
23
+ def llama_cpp_error_message(response: requests.Response) -> str:
24
+ body = response.text.strip()
25
+ try:
26
+ parsed = response.json()
27
+ body = parsed.get("error") or parsed.get("message") or body
28
+ except Exception:
29
+ pass
30
+ lowered = str(body).lower()
31
+ if "out of memory" in lowered or "cuda" in lowered or "memory" in lowered:
32
+ return f"llama.cpp reported a memory-related failure. Lower context/output limits or reduce GPU layers. Details: {body}"
33
+ return f"llama.cpp request failed with HTTP {response.status_code}. Details: {body}"
34
+
35
+
36
+ def llama_completion_prompt(system_prompt: str, prompt: str) -> str:
37
+ parts: List[str] = []
38
+ if system_prompt:
39
+ parts.append(f"System:\n{system_prompt}")
40
+ parts.append(f"User:\n{prompt}")
41
+ parts.append("Assistant:")
42
+ return "\n\n".join(parts)
43
+
44
+
45
+ def call_llamacpp(prepared: Dict[str, Any]) -> str:
46
+ profile = prepared["profile"]
47
+ model_id = prepared["model"]
48
+ if not gguf_path_exists(model_id):
49
+ raise BackendCallError(f"GGUF file path not found: {model_id}. Update config.json or start the matching local llama-server.")
50
+
51
+ base_url = get_backend_base_url(profile, "llamacpp")
52
+ messages: List[Dict[str, str]] = []
53
+ if prepared["system_prompt"]:
54
+ messages.append({"role": "system", "content": prepared["system_prompt"]})
55
+ messages.append({"role": "user", "content": prepared["prompt"]})
56
+
57
+ openai_payload: Dict[str, Any] = {
58
+ "model": Path(model_id).stem if model_id.lower().endswith(".gguf") else model_id,
59
+ "messages": messages,
60
+ "temperature": prepared["temperature"],
61
+ "top_p": prepared["top_p"],
62
+ "max_tokens": prepared["max_output_tokens_used"],
63
+ "stream": False,
64
+ }
65
+ if prepared.get("repeat_penalty") is not None:
66
+ openai_payload["repeat_penalty"] = prepared["repeat_penalty"]
67
+ if prepared.get("response_format"):
68
+ openai_payload["response_format"] = prepared["response_format"]
69
+
70
+ try:
71
+ response = requests.post(
72
+ f"{base_url}/v1/chat/completions",
73
+ json=openai_payload,
74
+ timeout=prepared["timeout_seconds"],
75
+ )
76
+ except requests.Timeout as exc:
77
+ raise BackendCallError(
78
+ f"llama.cpp request timed out after {prepared['timeout_seconds']} seconds. Lower context/output limits or GPU layers."
79
+ ) from exc
80
+ except requests.ConnectionError as exc:
81
+ raise BackendCallError(f"llama.cpp server is not running or is unreachable at {base_url}.") from exc
82
+
83
+ if response.status_code != 404:
84
+ if not response.ok:
85
+ raise BackendCallError(llama_cpp_error_message(response))
86
+ try:
87
+ data = response.json()
88
+ except Exception as exc:
89
+ raise BackendCallError(f"llama.cpp returned invalid JSON: {response.text[:500]}") from exc
90
+ try:
91
+ return str(data["choices"][0]["message"]["content"])
92
+ except Exception:
93
+ return json.dumps(data, ensure_ascii=False)
94
+
95
+ completion_payload = {
96
+ "prompt": llama_completion_prompt(prepared["system_prompt"], prepared["prompt"]),
97
+ "n_predict": prepared["max_output_tokens_used"],
98
+ "temperature": prepared["temperature"],
99
+ "top_p": prepared["top_p"],
100
+ "repeat_penalty": prepared["repeat_penalty"],
101
+ "stream": False,
102
+ }
103
+ try:
104
+ response = requests.post(
105
+ f"{base_url}/completion",
106
+ json=completion_payload,
107
+ timeout=prepared["timeout_seconds"],
108
+ )
109
+ except requests.Timeout as exc:
110
+ raise BackendCallError(
111
+ f"llama.cpp completion timed out after {prepared['timeout_seconds']} seconds. Lower context/output limits."
112
+ ) from exc
113
+ except requests.ConnectionError as exc:
114
+ raise BackendCallError(f"llama.cpp server is not running or is unreachable at {base_url}.") from exc
115
+
116
+ if not response.ok:
117
+ raise BackendCallError(llama_cpp_error_message(response))
118
+
119
+ try:
120
+ data = response.json()
121
+ except Exception as exc:
122
+ raise BackendCallError(f"llama.cpp returned invalid JSON: {response.text[:500]}") from exc
123
+ if "content" in data:
124
+ return str(data.get("content", ""))
125
+ if "response" in data:
126
+ return str(data.get("response", ""))
127
+ return json.dumps(data, ensure_ascii=False)
128
+
129
+
130
+ def llama_health(base_url: str) -> Dict[str, Any]:
131
+ if not is_loopback_url(base_url):
132
+ return {"reachable": False, "error": f"Refusing non-local llama.cpp URL: {base_url}"}
133
+ base_url = strip_trailing_slash(base_url)
134
+ for path in ("/health", "/v1/models"):
135
+ try:
136
+ response = requests.get(f"{base_url}{path}", timeout=2)
137
+ if response.ok:
138
+ return {"reachable": True, "endpoint": f"{base_url}{path}"}
139
+ except requests.ConnectionError:
140
+ return {"reachable": False, "error": f"llama.cpp is not reachable at {base_url}."}
141
+ except requests.Timeout:
142
+ return {"reachable": False, "error": f"llama.cpp health check timed out at {base_url}."}
143
+ return {"reachable": False, "error": f"llama.cpp did not answer /health or /v1/models at {base_url}."}
@@ -0,0 +1,374 @@
1
+ from __future__ import annotations
2
+
3
+ import base64
4
+ import json
5
+ import struct
6
+ from typing import Any, Dict, Iterator, List, Optional, Tuple
7
+
8
+ import requests
9
+
10
+ from ..utils import (
11
+ BackendCallError,
12
+ get_backend_base_url,
13
+ is_loopback_url,
14
+ require_gpu_only,
15
+ strip_trailing_slash,
16
+ )
17
+
18
+
19
+ def ollama_format(response_format: Any) -> Any:
20
+ """Translate OpenAI response_format into Ollama's native `format` value."""
21
+ if not isinstance(response_format, dict):
22
+ return None
23
+ format_type = response_format.get("type")
24
+ if format_type == "json_object":
25
+ return "json"
26
+ if format_type == "json_schema":
27
+ wrapper = response_format.get("json_schema")
28
+ if isinstance(wrapper, dict) and isinstance(wrapper.get("schema"), dict):
29
+ return wrapper["schema"]
30
+ return None
31
+
32
+
33
+ def options_payload(prepared: Dict[str, Any]) -> Dict[str, Any]:
34
+ options: Dict[str, Any] = {
35
+ "num_ctx": prepared["context_limit_used"],
36
+ "num_predict": prepared["max_output_tokens_used"],
37
+ }
38
+ for key in ("temperature", "top_p", "repeat_penalty"):
39
+ if prepared.get(key) is not None:
40
+ options[key] = prepared[key]
41
+ # Optional CPU/GPU placement: num_gpu is the number of layers offloaded to GPU
42
+ # (0 = force CPU). Absent -> Ollama's default (auto), preserving prior behaviour.
43
+ if prepared.get("num_gpu") is not None:
44
+ options["num_gpu"] = prepared["num_gpu"]
45
+ return options
46
+
47
+
48
+ def _message_content_for_ollama(content: Any) -> Tuple[str, List[str]]:
49
+ if isinstance(content, str):
50
+ return content, []
51
+ if not isinstance(content, list):
52
+ return str(content or ""), []
53
+ text: List[str] = []
54
+ images: List[str] = []
55
+ for part in content:
56
+ if not isinstance(part, dict):
57
+ continue
58
+ if part.get("type") in {"text", "input_text"}:
59
+ text.append(str(part.get("text") or ""))
60
+ elif part.get("type") in {"image_url", "input_image"}:
61
+ image = part.get("image_url") or part.get("image")
62
+ url = image.get("url") if isinstance(image, dict) else image
63
+ if isinstance(url, str):
64
+ images.append(url.split(",", 1)[1] if url.startswith("data:") and "," in url else url)
65
+ return "\n".join(value for value in text if value), images
66
+
67
+
68
+ def _ollama_messages(prepared: Dict[str, Any]) -> List[Dict[str, Any]]:
69
+ supplied = prepared.get("messages") or []
70
+ if supplied:
71
+ messages: List[Dict[str, Any]] = []
72
+ tool_names: Dict[str, str] = {}
73
+ for raw in supplied:
74
+ if not isinstance(raw, dict):
75
+ continue
76
+ content, images = _message_content_for_ollama(raw.get("content"))
77
+ message: Dict[str, Any] = {"role": str(raw.get("role") or "user"), "content": content}
78
+ if images:
79
+ message["images"] = images
80
+ if raw.get("tool_calls"):
81
+ normalized_calls = []
82
+ for call in raw["tool_calls"]:
83
+ if not isinstance(call, dict):
84
+ continue
85
+ normalized = dict(call)
86
+ function = dict(call.get("function") or {})
87
+ if isinstance(function.get("arguments"), str):
88
+ try:
89
+ function["arguments"] = json.loads(function["arguments"])
90
+ except ValueError:
91
+ pass
92
+ normalized["function"] = function
93
+ normalized_calls.append(normalized)
94
+ if call.get("id") and function.get("name"):
95
+ tool_names[str(call["id"])] = str(function["name"])
96
+ message["tool_calls"] = normalized_calls
97
+ if raw.get("tool_call_id"):
98
+ call_id = str(raw.get("tool_call_id"))
99
+ message["tool_name"] = raw.get("name") or tool_names.get(call_id) or call_id
100
+ messages.append(message)
101
+ return messages
102
+
103
+ messages = []
104
+ if prepared["system_prompt"]:
105
+ messages.append({"role": "system", "content": prepared["system_prompt"]})
106
+ user_message: Dict[str, Any] = {"role": "user", "content": prepared["prompt"]}
107
+ if prepared["images_base64"]:
108
+ user_message["images"] = prepared["images_base64"]
109
+ messages.append(user_message)
110
+ return messages
111
+
112
+
113
+ def _selected_tools(prepared: Dict[str, Any]) -> List[Dict[str, Any]]:
114
+ tools = [item for item in (prepared.get("tools") or []) if isinstance(item, dict)]
115
+ choice = prepared.get("tool_choice")
116
+ if choice == "none":
117
+ return []
118
+ if isinstance(choice, dict):
119
+ function = choice.get("function") or {}
120
+ selected_name = function.get("name")
121
+ if selected_name:
122
+ selected = [item for item in tools if (item.get("function") or {}).get("name") == selected_name]
123
+ return selected or tools
124
+ return tools
125
+
126
+
127
+ def ollama_metrics(data: Dict[str, Any]) -> Dict[str, Any]:
128
+ def seconds(name: str) -> Optional[float]:
129
+ value = data.get(name)
130
+ return round(float(value) / 1_000_000_000, 6) if value is not None else None
131
+
132
+ eval_count = _safe_int(data.get("eval_count"))
133
+ prompt_count = _safe_int(data.get("prompt_eval_count"))
134
+ load_seconds = seconds("load_duration")
135
+ eval_seconds = seconds("eval_duration")
136
+ prompt_seconds = seconds("prompt_eval_duration")
137
+ # Time to first token: Ollama's non-streaming /api/chat response has no
138
+ # explicit first-token timestamp, but load_duration + prompt_eval_duration
139
+ # IS the wall-clock time before the first generated token - the model
140
+ # finishes loading (if cold), processes the prompt, then starts emitting.
141
+ # Present whenever at least one of the two components is known.
142
+ ttft_ms = None
143
+ if load_seconds is not None or prompt_seconds is not None:
144
+ ttft_ms = round(((load_seconds or 0.0) + (prompt_seconds or 0.0)) * 1000, 1)
145
+ return {
146
+ "total_duration_seconds": seconds("total_duration"),
147
+ "load_duration_seconds": load_seconds,
148
+ "prompt_eval_count": prompt_count,
149
+ "prompt_eval_duration_seconds": prompt_seconds,
150
+ "prompt_tokens_per_second": round(prompt_count / prompt_seconds, 3) if prompt_count and prompt_seconds else None,
151
+ "eval_count": eval_count,
152
+ "eval_duration_seconds": eval_seconds,
153
+ "tokens_per_second": round(eval_count / eval_seconds, 3) if eval_count and eval_seconds else None,
154
+ "ttft_ms": ttft_ms,
155
+ }
156
+
157
+
158
+ def _safe_int(value: Any) -> Optional[int]:
159
+ try:
160
+ return int(value) if value is not None else None
161
+ except (TypeError, ValueError):
162
+ return None
163
+
164
+
165
+ def ollama_error_message(response: requests.Response, model_id: str) -> str:
166
+ body = response.text.strip()
167
+ try:
168
+ parsed = response.json()
169
+ body = parsed.get("error") or body
170
+ except Exception:
171
+ pass
172
+ lowered = body.lower()
173
+ if response.status_code == 404 or "not found" in lowered or "pull" in lowered:
174
+ return f"Ollama model '{model_id}' is not available. Run: ollama pull {model_id}. Details: {body}"
175
+ if "out of memory" in lowered or "cuda" in lowered or "memory" in lowered:
176
+ return f"Ollama reported a memory-related failure. Lower context/output limits or use a smaller profile. Details: {body}"
177
+ return f"Ollama request failed with HTTP {response.status_code}. Details: {body}"
178
+
179
+
180
+ def call_ollama_detailed(prepared: Dict[str, Any]) -> Dict[str, Any]:
181
+ if require_gpu_only():
182
+ raise BackendCallError("GPU-only mode is enabled; refusing to call Ollama.")
183
+ profile = prepared["profile"]
184
+ base_url = get_backend_base_url(profile, "ollama")
185
+ payload = {
186
+ "model": prepared["model"],
187
+ "messages": _ollama_messages(prepared),
188
+ "stream": False,
189
+ "options": options_payload(prepared),
190
+ }
191
+ if prepared.get("keep_alive"):
192
+ payload["keep_alive"] = prepared["keep_alive"]
193
+ tools = _selected_tools(prepared)
194
+ if tools:
195
+ payload["tools"] = tools
196
+ native_format = ollama_format(prepared.get("response_format"))
197
+ if native_format is not None:
198
+ payload["format"] = native_format
199
+ # Thinking models (Qwen3, DeepSeek-R1) put reasoning into a separate `thinking`
200
+ # field; `content` is empty until the model finishes. `think: false` disables
201
+ # the reasoning pass entirely for cases where we only want the final answer.
202
+ if profile.get("think") is not None:
203
+ payload["think"] = bool(profile["think"])
204
+ try:
205
+ response = requests.post(f"{base_url}/api/chat", json=payload, timeout=prepared["timeout_seconds"])
206
+ except requests.Timeout as exc:
207
+ raise BackendCallError(
208
+ f"Ollama request timed out after {prepared['timeout_seconds']} seconds. Lower context/output limits or try a smaller profile."
209
+ ) from exc
210
+ except requests.ConnectionError as exc:
211
+ raise BackendCallError(f"Ollama is not running or is unreachable at {base_url}. Start Ollama and retry.") from exc
212
+
213
+ if not response.ok:
214
+ raise BackendCallError(ollama_error_message(response, prepared["model"]))
215
+
216
+ try:
217
+ data = response.json()
218
+ except Exception as exc:
219
+ raise BackendCallError(f"Ollama returned invalid JSON: {response.text[:500]}") from exc
220
+
221
+ message = data.get("message") if isinstance(data.get("message"), dict) else {}
222
+ content = str(message.get("content", data.get("response", "")))
223
+ return {
224
+ "content": content,
225
+ "tool_calls": message.get("tool_calls") or [],
226
+ "metrics": ollama_metrics(data),
227
+ "done_reason": data.get("done_reason"),
228
+ }
229
+
230
+
231
+ def call_ollama(prepared: Dict[str, Any]) -> str:
232
+ return str(call_ollama_detailed(prepared).get("content") or "")
233
+
234
+
235
+ def stream_ollama_events(prepared: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
236
+ """Yield native content/tool/metrics events from an Ollama chat call."""
237
+ if require_gpu_only():
238
+ raise BackendCallError("GPU-only mode is enabled; refusing to call Ollama.")
239
+ profile = prepared["profile"]
240
+ base_url = get_backend_base_url(profile, "ollama")
241
+ payload = {
242
+ "model": prepared["model"],
243
+ "messages": _ollama_messages(prepared),
244
+ "stream": True,
245
+ "options": options_payload(prepared),
246
+ }
247
+ if prepared.get("keep_alive"):
248
+ payload["keep_alive"] = prepared["keep_alive"]
249
+ tools = _selected_tools(prepared)
250
+ if tools:
251
+ payload["tools"] = tools
252
+ native_format = ollama_format(prepared.get("response_format"))
253
+ if native_format is not None:
254
+ payload["format"] = native_format
255
+ if profile.get("think") is not None:
256
+ payload["think"] = bool(profile["think"])
257
+ try:
258
+ response = requests.post(
259
+ f"{base_url}/api/chat",
260
+ json=payload,
261
+ timeout=prepared["timeout_seconds"],
262
+ stream=True,
263
+ )
264
+ except requests.Timeout as exc:
265
+ raise BackendCallError(
266
+ f"Ollama request timed out after {prepared['timeout_seconds']} seconds. Lower context/output limits or try a smaller profile."
267
+ ) from exc
268
+ except requests.ConnectionError as exc:
269
+ raise BackendCallError(f"Ollama is not running or is unreachable at {base_url}. Start Ollama and retry.") from exc
270
+
271
+ # Use the response as a context manager so the underlying connection is
272
+ # released even if the SSE consumer disconnects mid-stream (GeneratorExit).
273
+ with response:
274
+ if not response.ok:
275
+ raise BackendCallError(ollama_error_message(response, prepared["model"]))
276
+
277
+ try:
278
+ for line in response.iter_lines():
279
+ if not line:
280
+ continue
281
+ try:
282
+ data = json.loads(line)
283
+ except Exception:
284
+ continue
285
+ message = data.get("message") if isinstance(data.get("message"), dict) else {}
286
+ event = {
287
+ "content": str(message.get("content", data.get("response", ""))),
288
+ "tool_calls": message.get("tool_calls") or [],
289
+ "done": bool(data.get("done")),
290
+ "done_reason": data.get("done_reason"),
291
+ "metrics": ollama_metrics(data) if data.get("done") else None,
292
+ }
293
+ if event["content"] or event["tool_calls"] or event["done"]:
294
+ yield event
295
+ if data.get("done"):
296
+ break
297
+ except requests.exceptions.RequestException as exc:
298
+ raise BackendCallError(f"Ollama connection was lost mid-response: {exc}") from exc
299
+
300
+
301
+ def stream_ollama(prepared: Dict[str, Any]) -> Iterator[str]:
302
+ """Backward-compatible content-only stream."""
303
+ for event in stream_ollama_events(prepared):
304
+ if event.get("content"):
305
+ yield str(event["content"])
306
+
307
+
308
+ def embed_ollama(base_url: str, model: str, inputs: List[str], timeout: int = 180) -> Dict[str, Any]:
309
+ if require_gpu_only():
310
+ raise BackendCallError("GPU-only mode is enabled; refusing to call Ollama.")
311
+ if not is_loopback_url(base_url):
312
+ raise BackendCallError(f"Refusing non-local Ollama URL: {base_url}")
313
+ base_url = strip_trailing_slash(base_url)
314
+ try:
315
+ response = requests.post(
316
+ f"{base_url}/api/embed",
317
+ json={"model": model, "input": inputs, "truncate": True},
318
+ timeout=timeout,
319
+ )
320
+ except requests.Timeout as exc:
321
+ raise BackendCallError(f"Ollama embedding request timed out after {timeout} seconds.") from exc
322
+ except requests.ConnectionError as exc:
323
+ raise BackendCallError(f"Ollama is not running or is unreachable at {base_url}.") from exc
324
+ if response.status_code == 404:
325
+ embeddings = []
326
+ counts = 0
327
+ for value in inputs:
328
+ try:
329
+ fallback = requests.post(
330
+ f"{base_url}/api/embeddings",
331
+ json={"model": model, "prompt": value},
332
+ timeout=timeout,
333
+ )
334
+ except (requests.Timeout, requests.ConnectionError) as exc:
335
+ raise BackendCallError(f"Ollama legacy embedding request failed at {base_url}.") from exc
336
+ if not fallback.ok:
337
+ raise BackendCallError(ollama_error_message(fallback, model))
338
+ try:
339
+ body = fallback.json()
340
+ except ValueError as exc:
341
+ raise BackendCallError("Ollama returned invalid JSON for a legacy embedding request.") from exc
342
+ embeddings.append(body.get("embedding") or [])
343
+ counts += max(1, len(value) // 4)
344
+ return {"embeddings": embeddings, "prompt_eval_count": counts, "metrics": {}}
345
+ if not response.ok:
346
+ raise BackendCallError(ollama_error_message(response, model))
347
+ try:
348
+ data = response.json()
349
+ except ValueError as exc:
350
+ raise BackendCallError("Ollama returned invalid JSON for an embedding request.") from exc
351
+ return {
352
+ "embeddings": data.get("embeddings") or [],
353
+ "prompt_eval_count": _safe_int(data.get("prompt_eval_count")),
354
+ "metrics": ollama_metrics(data),
355
+ }
356
+
357
+
358
+ def encode_embedding_base64(values: List[float]) -> str:
359
+ packed = struct.pack(f"<{len(values)}f", *(float(value) for value in values))
360
+ return base64.b64encode(packed).decode("ascii")
361
+
362
+
363
+ def ollama_models(base_url: str) -> Tuple[List[str], Optional[str]]:
364
+ if not is_loopback_url(base_url):
365
+ return [], f"Refusing non-local Ollama URL: {base_url}"
366
+ try:
367
+ response = requests.get(f"{strip_trailing_slash(base_url)}/api/tags", timeout=3)
368
+ response.raise_for_status()
369
+ data = response.json()
370
+ return [item.get("name", "") for item in data.get("models", []) if item.get("name")], None
371
+ except requests.ConnectionError:
372
+ return [], f"Ollama is not reachable at {base_url}."
373
+ except Exception as exc:
374
+ return [], str(exc)