localdeploy 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api_server.py +15 -0
- benchmark.py +15 -0
- chat_cli.py +15 -0
- compare_models.py +15 -0
- localdeploy/__init__.py +3 -0
- localdeploy/backends/__init__.py +0 -0
- localdeploy/backends/llamacpp.py +143 -0
- localdeploy/backends/ollama.py +374 -0
- localdeploy/backends/openai_compatible.py +179 -0
- localdeploy/benchmark.py +1996 -0
- localdeploy/chat_cli.py +249 -0
- localdeploy/cli.py +59 -0
- localdeploy/compare_models.py +481 -0
- localdeploy/control/__init__.py +75 -0
- localdeploy/control/_config.py +180 -0
- localdeploy/control/_ollama.py +253 -0
- localdeploy/control/bakeoff.py +216 -0
- localdeploy/control/bench.py +516 -0
- localdeploy/control/calibration.py +191 -0
- localdeploy/control/community.py +161 -0
- localdeploy/control/fit.py +651 -0
- localdeploy/control/hardware.py +602 -0
- localdeploy/control/history.py +125 -0
- localdeploy/control/manifest.py +452 -0
- localdeploy/control/models.py +833 -0
- localdeploy/control/monitor.py +378 -0
- localdeploy/control/profiles.py +161 -0
- localdeploy/control/recommend.py +322 -0
- localdeploy/control/registry.py +956 -0
- localdeploy/control/report.py +407 -0
- localdeploy/control/starter.py +520 -0
- localdeploy/control/updates.py +118 -0
- localdeploy/grader_sandbox.py +268 -0
- localdeploy/server.py +1873 -0
- localdeploy/utils.py +155 -0
- localdeploy/web/favicon.png +0 -0
- localdeploy/web/index.html +891 -0
- localdeploy/web/js/app.js +63 -0
- localdeploy/web/js/benchmark-views.js +73 -0
- localdeploy/web/js/benchmark.js +1691 -0
- localdeploy/web/js/chat.js +779 -0
- localdeploy/web/js/models.js +2758 -0
- localdeploy/web/js/shared.js +454 -0
- localdeploy/web/js/system.js +626 -0
- localdeploy/web/logo.svg +22 -0
- localdeploy/web/styles.css +2032 -0
- localdeploy-0.6.0.dist-info/METADATA +202 -0
- localdeploy-0.6.0.dist-info/RECORD +52 -0
- localdeploy-0.6.0.dist-info/WHEEL +5 -0
- localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
- localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
- localdeploy-0.6.0.dist-info/top_level.txt +5 -0
api_server.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Compatibility shim - the implementation lives in ``localdeploy.server``.
|
|
2
|
+
|
|
3
|
+
``sys.modules`` aliasing makes ``import api_server`` yield the *same* module object
|
|
4
|
+
as ``localdeploy.server``, so attribute access, monkeypatching, and ``from
|
|
5
|
+
api_server import X`` all behave exactly as before the module moved into the
|
|
6
|
+
package. Run directly (``python api_server.py``) to serve the API.
|
|
7
|
+
"""
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
from localdeploy import server as _mod
|
|
11
|
+
|
|
12
|
+
sys.modules[__name__] = _mod
|
|
13
|
+
|
|
14
|
+
if __name__ == "__main__":
|
|
15
|
+
_mod.serve()
|
benchmark.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Compatibility shim - the implementation lives in ``localdeploy.benchmark``.
|
|
2
|
+
|
|
3
|
+
``sys.modules`` aliasing makes ``import benchmark`` yield the *same* module object
|
|
4
|
+
as ``localdeploy.benchmark``, so attribute access, monkeypatching, and ``from
|
|
5
|
+
benchmark import X`` all behave exactly as before the module moved into the
|
|
6
|
+
package.
|
|
7
|
+
"""
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
from localdeploy import benchmark as _mod
|
|
11
|
+
|
|
12
|
+
sys.modules[__name__] = _mod
|
|
13
|
+
|
|
14
|
+
if __name__ == "__main__":
|
|
15
|
+
raise SystemExit(_mod.main())
|
chat_cli.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Compatibility shim - the implementation lives in ``localdeploy.chat_cli``.
|
|
2
|
+
|
|
3
|
+
``sys.modules`` aliasing makes ``import chat_cli`` yield the *same* module object
|
|
4
|
+
as ``localdeploy.chat_cli``, so attribute access, monkeypatching, and ``from
|
|
5
|
+
chat_cli import X`` all behave exactly as before the module moved into the
|
|
6
|
+
package.
|
|
7
|
+
"""
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
from localdeploy import chat_cli as _mod
|
|
11
|
+
|
|
12
|
+
sys.modules[__name__] = _mod
|
|
13
|
+
|
|
14
|
+
if __name__ == "__main__":
|
|
15
|
+
raise SystemExit(_mod.main())
|
compare_models.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Compatibility shim - the implementation lives in ``localdeploy.compare_models``.
|
|
2
|
+
|
|
3
|
+
``sys.modules`` aliasing makes ``import compare_models`` yield the *same* module object
|
|
4
|
+
as ``localdeploy.compare_models``, so attribute access, monkeypatching, and ``from
|
|
5
|
+
compare_models import X`` all behave exactly as before the module moved into the
|
|
6
|
+
package.
|
|
7
|
+
"""
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
from localdeploy import compare_models as _mod
|
|
11
|
+
|
|
12
|
+
sys.modules[__name__] = _mod
|
|
13
|
+
|
|
14
|
+
if __name__ == "__main__":
|
|
15
|
+
raise SystemExit(_mod.main())
|
localdeploy/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, Dict, List
|
|
6
|
+
|
|
7
|
+
import requests
|
|
8
|
+
|
|
9
|
+
from ..utils import (
|
|
10
|
+
BackendCallError,
|
|
11
|
+
get_backend_base_url,
|
|
12
|
+
is_loopback_url,
|
|
13
|
+
strip_trailing_slash,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def gguf_path_exists(model_id: str) -> bool:
|
|
18
|
+
if not model_id.lower().endswith(".gguf"):
|
|
19
|
+
return True
|
|
20
|
+
return Path(model_id).expanduser().exists()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def llama_cpp_error_message(response: requests.Response) -> str:
|
|
24
|
+
body = response.text.strip()
|
|
25
|
+
try:
|
|
26
|
+
parsed = response.json()
|
|
27
|
+
body = parsed.get("error") or parsed.get("message") or body
|
|
28
|
+
except Exception:
|
|
29
|
+
pass
|
|
30
|
+
lowered = str(body).lower()
|
|
31
|
+
if "out of memory" in lowered or "cuda" in lowered or "memory" in lowered:
|
|
32
|
+
return f"llama.cpp reported a memory-related failure. Lower context/output limits or reduce GPU layers. Details: {body}"
|
|
33
|
+
return f"llama.cpp request failed with HTTP {response.status_code}. Details: {body}"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def llama_completion_prompt(system_prompt: str, prompt: str) -> str:
|
|
37
|
+
parts: List[str] = []
|
|
38
|
+
if system_prompt:
|
|
39
|
+
parts.append(f"System:\n{system_prompt}")
|
|
40
|
+
parts.append(f"User:\n{prompt}")
|
|
41
|
+
parts.append("Assistant:")
|
|
42
|
+
return "\n\n".join(parts)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def call_llamacpp(prepared: Dict[str, Any]) -> str:
|
|
46
|
+
profile = prepared["profile"]
|
|
47
|
+
model_id = prepared["model"]
|
|
48
|
+
if not gguf_path_exists(model_id):
|
|
49
|
+
raise BackendCallError(f"GGUF file path not found: {model_id}. Update config.json or start the matching local llama-server.")
|
|
50
|
+
|
|
51
|
+
base_url = get_backend_base_url(profile, "llamacpp")
|
|
52
|
+
messages: List[Dict[str, str]] = []
|
|
53
|
+
if prepared["system_prompt"]:
|
|
54
|
+
messages.append({"role": "system", "content": prepared["system_prompt"]})
|
|
55
|
+
messages.append({"role": "user", "content": prepared["prompt"]})
|
|
56
|
+
|
|
57
|
+
openai_payload: Dict[str, Any] = {
|
|
58
|
+
"model": Path(model_id).stem if model_id.lower().endswith(".gguf") else model_id,
|
|
59
|
+
"messages": messages,
|
|
60
|
+
"temperature": prepared["temperature"],
|
|
61
|
+
"top_p": prepared["top_p"],
|
|
62
|
+
"max_tokens": prepared["max_output_tokens_used"],
|
|
63
|
+
"stream": False,
|
|
64
|
+
}
|
|
65
|
+
if prepared.get("repeat_penalty") is not None:
|
|
66
|
+
openai_payload["repeat_penalty"] = prepared["repeat_penalty"]
|
|
67
|
+
if prepared.get("response_format"):
|
|
68
|
+
openai_payload["response_format"] = prepared["response_format"]
|
|
69
|
+
|
|
70
|
+
try:
|
|
71
|
+
response = requests.post(
|
|
72
|
+
f"{base_url}/v1/chat/completions",
|
|
73
|
+
json=openai_payload,
|
|
74
|
+
timeout=prepared["timeout_seconds"],
|
|
75
|
+
)
|
|
76
|
+
except requests.Timeout as exc:
|
|
77
|
+
raise BackendCallError(
|
|
78
|
+
f"llama.cpp request timed out after {prepared['timeout_seconds']} seconds. Lower context/output limits or GPU layers."
|
|
79
|
+
) from exc
|
|
80
|
+
except requests.ConnectionError as exc:
|
|
81
|
+
raise BackendCallError(f"llama.cpp server is not running or is unreachable at {base_url}.") from exc
|
|
82
|
+
|
|
83
|
+
if response.status_code != 404:
|
|
84
|
+
if not response.ok:
|
|
85
|
+
raise BackendCallError(llama_cpp_error_message(response))
|
|
86
|
+
try:
|
|
87
|
+
data = response.json()
|
|
88
|
+
except Exception as exc:
|
|
89
|
+
raise BackendCallError(f"llama.cpp returned invalid JSON: {response.text[:500]}") from exc
|
|
90
|
+
try:
|
|
91
|
+
return str(data["choices"][0]["message"]["content"])
|
|
92
|
+
except Exception:
|
|
93
|
+
return json.dumps(data, ensure_ascii=False)
|
|
94
|
+
|
|
95
|
+
completion_payload = {
|
|
96
|
+
"prompt": llama_completion_prompt(prepared["system_prompt"], prepared["prompt"]),
|
|
97
|
+
"n_predict": prepared["max_output_tokens_used"],
|
|
98
|
+
"temperature": prepared["temperature"],
|
|
99
|
+
"top_p": prepared["top_p"],
|
|
100
|
+
"repeat_penalty": prepared["repeat_penalty"],
|
|
101
|
+
"stream": False,
|
|
102
|
+
}
|
|
103
|
+
try:
|
|
104
|
+
response = requests.post(
|
|
105
|
+
f"{base_url}/completion",
|
|
106
|
+
json=completion_payload,
|
|
107
|
+
timeout=prepared["timeout_seconds"],
|
|
108
|
+
)
|
|
109
|
+
except requests.Timeout as exc:
|
|
110
|
+
raise BackendCallError(
|
|
111
|
+
f"llama.cpp completion timed out after {prepared['timeout_seconds']} seconds. Lower context/output limits."
|
|
112
|
+
) from exc
|
|
113
|
+
except requests.ConnectionError as exc:
|
|
114
|
+
raise BackendCallError(f"llama.cpp server is not running or is unreachable at {base_url}.") from exc
|
|
115
|
+
|
|
116
|
+
if not response.ok:
|
|
117
|
+
raise BackendCallError(llama_cpp_error_message(response))
|
|
118
|
+
|
|
119
|
+
try:
|
|
120
|
+
data = response.json()
|
|
121
|
+
except Exception as exc:
|
|
122
|
+
raise BackendCallError(f"llama.cpp returned invalid JSON: {response.text[:500]}") from exc
|
|
123
|
+
if "content" in data:
|
|
124
|
+
return str(data.get("content", ""))
|
|
125
|
+
if "response" in data:
|
|
126
|
+
return str(data.get("response", ""))
|
|
127
|
+
return json.dumps(data, ensure_ascii=False)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def llama_health(base_url: str) -> Dict[str, Any]:
|
|
131
|
+
if not is_loopback_url(base_url):
|
|
132
|
+
return {"reachable": False, "error": f"Refusing non-local llama.cpp URL: {base_url}"}
|
|
133
|
+
base_url = strip_trailing_slash(base_url)
|
|
134
|
+
for path in ("/health", "/v1/models"):
|
|
135
|
+
try:
|
|
136
|
+
response = requests.get(f"{base_url}{path}", timeout=2)
|
|
137
|
+
if response.ok:
|
|
138
|
+
return {"reachable": True, "endpoint": f"{base_url}{path}"}
|
|
139
|
+
except requests.ConnectionError:
|
|
140
|
+
return {"reachable": False, "error": f"llama.cpp is not reachable at {base_url}."}
|
|
141
|
+
except requests.Timeout:
|
|
142
|
+
return {"reachable": False, "error": f"llama.cpp health check timed out at {base_url}."}
|
|
143
|
+
return {"reachable": False, "error": f"llama.cpp did not answer /health or /v1/models at {base_url}."}
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import json
|
|
5
|
+
import struct
|
|
6
|
+
from typing import Any, Dict, Iterator, List, Optional, Tuple
|
|
7
|
+
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from ..utils import (
|
|
11
|
+
BackendCallError,
|
|
12
|
+
get_backend_base_url,
|
|
13
|
+
is_loopback_url,
|
|
14
|
+
require_gpu_only,
|
|
15
|
+
strip_trailing_slash,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def ollama_format(response_format: Any) -> Any:
|
|
20
|
+
"""Translate OpenAI response_format into Ollama's native `format` value."""
|
|
21
|
+
if not isinstance(response_format, dict):
|
|
22
|
+
return None
|
|
23
|
+
format_type = response_format.get("type")
|
|
24
|
+
if format_type == "json_object":
|
|
25
|
+
return "json"
|
|
26
|
+
if format_type == "json_schema":
|
|
27
|
+
wrapper = response_format.get("json_schema")
|
|
28
|
+
if isinstance(wrapper, dict) and isinstance(wrapper.get("schema"), dict):
|
|
29
|
+
return wrapper["schema"]
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def options_payload(prepared: Dict[str, Any]) -> Dict[str, Any]:
|
|
34
|
+
options: Dict[str, Any] = {
|
|
35
|
+
"num_ctx": prepared["context_limit_used"],
|
|
36
|
+
"num_predict": prepared["max_output_tokens_used"],
|
|
37
|
+
}
|
|
38
|
+
for key in ("temperature", "top_p", "repeat_penalty"):
|
|
39
|
+
if prepared.get(key) is not None:
|
|
40
|
+
options[key] = prepared[key]
|
|
41
|
+
# Optional CPU/GPU placement: num_gpu is the number of layers offloaded to GPU
|
|
42
|
+
# (0 = force CPU). Absent -> Ollama's default (auto), preserving prior behaviour.
|
|
43
|
+
if prepared.get("num_gpu") is not None:
|
|
44
|
+
options["num_gpu"] = prepared["num_gpu"]
|
|
45
|
+
return options
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _message_content_for_ollama(content: Any) -> Tuple[str, List[str]]:
|
|
49
|
+
if isinstance(content, str):
|
|
50
|
+
return content, []
|
|
51
|
+
if not isinstance(content, list):
|
|
52
|
+
return str(content or ""), []
|
|
53
|
+
text: List[str] = []
|
|
54
|
+
images: List[str] = []
|
|
55
|
+
for part in content:
|
|
56
|
+
if not isinstance(part, dict):
|
|
57
|
+
continue
|
|
58
|
+
if part.get("type") in {"text", "input_text"}:
|
|
59
|
+
text.append(str(part.get("text") or ""))
|
|
60
|
+
elif part.get("type") in {"image_url", "input_image"}:
|
|
61
|
+
image = part.get("image_url") or part.get("image")
|
|
62
|
+
url = image.get("url") if isinstance(image, dict) else image
|
|
63
|
+
if isinstance(url, str):
|
|
64
|
+
images.append(url.split(",", 1)[1] if url.startswith("data:") and "," in url else url)
|
|
65
|
+
return "\n".join(value for value in text if value), images
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _ollama_messages(prepared: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
69
|
+
supplied = prepared.get("messages") or []
|
|
70
|
+
if supplied:
|
|
71
|
+
messages: List[Dict[str, Any]] = []
|
|
72
|
+
tool_names: Dict[str, str] = {}
|
|
73
|
+
for raw in supplied:
|
|
74
|
+
if not isinstance(raw, dict):
|
|
75
|
+
continue
|
|
76
|
+
content, images = _message_content_for_ollama(raw.get("content"))
|
|
77
|
+
message: Dict[str, Any] = {"role": str(raw.get("role") or "user"), "content": content}
|
|
78
|
+
if images:
|
|
79
|
+
message["images"] = images
|
|
80
|
+
if raw.get("tool_calls"):
|
|
81
|
+
normalized_calls = []
|
|
82
|
+
for call in raw["tool_calls"]:
|
|
83
|
+
if not isinstance(call, dict):
|
|
84
|
+
continue
|
|
85
|
+
normalized = dict(call)
|
|
86
|
+
function = dict(call.get("function") or {})
|
|
87
|
+
if isinstance(function.get("arguments"), str):
|
|
88
|
+
try:
|
|
89
|
+
function["arguments"] = json.loads(function["arguments"])
|
|
90
|
+
except ValueError:
|
|
91
|
+
pass
|
|
92
|
+
normalized["function"] = function
|
|
93
|
+
normalized_calls.append(normalized)
|
|
94
|
+
if call.get("id") and function.get("name"):
|
|
95
|
+
tool_names[str(call["id"])] = str(function["name"])
|
|
96
|
+
message["tool_calls"] = normalized_calls
|
|
97
|
+
if raw.get("tool_call_id"):
|
|
98
|
+
call_id = str(raw.get("tool_call_id"))
|
|
99
|
+
message["tool_name"] = raw.get("name") or tool_names.get(call_id) or call_id
|
|
100
|
+
messages.append(message)
|
|
101
|
+
return messages
|
|
102
|
+
|
|
103
|
+
messages = []
|
|
104
|
+
if prepared["system_prompt"]:
|
|
105
|
+
messages.append({"role": "system", "content": prepared["system_prompt"]})
|
|
106
|
+
user_message: Dict[str, Any] = {"role": "user", "content": prepared["prompt"]}
|
|
107
|
+
if prepared["images_base64"]:
|
|
108
|
+
user_message["images"] = prepared["images_base64"]
|
|
109
|
+
messages.append(user_message)
|
|
110
|
+
return messages
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _selected_tools(prepared: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
114
|
+
tools = [item for item in (prepared.get("tools") or []) if isinstance(item, dict)]
|
|
115
|
+
choice = prepared.get("tool_choice")
|
|
116
|
+
if choice == "none":
|
|
117
|
+
return []
|
|
118
|
+
if isinstance(choice, dict):
|
|
119
|
+
function = choice.get("function") or {}
|
|
120
|
+
selected_name = function.get("name")
|
|
121
|
+
if selected_name:
|
|
122
|
+
selected = [item for item in tools if (item.get("function") or {}).get("name") == selected_name]
|
|
123
|
+
return selected or tools
|
|
124
|
+
return tools
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def ollama_metrics(data: Dict[str, Any]) -> Dict[str, Any]:
|
|
128
|
+
def seconds(name: str) -> Optional[float]:
|
|
129
|
+
value = data.get(name)
|
|
130
|
+
return round(float(value) / 1_000_000_000, 6) if value is not None else None
|
|
131
|
+
|
|
132
|
+
eval_count = _safe_int(data.get("eval_count"))
|
|
133
|
+
prompt_count = _safe_int(data.get("prompt_eval_count"))
|
|
134
|
+
load_seconds = seconds("load_duration")
|
|
135
|
+
eval_seconds = seconds("eval_duration")
|
|
136
|
+
prompt_seconds = seconds("prompt_eval_duration")
|
|
137
|
+
# Time to first token: Ollama's non-streaming /api/chat response has no
|
|
138
|
+
# explicit first-token timestamp, but load_duration + prompt_eval_duration
|
|
139
|
+
# IS the wall-clock time before the first generated token - the model
|
|
140
|
+
# finishes loading (if cold), processes the prompt, then starts emitting.
|
|
141
|
+
# Present whenever at least one of the two components is known.
|
|
142
|
+
ttft_ms = None
|
|
143
|
+
if load_seconds is not None or prompt_seconds is not None:
|
|
144
|
+
ttft_ms = round(((load_seconds or 0.0) + (prompt_seconds or 0.0)) * 1000, 1)
|
|
145
|
+
return {
|
|
146
|
+
"total_duration_seconds": seconds("total_duration"),
|
|
147
|
+
"load_duration_seconds": load_seconds,
|
|
148
|
+
"prompt_eval_count": prompt_count,
|
|
149
|
+
"prompt_eval_duration_seconds": prompt_seconds,
|
|
150
|
+
"prompt_tokens_per_second": round(prompt_count / prompt_seconds, 3) if prompt_count and prompt_seconds else None,
|
|
151
|
+
"eval_count": eval_count,
|
|
152
|
+
"eval_duration_seconds": eval_seconds,
|
|
153
|
+
"tokens_per_second": round(eval_count / eval_seconds, 3) if eval_count and eval_seconds else None,
|
|
154
|
+
"ttft_ms": ttft_ms,
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _safe_int(value: Any) -> Optional[int]:
|
|
159
|
+
try:
|
|
160
|
+
return int(value) if value is not None else None
|
|
161
|
+
except (TypeError, ValueError):
|
|
162
|
+
return None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def ollama_error_message(response: requests.Response, model_id: str) -> str:
|
|
166
|
+
body = response.text.strip()
|
|
167
|
+
try:
|
|
168
|
+
parsed = response.json()
|
|
169
|
+
body = parsed.get("error") or body
|
|
170
|
+
except Exception:
|
|
171
|
+
pass
|
|
172
|
+
lowered = body.lower()
|
|
173
|
+
if response.status_code == 404 or "not found" in lowered or "pull" in lowered:
|
|
174
|
+
return f"Ollama model '{model_id}' is not available. Run: ollama pull {model_id}. Details: {body}"
|
|
175
|
+
if "out of memory" in lowered or "cuda" in lowered or "memory" in lowered:
|
|
176
|
+
return f"Ollama reported a memory-related failure. Lower context/output limits or use a smaller profile. Details: {body}"
|
|
177
|
+
return f"Ollama request failed with HTTP {response.status_code}. Details: {body}"
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def call_ollama_detailed(prepared: Dict[str, Any]) -> Dict[str, Any]:
|
|
181
|
+
if require_gpu_only():
|
|
182
|
+
raise BackendCallError("GPU-only mode is enabled; refusing to call Ollama.")
|
|
183
|
+
profile = prepared["profile"]
|
|
184
|
+
base_url = get_backend_base_url(profile, "ollama")
|
|
185
|
+
payload = {
|
|
186
|
+
"model": prepared["model"],
|
|
187
|
+
"messages": _ollama_messages(prepared),
|
|
188
|
+
"stream": False,
|
|
189
|
+
"options": options_payload(prepared),
|
|
190
|
+
}
|
|
191
|
+
if prepared.get("keep_alive"):
|
|
192
|
+
payload["keep_alive"] = prepared["keep_alive"]
|
|
193
|
+
tools = _selected_tools(prepared)
|
|
194
|
+
if tools:
|
|
195
|
+
payload["tools"] = tools
|
|
196
|
+
native_format = ollama_format(prepared.get("response_format"))
|
|
197
|
+
if native_format is not None:
|
|
198
|
+
payload["format"] = native_format
|
|
199
|
+
# Thinking models (Qwen3, DeepSeek-R1) put reasoning into a separate `thinking`
|
|
200
|
+
# field; `content` is empty until the model finishes. `think: false` disables
|
|
201
|
+
# the reasoning pass entirely for cases where we only want the final answer.
|
|
202
|
+
if profile.get("think") is not None:
|
|
203
|
+
payload["think"] = bool(profile["think"])
|
|
204
|
+
try:
|
|
205
|
+
response = requests.post(f"{base_url}/api/chat", json=payload, timeout=prepared["timeout_seconds"])
|
|
206
|
+
except requests.Timeout as exc:
|
|
207
|
+
raise BackendCallError(
|
|
208
|
+
f"Ollama request timed out after {prepared['timeout_seconds']} seconds. Lower context/output limits or try a smaller profile."
|
|
209
|
+
) from exc
|
|
210
|
+
except requests.ConnectionError as exc:
|
|
211
|
+
raise BackendCallError(f"Ollama is not running or is unreachable at {base_url}. Start Ollama and retry.") from exc
|
|
212
|
+
|
|
213
|
+
if not response.ok:
|
|
214
|
+
raise BackendCallError(ollama_error_message(response, prepared["model"]))
|
|
215
|
+
|
|
216
|
+
try:
|
|
217
|
+
data = response.json()
|
|
218
|
+
except Exception as exc:
|
|
219
|
+
raise BackendCallError(f"Ollama returned invalid JSON: {response.text[:500]}") from exc
|
|
220
|
+
|
|
221
|
+
message = data.get("message") if isinstance(data.get("message"), dict) else {}
|
|
222
|
+
content = str(message.get("content", data.get("response", "")))
|
|
223
|
+
return {
|
|
224
|
+
"content": content,
|
|
225
|
+
"tool_calls": message.get("tool_calls") or [],
|
|
226
|
+
"metrics": ollama_metrics(data),
|
|
227
|
+
"done_reason": data.get("done_reason"),
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def call_ollama(prepared: Dict[str, Any]) -> str:
|
|
232
|
+
return str(call_ollama_detailed(prepared).get("content") or "")
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def stream_ollama_events(prepared: Dict[str, Any]) -> Iterator[Dict[str, Any]]:
|
|
236
|
+
"""Yield native content/tool/metrics events from an Ollama chat call."""
|
|
237
|
+
if require_gpu_only():
|
|
238
|
+
raise BackendCallError("GPU-only mode is enabled; refusing to call Ollama.")
|
|
239
|
+
profile = prepared["profile"]
|
|
240
|
+
base_url = get_backend_base_url(profile, "ollama")
|
|
241
|
+
payload = {
|
|
242
|
+
"model": prepared["model"],
|
|
243
|
+
"messages": _ollama_messages(prepared),
|
|
244
|
+
"stream": True,
|
|
245
|
+
"options": options_payload(prepared),
|
|
246
|
+
}
|
|
247
|
+
if prepared.get("keep_alive"):
|
|
248
|
+
payload["keep_alive"] = prepared["keep_alive"]
|
|
249
|
+
tools = _selected_tools(prepared)
|
|
250
|
+
if tools:
|
|
251
|
+
payload["tools"] = tools
|
|
252
|
+
native_format = ollama_format(prepared.get("response_format"))
|
|
253
|
+
if native_format is not None:
|
|
254
|
+
payload["format"] = native_format
|
|
255
|
+
if profile.get("think") is not None:
|
|
256
|
+
payload["think"] = bool(profile["think"])
|
|
257
|
+
try:
|
|
258
|
+
response = requests.post(
|
|
259
|
+
f"{base_url}/api/chat",
|
|
260
|
+
json=payload,
|
|
261
|
+
timeout=prepared["timeout_seconds"],
|
|
262
|
+
stream=True,
|
|
263
|
+
)
|
|
264
|
+
except requests.Timeout as exc:
|
|
265
|
+
raise BackendCallError(
|
|
266
|
+
f"Ollama request timed out after {prepared['timeout_seconds']} seconds. Lower context/output limits or try a smaller profile."
|
|
267
|
+
) from exc
|
|
268
|
+
except requests.ConnectionError as exc:
|
|
269
|
+
raise BackendCallError(f"Ollama is not running or is unreachable at {base_url}. Start Ollama and retry.") from exc
|
|
270
|
+
|
|
271
|
+
# Use the response as a context manager so the underlying connection is
|
|
272
|
+
# released even if the SSE consumer disconnects mid-stream (GeneratorExit).
|
|
273
|
+
with response:
|
|
274
|
+
if not response.ok:
|
|
275
|
+
raise BackendCallError(ollama_error_message(response, prepared["model"]))
|
|
276
|
+
|
|
277
|
+
try:
|
|
278
|
+
for line in response.iter_lines():
|
|
279
|
+
if not line:
|
|
280
|
+
continue
|
|
281
|
+
try:
|
|
282
|
+
data = json.loads(line)
|
|
283
|
+
except Exception:
|
|
284
|
+
continue
|
|
285
|
+
message = data.get("message") if isinstance(data.get("message"), dict) else {}
|
|
286
|
+
event = {
|
|
287
|
+
"content": str(message.get("content", data.get("response", ""))),
|
|
288
|
+
"tool_calls": message.get("tool_calls") or [],
|
|
289
|
+
"done": bool(data.get("done")),
|
|
290
|
+
"done_reason": data.get("done_reason"),
|
|
291
|
+
"metrics": ollama_metrics(data) if data.get("done") else None,
|
|
292
|
+
}
|
|
293
|
+
if event["content"] or event["tool_calls"] or event["done"]:
|
|
294
|
+
yield event
|
|
295
|
+
if data.get("done"):
|
|
296
|
+
break
|
|
297
|
+
except requests.exceptions.RequestException as exc:
|
|
298
|
+
raise BackendCallError(f"Ollama connection was lost mid-response: {exc}") from exc
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def stream_ollama(prepared: Dict[str, Any]) -> Iterator[str]:
|
|
302
|
+
"""Backward-compatible content-only stream."""
|
|
303
|
+
for event in stream_ollama_events(prepared):
|
|
304
|
+
if event.get("content"):
|
|
305
|
+
yield str(event["content"])
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def embed_ollama(base_url: str, model: str, inputs: List[str], timeout: int = 180) -> Dict[str, Any]:
|
|
309
|
+
if require_gpu_only():
|
|
310
|
+
raise BackendCallError("GPU-only mode is enabled; refusing to call Ollama.")
|
|
311
|
+
if not is_loopback_url(base_url):
|
|
312
|
+
raise BackendCallError(f"Refusing non-local Ollama URL: {base_url}")
|
|
313
|
+
base_url = strip_trailing_slash(base_url)
|
|
314
|
+
try:
|
|
315
|
+
response = requests.post(
|
|
316
|
+
f"{base_url}/api/embed",
|
|
317
|
+
json={"model": model, "input": inputs, "truncate": True},
|
|
318
|
+
timeout=timeout,
|
|
319
|
+
)
|
|
320
|
+
except requests.Timeout as exc:
|
|
321
|
+
raise BackendCallError(f"Ollama embedding request timed out after {timeout} seconds.") from exc
|
|
322
|
+
except requests.ConnectionError as exc:
|
|
323
|
+
raise BackendCallError(f"Ollama is not running or is unreachable at {base_url}.") from exc
|
|
324
|
+
if response.status_code == 404:
|
|
325
|
+
embeddings = []
|
|
326
|
+
counts = 0
|
|
327
|
+
for value in inputs:
|
|
328
|
+
try:
|
|
329
|
+
fallback = requests.post(
|
|
330
|
+
f"{base_url}/api/embeddings",
|
|
331
|
+
json={"model": model, "prompt": value},
|
|
332
|
+
timeout=timeout,
|
|
333
|
+
)
|
|
334
|
+
except (requests.Timeout, requests.ConnectionError) as exc:
|
|
335
|
+
raise BackendCallError(f"Ollama legacy embedding request failed at {base_url}.") from exc
|
|
336
|
+
if not fallback.ok:
|
|
337
|
+
raise BackendCallError(ollama_error_message(fallback, model))
|
|
338
|
+
try:
|
|
339
|
+
body = fallback.json()
|
|
340
|
+
except ValueError as exc:
|
|
341
|
+
raise BackendCallError("Ollama returned invalid JSON for a legacy embedding request.") from exc
|
|
342
|
+
embeddings.append(body.get("embedding") or [])
|
|
343
|
+
counts += max(1, len(value) // 4)
|
|
344
|
+
return {"embeddings": embeddings, "prompt_eval_count": counts, "metrics": {}}
|
|
345
|
+
if not response.ok:
|
|
346
|
+
raise BackendCallError(ollama_error_message(response, model))
|
|
347
|
+
try:
|
|
348
|
+
data = response.json()
|
|
349
|
+
except ValueError as exc:
|
|
350
|
+
raise BackendCallError("Ollama returned invalid JSON for an embedding request.") from exc
|
|
351
|
+
return {
|
|
352
|
+
"embeddings": data.get("embeddings") or [],
|
|
353
|
+
"prompt_eval_count": _safe_int(data.get("prompt_eval_count")),
|
|
354
|
+
"metrics": ollama_metrics(data),
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def encode_embedding_base64(values: List[float]) -> str:
|
|
359
|
+
packed = struct.pack(f"<{len(values)}f", *(float(value) for value in values))
|
|
360
|
+
return base64.b64encode(packed).decode("ascii")
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def ollama_models(base_url: str) -> Tuple[List[str], Optional[str]]:
|
|
364
|
+
if not is_loopback_url(base_url):
|
|
365
|
+
return [], f"Refusing non-local Ollama URL: {base_url}"
|
|
366
|
+
try:
|
|
367
|
+
response = requests.get(f"{strip_trailing_slash(base_url)}/api/tags", timeout=3)
|
|
368
|
+
response.raise_for_status()
|
|
369
|
+
data = response.json()
|
|
370
|
+
return [item.get("name", "") for item in data.get("models", []) if item.get("name")], None
|
|
371
|
+
except requests.ConnectionError:
|
|
372
|
+
return [], f"Ollama is not reachable at {base_url}."
|
|
373
|
+
except Exception as exc:
|
|
374
|
+
return [], str(exc)
|