localdeploy 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- api_server.py +15 -0
- benchmark.py +15 -0
- chat_cli.py +15 -0
- compare_models.py +15 -0
- localdeploy/__init__.py +3 -0
- localdeploy/backends/__init__.py +0 -0
- localdeploy/backends/llamacpp.py +143 -0
- localdeploy/backends/ollama.py +374 -0
- localdeploy/backends/openai_compatible.py +179 -0
- localdeploy/benchmark.py +1996 -0
- localdeploy/chat_cli.py +249 -0
- localdeploy/cli.py +59 -0
- localdeploy/compare_models.py +481 -0
- localdeploy/control/__init__.py +75 -0
- localdeploy/control/_config.py +180 -0
- localdeploy/control/_ollama.py +253 -0
- localdeploy/control/bakeoff.py +216 -0
- localdeploy/control/bench.py +516 -0
- localdeploy/control/calibration.py +191 -0
- localdeploy/control/community.py +161 -0
- localdeploy/control/fit.py +651 -0
- localdeploy/control/hardware.py +602 -0
- localdeploy/control/history.py +125 -0
- localdeploy/control/manifest.py +452 -0
- localdeploy/control/models.py +833 -0
- localdeploy/control/monitor.py +378 -0
- localdeploy/control/profiles.py +161 -0
- localdeploy/control/recommend.py +322 -0
- localdeploy/control/registry.py +956 -0
- localdeploy/control/report.py +407 -0
- localdeploy/control/starter.py +520 -0
- localdeploy/control/updates.py +118 -0
- localdeploy/grader_sandbox.py +268 -0
- localdeploy/server.py +1873 -0
- localdeploy/utils.py +155 -0
- localdeploy/web/favicon.png +0 -0
- localdeploy/web/index.html +891 -0
- localdeploy/web/js/app.js +63 -0
- localdeploy/web/js/benchmark-views.js +73 -0
- localdeploy/web/js/benchmark.js +1691 -0
- localdeploy/web/js/chat.js +779 -0
- localdeploy/web/js/models.js +2758 -0
- localdeploy/web/js/shared.js +454 -0
- localdeploy/web/js/system.js +626 -0
- localdeploy/web/logo.svg +22 -0
- localdeploy/web/styles.css +2032 -0
- localdeploy-0.6.0.dist-info/METADATA +202 -0
- localdeploy-0.6.0.dist-info/RECORD +52 -0
- localdeploy-0.6.0.dist-info/WHEEL +5 -0
- localdeploy-0.6.0.dist-info/entry_points.txt +2 -0
- localdeploy-0.6.0.dist-info/licenses/LICENSE +21 -0
- localdeploy-0.6.0.dist-info/top_level.txt +5 -0
localdeploy/chat_cli.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import subprocess
|
|
7
|
+
import sys
|
|
8
|
+
import time
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any, Dict, Optional
|
|
11
|
+
|
|
12
|
+
import requests
|
|
13
|
+
from dotenv import load_dotenv
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
from localdeploy.utils import api_auth_headers, api_client_base_url, app_home
|
|
17
|
+
|
|
18
|
+
APP_DIR = Path(__file__).resolve().parents[1] # repo root in a checkout
|
|
19
|
+
APP_HOME = app_home()
|
|
20
|
+
load_dotenv(APP_HOME / ".env")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def default_base_url() -> str:
|
|
24
|
+
return api_client_base_url()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def server_health(base_url: str) -> Optional[Dict[str, Any]]:
|
|
28
|
+
try:
|
|
29
|
+
response = requests.get(f"{base_url.rstrip('/')}/health", headers=api_auth_headers(), timeout=5)
|
|
30
|
+
if response.ok:
|
|
31
|
+
return dict(response.json())
|
|
32
|
+
except requests.RequestException:
|
|
33
|
+
return None
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def start_server(base_url: str) -> Optional[subprocess.Popen[Any]]:
|
|
38
|
+
if server_health(base_url):
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
python_exe = APP_DIR / ".venv" / "Scripts" / "python.exe"
|
|
42
|
+
if not python_exe.exists():
|
|
43
|
+
python_exe = Path(sys.executable)
|
|
44
|
+
|
|
45
|
+
logs_dir = APP_HOME / "logs"
|
|
46
|
+
logs_dir.mkdir(exist_ok=True)
|
|
47
|
+
stdout = (logs_dir / "chat_cli_api_server.out.log").open("a", encoding="utf-8")
|
|
48
|
+
stderr = (logs_dir / "chat_cli_api_server.err.log").open("a", encoding="utf-8")
|
|
49
|
+
process = subprocess.Popen(
|
|
50
|
+
[str(python_exe), "-m", "localdeploy.cli", "--no-browser"],
|
|
51
|
+
cwd=str(APP_HOME),
|
|
52
|
+
stdout=stdout,
|
|
53
|
+
stderr=stderr,
|
|
54
|
+
creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
for _ in range(40):
|
|
58
|
+
time.sleep(0.5)
|
|
59
|
+
if server_health(base_url):
|
|
60
|
+
return process
|
|
61
|
+
return process
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def print_profiles(base_url: str) -> int:
|
|
65
|
+
try:
|
|
66
|
+
response = requests.get(f"{base_url.rstrip('/')}/profiles", headers=api_auth_headers(), timeout=10)
|
|
67
|
+
response.raise_for_status()
|
|
68
|
+
data = response.json()
|
|
69
|
+
except requests.RequestException as exc:
|
|
70
|
+
print(f"Could not load profiles from {base_url}: {exc}")
|
|
71
|
+
return 1
|
|
72
|
+
|
|
73
|
+
print(f"Default profile: {data.get('default_profile')}")
|
|
74
|
+
print()
|
|
75
|
+
for name, profile in data.get("profiles", {}).items():
|
|
76
|
+
status = "enabled" if profile.get("enabled") else "disabled"
|
|
77
|
+
recommended = profile.get("recommended_for_8gb_vram")
|
|
78
|
+
model_id = profile.get("model_id")
|
|
79
|
+
backend = profile.get("backend")
|
|
80
|
+
print(f"- {name}: {status}, {backend}, {model_id}, 8GB={recommended}")
|
|
81
|
+
return 0
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def send_chat(
|
|
85
|
+
base_url: str,
|
|
86
|
+
profile: str,
|
|
87
|
+
prompt: str,
|
|
88
|
+
max_output_tokens: int,
|
|
89
|
+
temperature: Optional[float],
|
|
90
|
+
timeout: int,
|
|
91
|
+
raw: bool = False,
|
|
92
|
+
) -> int:
|
|
93
|
+
payload: Dict[str, Any] = {
|
|
94
|
+
"profile": profile,
|
|
95
|
+
"prompt": prompt,
|
|
96
|
+
"safe_mode": True,
|
|
97
|
+
"max_output_tokens": max_output_tokens,
|
|
98
|
+
}
|
|
99
|
+
if temperature is not None:
|
|
100
|
+
payload["temperature"] = temperature
|
|
101
|
+
|
|
102
|
+
started = time.perf_counter()
|
|
103
|
+
try:
|
|
104
|
+
response = requests.post(
|
|
105
|
+
f"{base_url.rstrip('/')}/chat",
|
|
106
|
+
json=payload,
|
|
107
|
+
headers=api_auth_headers(),
|
|
108
|
+
timeout=timeout,
|
|
109
|
+
)
|
|
110
|
+
except requests.Timeout:
|
|
111
|
+
print(f"Request timed out after {timeout} seconds.")
|
|
112
|
+
return 1
|
|
113
|
+
except requests.ConnectionError:
|
|
114
|
+
print(f"LocalDeploy is not reachable at {base_url}. Start it with: .\\scripts\\start.ps1")
|
|
115
|
+
return 1
|
|
116
|
+
|
|
117
|
+
elapsed = time.perf_counter() - started
|
|
118
|
+
try:
|
|
119
|
+
data = response.json()
|
|
120
|
+
except ValueError:
|
|
121
|
+
print(f"Server returned non-JSON HTTP {response.status_code}:")
|
|
122
|
+
print(response.text[:1000])
|
|
123
|
+
return 1
|
|
124
|
+
|
|
125
|
+
if raw:
|
|
126
|
+
print(json.dumps(data, indent=2, ensure_ascii=False))
|
|
127
|
+
return 0 if data.get("success") else 1
|
|
128
|
+
|
|
129
|
+
if not data.get("success"):
|
|
130
|
+
print(f"Error: {data.get('error')}")
|
|
131
|
+
if data.get("warning"):
|
|
132
|
+
print(f"Warning: {data.get('warning')}")
|
|
133
|
+
return 1
|
|
134
|
+
|
|
135
|
+
print()
|
|
136
|
+
print(data.get("response", ""))
|
|
137
|
+
print()
|
|
138
|
+
print(
|
|
139
|
+
f"[{data.get('profile')} | {data.get('model')} | "
|
|
140
|
+
f"{data.get('elapsed_seconds', round(elapsed, 3))}s | "
|
|
141
|
+
f"ctx={data.get('context_limit_used')} | out={data.get('max_output_tokens_used')}]"
|
|
142
|
+
)
|
|
143
|
+
if data.get("warning"):
|
|
144
|
+
print(f"Warning: {data['warning']}")
|
|
145
|
+
return 0
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def repl(args: argparse.Namespace) -> int:
|
|
149
|
+
print("LocalDeploy terminal chat")
|
|
150
|
+
print("Type a prompt and press Enter. Commands: :quit, :profiles, :profile NAME, :tokens N, :raw, :help")
|
|
151
|
+
print(f"Server: {args.base_url}")
|
|
152
|
+
print(f"Profile: {args.profile or '(server default)'}")
|
|
153
|
+
print()
|
|
154
|
+
|
|
155
|
+
profile = args.profile
|
|
156
|
+
max_tokens = args.max_output_tokens
|
|
157
|
+
raw = args.raw
|
|
158
|
+
|
|
159
|
+
while True:
|
|
160
|
+
try:
|
|
161
|
+
prompt = input("You> ").strip()
|
|
162
|
+
except (EOFError, KeyboardInterrupt):
|
|
163
|
+
print()
|
|
164
|
+
return 0
|
|
165
|
+
|
|
166
|
+
if not prompt:
|
|
167
|
+
continue
|
|
168
|
+
if prompt in {":quit", ":exit", "quit", "exit"}:
|
|
169
|
+
return 0
|
|
170
|
+
if prompt == ":help":
|
|
171
|
+
print("Commands: :quit, :profiles, :profile NAME, :tokens N, :raw")
|
|
172
|
+
continue
|
|
173
|
+
if prompt == ":profiles":
|
|
174
|
+
print_profiles(args.base_url)
|
|
175
|
+
continue
|
|
176
|
+
if prompt.startswith(":profile "):
|
|
177
|
+
profile = prompt.split(" ", 1)[1].strip()
|
|
178
|
+
print(f"Profile set to {profile}")
|
|
179
|
+
continue
|
|
180
|
+
if prompt.startswith(":tokens "):
|
|
181
|
+
try:
|
|
182
|
+
max_tokens = int(prompt.split(" ", 1)[1].strip())
|
|
183
|
+
except ValueError:
|
|
184
|
+
print("Usage: :tokens 256")
|
|
185
|
+
continue
|
|
186
|
+
print(f"max_output_tokens set to {max_tokens}")
|
|
187
|
+
continue
|
|
188
|
+
if prompt == ":raw":
|
|
189
|
+
raw = not raw
|
|
190
|
+
print(f"Raw JSON output: {raw}")
|
|
191
|
+
continue
|
|
192
|
+
|
|
193
|
+
send_chat(
|
|
194
|
+
args.base_url,
|
|
195
|
+
profile,
|
|
196
|
+
prompt,
|
|
197
|
+
max_tokens,
|
|
198
|
+
args.temperature,
|
|
199
|
+
args.timeout,
|
|
200
|
+
raw,
|
|
201
|
+
)
|
|
202
|
+
print()
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def main() -> int:
|
|
206
|
+
parser = argparse.ArgumentParser(description="Simple terminal chat client for LocalDeploy.")
|
|
207
|
+
parser.add_argument("--base-url", default=default_base_url(), help="LocalDeploy API base URL.")
|
|
208
|
+
parser.add_argument(
|
|
209
|
+
"--profile",
|
|
210
|
+
default=os.getenv("DEFAULT_MODEL_PROFILE"),
|
|
211
|
+
help="Profile name from config.json. If omitted, the server selects its configured default.",
|
|
212
|
+
)
|
|
213
|
+
parser.add_argument("--prompt", help="One-shot prompt. If omitted, starts interactive chat.")
|
|
214
|
+
parser.add_argument("--max-output-tokens", type=int, default=256)
|
|
215
|
+
parser.add_argument("--temperature", type=float)
|
|
216
|
+
parser.add_argument("--timeout", type=int, default=180)
|
|
217
|
+
parser.add_argument("--raw", action="store_true", help="Print full JSON response.")
|
|
218
|
+
parser.add_argument("--profiles", action="store_true", help="List configured profiles and exit.")
|
|
219
|
+
parser.add_argument("--start-server", action="store_true", help="Start api_server.py if it is not already running.")
|
|
220
|
+
args = parser.parse_args()
|
|
221
|
+
|
|
222
|
+
if args.start_server:
|
|
223
|
+
start_server(args.base_url)
|
|
224
|
+
|
|
225
|
+
if args.profiles:
|
|
226
|
+
return print_profiles(args.base_url)
|
|
227
|
+
|
|
228
|
+
if not server_health(args.base_url):
|
|
229
|
+
print(f"LocalDeploy is not reachable at {args.base_url}.")
|
|
230
|
+
print("Start it with:")
|
|
231
|
+
print(" .\\.venv\\Scripts\\python.exe api_server.py")
|
|
232
|
+
print("Or rerun this client with --start-server.")
|
|
233
|
+
return 1
|
|
234
|
+
|
|
235
|
+
if args.prompt:
|
|
236
|
+
return send_chat(
|
|
237
|
+
args.base_url,
|
|
238
|
+
args.profile,
|
|
239
|
+
args.prompt,
|
|
240
|
+
args.max_output_tokens,
|
|
241
|
+
args.temperature,
|
|
242
|
+
args.timeout,
|
|
243
|
+
args.raw,
|
|
244
|
+
)
|
|
245
|
+
return repl(args)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
if __name__ == "__main__":
|
|
249
|
+
raise SystemExit(main())
|
localdeploy/cli.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Console entry point: ``localdeploy`` serves the API + web UI.
|
|
2
|
+
|
|
3
|
+
For pip/pipx installs there is no repo checkout to anchor to, so runtime state
|
|
4
|
+
(.env, config.json, logs/, reports/) lives in the app home - ~/.localdeploy by
|
|
5
|
+
default, or wherever LOCALDEPLOY_HOME points (see ``localdeploy.utils.app_home``).
|
|
6
|
+
Running from a source checkout keeps using the repo root, same as the start
|
|
7
|
+
scripts.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import os
|
|
13
|
+
import threading
|
|
14
|
+
import webbrowser
|
|
15
|
+
|
|
16
|
+
from . import __version__
|
|
17
|
+
from .utils import api_client_base_url, app_home
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def main(argv: list[str] | None = None) -> int:
|
|
21
|
+
parser = argparse.ArgumentParser(
|
|
22
|
+
prog="localdeploy",
|
|
23
|
+
description="Pick, deploy & benchmark the best local AI model for your machine.",
|
|
24
|
+
)
|
|
25
|
+
parser.add_argument("--host", help="bind address (default: API_HOST or 127.0.0.1)")
|
|
26
|
+
parser.add_argument("--port", type=int, help="port (default: API_PORT or 8000)")
|
|
27
|
+
parser.add_argument("--no-browser", action="store_true", help="don't open the UI in a browser")
|
|
28
|
+
parser.add_argument("--version", action="version", version=f"localdeploy {__version__}")
|
|
29
|
+
args = parser.parse_args(argv)
|
|
30
|
+
|
|
31
|
+
home = app_home()
|
|
32
|
+
home.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
# CLI flags beat .env: they land in the environment before api_server's
|
|
34
|
+
# load_dotenv runs, and load_dotenv never overrides existing variables.
|
|
35
|
+
if args.host:
|
|
36
|
+
os.environ["API_HOST"] = args.host
|
|
37
|
+
if args.port:
|
|
38
|
+
os.environ["API_PORT"] = str(args.port)
|
|
39
|
+
|
|
40
|
+
import uvicorn
|
|
41
|
+
|
|
42
|
+
from localdeploy.server import app # noqa: PLC0415 - import after env is settled
|
|
43
|
+
|
|
44
|
+
host = os.getenv("API_HOST", "127.0.0.1")
|
|
45
|
+
port = int(os.getenv("API_PORT", "8000"))
|
|
46
|
+
client_base = api_client_base_url(host, port)
|
|
47
|
+
url = f"{client_base}/ui"
|
|
48
|
+
print(f"LocalDeploy {__version__}")
|
|
49
|
+
print(f" home: {home}")
|
|
50
|
+
print(f" UI: {url}")
|
|
51
|
+
print(f" docs: {client_base}/docs")
|
|
52
|
+
if not args.no_browser:
|
|
53
|
+
threading.Timer(1.5, lambda: webbrowser.open(url)).start()
|
|
54
|
+
uvicorn.run(app, host=host, port=port)
|
|
55
|
+
return 0
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
if __name__ == "__main__":
|
|
59
|
+
raise SystemExit(main())
|