localdeploy 0.6.2__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {localdeploy-0.6.2 → localdeploy-0.7.0}/.env.example +1 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/CHANGELOG.md +98 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/MANIFEST.in +1 -1
- localdeploy-0.7.0/PKG-INFO +189 -0
- localdeploy-0.7.0/README.md +143 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/SECURITY.md +1 -1
- {localdeploy-0.6.2 → localdeploy-0.7.0}/config.example.json +249 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/__init__.py +1 -1
- localdeploy-0.7.0/localdeploy/backends/decision.py +732 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/backends/llamacpp.py +37 -2
- localdeploy-0.7.0/localdeploy/backends/llamacpp_managed.py +290 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/backends/ollama.py +25 -1
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/cli.py +41 -1
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/bakeoff.py +8 -4
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/hardware.py +23 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/models.py +51 -10
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/monitor.py +18 -1
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/profiles.py +4 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/starter.py +4 -4
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/server.py +117 -11
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/index.html +8 -5
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/js/app.js +5 -5
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/js/benchmark-views.js +1 -1
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/js/benchmark.js +2 -2
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/js/chat.js +57 -18
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/js/models.js +97 -24
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/js/shared.js +4 -1
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/js/system.js +28 -1
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/styles.css +6 -6
- localdeploy-0.7.0/localdeploy.egg-info/PKG-INFO +189 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy.egg-info/SOURCES.txt +3 -0
- localdeploy-0.7.0/localdeploy.egg-info/requires.txt +22 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/pyproject.toml +21 -11
- localdeploy-0.7.0/requirements-decision.txt +12 -0
- localdeploy-0.7.0/requirements.txt +9 -0
- localdeploy-0.6.2/PKG-INFO +0 -269
- localdeploy-0.6.2/README.md +0 -221
- localdeploy-0.6.2/localdeploy.egg-info/PKG-INFO +0 -269
- localdeploy-0.6.2/localdeploy.egg-info/requires.txt +0 -23
- localdeploy-0.6.2/requirements.txt +0 -9
- {localdeploy-0.6.2 → localdeploy-0.7.0}/LICENSE +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/api_server.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/benchmark.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/chat_cli.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/compare_models.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/backends/__init__.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/backends/openai_compatible.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/benchmark.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/chat_cli.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/compare_models.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/__init__.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/_config.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/_ollama.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/bench.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/calibration.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/community.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/fit.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/history.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/manifest.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/recommend.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/registry.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/report.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/control/updates.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/grader_sandbox.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/utils.py +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/favicon.png +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy/web/logo.svg +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy.egg-info/dependency_links.txt +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy.egg-info/entry_points.txt +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/localdeploy.egg-info/top_level.txt +0 -0
- {localdeploy-0.6.2 → localdeploy-0.7.0}/setup.cfg +0 -0
|
@@ -30,6 +30,7 @@ SLOW_RESPONSE_SECONDS=60
|
|
|
30
30
|
# Upper bound (GB) on a direct-URL GGUF import (Get a model > Pull / Import).
|
|
31
31
|
# MAX_GGUF_IMPORT_GB=128
|
|
32
32
|
GLOBAL_MAX_PROMPT_CHARS=20000
|
|
33
|
+
GLOBAL_MAX_RESPONSE_SCHEMA_CHARS=400000
|
|
33
34
|
GLOBAL_MAX_OUTPUT_TOKENS=2048
|
|
34
35
|
GLOBAL_MAX_IMAGES=8
|
|
35
36
|
GLOBAL_MAX_IMAGE_MB=10
|
|
@@ -2,6 +2,104 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project should be documented here.
|
|
4
4
|
|
|
5
|
+
## 0.7.0 - 2026-11-20
|
|
6
|
+
|
|
7
|
+
- The README, the API guide and the package description now describe every way to use LocalDeploy, not only the OpenAI-compatible
|
|
8
|
+
API: the web UI, the OpenAI-compatible API, the native API (`/chat`, `/vision`, `/estimate`, `/benchmark`), the control API,
|
|
9
|
+
the decision API, generated client setups for Open WebUI, AnythingLLM, Continue, Cline, curl, Python, JavaScript and Docker
|
|
10
|
+
Compose, and deployment manifests. The README also carries the measured decision-model result.
|
|
11
|
+
- Updated runtime dependencies: FastAPI 0.162.5, pydantic 2.14.3, python-multipart 0.0.33 and huggingface-hub 2.4.0.
|
|
12
|
+
- The setup screen now opens on the answer to "which model should I run?". Once the hardware is read, the three recommended
|
|
13
|
+
models appear without a click, and the hardware detail and fit budget fold under one line that names the GPU, CPU, RAM and
|
|
14
|
+
budget. A slow automatic request can no longer replace recommendations the user asked for: only the latest request writes
|
|
15
|
+
the panel. The CPU is now shown by its name ("AMD Ryzen 9 5900HS with Radeon Graphics") on Windows, read from the registry,
|
|
16
|
+
and on macOS, instead of the raw identifier "AMD64 Family 25 Model 80 Stepping 0, AuthenticAMD". Web asset version `ui35`.
|
|
17
|
+
- `POST /models/free` now stops the llama.cpp server LocalDeploy manages as well as unloading Ollama's models. It only unloaded Ollama's, so
|
|
18
|
+
freeing the GPU for something else (a game) left the main language model in video memory when it was served by llama.cpp. The reply names
|
|
19
|
+
the profile that was stopped (`llamacpp`) and counts it in `unloaded`; when Ollama cannot be reached but a llama.cpp server was stopped,
|
|
20
|
+
memory was freed and that is a success. The server starts again on the next request that needs it.
|
|
21
|
+
- `/health` no longer waits for llama.cpp either. The same refused-connection wait (about two seconds on Windows) applied to the
|
|
22
|
+
llama.cpp look when `ENABLE_LLAMA_CPP` is on and no `llama-server` is running on the default port (a managed profile starts its own
|
|
23
|
+
on another port), so `/health` still took two seconds after the Ollama fix. The connect step is half a second and the answer is
|
|
24
|
+
remembered for ten seconds (`LLAMA_PROBE_TTL_SECONDS`).
|
|
25
|
+
- `/health` no longer waits for an Ollama that is not running. A refused connection to a loopback port takes about two seconds on
|
|
26
|
+
Windows, and `/health` asked Ollama on every call, so it took four seconds; its callers (the YBM console on every page, `ybm status`, the
|
|
27
|
+
launchers) waited, and `ybm status` showed LocalDeploy as down while it was up. The connect step of the Ollama look now has half a
|
|
28
|
+
second, and `/health` remembers the answer for ten seconds (`OLLAMA_PROBE_TTL_SECONDS`).
|
|
29
|
+
- A `response_format` JSON schema is no longer counted as prompt text. The backend enforces it while the model writes
|
|
30
|
+
(llama.cpp builds a grammar from it, Ollama takes it as `format`), so a 75,000-character schema, such as one that gives
|
|
31
|
+
every tool call its own argument schema, was refused as "Prompt is too large" at the 30,000-character prompt limit.
|
|
32
|
+
It has its own bound, `GLOBAL_MAX_RESPONSE_SCHEMA_CHARS` (default 400,000).
|
|
33
|
+
- Corrected `benchmarks/agent-operator` and the matching section of `docs/MODELS.md`. They had been measured on prompts YBM stores
|
|
34
|
+
cut to 8,000 characters (the real ones are about 17,000), so the models did not see the whole request. Re-run on full prompts,
|
|
35
|
+
Qwen3.5-9B and Qwen3-VL 8B are not clearly different on the strict score (49% against 44%), steps take about twice as long as
|
|
36
|
+
first reported, and none of the percentages is the model's accuracy. The corrected numbers are in `summary-full-prompts.json`.
|
|
37
|
+
- A `llamacpp` profile with `"managed": true` is run by LocalDeploy itself: the first request starts `llama-server` on a free
|
|
38
|
+
local port, one managed chat model stays loaded and asking for another switches, and it stops when LocalDeploy does (or
|
|
39
|
+
after `idle_stop_seconds`). The profile's `server_args`, context, flash attention, KV cache and GPU-layer fields become
|
|
40
|
+
the command line; `/health` shows `managed_llamacpp`; `scripts/start_llamacpp.ps1` leaves managed profiles to the server;
|
|
41
|
+
`scripts/stop.ps1` clears the pid files. The example `qwen35_9b_llamacpp` and `qwen3vl_8b_llamacpp` profiles use it.
|
|
42
|
+
- `benchmarks/agent-operator/` publishes a model comparison for agent work: 599 decisions a real YBM install had made,
|
|
43
|
+
replayed against Qwen3.5 9B and 4B, Qwen3-VL 8B, Qwen3.6, Gemma 4, GLM-4.6V-Flash and gpt-oss-20b, with the settings, model
|
|
44
|
+
checksums and a paired test. The same Qwen3-VL 8B was about five times faster through `llama-server` than through Ollama
|
|
45
|
+
(2.8 s against 13.4 s per step). `docs/MODELS.md` has the short version and `config.example.json` gains disabled
|
|
46
|
+
`qwen35_9b_llamacpp` and `qwen3vl_8b_llamacpp` profiles.
|
|
47
|
+
- A profile can set `server_args`, a list of extra `llama-server` flags (for example `"--reasoning", "off"` for a thinking
|
|
48
|
+
model, or `"--n-cpu-moe", "30"` to keep a mixture-of-experts model's experts in RAM). The profile editor accepts the field.
|
|
49
|
+
- `scripts/stop.ps1` and a restart through `scripts/start.ps1` stop the process in `logs\api_server.pid` or
|
|
50
|
+
`logs\llama_server.pid` only while that PID still runs the API server or llama-server. A PID file outlives a server
|
|
51
|
+
that was killed some other way, and Windows reuses PIDs, so the old scripts could force-stop an unrelated program.
|
|
52
|
+
- A decision profile with `"device": "cpu"` now starts llama-server with `--device none`. With a Vulkan or CUDA
|
|
53
|
+
build of llama.cpp (the WinGet package is Vulkan), `-ngl 0` alone still sent prompt work to the GPU: about 300 MB of
|
|
54
|
+
the chat model's GPU memory and about 10 s of shader compiling on the first request.
|
|
55
|
+
- LocalDeploy can now serve **decision models** at `POST /v1/systemone`. A decision model does not write text: you
|
|
56
|
+
give it a piece of text and named questions ("is this chat or a task, and which route?") and it returns a typed
|
|
57
|
+
answer with a probability for every option. Applications can act on a confident answer and call a language model
|
|
58
|
+
only when it is not. The `llamacpp` adapter starts a `llama-server` for a GGUF file (no Python packages, no model
|
|
59
|
+
code) and answered in 0.6 to 0.7 s on a CPU with the vLLM Semantic Router `d3-lite` model; adapters for LiquidAI d1
|
|
60
|
+
and Laya (PyTorch, optional packages) and a loopback forwarder for any other `/v1/systemone` server are included.
|
|
61
|
+
Profiles are off by default. `docs/MODELS.md` compares 18 model and quantization choices on CPU and GPU on a routing
|
|
62
|
+
task, `scripts/decision_bench.py` reproduces it, and a profile's `"quantize": "Q4_K_M"` makes the faster file on
|
|
63
|
+
first use (`scripts/make_decision_gguf.py` does it ahead of time). `llama-server` is found on `PATH`, in a WinGet
|
|
64
|
+
install or at `LLAMACPP_SERVER_PATH`, and one too old to serve `/v1/systemone` is named with its build number.
|
|
65
|
+
See "Decision models" in `docs/API_OPTIONS.md`.
|
|
66
|
+
- `benchmarks/decision-routing/` publishes the routing comparison: the labeled and held-out message sets, the raw result
|
|
67
|
+
of every run, the environment and model-file hashes, and `analyze.py`, which recomputes the intervals, the paired test
|
|
68
|
+
against an 8B LLM and a replay of YBM's routing modes from those files.
|
|
69
|
+
- The launcher now notices a `.venv` whose Python no longer runs (for example one built from a removed Anaconda) and
|
|
70
|
+
rebuilds it, keeping the broken one under `.setup`. It used to check only that `python.exe` existed, so the server
|
|
71
|
+
failed with "No Python at ..." and every repair retried the dead interpreter.
|
|
72
|
+
- Guided recommendations and automated comparisons now include the requested
|
|
73
|
+
context length in the estimated memory budget. The download budget is also
|
|
74
|
+
respected when even the first candidate is too large.
|
|
75
|
+
- The detected VRAM budget is shared with model and benchmark screens as soon
|
|
76
|
+
as the hardware check completes, instead of waiting for unrelated probes.
|
|
77
|
+
- Python 3.11 or newer is now required. Python 3.10 reaches end of life this month, and current Pillow releases
|
|
78
|
+
need 3.11. On Windows the launcher used to accept any Python 3; an older one now leads to the guided Python 3.12
|
|
79
|
+
install instead of a failed setup.
|
|
80
|
+
|
|
81
|
+
## 0.6.3 - 2026-10-22
|
|
82
|
+
|
|
83
|
+
- `localdeploy` (the pip and uvx entry point) on a taken port 8000 printed that port's UI address, opened the
|
|
84
|
+
browser on the other program, and ended with one bind error and exit code 0. It now checks the port first:
|
|
85
|
+
without a port given it uses the next free one and says so; with `--port` or `API_PORT` it stops with a clear
|
|
86
|
+
message and exit code 2.
|
|
87
|
+
- Model fit badges could disagree for the same hardware. A check that started with the early "live free VRAM" number
|
|
88
|
+
(for example 5.6 GB) could finish after the corrected 8 GB budget had been applied and overwrite it, so one row said
|
|
89
|
+
"cpu only" while its neighbour said "comfortable". Answers for an earlier budget are now discarded.
|
|
90
|
+
- Deploying a model from "Your models" now enables chat for it. Models pulled outside LocalDeploy had no chat profile,
|
|
91
|
+
so Chat stayed locked behind a second "Enable chat" step. Chat also follows the model you just loaded instead of
|
|
92
|
+
staying on the first one in the list, and its messages no longer say "Load an installed model" for a loaded model.
|
|
93
|
+
- Monitor showed 8.5 GB of VRAM on a laptop where every other screen said 8.0 GB, because it added the integrated
|
|
94
|
+
GPU's 512 MB to the discrete card. It now uses the same GPU pool as the fit checks.
|
|
95
|
+
- The UI's cache-busting token had not changed since July, so a browser that already had the page could keep
|
|
96
|
+
showing old styles and scripts after an upgrade. The token is bumped.
|
|
97
|
+
- Updated runtime dependencies: FastAPI 0.143.1, uvicorn 0.54.0, pydantic 2.14.1, python-dotenv 1.2.4, and
|
|
98
|
+
huggingface-hub 2.2.0 (a new major version, used for Hugging Face model search and GGUF import).
|
|
99
|
+
- Benchmark leaderboard: the first chip rendered as "90/90 comple..." in 0.6.2, because the longer word
|
|
100
|
+
"completed" did not fit its fixed-width column. The chips now size to their text and wrap onto another row on
|
|
101
|
+
narrow screens, so no label is cut off.
|
|
102
|
+
|
|
5
103
|
## 0.6.2 - 2026-10-03
|
|
6
104
|
|
|
7
105
|
- Benchmark screen: the leaderboard and comparison tables said "N/N passed" even for a model with low
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: localdeploy
|
|
3
|
+
Version: 0.7.0
|
|
4
|
+
Summary: Find the local AI models that fit your hardware, test them on your own questions, and connect the winner to your apps through OpenAI-compatible, native, and decision APIs.
|
|
5
|
+
Author: oney-erge
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/oney-erge/LocalDeploy
|
|
8
|
+
Project-URL: Documentation, https://github.com/oney-erge/LocalDeploy#readme
|
|
9
|
+
Project-URL: Changelog, https://github.com/oney-erge/LocalDeploy/blob/main/CHANGELOG.md
|
|
10
|
+
Project-URL: Issues, https://github.com/oney-erge/LocalDeploy/issues
|
|
11
|
+
Keywords: ollama,llm,local-ai,benchmark,vram,self-hosted
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Web Environment
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: End Users/Desktop
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Requires-Python: >=3.11
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: fastapi==0.162.5
|
|
26
|
+
Requires-Dist: uvicorn==0.54.0
|
|
27
|
+
Requires-Dist: requests==2.34.2
|
|
28
|
+
Requires-Dist: python-dotenv==1.2.4
|
|
29
|
+
Requires-Dist: pydantic==2.14.3
|
|
30
|
+
Requires-Dist: python-multipart==0.0.33
|
|
31
|
+
Requires-Dist: huggingface_hub==2.4.0
|
|
32
|
+
Requires-Dist: psutil==7.2.2
|
|
33
|
+
Requires-Dist: pyyaml==6.0.3
|
|
34
|
+
Provides-Extra: decision
|
|
35
|
+
Requires-Dist: torch>=2.5; extra == "decision"
|
|
36
|
+
Requires-Dist: torchvision>=0.20; extra == "decision"
|
|
37
|
+
Requires-Dist: transformers>=5.15; extra == "decision"
|
|
38
|
+
Requires-Dist: pillow>=10.0; extra == "decision"
|
|
39
|
+
Requires-Dist: soundfile>=0.12; extra == "decision"
|
|
40
|
+
Requires-Dist: laya>=0.4.17; extra == "decision"
|
|
41
|
+
Provides-Extra: packaging
|
|
42
|
+
Requires-Dist: pyinstaller>=6.0; extra == "packaging"
|
|
43
|
+
Requires-Dist: pystray>=0.19; extra == "packaging"
|
|
44
|
+
Requires-Dist: pillow>=10.0; extra == "packaging"
|
|
45
|
+
Dynamic: license-file
|
|
46
|
+
|
|
47
|
+
<p align="center">
|
|
48
|
+
<img src="https://raw.githubusercontent.com/oney-erge/LocalDeploy/main/localdeploy/web/logo.svg" width="96" height="96" alt="LocalDeploy logo" />
|
|
49
|
+
</p>
|
|
50
|
+
|
|
51
|
+
<h1 align="center">LocalDeploy</h1>
|
|
52
|
+
|
|
53
|
+
<p align="center"><strong>Choose, run, and compare local AI models on your own hardware.</strong></p>
|
|
54
|
+
|
|
55
|
+
<p align="center">
|
|
56
|
+
<a href="https://github.com/oney-erge/LocalDeploy/actions/workflows/ci.yml"><img src="https://github.com/oney-erge/LocalDeploy/actions/workflows/ci.yml/badge.svg" alt="CI status" /></a>
|
|
57
|
+
<a href="https://pypi.org/project/localdeploy/"><img src="https://img.shields.io/pypi/v/localdeploy.svg" alt="PyPI version" /></a>
|
|
58
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT license" /></a>
|
|
59
|
+
<img src="https://img.shields.io/badge/python-3.11%2B-blue.svg" alt="Python 3.11+" />
|
|
60
|
+
</p>
|
|
61
|
+
|
|
62
|
+
<p align="center">
|
|
63
|
+
<img src="https://raw.githubusercontent.com/oney-erge/LocalDeploy/main/docs/assets/demo.gif" alt="LocalDeploy walkthrough showing hardware-aware model recommendations, deployment, local chat, benchmark comparisons, and live monitoring" width="820" />
|
|
64
|
+
</p>
|
|
65
|
+
|
|
66
|
+
LocalDeploy helps you answer a practical question before you download a model:
|
|
67
|
+
will it fit and run well on this machine? It finds the local models that fit
|
|
68
|
+
your hardware, tests them on your own questions, and connects the winner to your
|
|
69
|
+
apps. It detects hardware, estimates memory use, manages models across Ollama,
|
|
70
|
+
llama.cpp, LM Studio, and vLLM, records benchmarks, and serves the deployed
|
|
71
|
+
model through several APIs. It runs locally, has no telemetry, and does not
|
|
72
|
+
provide cloud inference.
|
|
73
|
+
|
|
74
|
+
## Quick start
|
|
75
|
+
|
|
76
|
+
With Python 3.11+ and [Ollama](https://ollama.com) installed:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
pip install localdeploy
|
|
80
|
+
localdeploy # opens http://localhost:8000/ui
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
The UI guides you through hardware detection, model search, a fit estimate,
|
|
84
|
+
deployment, chat, and benchmarking. If you use `uv`, `uvx localdeploy` runs it
|
|
85
|
+
without a permanent install.
|
|
86
|
+
|
|
87
|
+
Open **Get a model**, choose a use case and context length, and review the
|
|
88
|
+
recommended model's memory and download estimates. Start an installed model or
|
|
89
|
+
explicitly download one, then use **Chat** and **Benchmark** to see how it
|
|
90
|
+
performs on your machine. [A worked comparison](docs/FIT_COMPARISON.md) shows
|
|
91
|
+
how to inspect estimates against another tool.
|
|
92
|
+
|
|
93
|
+
Need help setting up Python or Ollama? Clone the repository and run its launcher:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
git clone https://github.com/oney-erge/LocalDeploy.git
|
|
97
|
+
cd LocalDeploy
|
|
98
|
+
./run.sh # Linux; macOS: ./run.command
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
On Windows, use `run.bat`. The launchers can prepare the environment and open
|
|
102
|
+
the UI; [Installation](docs/INSTALL.md) covers prerequisites, Docker, and the
|
|
103
|
+
Windows setup script. Desktop builds are experimental and unsigned; see
|
|
104
|
+
[Packaging](docs/PACKAGING.md).
|
|
105
|
+
|
|
106
|
+
## What it helps with
|
|
107
|
+
|
|
108
|
+
- Find models from Ollama, Hugging Face, ModelScope GGUF repositories, local
|
|
109
|
+
GGUF files, or direct URLs.
|
|
110
|
+
- Estimate model fit before download, including likely CPU offload, and compare
|
|
111
|
+
quantizations with the quant advisor.
|
|
112
|
+
- Pull, deploy, switch, unload, and remove local models. A llama.cpp chat model
|
|
113
|
+
can be started, switched, and stopped by LocalDeploy itself.
|
|
114
|
+
- Chat with text, images, and documents, then compare installed models on the
|
|
115
|
+
built-in or your own question sets using measured latency, throughput, memory,
|
|
116
|
+
and accuracy results.
|
|
117
|
+
- Monitor CPU, RAM, VRAM, model placement, and request timing while a model runs.
|
|
118
|
+
- Export a deployment as a manifest, check it on another machine, and recreate
|
|
119
|
+
it there.
|
|
120
|
+
|
|
121
|
+
## Ways to use it
|
|
122
|
+
|
|
123
|
+
Once a model is running, your tools can reach it in several ways:
|
|
124
|
+
|
|
125
|
+
| Way in | What it is for |
|
|
126
|
+
|---|---|
|
|
127
|
+
| Web UI at `/ui` | Check hardware, find, pull, and deploy models, chat, benchmark and compare, monitor |
|
|
128
|
+
| OpenAI-compatible API: `/v1/chat/completions`, `/v1/responses`, `/v1/embeddings`, `/v1/models` | Any app or SDK that speaks the OpenAI API, with streaming, JSON-schema output, and tool calls returned to your app |
|
|
129
|
+
| Native API: `/chat`, `/vision`, `/estimate`, `/benchmark` | Requests by profile, with per-request context limits, safe mode, image input, and a safety estimate before you send |
|
|
130
|
+
| Control API: about 50 endpoints | Script what the UI does: hardware and fit checks, recommendations, search, pull, serve, switch, free the GPU, benchmark, compare, export. The interactive reference is at `/docs` |
|
|
131
|
+
| Decision API: `/v1/systemone` | Small classifiers that answer typed questions about a piece of text with a probability for every option |
|
|
132
|
+
| Client setups | Ready configuration for Open WebUI, AnythingLLM, Continue, Cline, curl, Python, JavaScript, and Docker Compose, generated from a running model |
|
|
133
|
+
| Deployment manifests | Record the model digest, quantization, runtime settings, hardware, and measurements, then recreate the deployment elsewhere |
|
|
134
|
+
| Terminal | `scripts/chat.ps1` for chat and `compare_models.py` for side-by-side runs; see [CLI](docs/CLI.md) |
|
|
135
|
+
|
|
136
|
+
[API options](docs/API_OPTIONS.md) has request examples for the native,
|
|
137
|
+
OpenAI-compatible, and decision APIs. The control API is described at `/docs`.
|
|
138
|
+
|
|
139
|
+
The **decision API** is optional. It runs small models (vLLM Semantic Router d3,
|
|
140
|
+
LiquidAI d1, Laya) that return a probability for every option in one pass, so an
|
|
141
|
+
app can classify or route locally and call a language model only when the answer
|
|
142
|
+
is not confident. On a laptop CPU, the 0.85B d3-lite model routed agent messages
|
|
143
|
+
to 15 routes as accurately as an 8B language model on one set (93% against 92%
|
|
144
|
+
of 98 messages), slightly less on another (84% against 88% of 43), in 0.5 s
|
|
145
|
+
instead of 15 s. The [raw runs and method](benchmarks/decision-routing/README.md)
|
|
146
|
+
are in the repository, and [Decision models](docs/API_OPTIONS.md#decision-models)
|
|
147
|
+
shows how to enable one.
|
|
148
|
+
|
|
149
|
+
## Where it fits
|
|
150
|
+
|
|
151
|
+
- If you already know which model you want and only need terminal chat, Ollama
|
|
152
|
+
may be enough.
|
|
153
|
+
- If you use a front end such as Open WebUI, Continue, or Cline, LocalDeploy is
|
|
154
|
+
the layer underneath: it helps choose and run the model and generates those
|
|
155
|
+
tools' connection settings.
|
|
156
|
+
- If you are choosing between models for a particular machine, it runs the
|
|
157
|
+
candidates on your own questions and records what you measured.
|
|
158
|
+
|
|
159
|
+
## What to expect
|
|
160
|
+
|
|
161
|
+
Fit estimates are guidance, not guarantees. Driver versions, context length,
|
|
162
|
+
quantization, desktop GPU use, and the runtime all affect actual memory use.
|
|
163
|
+
LocalDeploy records what happened after deployment so later estimates can be
|
|
164
|
+
adjusted for that machine.
|
|
165
|
+
|
|
166
|
+
It works with Ollama, llama.cpp, LM Studio, vLLM, and other loopback
|
|
167
|
+
OpenAI-compatible runtimes. Runtime URLs must remain local. An `API_TOKEN`
|
|
168
|
+
is an optional shared token, not full authentication. Keep the service on
|
|
169
|
+
localhost; it has no TLS, user isolation, or rate limiting. Tool calls are
|
|
170
|
+
returned to clients and are never executed by LocalDeploy.
|
|
171
|
+
|
|
172
|
+
Model search and the UI's release check can contact external services.
|
|
173
|
+
Set `OFFLINE=true` to disable those requests. Separately installed runtimes
|
|
174
|
+
have their own network settings; see [Security](SECURITY.md).
|
|
175
|
+
|
|
176
|
+
## Documentation
|
|
177
|
+
|
|
178
|
+
| Guide | Use it for |
|
|
179
|
+
|---|---|
|
|
180
|
+
| [Installation](docs/INSTALL.md) | Launcher commands, Docker, and setup alternatives |
|
|
181
|
+
| [UI guide](docs/UI.md) | Choosing, deploying, chatting with, and benchmarking models |
|
|
182
|
+
| [API options](docs/API_OPTIONS.md) | Native, OpenAI-compatible, and decision APIs, with request examples |
|
|
183
|
+
| [CLI](docs/CLI.md) | Commands and automation |
|
|
184
|
+
| [Models](docs/MODELS.md) | Runtimes, model discovery, and fit estimates |
|
|
185
|
+
| [Screenshots](docs/SCREENSHOTS.md) | The complete UI tour |
|
|
186
|
+
| [Packaging](docs/PACKAGING.md) | Experimental desktop builds and signing limitations |
|
|
187
|
+
| [Contributing](CONTRIBUTING.md) | Development setup and verification |
|
|
188
|
+
|
|
189
|
+
MIT licensed. Built by [Oney Erge](https://oneyerge.com).
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/oney-erge/LocalDeploy/main/localdeploy/web/logo.svg" width="96" height="96" alt="LocalDeploy logo" />
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<h1 align="center">LocalDeploy</h1>
|
|
6
|
+
|
|
7
|
+
<p align="center"><strong>Choose, run, and compare local AI models on your own hardware.</strong></p>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<a href="https://github.com/oney-erge/LocalDeploy/actions/workflows/ci.yml"><img src="https://github.com/oney-erge/LocalDeploy/actions/workflows/ci.yml/badge.svg" alt="CI status" /></a>
|
|
11
|
+
<a href="https://pypi.org/project/localdeploy/"><img src="https://img.shields.io/pypi/v/localdeploy.svg" alt="PyPI version" /></a>
|
|
12
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT license" /></a>
|
|
13
|
+
<img src="https://img.shields.io/badge/python-3.11%2B-blue.svg" alt="Python 3.11+" />
|
|
14
|
+
</p>
|
|
15
|
+
|
|
16
|
+
<p align="center">
|
|
17
|
+
<img src="https://raw.githubusercontent.com/oney-erge/LocalDeploy/main/docs/assets/demo.gif" alt="LocalDeploy walkthrough showing hardware-aware model recommendations, deployment, local chat, benchmark comparisons, and live monitoring" width="820" />
|
|
18
|
+
</p>
|
|
19
|
+
|
|
20
|
+
LocalDeploy helps you answer a practical question before you download a model:
|
|
21
|
+
will it fit and run well on this machine? It finds the local models that fit
|
|
22
|
+
your hardware, tests them on your own questions, and connects the winner to your
|
|
23
|
+
apps. It detects hardware, estimates memory use, manages models across Ollama,
|
|
24
|
+
llama.cpp, LM Studio, and vLLM, records benchmarks, and serves the deployed
|
|
25
|
+
model through several APIs. It runs locally, has no telemetry, and does not
|
|
26
|
+
provide cloud inference.
|
|
27
|
+
|
|
28
|
+
## Quick start
|
|
29
|
+
|
|
30
|
+
With Python 3.11+ and [Ollama](https://ollama.com) installed:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install localdeploy
|
|
34
|
+
localdeploy # opens http://localhost:8000/ui
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
The UI guides you through hardware detection, model search, a fit estimate,
|
|
38
|
+
deployment, chat, and benchmarking. If you use `uv`, `uvx localdeploy` runs it
|
|
39
|
+
without a permanent install.
|
|
40
|
+
|
|
41
|
+
Open **Get a model**, choose a use case and context length, and review the
|
|
42
|
+
recommended model's memory and download estimates. Start an installed model or
|
|
43
|
+
explicitly download one, then use **Chat** and **Benchmark** to see how it
|
|
44
|
+
performs on your machine. [A worked comparison](docs/FIT_COMPARISON.md) shows
|
|
45
|
+
how to inspect estimates against another tool.
|
|
46
|
+
|
|
47
|
+
Need help setting up Python or Ollama? Clone the repository and run its launcher:
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
git clone https://github.com/oney-erge/LocalDeploy.git
|
|
51
|
+
cd LocalDeploy
|
|
52
|
+
./run.sh # Linux; macOS: ./run.command
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
On Windows, use `run.bat`. The launchers can prepare the environment and open
|
|
56
|
+
the UI; [Installation](docs/INSTALL.md) covers prerequisites, Docker, and the
|
|
57
|
+
Windows setup script. Desktop builds are experimental and unsigned; see
|
|
58
|
+
[Packaging](docs/PACKAGING.md).
|
|
59
|
+
|
|
60
|
+
## What it helps with
|
|
61
|
+
|
|
62
|
+
- Find models from Ollama, Hugging Face, ModelScope GGUF repositories, local
|
|
63
|
+
GGUF files, or direct URLs.
|
|
64
|
+
- Estimate model fit before download, including likely CPU offload, and compare
|
|
65
|
+
quantizations with the quant advisor.
|
|
66
|
+
- Pull, deploy, switch, unload, and remove local models. A llama.cpp chat model
|
|
67
|
+
can be started, switched, and stopped by LocalDeploy itself.
|
|
68
|
+
- Chat with text, images, and documents, then compare installed models on the
|
|
69
|
+
built-in or your own question sets using measured latency, throughput, memory,
|
|
70
|
+
and accuracy results.
|
|
71
|
+
- Monitor CPU, RAM, VRAM, model placement, and request timing while a model runs.
|
|
72
|
+
- Export a deployment as a manifest, check it on another machine, and recreate
|
|
73
|
+
it there.
|
|
74
|
+
|
|
75
|
+
## Ways to use it
|
|
76
|
+
|
|
77
|
+
Once a model is running, your tools can reach it in several ways:
|
|
78
|
+
|
|
79
|
+
| Way in | What it is for |
|
|
80
|
+
|---|---|
|
|
81
|
+
| Web UI at `/ui` | Check hardware, find, pull, and deploy models, chat, benchmark and compare, monitor |
|
|
82
|
+
| OpenAI-compatible API: `/v1/chat/completions`, `/v1/responses`, `/v1/embeddings`, `/v1/models` | Any app or SDK that speaks the OpenAI API, with streaming, JSON-schema output, and tool calls returned to your app |
|
|
83
|
+
| Native API: `/chat`, `/vision`, `/estimate`, `/benchmark` | Requests by profile, with per-request context limits, safe mode, image input, and a safety estimate before you send |
|
|
84
|
+
| Control API: about 50 endpoints | Script what the UI does: hardware and fit checks, recommendations, search, pull, serve, switch, free the GPU, benchmark, compare, export. The interactive reference is at `/docs` |
|
|
85
|
+
| Decision API: `/v1/systemone` | Small classifiers that answer typed questions about a piece of text with a probability for every option |
|
|
86
|
+
| Client setups | Ready configuration for Open WebUI, AnythingLLM, Continue, Cline, curl, Python, JavaScript, and Docker Compose, generated from a running model |
|
|
87
|
+
| Deployment manifests | Record the model digest, quantization, runtime settings, hardware, and measurements, then recreate the deployment elsewhere |
|
|
88
|
+
| Terminal | `scripts/chat.ps1` for chat and `compare_models.py` for side-by-side runs; see [CLI](docs/CLI.md) |
|
|
89
|
+
|
|
90
|
+
[API options](docs/API_OPTIONS.md) has request examples for the native,
|
|
91
|
+
OpenAI-compatible, and decision APIs. The control API is described at `/docs`.
|
|
92
|
+
|
|
93
|
+
The **decision API** is optional. It runs small models (vLLM Semantic Router d3,
|
|
94
|
+
LiquidAI d1, Laya) that return a probability for every option in one pass, so an
|
|
95
|
+
app can classify or route locally and call a language model only when the answer
|
|
96
|
+
is not confident. On a laptop CPU, the 0.85B d3-lite model routed agent messages
|
|
97
|
+
to 15 routes as accurately as an 8B language model on one set (93% against 92%
|
|
98
|
+
of 98 messages), slightly less on another (84% against 88% of 43), in 0.5 s
|
|
99
|
+
instead of 15 s. The [raw runs and method](benchmarks/decision-routing/README.md)
|
|
100
|
+
are in the repository, and [Decision models](docs/API_OPTIONS.md#decision-models)
|
|
101
|
+
shows how to enable one.
|
|
102
|
+
|
|
103
|
+
## Where it fits
|
|
104
|
+
|
|
105
|
+
- If you already know which model you want and only need terminal chat, Ollama
|
|
106
|
+
may be enough.
|
|
107
|
+
- If you use a front end such as Open WebUI, Continue, or Cline, LocalDeploy is
|
|
108
|
+
the layer underneath: it helps choose and run the model and generates those
|
|
109
|
+
tools' connection settings.
|
|
110
|
+
- If you are choosing between models for a particular machine, it runs the
|
|
111
|
+
candidates on your own questions and records what you measured.
|
|
112
|
+
|
|
113
|
+
## What to expect
|
|
114
|
+
|
|
115
|
+
Fit estimates are guidance, not guarantees. Driver versions, context length,
|
|
116
|
+
quantization, desktop GPU use, and the runtime all affect actual memory use.
|
|
117
|
+
LocalDeploy records what happened after deployment so later estimates can be
|
|
118
|
+
adjusted for that machine.
|
|
119
|
+
|
|
120
|
+
It works with Ollama, llama.cpp, LM Studio, vLLM, and other loopback
|
|
121
|
+
OpenAI-compatible runtimes. Runtime URLs must remain local. An `API_TOKEN`
|
|
122
|
+
is an optional shared token, not full authentication. Keep the service on
|
|
123
|
+
localhost; it has no TLS, user isolation, or rate limiting. Tool calls are
|
|
124
|
+
returned to clients and are never executed by LocalDeploy.
|
|
125
|
+
|
|
126
|
+
Model search and the UI's release check can contact external services.
|
|
127
|
+
Set `OFFLINE=true` to disable those requests. Separately installed runtimes
|
|
128
|
+
have their own network settings; see [Security](SECURITY.md).
|
|
129
|
+
|
|
130
|
+
## Documentation
|
|
131
|
+
|
|
132
|
+
| Guide | Use it for |
|
|
133
|
+
|---|---|
|
|
134
|
+
| [Installation](docs/INSTALL.md) | Launcher commands, Docker, and setup alternatives |
|
|
135
|
+
| [UI guide](docs/UI.md) | Choosing, deploying, chatting with, and benchmarking models |
|
|
136
|
+
| [API options](docs/API_OPTIONS.md) | Native, OpenAI-compatible, and decision APIs, with request examples |
|
|
137
|
+
| [CLI](docs/CLI.md) | Commands and automation |
|
|
138
|
+
| [Models](docs/MODELS.md) | Runtimes, model discovery, and fit estimates |
|
|
139
|
+
| [Screenshots](docs/SCREENSHOTS.md) | The complete UI tour |
|
|
140
|
+
| [Packaging](docs/PACKAGING.md) | Experimental desktop builds and signing limitations |
|
|
141
|
+
| [Contributing](CONTRIBUTING.md) | Development setup and verification |
|
|
142
|
+
|
|
143
|
+
MIT licensed. Built by [Oney Erge](https://oneyerge.com).
|
|
@@ -39,7 +39,7 @@ Do not commit `.env`, `config.json`, `calibration.json`, model files, private sc
|
|
|
39
39
|
|
|
40
40
|
## Reporting a vulnerability
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
Use the **Report a vulnerability** button on the repository's Security tab instead of opening a public issue. GitHub keeps the report private between you and the maintainer.
|
|
43
43
|
|
|
44
44
|
If that button is not available, open a minimal issue asking for a private contact method. Do not put exploit details, tokens, private prompts, images, model files, or local paths in the public issue.
|
|
45
45
|
|