localdeploy 0.6.3__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. {localdeploy-0.6.3 → localdeploy-0.7.0}/.env.example +1 -0
  2. {localdeploy-0.6.3 → localdeploy-0.7.0}/CHANGELOG.md +76 -0
  3. {localdeploy-0.6.3 → localdeploy-0.7.0}/MANIFEST.in +1 -1
  4. {localdeploy-0.6.3 → localdeploy-0.7.0}/PKG-INFO +75 -32
  5. localdeploy-0.7.0/README.md +143 -0
  6. {localdeploy-0.6.3 → localdeploy-0.7.0}/SECURITY.md +1 -1
  7. {localdeploy-0.6.3 → localdeploy-0.7.0}/config.example.json +249 -0
  8. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/__init__.py +1 -1
  9. localdeploy-0.7.0/localdeploy/backends/decision.py +732 -0
  10. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/backends/llamacpp.py +37 -2
  11. localdeploy-0.7.0/localdeploy/backends/llamacpp_managed.py +290 -0
  12. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/backends/ollama.py +25 -1
  13. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/bakeoff.py +8 -4
  14. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/hardware.py +23 -0
  15. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/models.py +51 -10
  16. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/profiles.py +4 -0
  17. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/starter.py +4 -4
  18. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/server.py +117 -11
  19. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/index.html +8 -5
  20. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/js/app.js +5 -5
  21. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/js/benchmark-views.js +1 -1
  22. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/js/benchmark.js +2 -2
  23. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/js/chat.js +38 -13
  24. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/js/models.js +73 -23
  25. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/js/shared.js +4 -1
  26. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/js/system.js +28 -1
  27. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/styles.css +2 -0
  28. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy.egg-info/PKG-INFO +75 -32
  29. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy.egg-info/SOURCES.txt +3 -0
  30. localdeploy-0.7.0/localdeploy.egg-info/requires.txt +22 -0
  31. {localdeploy-0.6.3 → localdeploy-0.7.0}/pyproject.toml +21 -11
  32. localdeploy-0.7.0/requirements-decision.txt +12 -0
  33. {localdeploy-0.6.3 → localdeploy-0.7.0}/requirements.txt +4 -4
  34. localdeploy-0.6.3/README.md +0 -98
  35. localdeploy-0.6.3/localdeploy.egg-info/requires.txt +0 -23
  36. {localdeploy-0.6.3 → localdeploy-0.7.0}/LICENSE +0 -0
  37. {localdeploy-0.6.3 → localdeploy-0.7.0}/api_server.py +0 -0
  38. {localdeploy-0.6.3 → localdeploy-0.7.0}/benchmark.py +0 -0
  39. {localdeploy-0.6.3 → localdeploy-0.7.0}/chat_cli.py +0 -0
  40. {localdeploy-0.6.3 → localdeploy-0.7.0}/compare_models.py +0 -0
  41. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/backends/__init__.py +0 -0
  42. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/backends/openai_compatible.py +0 -0
  43. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/benchmark.py +0 -0
  44. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/chat_cli.py +0 -0
  45. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/cli.py +0 -0
  46. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/compare_models.py +0 -0
  47. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/__init__.py +0 -0
  48. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/_config.py +0 -0
  49. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/_ollama.py +0 -0
  50. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/bench.py +0 -0
  51. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/calibration.py +0 -0
  52. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/community.py +0 -0
  53. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/fit.py +0 -0
  54. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/history.py +0 -0
  55. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/manifest.py +0 -0
  56. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/monitor.py +0 -0
  57. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/recommend.py +0 -0
  58. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/registry.py +0 -0
  59. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/report.py +0 -0
  60. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/control/updates.py +0 -0
  61. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/grader_sandbox.py +0 -0
  62. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/utils.py +0 -0
  63. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/favicon.png +0 -0
  64. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy/web/logo.svg +0 -0
  65. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy.egg-info/dependency_links.txt +0 -0
  66. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy.egg-info/entry_points.txt +0 -0
  67. {localdeploy-0.6.3 → localdeploy-0.7.0}/localdeploy.egg-info/top_level.txt +0 -0
  68. {localdeploy-0.6.3 → localdeploy-0.7.0}/setup.cfg +0 -0
@@ -30,6 +30,7 @@ SLOW_RESPONSE_SECONDS=60
30
30
  # Upper bound (GB) on a direct-URL GGUF import (Get a model > Pull / Import).
31
31
  # MAX_GGUF_IMPORT_GB=128
32
32
  GLOBAL_MAX_PROMPT_CHARS=20000
33
+ GLOBAL_MAX_RESPONSE_SCHEMA_CHARS=400000
33
34
  GLOBAL_MAX_OUTPUT_TOKENS=2048
34
35
  GLOBAL_MAX_IMAGES=8
35
36
  GLOBAL_MAX_IMAGE_MB=10
@@ -2,6 +2,82 @@
2
2
 
3
3
  All notable changes to this project should be documented here.
4
4
 
5
+ ## 0.7.0 - 2026-11-20
6
+
7
+ - The README, the API guide and the package description now describe every way to use LocalDeploy, not only the OpenAI-compatible
8
+ API: the web UI, the OpenAI-compatible API, the native API (`/chat`, `/vision`, `/estimate`, `/benchmark`), the control API,
9
+ the decision API, generated client setups for Open WebUI, AnythingLLM, Continue, Cline, curl, Python, JavaScript and Docker
10
+ Compose, and deployment manifests. The README also carries the measured decision-model result.
11
+ - Updated runtime dependencies: FastAPI 0.162.5, pydantic 2.14.3, python-multipart 0.0.33 and huggingface-hub 2.4.0.
12
+ - The setup screen now opens on the answer to "which model should I run?". Once the hardware is read, the three recommended
13
+ models appear without a click, and the hardware detail and fit budget fold under one line that names the GPU, CPU, RAM and
14
+ budget. A slow automatic request can no longer replace recommendations the user asked for: only the latest request writes
15
+ the panel. The CPU is now shown by its name ("AMD Ryzen 9 5900HS with Radeon Graphics") on Windows, read from the registry,
16
+ and on macOS, instead of the raw identifier "AMD64 Family 25 Model 80 Stepping 0, AuthenticAMD". Web asset version `ui35`.
17
+ - `POST /models/free` now stops the llama.cpp server LocalDeploy manages as well as unloading Ollama's models. It only unloaded Ollama's, so
18
+ freeing the GPU for something else (a game) left the main language model in video memory when it was served by llama.cpp. The reply names
19
+ the profile that was stopped (`llamacpp`) and counts it in `unloaded`; when Ollama cannot be reached but a llama.cpp server was stopped,
20
+ memory was freed and that is a success. The server starts again on the next request that needs it.
21
+ - `/health` no longer waits for llama.cpp either. The same refused-connection wait (about two seconds on Windows) applied to the
22
+ llama.cpp look when `ENABLE_LLAMA_CPP` is on and no `llama-server` is running on the default port (a managed profile starts its own
23
+ on another port), so `/health` still took two seconds after the Ollama fix. The connect step is half a second and the answer is
24
+ remembered for ten seconds (`LLAMA_PROBE_TTL_SECONDS`).
25
+ - `/health` no longer waits for an Ollama that is not running. A refused connection to a loopback port takes about two seconds on
26
+ Windows, and `/health` asked Ollama on every call, so it took four seconds; its callers (the YBM console on every page, `ybm status`, the
27
+ launchers) waited, and `ybm status` showed LocalDeploy as down while it was up. The connect step of the Ollama look now has half a
28
+ second, and `/health` remembers the answer for ten seconds (`OLLAMA_PROBE_TTL_SECONDS`).
29
+ - A `response_format` JSON schema is no longer counted as prompt text. The backend enforces it while the model writes
30
+ (llama.cpp builds a grammar from it, Ollama takes it as `format`), so a 75,000-character schema, such as one that gives
31
+ every tool call its own argument schema, was refused as "Prompt is too large" at the 30,000-character prompt limit.
32
+ It has its own bound, `GLOBAL_MAX_RESPONSE_SCHEMA_CHARS` (default 400,000).
33
+ - Corrected `benchmarks/agent-operator` and the matching section of `docs/MODELS.md`. They had been measured on prompts YBM stores
34
+ cut to 8,000 characters (the real ones are about 17,000), so the models did not see the whole request. Re-run on full prompts,
35
+ Qwen3.5-9B and Qwen3-VL 8B are not clearly different on the strict score (49% against 44%), steps take about twice as long as
36
+ first reported, and none of the percentages is the model's accuracy. The corrected numbers are in `summary-full-prompts.json`.
37
+ - A `llamacpp` profile with `"managed": true` is run by LocalDeploy itself: the first request starts `llama-server` on a free
38
+ local port, one managed chat model stays loaded and asking for another switches, and it stops when LocalDeploy does (or
39
+ after `idle_stop_seconds`). The profile's `server_args`, context, flash attention, KV cache and GPU-layer fields become
40
+ the command line; `/health` shows `managed_llamacpp`; `scripts/start_llamacpp.ps1` leaves managed profiles to the server;
41
+ `scripts/stop.ps1` clears the pid files. The example `qwen35_9b_llamacpp` and `qwen3vl_8b_llamacpp` profiles use it.
42
+ - `benchmarks/agent-operator/` publishes a model comparison for agent work: 599 decisions a real YBM install had made,
43
+ replayed against Qwen3.5 9B and 4B, Qwen3-VL 8B, Qwen3.6, Gemma 4, GLM-4.6V-Flash and gpt-oss-20b, with the settings, model
44
+ checksums and a paired test. The same Qwen3-VL 8B was about five times faster through `llama-server` than through Ollama
45
+ (2.8 s against 13.4 s per step). `docs/MODELS.md` has the short version and `config.example.json` gains disabled
46
+ `qwen35_9b_llamacpp` and `qwen3vl_8b_llamacpp` profiles.
47
+ - A profile can set `server_args`, a list of extra `llama-server` flags (for example `"--reasoning", "off"` for a thinking
48
+ model, or `"--n-cpu-moe", "30"` to keep a mixture-of-experts model's experts in RAM). The profile editor accepts the field.
49
+ - `scripts/stop.ps1` and a restart through `scripts/start.ps1` stop the process in `logs\api_server.pid` or
50
+ `logs\llama_server.pid` only while that PID still runs the API server or llama-server. A PID file outlives a server
51
+ that was killed some other way, and Windows reuses PIDs, so the old scripts could force-stop an unrelated program.
52
+ - A decision profile with `"device": "cpu"` now starts llama-server with `--device none`. With a Vulkan or CUDA
53
+ build of llama.cpp (the WinGet package is Vulkan), `-ngl 0` alone still sent prompt work to the GPU: about 300 MB of
54
+ the chat model's GPU memory and about 10 s of shader compiling on the first request.
55
+ - LocalDeploy can now serve **decision models** at `POST /v1/systemone`. A decision model does not write text: you
56
+ give it a piece of text and named questions ("is this chat or a task, and which route?") and it returns a typed
57
+ answer with a probability for every option. Applications can act on a confident answer and call a language model
58
+ only when it is not. The `llamacpp` adapter starts a `llama-server` for a GGUF file (no Python packages, no model
59
+ code) and answered in 0.6 to 0.7 s on a CPU with the vLLM Semantic Router `d3-lite` model; adapters for LiquidAI d1
60
+ and Laya (PyTorch, optional packages) and a loopback forwarder for any other `/v1/systemone` server are included.
61
+ Profiles are off by default. `docs/MODELS.md` compares 18 model and quantization choices on CPU and GPU on a routing
62
+ task, `scripts/decision_bench.py` reproduces it, and a profile's `"quantize": "Q4_K_M"` makes the faster file on
63
+ first use (`scripts/make_decision_gguf.py` does it ahead of time). `llama-server` is found on `PATH`, in a WinGet
64
+ install or at `LLAMACPP_SERVER_PATH`, and one too old to serve `/v1/systemone` is named with its build number.
65
+ See "Decision models" in `docs/API_OPTIONS.md`.
66
+ - `benchmarks/decision-routing/` publishes the routing comparison: the labeled and held-out message sets, the raw result
67
+ of every run, the environment and model-file hashes, and `analyze.py`, which recomputes the intervals, the paired test
68
+ against an 8B LLM and a replay of YBM's routing modes from those files.
69
+ - The launcher now notices a `.venv` whose Python no longer runs (for example one built from a removed Anaconda) and
70
+ rebuilds it, keeping the broken one under `.setup`. It used to check only that `python.exe` existed, so the server
71
+ failed with "No Python at ..." and every repair retried the dead interpreter.
72
+ - Guided recommendations and automated comparisons now include the requested
73
+ context length in the estimated memory budget. The download budget is also
74
+ respected when even the first candidate is too large.
75
+ - The detected VRAM budget is shared with model and benchmark screens as soon
76
+ as the hardware check completes, instead of waiting for unrelated probes.
77
+ - Python 3.11 or newer is now required. Python 3.10 reaches end of life this month, and current Pillow releases
78
+ need 3.11. On Windows the launcher used to accept any Python 3; an older one now leads to the guided Python 3.12
79
+ install instead of a failed setup.
80
+
5
81
  ## 0.6.3 - 2026-10-22
6
82
 
7
83
  - `localdeploy` (the pip and uvx entry point) on a taken port 8000 printed that port's UI address, opened the
@@ -1,6 +1,6 @@
1
1
  include LICENSE README.md CHANGELOG.md SECURITY.md
2
2
  include config.example.json .env.example
3
- include requirements.txt
3
+ include requirements.txt requirements-decision.txt
4
4
  recursive-include localdeploy/web *
5
5
  prune docs
6
6
  prune tests
@@ -1,7 +1,7 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: localdeploy
3
- Version: 0.6.3
4
- Summary: Find, deploy, manage, and benchmark local AI models for your hardware across Ollama, llama.cpp, and compatible runtimes.
3
+ Version: 0.7.0
4
+ Summary: Find the local AI models that fit your hardware, test them on your own questions, and connect the winner to your apps through OpenAI-compatible, native, and decision APIs.
5
5
  Author: oney-erge
6
6
  License-Expression: MIT
7
7
  Project-URL: Homepage, https://github.com/oney-erge/LocalDeploy
@@ -15,31 +15,29 @@ Classifier: Intended Audience :: Developers
15
15
  Classifier: Intended Audience :: End Users/Desktop
16
16
  Classifier: Operating System :: OS Independent
17
17
  Classifier: Programming Language :: Python :: 3
18
- Classifier: Programming Language :: Python :: 3.10
19
18
  Classifier: Programming Language :: Python :: 3.11
20
19
  Classifier: Programming Language :: Python :: 3.12
21
20
  Classifier: Programming Language :: Python :: 3.13
22
21
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
23
- Requires-Python: >=3.10
22
+ Requires-Python: >=3.11
24
23
  Description-Content-Type: text/markdown
25
24
  License-File: LICENSE
26
- Requires-Dist: fastapi==0.143.1
25
+ Requires-Dist: fastapi==0.162.5
27
26
  Requires-Dist: uvicorn==0.54.0
28
27
  Requires-Dist: requests==2.34.2
29
28
  Requires-Dist: python-dotenv==1.2.4
30
- Requires-Dist: pydantic==2.14.1
31
- Requires-Dist: python-multipart==0.0.32
32
- Requires-Dist: huggingface_hub==2.2.0
29
+ Requires-Dist: pydantic==2.14.3
30
+ Requires-Dist: python-multipart==0.0.33
31
+ Requires-Dist: huggingface_hub==2.4.0
33
32
  Requires-Dist: psutil==7.2.2
34
33
  Requires-Dist: pyyaml==6.0.3
35
- Provides-Extra: dev
36
- Requires-Dist: pytest>=7.0; extra == "dev"
37
- Requires-Dist: httpx>=0.27; extra == "dev"
38
- Requires-Dist: ruff==0.17.0; extra == "dev"
39
- Requires-Dist: build>=1.2; extra == "dev"
40
- Requires-Dist: twine>=5.0; extra == "dev"
41
- Requires-Dist: playwright>=1.40; extra == "dev"
42
- Requires-Dist: pillow>=10.0; extra == "dev"
34
+ Provides-Extra: decision
35
+ Requires-Dist: torch>=2.5; extra == "decision"
36
+ Requires-Dist: torchvision>=0.20; extra == "decision"
37
+ Requires-Dist: transformers>=5.15; extra == "decision"
38
+ Requires-Dist: pillow>=10.0; extra == "decision"
39
+ Requires-Dist: soundfile>=0.12; extra == "decision"
40
+ Requires-Dist: laya>=0.4.17; extra == "decision"
43
41
  Provides-Extra: packaging
44
42
  Requires-Dist: pyinstaller>=6.0; extra == "packaging"
45
43
  Requires-Dist: pystray>=0.19; extra == "packaging"
@@ -58,7 +56,7 @@ Dynamic: license-file
58
56
  <a href="https://github.com/oney-erge/LocalDeploy/actions/workflows/ci.yml"><img src="https://github.com/oney-erge/LocalDeploy/actions/workflows/ci.yml/badge.svg" alt="CI status" /></a>
59
57
  <a href="https://pypi.org/project/localdeploy/"><img src="https://img.shields.io/pypi/v/localdeploy.svg" alt="PyPI version" /></a>
60
58
  <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT license" /></a>
61
- <img src="https://img.shields.io/badge/python-3.10%2B-blue.svg" alt="Python 3.10+" />
59
+ <img src="https://img.shields.io/badge/python-3.11%2B-blue.svg" alt="Python 3.11+" />
62
60
  </p>
63
61
 
64
62
  <p align="center">
@@ -66,14 +64,16 @@ Dynamic: license-file
66
64
  </p>
67
65
 
68
66
  LocalDeploy helps you answer a practical question before you download a model:
69
- will it fit and run well on this machine? It detects hardware, estimates memory
70
- use, manages local models, records benchmarks, and serves the deployed model
71
- through an OpenAI-compatible API. It runs locally, has no telemetry, and does
72
- not provide cloud inference.
67
+ will it fit and run well on this machine? It finds the local models that fit
68
+ your hardware, tests them on your own questions, and connects the winner to your
69
+ apps. It detects hardware, estimates memory use, manages models across Ollama,
70
+ llama.cpp, LM Studio, and vLLM, records benchmarks, and serves the deployed
71
+ model through several APIs. It runs locally, has no telemetry, and does not
72
+ provide cloud inference.
73
73
 
74
74
  ## Quick start
75
75
 
76
- With Python 3.10+ and [Ollama](https://ollama.com) installed:
76
+ With Python 3.11+ and [Ollama](https://ollama.com) installed:
77
77
 
78
78
  ```bash
79
79
  pip install localdeploy
@@ -84,6 +84,12 @@ The UI guides you through hardware detection, model search, a fit estimate,
84
84
  deployment, chat, and benchmarking. If you use `uv`, `uvx localdeploy` runs it
85
85
  without a permanent install.
86
86
 
87
+ Open **Get a model**, choose a use case and context length, and review the
88
+ recommended model's memory and download estimates. Start an installed model or
89
+ explicitly download one, then use **Chat** and **Benchmark** to see how it
90
+ performs on your machine. [A worked comparison](docs/FIT_COMPARISON.md) shows
91
+ how to inspect estimates against another tool.
92
+
87
93
  Need help setting up Python or Ollama? Clone the repository and run its launcher:
88
94
 
89
95
  ```bash
@@ -101,17 +107,54 @@ Windows setup script. Desktop builds are experimental and unsigned; see
101
107
 
102
108
  - Find models from Ollama, Hugging Face, ModelScope GGUF repositories, local
103
109
  GGUF files, or direct URLs.
104
- - Estimate model fit before download, including likely CPU offload.
105
- - Pull, deploy, switch, unload, and remove local models.
106
- - Chat with text, images, and documents, then compare installed models using
107
- measured latency, throughput, memory, and accuracy results.
110
+ - Estimate model fit before download, including likely CPU offload, and compare
111
+ quantizations with the quant advisor.
112
+ - Pull, deploy, switch, unload, and remove local models. A llama.cpp chat model
113
+ can be started, switched, and stopped by LocalDeploy itself.
114
+ - Chat with text, images, and documents, then compare installed models on the
115
+ built-in or your own question sets using measured latency, throughput, memory,
116
+ and accuracy results.
108
117
  - Monitor CPU, RAM, VRAM, model placement, and request timing while a model runs.
109
- - Connect compatible tools through `/v1/chat/completions`, `/v1/responses`,
110
- `/v1/embeddings`, and `/v1/models`.
118
+ - Export a deployment as a manifest, check it on another machine, and recreate
119
+ it there.
120
+
121
+ ## Ways to use it
111
122
 
112
- If you already know which Ollama model you want and only need terminal chat,
113
- Ollama may be enough. LocalDeploy is for selecting and evaluating models on a
114
- particular machine.
123
+ Once a model is running, your tools can reach it in several ways:
124
+
125
+ | Way in | What it is for |
126
+ |---|---|
127
+ | Web UI at `/ui` | Check hardware, find, pull, and deploy models, chat, benchmark and compare, monitor |
128
+ | OpenAI-compatible API: `/v1/chat/completions`, `/v1/responses`, `/v1/embeddings`, `/v1/models` | Any app or SDK that speaks the OpenAI API, with streaming, JSON-schema output, and tool calls returned to your app |
129
+ | Native API: `/chat`, `/vision`, `/estimate`, `/benchmark` | Requests by profile, with per-request context limits, safe mode, image input, and a safety estimate before you send |
130
+ | Control API: about 50 endpoints | Script what the UI does: hardware and fit checks, recommendations, search, pull, serve, switch, free the GPU, benchmark, compare, export. The interactive reference is at `/docs` |
131
+ | Decision API: `/v1/systemone` | Small classifiers that answer typed questions about a piece of text with a probability for every option |
132
+ | Client setups | Ready configuration for Open WebUI, AnythingLLM, Continue, Cline, curl, Python, JavaScript, and Docker Compose, generated from a running model |
133
+ | Deployment manifests | Record the model digest, quantization, runtime settings, hardware, and measurements, then recreate the deployment elsewhere |
134
+ | Terminal | `scripts/chat.ps1` for chat and `compare_models.py` for side-by-side runs; see [CLI](docs/CLI.md) |
135
+
136
+ [API options](docs/API_OPTIONS.md) has request examples for the native,
137
+ OpenAI-compatible, and decision APIs. The control API is described at `/docs`.
138
+
139
+ The **decision API** is optional. It runs small models (vLLM Semantic Router d3,
140
+ LiquidAI d1, Laya) that return a probability for every option in one pass, so an
141
+ app can classify or route locally and call a language model only when the answer
142
+ is not confident. On a laptop CPU, the 0.85B d3-lite model routed agent messages
143
+ to 15 routes as accurately as an 8B language model on one set (93% against 92%
144
+ of 98 messages), slightly less on another (84% against 88% of 43), in 0.5 s
145
+ instead of 15 s. The [raw runs and method](benchmarks/decision-routing/README.md)
146
+ are in the repository, and [Decision models](docs/API_OPTIONS.md#decision-models)
147
+ shows how to enable one.
148
+
149
+ ## Where it fits
150
+
151
+ - If you already know which model you want and only need terminal chat, Ollama
152
+ may be enough.
153
+ - If you use a front end such as Open WebUI, Continue, or Cline, LocalDeploy is
154
+ the layer underneath: it helps choose and run the model and generates those
155
+ tools' connection settings.
156
+ - If you are choosing between models for a particular machine, it runs the
157
+ candidates on your own questions and records what you measured.
115
158
 
116
159
  ## What to expect
117
160
 
@@ -136,7 +179,7 @@ have their own network settings; see [Security](SECURITY.md).
136
179
  |---|---|
137
180
  | [Installation](docs/INSTALL.md) | Launcher commands, Docker, and setup alternatives |
138
181
  | [UI guide](docs/UI.md) | Choosing, deploying, chatting with, and benchmarking models |
139
- | [API options](docs/API_OPTIONS.md) | OpenAI-compatible API and local integrations |
182
+ | [API options](docs/API_OPTIONS.md) | Native, OpenAI-compatible, and decision APIs, with request examples |
140
183
  | [CLI](docs/CLI.md) | Commands and automation |
141
184
  | [Models](docs/MODELS.md) | Runtimes, model discovery, and fit estimates |
142
185
  | [Screenshots](docs/SCREENSHOTS.md) | The complete UI tour |
@@ -0,0 +1,143 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/oney-erge/LocalDeploy/main/localdeploy/web/logo.svg" width="96" height="96" alt="LocalDeploy logo" />
3
+ </p>
4
+
5
+ <h1 align="center">LocalDeploy</h1>
6
+
7
+ <p align="center"><strong>Choose, run, and compare local AI models on your own hardware.</strong></p>
8
+
9
+ <p align="center">
10
+ <a href="https://github.com/oney-erge/LocalDeploy/actions/workflows/ci.yml"><img src="https://github.com/oney-erge/LocalDeploy/actions/workflows/ci.yml/badge.svg" alt="CI status" /></a>
11
+ <a href="https://pypi.org/project/localdeploy/"><img src="https://img.shields.io/pypi/v/localdeploy.svg" alt="PyPI version" /></a>
12
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT license" /></a>
13
+ <img src="https://img.shields.io/badge/python-3.11%2B-blue.svg" alt="Python 3.11+" />
14
+ </p>
15
+
16
+ <p align="center">
17
+ <img src="https://raw.githubusercontent.com/oney-erge/LocalDeploy/main/docs/assets/demo.gif" alt="LocalDeploy walkthrough showing hardware-aware model recommendations, deployment, local chat, benchmark comparisons, and live monitoring" width="820" />
18
+ </p>
19
+
20
+ LocalDeploy helps you answer a practical question before you download a model:
21
+ will it fit and run well on this machine? It finds the local models that fit
22
+ your hardware, tests them on your own questions, and connects the winner to your
23
+ apps. It detects hardware, estimates memory use, manages models across Ollama,
24
+ llama.cpp, LM Studio, and vLLM, records benchmarks, and serves the deployed
25
+ model through several APIs. It runs locally, has no telemetry, and does not
26
+ provide cloud inference.
27
+
28
+ ## Quick start
29
+
30
+ With Python 3.11+ and [Ollama](https://ollama.com) installed:
31
+
32
+ ```bash
33
+ pip install localdeploy
34
+ localdeploy # opens http://localhost:8000/ui
35
+ ```
36
+
37
+ The UI guides you through hardware detection, model search, a fit estimate,
38
+ deployment, chat, and benchmarking. If you use `uv`, `uvx localdeploy` runs it
39
+ without a permanent install.
40
+
41
+ Open **Get a model**, choose a use case and context length, and review the
42
+ recommended model's memory and download estimates. Start an installed model or
43
+ explicitly download one, then use **Chat** and **Benchmark** to see how it
44
+ performs on your machine. [A worked comparison](docs/FIT_COMPARISON.md) shows
45
+ how to inspect estimates against another tool.
46
+
47
+ Need help setting up Python or Ollama? Clone the repository and run its launcher:
48
+
49
+ ```bash
50
+ git clone https://github.com/oney-erge/LocalDeploy.git
51
+ cd LocalDeploy
52
+ ./run.sh # Linux; macOS: ./run.command
53
+ ```
54
+
55
+ On Windows, use `run.bat`. The launchers can prepare the environment and open
56
+ the UI; [Installation](docs/INSTALL.md) covers prerequisites, Docker, and the
57
+ Windows setup script. Desktop builds are experimental and unsigned; see
58
+ [Packaging](docs/PACKAGING.md).
59
+
60
+ ## What it helps with
61
+
62
+ - Find models from Ollama, Hugging Face, ModelScope GGUF repositories, local
63
+ GGUF files, or direct URLs.
64
+ - Estimate model fit before download, including likely CPU offload, and compare
65
+ quantizations with the quant advisor.
66
+ - Pull, deploy, switch, unload, and remove local models. A llama.cpp chat model
67
+ can be started, switched, and stopped by LocalDeploy itself.
68
+ - Chat with text, images, and documents, then compare installed models on the
69
+ built-in or your own question sets using measured latency, throughput, memory,
70
+ and accuracy results.
71
+ - Monitor CPU, RAM, VRAM, model placement, and request timing while a model runs.
72
+ - Export a deployment as a manifest, check it on another machine, and recreate
73
+ it there.
74
+
75
+ ## Ways to use it
76
+
77
+ Once a model is running, your tools can reach it in several ways:
78
+
79
+ | Way in | What it is for |
80
+ |---|---|
81
+ | Web UI at `/ui` | Check hardware, find, pull, and deploy models, chat, benchmark and compare, monitor |
82
+ | OpenAI-compatible API: `/v1/chat/completions`, `/v1/responses`, `/v1/embeddings`, `/v1/models` | Any app or SDK that speaks the OpenAI API, with streaming, JSON-schema output, and tool calls returned to your app |
83
+ | Native API: `/chat`, `/vision`, `/estimate`, `/benchmark` | Requests by profile, with per-request context limits, safe mode, image input, and a safety estimate before you send |
84
+ | Control API: about 50 endpoints | Script what the UI does: hardware and fit checks, recommendations, search, pull, serve, switch, free the GPU, benchmark, compare, export. The interactive reference is at `/docs` |
85
+ | Decision API: `/v1/systemone` | Small classifiers that answer typed questions about a piece of text with a probability for every option |
86
+ | Client setups | Ready configuration for Open WebUI, AnythingLLM, Continue, Cline, curl, Python, JavaScript, and Docker Compose, generated from a running model |
87
+ | Deployment manifests | Record the model digest, quantization, runtime settings, hardware, and measurements, then recreate the deployment elsewhere |
88
+ | Terminal | `scripts/chat.ps1` for chat and `compare_models.py` for side-by-side runs; see [CLI](docs/CLI.md) |
89
+
90
+ [API options](docs/API_OPTIONS.md) has request examples for the native,
91
+ OpenAI-compatible, and decision APIs. The control API is described at `/docs`.
92
+
93
+ The **decision API** is optional. It runs small models (vLLM Semantic Router d3,
94
+ LiquidAI d1, Laya) that return a probability for every option in one pass, so an
95
+ app can classify or route locally and call a language model only when the answer
96
+ is not confident. On a laptop CPU, the 0.85B d3-lite model routed agent messages
97
+ to 15 routes as accurately as an 8B language model on one set (93% against 92%
98
+ of 98 messages), slightly less on another (84% against 88% of 43), in 0.5 s
99
+ instead of 15 s. The [raw runs and method](benchmarks/decision-routing/README.md)
100
+ are in the repository, and [Decision models](docs/API_OPTIONS.md#decision-models)
101
+ shows how to enable one.
102
+
103
+ ## Where it fits
104
+
105
+ - If you already know which model you want and only need terminal chat, Ollama
106
+ may be enough.
107
+ - If you use a front end such as Open WebUI, Continue, or Cline, LocalDeploy is
108
+ the layer underneath: it helps choose and run the model and generates those
109
+ tools' connection settings.
110
+ - If you are choosing between models for a particular machine, it runs the
111
+ candidates on your own questions and records what you measured.
112
+
113
+ ## What to expect
114
+
115
+ Fit estimates are guidance, not guarantees. Driver versions, context length,
116
+ quantization, desktop GPU use, and the runtime all affect actual memory use.
117
+ LocalDeploy records what happened after deployment so later estimates can be
118
+ adjusted for that machine.
119
+
120
+ It works with Ollama, llama.cpp, LM Studio, vLLM, and other loopback
121
+ OpenAI-compatible runtimes. Runtime URLs must remain local. An `API_TOKEN`
122
+ is an optional shared token, not full authentication. Keep the service on
123
+ localhost; it has no TLS, user isolation, or rate limiting. Tool calls are
124
+ returned to clients and are never executed by LocalDeploy.
125
+
126
+ Model search and the UI's release check can contact external services.
127
+ Set `OFFLINE=true` to disable those requests. Separately installed runtimes
128
+ have their own network settings; see [Security](SECURITY.md).
129
+
130
+ ## Documentation
131
+
132
+ | Guide | Use it for |
133
+ |---|---|
134
+ | [Installation](docs/INSTALL.md) | Launcher commands, Docker, and setup alternatives |
135
+ | [UI guide](docs/UI.md) | Choosing, deploying, chatting with, and benchmarking models |
136
+ | [API options](docs/API_OPTIONS.md) | Native, OpenAI-compatible, and decision APIs, with request examples |
137
+ | [CLI](docs/CLI.md) | Commands and automation |
138
+ | [Models](docs/MODELS.md) | Runtimes, model discovery, and fit estimates |
139
+ | [Screenshots](docs/SCREENSHOTS.md) | The complete UI tour |
140
+ | [Packaging](docs/PACKAGING.md) | Experimental desktop builds and signing limitations |
141
+ | [Contributing](CONTRIBUTING.md) | Development setup and verification |
142
+
143
+ MIT licensed. Built by [Oney Erge](https://oneyerge.com).
@@ -39,7 +39,7 @@ Do not commit `.env`, `config.json`, `calibration.json`, model files, private sc
39
39
 
40
40
  ## Reporting a vulnerability
41
41
 
42
- After the repository is public, maintainers must enable GitHub private vulnerability reporting under Settings / Security / Advanced Security. When the Report a vulnerability button is available, use it instead of opening a public issue.
42
+ Use the **Report a vulnerability** button on the repository's Security tab instead of opening a public issue. GitHub keeps the report private between you and the maintainer.
43
43
 
44
44
  If that button is not available, open a minimal issue asking for a private contact method. Do not put exploit details, tokens, private prompts, images, model files, or local paths in the public issue.
45
45