shard-engine 1.8.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. shard_engine-1.8.1/LICENSE +17 -0
  2. shard_engine-1.8.1/MANIFEST.in +16 -0
  3. shard_engine-1.8.1/PKG-INFO +549 -0
  4. shard_engine-1.8.1/README.kz.md +452 -0
  5. shard_engine-1.8.1/README.md +490 -0
  6. shard_engine-1.8.1/README.ru.md +454 -0
  7. shard_engine-1.8.1/pyproject.toml +81 -0
  8. shard_engine-1.8.1/setup.cfg +4 -0
  9. shard_engine-1.8.1/setup.py +2 -0
  10. shard_engine-1.8.1/shard_engine/__init__.py +3 -0
  11. shard_engine-1.8.1/shard_engine/__main__.py +3 -0
  12. shard_engine-1.8.1/shard_engine/cache_index.py +37 -0
  13. shard_engine-1.8.1/shard_engine/cli.py +1994 -0
  14. shard_engine-1.8.1/shard_engine/constants.py +23 -0
  15. shard_engine-1.8.1/shard_engine/dashboard.html +4993 -0
  16. shard_engine-1.8.1/shard_engine/engine.py +1518 -0
  17. shard_engine-1.8.1/shard_engine/engine_optimizations.json +428 -0
  18. shard_engine-1.8.1/shard_engine/feedback.py +942 -0
  19. shard_engine-1.8.1/shard_engine/fit.py +289 -0
  20. shard_engine-1.8.1/shard_engine/formatters.py +30 -0
  21. shard_engine-1.8.1/shard_engine/gguf_backend.py +769 -0
  22. shard_engine-1.8.1/shard_engine/hardware.py +165 -0
  23. shard_engine-1.8.1/shard_engine/hf_hub.py +290 -0
  24. shard_engine-1.8.1/shard_engine/lan.py +1613 -0
  25. shard_engine-1.8.1/shard_engine/main.py +32 -0
  26. shard_engine-1.8.1/shard_engine/model_browser.py +687 -0
  27. shard_engine-1.8.1/shard_engine/net_discovery.py +1145 -0
  28. shard_engine-1.8.1/shard_engine/net_identity.py +1072 -0
  29. shard_engine-1.8.1/shard_engine/net_platform_linux.py +910 -0
  30. shard_engine-1.8.1/shard_engine/net_platform_win.py +930 -0
  31. shard_engine-1.8.1/shard_engine/net_strategy.py +2169 -0
  32. shard_engine-1.8.1/shard_engine/net_tunnel.py +2234 -0
  33. shard_engine-1.8.1/shard_engine/net_wireguard.py +780 -0
  34. shard_engine-1.8.1/shard_engine/paths.py +173 -0
  35. shard_engine-1.8.1/shard_engine/process.py +231 -0
  36. shard_engine-1.8.1/shard_engine/server.py +2652 -0
  37. shard_engine-1.8.1/shard_engine/session_registry.py +105 -0
  38. shard_engine-1.8.1/shard_engine/settings.py +142 -0
  39. shard_engine-1.8.1/shard_engine/shard_bridge.py +289 -0
  40. shard_engine-1.8.1/shard_engine/soup_adapter.py +1202 -0
  41. shard_engine-1.8.1/shard_engine/storage.py +457 -0
  42. shard_engine-1.8.1/shard_engine/telemetry.py +43 -0
  43. shard_engine-1.8.1/shard_engine/training.py +1137 -0
  44. shard_engine-1.8.1/shard_engine/updater.py +861 -0
  45. shard_engine-1.8.1/shard_engine/validators.py +16 -0
  46. shard_engine-1.8.1/shard_engine.egg-info/PKG-INFO +549 -0
  47. shard_engine-1.8.1/shard_engine.egg-info/SOURCES.txt +49 -0
  48. shard_engine-1.8.1/shard_engine.egg-info/dependency_links.txt +1 -0
  49. shard_engine-1.8.1/shard_engine.egg-info/entry_points.txt +2 -0
  50. shard_engine-1.8.1/shard_engine.egg-info/requires.txt +37 -0
  51. shard_engine-1.8.1/shard_engine.egg-info/top_level.txt +1 -0
@@ -0,0 +1,17 @@
1
+ MIT License
2
+ Copyright (c) 2026 avelone123123
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ of this software and associated documentation files (the "Software"), to deal
5
+ in the Software without restriction, including without limitation the rights
6
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ copies of the Software, and to permit persons to whom the Software is
8
+ furnished to do so, subject to the following conditions:
9
+ The above copyright notice and this permission notice shall be included in all
10
+ copies or substantial portions of the Software.
11
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
12
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
13
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
14
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
15
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
16
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
17
+ SOFTWARE.
@@ -0,0 +1,16 @@
1
+ include LICENSE
2
+ include README.md
3
+ include README.ru.md
4
+ include README.kz.md
5
+ include pyproject.toml
6
+ include setup.py
7
+ include shard_engine/dashboard.html
8
+ include shard_engine/engine_optimizations.json
9
+ prune assets
10
+ prune website
11
+ prune docs
12
+ prune native
13
+ prune dist
14
+ prune build
15
+ global-exclude *.pyc *.pyo __pycache__ *.exe *.dll *.so *.dylib *.lib *.exp *.obj keys_db.json
16
+
@@ -0,0 +1,549 @@
1
+ Metadata-Version: 2.4
2
+ Name: shard-engine
3
+ Version: 1.8.1
4
+ Summary: Distributed local LLM inference engine with OpenAI/Anthropic API, GGUF/llama.cpp, and LAN memory pooling
5
+ Author-email: avelone123123 <avelone.dev@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://avelone123123.github.io/shard/
8
+ Project-URL: Repository, https://github.com/avelone123123/shard
9
+ Project-URL: Issues, https://github.com/avelone123123/shard/issues
10
+ Keywords: llm,inference,local-ai,gguf,llama-cpp,distributed-inference,openai-api,cli
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Environment :: Console
13
+ Classifier: Framework :: FastAPI
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Operating System :: Microsoft :: Windows
17
+ Classifier: Operating System :: POSIX :: Linux
18
+ Classifier: Operating System :: MacOS
19
+ Classifier: Programming Language :: Python :: 3
20
+ Classifier: Programming Language :: Python :: 3.8
21
+ Classifier: Programming Language :: Python :: 3.9
22
+ Classifier: Programming Language :: Python :: 3.10
23
+ Classifier: Programming Language :: Python :: 3.11
24
+ Classifier: Programming Language :: Python :: 3.12
25
+ Classifier: Programming Language :: Python :: 3.13
26
+ Classifier: Programming Language :: Python :: 3.14
27
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
28
+ Requires-Python: >=3.8
29
+ Description-Content-Type: text/markdown
30
+ License-File: LICENSE
31
+ Requires-Dist: fastapi>=0.100.0
32
+ Requires-Dist: uvicorn>=0.22.0
33
+ Requires-Dist: requests>=2.31.0
34
+ Requires-Dist: httpx>=0.24.0
35
+ Requires-Dist: psutil>=5.9.0
36
+ Requires-Dist: pydantic>=2.0.0
37
+ Requires-Dist: cryptography<43.0,>=41.0; python_version < "3.9"
38
+ Requires-Dist: cryptography>=42.0; python_version >= "3.9"
39
+ Requires-Dist: websockets>=12.0
40
+ Provides-Extra: ml
41
+ Requires-Dist: transformers>=4.40.0; extra == "ml"
42
+ Requires-Dist: accelerate>=0.28.0; extra == "ml"
43
+ Requires-Dist: safetensors>=0.4.0; extra == "ml"
44
+ Requires-Dist: huggingface_hub>=0.20.0; extra == "ml"
45
+ Requires-Dist: tqdm>=4.65.0; extra == "ml"
46
+ Provides-Extra: full
47
+ Requires-Dist: transformers>=4.40.0; extra == "full"
48
+ Requires-Dist: accelerate>=0.28.0; extra == "full"
49
+ Requires-Dist: safetensors>=0.4.0; extra == "full"
50
+ Requires-Dist: huggingface_hub>=0.20.0; extra == "full"
51
+ Requires-Dist: tqdm>=4.65.0; extra == "full"
52
+ Provides-Extra: training
53
+ Requires-Dist: soup-cli[train]>=0.72.0; (python_version >= "3.11" and python_version < "3.14") and extra == "training"
54
+ Provides-Extra: dev
55
+ Requires-Dist: build>=1.0.0; extra == "dev"
56
+ Requires-Dist: twine>=5.0.0; extra == "dev"
57
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
58
+ Dynamic: license-file
59
+
60
+ <p align="center">
61
+ <strong>English</strong> &nbsp;|&nbsp;
62
+ <a href="README.ru.md">Русская версия</a> &nbsp;|&nbsp;
63
+ <a href="README.kz.md">Қазақша нұсқасы</a>
64
+ </p>
65
+
66
+ <p align="center">
67
+ <picture>
68
+ <source media="(prefers-color-scheme: dark)" srcset="assets/branding/shard-logo-dark.svg">
69
+ <source media="(prefers-color-scheme: light)" srcset="assets/branding/shard-logo-light.svg">
70
+ <img src="assets/branding/shard-logo.svg" alt="SHARD — Distributed Local LLM Inference Engine" width="760">
71
+ </picture>
72
+ </p>
73
+
74
+ <h1 align="center">SHARD</h1>
75
+
76
+ <p align="center">
77
+ <strong>One Network. Many Servers. Shared AI Infrastructure.</strong><br>
78
+ <sub>Distributed local LLM inference engine pooling GPU VRAM and system RAM across LAN machines — powered by PyTorch, FastAPI, <code>llama.cpp</code> RPC, and native C++20 / MASM AVX2 SIMD kernels.</sub>
79
+ </p>
80
+
81
+ <p align="center">
82
+ <a href="https://avelone123123.github.io/shard/"><img src="https://img.shields.io/badge/WEBSITE-Live-00E5FF?style=for-the-badge&labelColor=0A1022" alt="Website"></a>
83
+ <a href="README.md"><img src="https://img.shields.io/badge/LANG-English-00E5FF?style=for-the-badge&labelColor=0A1022" alt="English README"></a>
84
+ <a href="README.ru.md"><img src="https://img.shields.io/badge/ЯЗЫК-Русский-38BDF8?style=for-the-badge&labelColor=0A1022" alt="Русская версия"></a>
85
+ <a href="README.kz.md"><img src="https://img.shields.io/badge/ТІЛ-Қазақша-A855F7?style=for-the-badge&labelColor=0A1022" alt="Қазақша версия"></a>
86
+ <a href="docs/LAN_SHARDING_DESIGN.md"><img src="https://img.shields.io/badge/DOCS-LAN_Sharding-34D399?style=for-the-badge&labelColor=0A1022" alt="Documentation"></a>
87
+ <a href="#installation--quick-start"><img src="https://img.shields.io/badge/START-Quick_Start-00E5FF?style=for-the-badge&labelColor=0A1022" alt="Getting Started"></a>
88
+ <a href="#installation"><img src="https://img.shields.io/badge/SETUP-Installation-38BDF8?style=for-the-badge&labelColor=0A1022" alt="Installation"></a>
89
+ <a href="https://github.com/avelone123123/shard/discussions"><img src="https://img.shields.io/badge/COMMUNITY-Discussions-A855F7?style=for-the-badge&labelColor=0A1022" alt="GitHub Discussions"></a>
90
+ <a href="https://github.com/avelone123123/shard/issues"><img src="https://img.shields.io/badge/TRACKER-Issues-F59E0B?style=for-the-badge&labelColor=0A1022" alt="Issues"></a>
91
+ <a href="https://github.com/avelone123123/shard/releases"><img src="https://img.shields.io/github/downloads/avelone123123/shard/total?style=for-the-badge&labelColor=0A1022&color=34D399&label=DOWNLOADS" alt="Total Downloads"></a>
92
+ </p>
93
+
94
+ <p align="center">
95
+ <a href="https://github.com/avelone123123/shard/releases"><img src="https://img.shields.io/github/v/release/avelone123123/shard?style=flat-square&labelColor=0D1326&color=00E5FF&label=release" alt="Latest Release"></a>
96
+ <a href="https://github.com/avelone123123/shard/releases/latest"><img src="https://img.shields.io/github/downloads/avelone123123/shard/latest/total?style=flat-square&labelColor=0D1326&color=34D399&label=latest%20downloads&logo=github&logoColor=white" alt="Latest Release Downloads"></a>
97
+ <a href="https://github.com/avelone123123/shard/stargazers"><img src="https://img.shields.io/github/stars/avelone123123/shard?style=flat-square&labelColor=0D1326&color=F59E0B&label=stars&logo=github&logoColor=white" alt="GitHub Stars"></a>
98
+ <a href="https://www.python.org/"><img src="https://img.shields.io/badge/python-3.8%2B%20%7C%203.10%2B-38BDF8?style=flat-square&labelColor=0D1326&logo=python&logoColor=white" alt="Python 3.8+"></a>
99
+ <a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-34D399?style=flat-square&labelColor=0D1326" alt="License: MIT"></a>
100
+ <a href="https://github.com/avelone123123/shard/actions/workflows/tests.yml"><img src="https://img.shields.io/github/actions/workflow/status/avelone123123/shard/tests.yml?branch=master&style=flat-square&labelColor=0D1326&label=tests" alt="Tests CI"></a>
101
+ <img src="https://img.shields.io/badge/backend-PyTorch%20%7C%20GGUF%20%7C%20AVX2-A855F7?style=flat-square&labelColor=0D1326" alt="Backends">
102
+ <img src="https://img.shields.io/badge/API-OpenAI%20%26%20Anthropic-00E5FF?style=flat-square&labelColor=0D1326" alt="OpenAI and Anthropic API">
103
+ </p>
104
+
105
+ <p align="center">
106
+ <img src="assets/animations/hero-network.gif" alt="SHARD Hero Network Animation — Distributed AI Inference Cluster" width="800">
107
+ <br>
108
+ <sub><b>SHARD Distributed Inference Cluster:</b> Central master coordinator pooling VRAM across connected LAN worker nodes over UDP discovery (<code>:8001</code>) and FastAPI SSE endpoints (<code>:8000</code>).</sub>
109
+ </p>
110
+
111
+ <p align="center">
112
+ <a href="#what-is-shard">Overview</a> &middot;
113
+ <a href="#how-shard-works">How It Works</a> &middot;
114
+ <a href="#features">Features</a> &middot;
115
+ <a href="#installation--quick-start">Quick Start</a> &middot;
116
+ <a href="#cli-reference--configuration">CLI Reference</a> &middot;
117
+ <a href="#architecture">Architecture</a> &middot;
118
+ <a href="#web-dashboard--demonstrations">Dashboard</a> &middot;
119
+ <a href="#project-status--community">Community</a>
120
+ </p>
121
+
122
+ ---
123
+
124
+ ## What Is SHARD?
125
+
126
+ **[SHARD](https://github.com/avelone123123/shard)** (`shard-engine`) is an open-source distributed local LLM inference engine written in Python ([FastAPI](https://fastapi.tiangolo.com/) + [PyTorch](https://pytorch.org/)) with a [GGUF / `llama.cpp` backend](https://github.com/ggml-org/llama.cpp) and native C++20 / MASM AVX2 SIMD acceleration.
127
+
128
+ ### The Problem It Solves
129
+ Running capable 14B, 32B, or 70B+ open-weight language models locally typically hits a hard hardware wall: **single-card VRAM limits**. Most developers, labs, and small teams own multiple machines—a desktop gaming GPU, a workstation laptop, an auxiliary mini-PC—whose memory sits fragmented across the local network while no single device can fit the full model weights and KV cache.
130
+
131
+ ### Why SHARD Exists
132
+ SHARD unites idle GPUs and system memory across your Local Area Network (LAN) into a single logical inference cluster:
133
+ - **Pool Hardware Without Cloud Lock-In:** Split transformer layers across multiple consumer GPUs and host RAM over standard gigabit LAN (`lan.py`, `gguf_backend.py`) with zero external cloud dependencies or recurring API fees.
134
+ - **Know Before You Download (`shard fit`):** Inspect whether any Hugging Face model will fit your single GPU, hybrid CPU+GPU setup, or pooled LAN cluster before downloading gigabytes of weights (`fit.py`).
135
+ - **Drop-In Agent & IDE Backend:** Expose both OpenAI (`/v1/chat/completions`, `/v1/completions`, `/v1/models`) and Anthropic (`/v1/messages`) compatible endpoints on port `8000` so coding agents and SDKs connect immediately.
136
+
137
+ ### Who Should Use SHARD?
138
+ - **Local AI Developers & Researchers** running 7B–70B GGUF or PyTorch SafeTensors models across home lab or office workstations.
139
+ - **Software Engineering Teams** hosting private, zero-telemetry coding assistants for [Cursor](https://www.cursor.com/), [Claude Code](https://docs.anthropic.com/en/docs/agents-and-tools/claude-code/overview), [Aider](https://aider.chat/), [Continue](https://www.continue.dev/), [Zed](https://zed.dev/), and [OpenCode](https://github.com/opencode-ai/opencode).
140
+ - **AI Agent Builders** orchestrating multi-agent pipelines via [OpenAI Python SDK](https://github.com/openai/openai-python), [LangChain](https://www.langchain.com/), or [LlamaIndex](https://www.llamaindex.ai/) against self-hosted hardware.
141
+
142
+ ---
143
+
144
+ ## How SHARD Works
145
+
146
+ SHARD turns distributed local AI infrastructure into a four-step workflow: **CREATE → SHARE → CONNECT → USE**.
147
+
148
+ <table>
149
+ <tr>
150
+ <td width="50%" valign="top">
151
+ <h3 align="center">1. CREATE — Launch Your Server</h3>
152
+ <p align="center">
153
+ <img src="assets/animations/create-server.gif" alt="Step 1: Create and Launch Your SHARD Server" width="100%">
154
+ </p>
155
+ <p>
156
+ Start a local SHARD node with <code>shard start</code> or <code>python main.py</code>. The engine probes your CPU, system RAM, and NVIDIA CUDA / Apple MPS GPU via <code>hardware.py</code>, loads your chosen GGUF or Hugging Face model, and brings up the FastAPI server and Web UI on <code>http://localhost:8000</code>.
157
+ </p>
158
+ </td>
159
+ <td width="50%" valign="top">
160
+ <h3 align="center">2. SHARE — Broadcast to the LAN</h3>
161
+ <p align="center">
162
+ <img src="assets/animations/share-network.gif" alt="Step 2: Share Server With the LAN Network" width="100%">
163
+ </p>
164
+ <p>
165
+ Open a LAN compute pool with <code>shard lan create --open</code> (or generate a pairing code via <code>shard lan create</code>). SHARD broadcasts UDP discovery beacons on port <code>8001</code> and exposes <code>/api/lan/*</code> endpoints so peer machines on your subnet can pool their VRAM and RAM.
166
+ </p>
167
+ </td>
168
+ </tr>
169
+ <tr>
170
+ <td width="50%" valign="top">
171
+ <h3 align="center">3. CONNECT — Pair Worker Nodes</h3>
172
+ <p align="center">
173
+ <img src="assets/animations/connect-server.gif" alt="Step 3: Connect and Pair to a SHARD Server" width="100%">
174
+ </p>
175
+ <p>
176
+ On any secondary computer on the network, run <code>shard lan join &lt;master-ip&gt; --share</code>. The worker completes an HTTP handshake (<code>POST /api/lan/pair</code>), launches its local RPC tensor worker, sends periodic health heartbeats, and contributes its GPU/RAM to the cluster.
177
+ </p>
178
+ </td>
179
+ <td width="50%" valign="top">
180
+ <h3 align="center">4. USE — Power AI Agents & Apps</h3>
181
+ <p align="center">
182
+ <img src="assets/animations/ai-agents.gif" alt="Step 4: Host and Access AI Agents Through SHARD" width="100%">
183
+ </p>
184
+ <p>
185
+ Connect coding IDEs, autonomous AI agents, or custom Python apps to <code>http://&lt;master-ip&gt;:8000/v1/chat/completions</code> or <code>/v1/messages</code>. Master coordinates layer execution across connected nodes and streams tokens back via Server-Sent Events (SSE).
186
+ </p>
187
+ </td>
188
+ </tr>
189
+ </table>
190
+
191
+ ---
192
+
193
+ ## Features
194
+
195
+ Every feature listed below is verified directly against the [SHARD source tree](https://github.com/avelone123123/shard) (`v1.8.1`).
196
+
197
+ ### Implemented Features
198
+
199
+ | Category | Capability | Source Module | Description |
200
+ | :--- | :--- | :--- | :--- |
201
+ | **LAN Cluster & Sharding** | **UDP Auto-Discovery & LAN Pooling** | [`lan.py`](lan.py), [`docs/LAN_SHARDING_DESIGN.md`](docs/LAN_SHARDING_DESIGN.md) | Broadcasts UDP beacons (`:8001`), pairs nodes via code or `--open` mode, monitors node heartbeats (`/api/lan/heartbeat`), and pools VRAM/RAM across machines. |
202
+ | **Inference Backends** | **Dual PyTorch & GGUF (`llama.cpp`) Engine** | [`engine.py`](engine.py), [`gguf_backend.py`](gguf_backend.py) | Runs Hugging Face `transformers` / `safetensors` models on CUDA/CPU/MPS and quantized GGUF weights via managed `llama-server` & `ggml-rpc-server` subprocesses with configurable GPU layer offload (`-ngl`). |
203
+ | **Hardware Planning** | **Pre-Flight Model Fit Advisor (`shard fit`)** | [`fit.py`](fit.py), [`hardware.py`](hardware.py) | Inspects model architecture parameters (`config.json` or parameter heuristics) against host VRAM and RAM before downloading, returning `GPU`, `HYBRID`, or `IMPOSSIBLE` verdicts, max context length, estimated tok/s, and quantization advice. |
204
+ | **Native Acceleration** | **C++20 & MASM AVX2 SIMD Kernels** | [`native/shard_core.cpp`](native/shard_core.cpp), [`native/shard_asm.asm`](native/shard_asm.asm), [`shard_bridge.py`](shard_bridge.py) | Custom C++20 and x86-64 AVX2 assembly routines for `RMSNorm`, `LayerNorm`, Rotary Position Embeddings (`RoPE`), `Softmax`, `GELU`, `SiLU`, and `MatMul` loaded via `ctypes`. |
205
+ | **API & Streaming** | **OpenAI & Anthropic Compatible Server** | [`server.py`](server.py) | Serves `/v1/chat/completions`, `/v1/completions`, `/v1/models`, `/v1/messages` (Anthropic protocol), `/api/generate`, and `/api/generate/stream` with real-time Server-Sent Events (SSE) and optional API key auth (`shard key`). |
206
+ | **Web Control Panel** | **Zero-Dependency Dashboard & Playground** | [`dashboard.html`](dashboard.html) | Built-in browser UI at `http://localhost:8000` with live GPU/VRAM/RAM telemetry, Hugging Face model search & download, quantization selector, streaming chat playground, LAN node manager, and one-click IDE integration snippets. |
207
+ | **Containerization** | **Docker & NVIDIA GPU Compose Stack** | [`Dockerfile`](Dockerfile), [`docker-compose.yml`](docker-compose.yml) | Production-ready container setup with persistent named volumes (`shard_models_volume`, `shard_data_volume`) and NVIDIA GPU reservation. |
208
+ | **Collaborative Training** | **Optional SOUP Fine-Tuning Ecosystem** | [`soup_adapter.py`](shard_engine/soup_adapter.py), [`feedback.py`](shard_engine/feedback.py), [`training.py`](shard_engine/training.py), [`docs/COLLABORATIVE_TRAINING.md`](docs/COLLABORATIVE_TRAINING.md) | Privacy-first consent feedback, secret/PII redaction, immutable SFT/DPO dataset snapshots, dynamic SOUP training orchestration with cancellation, Baseline vs Candidate eval, model versioning, deployment, and 1-click rollback. |
209
+ | **Unified Hosting & Privacy** | **Model-First Hosting & Private Sessions** | [`server.py`](shard_engine/server.py), [`lan.py`](shard_engine/lan.py) | Enforces loaded model verification before server creation/publishing (HTTP 409 `MODEL_NOT_READY`), guarantees zero-leakage participant-isolated private chats, and enforces compute worker VRAM/CPU quotas. |
210
+
211
+ ### Experimental Features
212
+ - **Multi-Host `llama.cpp` RPC Tensor Distribution (`gguf_backend.py`, `lan.py`):** Distributed layer execution over TCP/RPC across heterogeneous LAN GPUs works best on low-latency wired Gigabit/2.5GbE Ethernet subnets; Wi-Fi links may experience higher inter-token latency.
213
+ - **Native Win32 Desktop GUI (`native/shard_gui.cpp`):** Native C++ Windows desktop wrapper alongside the cross-platform web dashboard (`dashboard.html`).
214
+
215
+ ### Planned Features
216
+ - **Automated Dynamic Re-Sharding on Node Dropout:** Seamless mid-generation layer migration if a worker node disconnects unexpectedly (`docs/LAN_SHARDING_DESIGN.md`).
217
+ - **Linux / macOS ARM64 NEON Intrinsics:** Expanding the native SIMD kernel suite (`native/`) beyond x86-64 AVX2/FMA to native Apple Silicon / ARM64 assembly paths.
218
+
219
+ ---
220
+
221
+ ## Installation & Quick Start
222
+
223
+ ### Prerequisites
224
+ - **OS:** Windows 10/11 (64-bit), Linux (Ubuntu 20.04+ / Debian 12+), or macOS.
225
+ - **Python:** Python `3.8+` (`3.10+` recommended).
226
+ - **Hardware:** NVIDIA GPU with CUDA drivers (recommended for GPU acceleration), Apple Silicon (MPS), or x86-64 CPU with AVX2 support.
227
+ - **Network (Optional for LAN Sharding):** Two or more machines on the same LAN with TCP port `8000` (HTTP/API) and UDP/TCP port `8001` (LAN discovery & RPC) open.
228
+
229
+ ### Installation
230
+
231
+ #### Option 1: Install via PyPI (Recommended)
232
+ Install SHARD directly from PyPI in any environment:
233
+ ```bash
234
+ # Core install: CLI, FastAPI server, web dashboard, GGUF (llama.cpp) + LAN clustering
235
+ pip install shard-engine
236
+
237
+ # Full install: adds PyTorch/Transformers safetensors & Hugging Face extras
238
+ pip install "shard-engine[full]"
239
+ ```
240
+ To install the latest development snapshot directly from GitHub:
241
+ ```bash
242
+ # Core install from GitHub
243
+ pip install "git+https://github.com/avelone123123/shard.git"
244
+
245
+ # Full install from GitHub
246
+ pip install "shard-engine[full] @ git+https://github.com/avelone123123/shard.git"
247
+ ```
248
+
249
+ Verify your environment right after installation:
250
+ ```bash
251
+ shard doctor
252
+ ```
253
+ > **Windows PATH Note:** If Windows reports that `shard.exe` is not on your PATH, you can run SHARD directly through Python: `python -m shard_engine <command>` (for example, `python -m shard_engine help` or `python -m shard_engine start`).
254
+ > **GPU & Runtime Note:** GGUF inference auto-downloads platform binaries for `llama-server` / `ggml-rpc-server` on first run and needs no extra ML packages. For CUDA PyTorch inference, install a matching CUDA build of `torch` before `pip install "shard-engine[full]"`.
255
+
256
+ #### Option 2: Editable Install from Source (Developers)
257
+ ```bash
258
+ git clone https://github.com/avelone123123/shard.git
259
+ cd shard
260
+ pip install -e ".[dev]"
261
+ ```
262
+
263
+ #### Option 3: Docker Compose (with NVIDIA GPU Support)
264
+ ```bash
265
+ git clone https://github.com/avelone123123/shard.git
266
+ cd shard
267
+ docker-compose up --build
268
+ ```
269
+ This builds `shard-engine`, mounts persistent volumes (`/data/models` and `/data/shard`), exposes port `8000:8000`, and enables NVIDIA GPU passthrough.
270
+
271
+ #### Option 4: Compile Native C++20 / MASM AVX2 Kernels (Optional Speedup)
272
+ To compile `shard_core.dll` (Windows MSVC `cl.exe` + `ml64.exe`) or `shard_core.so` (Linux GCC/Clang):
273
+ ```bash
274
+ python native/build_native.py
275
+ ```
276
+
277
+ #### Upgrading & Uninstalling
278
+ ```bash
279
+ shard update --check
280
+ pip install --upgrade shard-engine
281
+ pip uninstall shard-engine
282
+ ```
283
+ Uninstalling the Python package keeps your downloaded models, settings, and API keys intact in your user data directory.
284
+
285
+ ---
286
+
287
+ ### Quick Start Workflow
288
+
289
+ #### 1. Verify Environment & Launch SHARD
290
+ ```bash
291
+ # Check Python runtime, dependencies, GPU VRAM, ports, and writable paths
292
+ shard doctor
293
+
294
+ # Start FastAPI server in background (--daemon / -d) and inspect logs/status
295
+ shard start --daemon
296
+ shard status
297
+
298
+ # Or open the interactive Web Dashboard at http://localhost:8000 immediately
299
+ shard gui
300
+
301
+ # Stop or restart background server cleanly
302
+ shard restart
303
+ shard stop
304
+ ```
305
+
306
+ #### 2. Check Hardware & Model Fit Before Downloading
307
+ ```bash
308
+ # Check if a model fits your local machine or LAN cluster before downloading
309
+ shard fit Qwen/Qwen2.5-Coder-7B-Instruct
310
+ shard fit meta-llama/Llama-3.1-8B-Instruct
311
+ ```
312
+
313
+ #### 3. Download & Run a Model
314
+ ```bash
315
+ # Download and start an interactive terminal chat (default: Qwen/Qwen2.5-Coder-1.5B-Instruct)
316
+ shard run Qwen/Qwen2.5-Coder-7B-Instruct
317
+
318
+ # List cached and catalog models
319
+ shard list
320
+ ```
321
+
322
+ #### 4. Create or Join a Distributed LAN Cluster
323
+ On the **Master Machine** (`shard lan create` prints the live LAN IPv4 addresses detected on this host):
324
+ ```bash
325
+ # Create an open LAN pool (auto-accepts subnet workers and sets target model)
326
+ shard lan create --open --share --model Qwen/Qwen2.5-Coder-7B-Instruct
327
+
328
+ # Or create a code-protected LAN pool
329
+ shard lan create --share
330
+ ```
331
+
332
+ On each **Worker Machine** on the same subnet, replace `<SERVER_IP>` with the IP printed by `shard lan create` on the Master Machine:
333
+ ```bash
334
+ # Join an open LAN pool and contribute local GPU/RAM
335
+ shard lan join <SERVER_IP> --share
336
+
337
+ # Or join a code-protected pool
338
+ shard lan join <SERVER_IP> <PAIR_CODE> --share
339
+
340
+ # Inspect cluster nodes, host IPv4 addresses, and pooled VRAM/RAM
341
+ shard lan status
342
+
343
+ # Leave or disable LAN pooling
344
+ shard lan off
345
+ ```
346
+
347
+ ---
348
+
349
+ ## CLI Reference & Configuration
350
+
351
+ ### Verified CLI Commands (`shard_engine/cli.py`)
352
+
353
+ | Command | Arguments & Flags | Description |
354
+ | :--- | :--- | :--- |
355
+ | `shard start` | `[--host HOST] [--port PORT] [-d\|--daemon] [--no-browser]` | Launches the SHARD FastAPI server and Uvicorn runtime (default `0.0.0.0:8000`). |
356
+ | `shard stop` | *(none)* | Safely stops the background SHARD server managed by PID tracking. |
357
+ | `shard restart` | `[--host HOST] [--port PORT] [--foreground] [--no-browser]` | Stops and restarts the SHARD server cleanly. |
358
+ | `shard gui` | *(none)* | Launches the server (or native Win32 desktop UI if present) and opens the Web Dashboard. |
359
+ | `shard status` | *(none)* | Displays detected CPU, RAM, GPU VRAM, active model, and detected LAN IPv4 addresses. |
360
+ | `shard doctor` | *(none)* | Audits Python, packaged assets, dependencies, CUDA/GPU, LAN IPs, ports, and user directories. |
361
+ | `shard version` | `[-V\|--version]` | Prints installed SHARD version, Python executable path, OS info, and active directories. |
362
+ | `shard update` | `[--check]` | Checks PyPI for newer `shard-engine` releases or upgrades non-destructively via `pip`. |
363
+ | `shard fit` | `<model_id> [--ctx N] [--no-fetch]` | Calculates VRAM/RAM requirements, quantization fit, and estimated throughput before downloading. |
364
+ | `shard run` | `[model_id]` | Downloads (if needed), loads the model, and opens interactive terminal chat REPL. |
365
+ | `shard list` | `[--category CAT]` | Lists cached and curated catalog models (`all`, `ideal`, `coding`, `reasoning`, `compact`). |
366
+ | `shard lan` | `[status\|create\|join\|model\|off] [extra] [--share] [--open] [--model ID] [--port N]` | Creates, joins, configures, inspects, or stops distributed LAN compute pooling across machines. |
367
+ | `shard training` | `[status\|doctor\|privacy\|feedback\|dataset\|start\|logs\|cancel\|resume\|models\|eval\|prepare\|approve\|deploy\|rollback]` | Manages consent feedback, dataset snapshots, SOUP fine-tuning jobs, evaluation, model deployment, and rollback ([Guide](docs/COLLABORATIVE_TRAINING.md)). |
368
+ | `shard key` | `[--gen]` | Views the current local API key or generates a new token (`--gen`). |
369
+ | `shard models-dir` | `[path \| reset]` | Views, updates, or resets the directory where model weights are stored. |
370
+ | `shard test` | `[--strict-native]` | Verifies tensor math kernels and AVX2/pure-Python fallback throughput. |
371
+ | `shard help` | `[-h\|--help]` | Prints full CLI command usage and examples. |
372
+
373
+ ### Cross-Platform Directories & Environment Variables (`shard_engine/paths.py`)
374
+
375
+ SHARD stores user data outside `site-packages` following OS conventions (and reads existing `~/.shard` data cleanly on upgrade):
376
+ - **Windows:** `%LOCALAPPDATA%\SHARD` (config, `keys_db.json`, `logs/`, `run/`, `models/`)
377
+ - **Linux:** `~/.local/share/shard`, `~/.config/shard`, `~/.cache/shard`, `~/.local/state/shard` (respects `XDG_*` variables)
378
+ - **macOS:** `~/Library/Application Support/SHARD`, `~/Library/Caches/SHARD`, `~/Library/Logs/SHARD`
379
+
380
+ | Variable / File | Default | Purpose |
381
+ | :--- | :--- | :--- |
382
+ | `SHARD_HOST` | `0.0.0.0` | Network interface IP address bound by `server.py`. |
383
+ | `SHARD_PORT` | `8000` | HTTP port for the API server and Web Dashboard. |
384
+ | `SHARD_DATA_DIR` | OS data directory (`/data/shard` in Docker) | Overrides root directory for `settings.json`, `keys_db.json`, `lan.json`, logs, and PID metadata. |
385
+ | `SHARD_MODELS_DIR` | `<data_dir>/models` (`/data/models` in Docker) | Storage directory for downloaded Hugging Face SafeTensors and GGUF weights. |
386
+ | `SHARD_NO_BROWSER` / `SHARD_DOCKER` | *(unset)* | Suppresses auto-opening a browser tab when starting SHARD. |
387
+ | `SHARD_NATIVE_LIB` | *(auto-discovered)* | Path override to `shard_core.dll`/`.so`/`.dylib` (set `none` to force Python fallback). |
388
+ | `SHARD_SOUP_PYTHON` / `SHARD_SOUP_CMD` | *(auto-discovered)* | Optional path to a Python 3.11–3.13 environment or binary running [`soup-cli`](https://github.com/MakazhanAlpamys/Soup) when training on Python 3.14+ hosts. |
389
+ | `HF_HOME` / `TRANSFORMERS_CACHE` | `~/.cache/huggingface` | Standard Hugging Face cache override respected by `engine.py` and `hf_hub.py`. |
390
+ | `engine_optimizations.json` | Packaged inside `shard_engine` | Runtime optimization profile loaded via `importlib.resources`. |
391
+
392
+ ---
393
+
394
+ ## Architecture
395
+
396
+ <p align="center">
397
+ <img src="assets/animations/network-ecosystem.gif" alt="SHARD Network Ecosystem Overview" width="800">
398
+ <br>
399
+ <sub><b>SHARD Ecosystem Topology:</b> Coding IDEs, agent frameworks, master coordinator, and distributed LAN worker nodes operating as a unified inference fabric.</sub>
400
+ </p>
401
+
402
+ ### System Architecture Diagram
403
+
404
+ ```mermaid
405
+ flowchart TB
406
+ subgraph Clients["AI Agents, IDEs & SDK Clients"]
407
+ IDE["Cursor / Claude Code / Aider / Continue / Zed"]
408
+ SDK["OpenAI SDK / Anthropic SDK / LangChain / LlamaIndex"]
409
+ Browser["Web Dashboard UI (dashboard.html)"]
410
+ end
411
+
412
+ subgraph Master["SHARD Master Node (FastAPI :8000)"]
413
+ Server["server.py\n(/v1/chat/completions, /v1/messages, /api/*)"]
414
+ Fit["fit.py & hardware.py\n(Pre-Flight VRAM/RAM Advisor)"]
415
+ Engine["engine.py\n(PyTorch Transformers & Token Streamer)"]
416
+ GGUF["gguf_backend.py\n(llama-server & Layer Offload -ngl)"]
417
+ Bridge["shard_bridge.py\n(ctypes C++20 / MASM AVX2 SIMD Core)"]
418
+ LANCoord["lan.py\n(UDP Discovery :8001 & Cluster Coordinator)"]
419
+ end
420
+
421
+ subgraph Workers["Distributed LAN Worker Nodes (:8001)"]
422
+ W1["Worker Node 1\nggml-rpc-server / GPU VRAM Shard"]
423
+ W2["Worker Node 2\nggml-rpc-server / GPU VRAM Shard"]
424
+ W3["Worker Node N\nggml-rpc-server / System RAM + GPU"]
425
+ end
426
+
427
+ IDE -->|HTTP / SSE :8000| Server
428
+ SDK -->|HTTP / SSE :8000| Server
429
+ Browser -->|REST / SSE :8000| Server
430
+
431
+ Server --> Fit
432
+ Server --> Engine
433
+ Server --> LANCoord
434
+ Engine --> GGUF
435
+ Engine --> Bridge
436
+
437
+ LANCoord <-->|UDP Broadcast & Heartbeat :8001| W1
438
+ LANCoord <-->|UDP Broadcast & Heartbeat :8001| W2
439
+ LANCoord <-->|UDP Broadcast & Heartbeat :8001| W3
440
+
441
+ GGUF <-->|TCP Tensor RPC Stream| W1
442
+ GGUF <-->|TCP Tensor RPC Stream| W2
443
+ GGUF <-->|TCP Tensor RPC Stream| W3
444
+ ```
445
+
446
+ ### Core Repository Modules
447
+ ```text
448
+ shard/
449
+ ├── assets/
450
+ │ ├── branding/ # Official SHARD SVG & PNG logos, icons, and favicon
451
+ │ ├── animations/ # Isometric pixel-art / voxel animated GIFs
452
+ │ └── screenshots/ # Dashboard and CLI fit advisor visuals
453
+ ├── docs/
454
+ │ └── LAN_SHARDING_DESIGN.md # Distributed LAN sharding protocol & architecture spec
455
+ ├── native/
456
+ │ ├── build_native.py # Cross-platform compiler for C++20 & MASM AVX2 kernels
457
+ │ ├── shard_core.cpp # C++20 RMSNorm, LayerNorm, RoPE, Softmax, GELU, SiLU, MatMul
458
+ │ ├── shard_asm.asm # x86-64 MASM AVX2 SIMD assembly kernels
459
+ │ └── shard_gui.cpp # Native Win32 desktop GUI wrapper
460
+ ├── shard_engine/ # Installable Python package (paths, process, CLI, API, inference)
461
+ │ ├── paths.py # Cross-platform XDG / AppData / macOS paths & packaged assets
462
+ │ ├── process.py # Detached daemon launcher & verified PID lifecycle management
463
+ │ ├── cli.py # `shard` command-line interface entry point (`shard_engine.cli:main`)
464
+ │ ├── main.py # Application launcher & Uvicorn bootstrapper
465
+ │ ├── server.py # FastAPI server (OpenAI, Anthropic, Model & LAN routes)
466
+ │ ├── engine.py # Core LLM inference orchestrator & SSE generator
467
+ │ ├── gguf_backend.py # Managed llama.cpp / GGUF server & RPC integration
468
+ │ ├── lan.py # Dynamic subnet IP discovery, UDP beacons & LAN pooling
469
+ │ ├── fit.py # Pre-flight model memory & throughput calculator
470
+ │ ├── hardware.py # GPU VRAM (nvidia-smi), CPU & RAM probing
471
+ │ ├── hf_hub.py # Hugging Face Hub search, metadata & downloader
472
+ │ ├── shard_bridge.py # Python ctypes bridge to native shard_core.dll / .so
473
+ │ ├── settings.py # Persistent configuration manager
474
+ │ └── dashboard.html # Packaged web control panel & agent config generator
475
+ ├── pyproject.toml # Single-source packaging configuration (version from __init__.py)
476
+ ├── tests_shard.py # Unit, API, fit advisor, LAN sharding & kernel verification suite
477
+ └── tests_cli_pkg.py # Packaging, OS path, CLI, PID safety & 2-node LAN cluster suite
478
+ ```
479
+
480
+ ---
481
+
482
+ ## Web Dashboard & Demonstrations
483
+
484
+ ### Interactive Web Control Panel (`dashboard.html`)
485
+ Navigating to `http://localhost:8000` opens the built-in SHARD Control Panel:
486
+
487
+ <p align="center">
488
+ <img src="assets/screenshots/dashboard-overview.png" alt="SHARD Web Dashboard & LAN Pool Telemetry" width="960">
489
+ <br>
490
+ <sub><b>SHARD Web Dashboard:</b> Real-time per-node VRAM/RAM gauges, Pre-Flight Model Fit Advisor, one-click IDE/agent config snippets, and streaming playground.</sub>
491
+ </p>
492
+
493
+ ### CLI Pre-Flight Advisor (`shard fit`) & LAN Cluster Status (`shard lan status`)
494
+
495
+ <p align="center">
496
+ <img src="assets/screenshots/cli-fit-advisor.png" alt="SHARD CLI Fit Advisor and LAN Status" width="900">
497
+ <br>
498
+ <sub><b>Pre-Flight Fit Verification:</b> Evaluate model quantization, context ceiling, and pooled LAN cluster fit before downloading model weights.</sub>
499
+ </p>
500
+
501
+ ### Connecting AI Agents & Coding Tools
502
+ Point any OpenAI- or Anthropic-compatible client to your SHARD instance:
503
+
504
+ ```python
505
+ from openai import OpenAI
506
+
507
+ client = OpenAI(
508
+ base_url="http://localhost:8000/v1",
509
+ api_key="shard-local", # Or generate a token via `shard key --gen`
510
+ )
511
+
512
+ stream = client.chat.completions.create(
513
+ model="Qwen/Qwen2.5-Coder-7B-Instruct",
514
+ messages=[{"role": "user", "content": "Explain tensor parallelism across LAN nodes."}],
515
+ stream=True,
516
+ )
517
+
518
+ for chunk in stream:
519
+ if chunk.choices[0].delta.content:
520
+ print(chunk.choices[0].delta.content, end="", flush=True)
521
+ ```
522
+
523
+ ---
524
+
525
+ ## Project Status & Community
526
+
527
+ ### Technology Stack
528
+ - **Runtime & API:** Python 3.8+ / 3.10+, [FastAPI](https://fastapi.tiangolo.com/), [Uvicorn](https://www.uvicorn.org/), [Pydantic v2](https://docs.pydantic.dev/)
529
+ - **ML & Quantization:** [PyTorch](https://pytorch.org/), Hugging Face [`transformers`](https://huggingface.co/docs/transformers), [`accelerate`](https://huggingface.co/docs/accelerate), [`safetensors`](https://huggingface.co/docs/safetensors), [`huggingface_hub`](https://huggingface.co/docs/huggingface_hub), [`llama.cpp` GGUF](https://github.com/ggml-org/llama.cpp)
530
+ - **Native Acceleration:** C++20 (`native/shard_core.cpp`) + x86-64 MASM AVX2 Assembly (`native/shard_asm.asm`)
531
+ - **System & Networking:** `psutil`, UDP broadcast discovery (`:8001`), HTTP/SSE streaming (`:8000`)
532
+
533
+ ### Community, Contribution & Governance
534
+ - **Architecture & LAN Design Spec:** [`docs/LAN_SHARDING_DESIGN.md`](docs/LAN_SHARDING_DESIGN.md)
535
+ - **Contribution Guidelines:** [`CONTRIBUTING.md`](CONTRIBUTING.md) — covers zero-copy memory rules, native C++20/AVX2 build steps, and running `shard test`.
536
+ - **Code of Conduct:** [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md)
537
+ - **Release History:** [`CHANGELOG.md`](CHANGELOG.md) (current release: `v1.8.1`)
538
+ - **Bug Reports & Feature Requests:** [GitHub Issues](https://github.com/avelone123123/shard/issues)
539
+ - **Community Q&A & Ideas:** [GitHub Discussions](https://github.com/avelone123123/shard/discussions)
540
+ - **Security Policy:** Please report security vulnerabilities privately via [GitHub Security Advisories](https://github.com/avelone123123/shard/security) or by contacting the maintainer before public disclosure.
541
+ - **License:** Distributed under the [MIT License](LICENSE).
542
+
543
+ ---
544
+
545
+ <p align="center">
546
+ <img src="assets/branding/shard-icon.svg" alt="SHARD Icon" width="64"><br>
547
+ <strong>SHARD — One Network. Many Servers. Shared AI Infrastructure.</strong><br>
548
+ <sub>Built for the open-source local AI community &middot; <a href="README.md">English</a> &middot; <a href="README.ru.md">Русский</a> &middot; <a href="README.kz.md">Қазақша</a></sub>
549
+ </p>