vserve 0.1.0__tar.gz

vserve-0.1.0/.githooks/pre-commit

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+cd "$(git rev-parse --show-toplevel)"
+
+echo "=== ruff ==="
+uv run ruff check src/ tests/
+
+echo "=== mypy ==="
+uv run mypy src/vserve/ --ignore-missing-imports
+
+echo "=== pytest ==="
+uv run pytest tests/ -q --tb=short
+
+echo "=== all checks passed ==="

vserve-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,19 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v7
+      - run: uv python install 3.13
+      - run: uv sync --dev
+      - run: uv run ruff check src/ tests/
+      - run: uv run mypy src/vserve/ --ignore-missing-imports
+      - run: uv run pytest tests/ -q --tb=short

vserve-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,46 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v7
+      - run: uv python install 3.13
+      - run: uv sync --dev
+      - run: uv run ruff check src/ tests/
+      - run: uv run mypy src/vserve/ --ignore-missing-imports
+      - run: uv run pytest tests/ -q --tb=short
+
+  build:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v7
+      - run: uv python install 3.13
+      - run: uv build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: build
+    environment:
+      name: pypi
+      url: https://pypi.org/p/vserve
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - uses: pypa/gh-action-pypi-publish@release/v1

vserve-0.1.0/.gitignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.pyc
+.venv/
+.pytest_cache/
+*.egg-info/
+dist/
+.serena/
+docs/superpowers/
+docs/plans/
+docs/specs/

vserve-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mohan Qiao
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vserve-0.1.0/PKG-INFO

@@ -0,0 +1,156 @@
+Metadata-Version: 2.4
+Name: vserve
+Version: 0.1.0
+Summary: CLI for managing vLLM inference on GPU workstations
+Project-URL: Homepage, https://github.com/Gavin-Qiao/vserve
+Project-URL: Repository, https://github.com/Gavin-Qiao/vserve
+Project-URL: Issues, https://github.com/Gavin-Qiao/vserve/issues
+Author: Mohan Qiao
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Environment :: GPU :: NVIDIA CUDA
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: huggingface-hub>=0.30
+Requires-Dist: nvidia-ml-py>=13.595.45
+Requires-Dist: pyyaml>=6
+Requires-Dist: rich>=14
+Requires-Dist: typer>=0.15
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+# vserve
+
+**A CLI for managing vLLM inference on GPU workstations.**
+
+Download models. Calculate limits. Serve with one command. Control fans.
+
+![Python 3.12+](https://img.shields.io/badge/python-3.12+-3776ab?style=flat-square&logo=python&logoColor=white)
+![vLLM 0.18+](https://img.shields.io/badge/vLLM-0.18+-ff6f00?style=flat-square)
+![Tests](https://img.shields.io/badge/tests-175%20passed-brightgreen?style=flat-square)
+![License](https://img.shields.io/badge/license-MIT-green?style=flat-square)
+
+</div>
+
+---
+
+## Install
+
+```bash
+uv tool install vserve
+```
+
+Or with pip:
+
+```bash
+pip install vserve
+```
+
+---
+
+## Quick Start
+
+```bash
+vserve init                  # scan GPU, vLLM, CUDA, systemd — write config
+vserve download              # search HuggingFace, pick variant, download
+vserve tune <model>          # calculate context/concurrency limits for your GPU
+vserve start <model>         # interactive config → serve via systemd
+```
+
+---
+
+## What It Does
+
+**vserve** manages the full lifecycle of serving LLMs with vLLM on a GPU workstation:
+
+- **Download** — search HuggingFace, see available weight variants (MXFP4, BF16, GGUF) with sizes, download only what you need
+- **Tune** — calculate exactly what context lengths and concurrency your GPU can handle, based on model architecture and available VRAM
+- **Start/Stop** — interactive config wizard, systemd service management, health check with timeout
+- **Fan control** — temperature-based curve daemon with quiet hours, or hold a fixed speed
+- **Multi-user** — file-based locking, terminal notifications when another user holds the GPU
+- **Doctor** — diagnose GPU, CUDA, vLLM, systemd issues with actionable fix suggestions
+
+---
+
+## Commands
+
+| Command | Description |
+|:--------|:------------|
+| `vserve` | Dashboard — GPU, models, status |
+| `vserve init` | Auto-discover vLLM and write config |
+| `vserve download [model]` | Search and download from HuggingFace with variant picker |
+| `vserve models [name]` | List models or show detail (fuzzy match) |
+| `vserve tune <model>` | Calculate context/concurrency limits for your GPU |
+| `vserve start [model]` | Configure and start serving |
+| `vserve stop` | Stop the vLLM service |
+| `vserve status` | Show current serving config |
+| `vserve fan [auto\|off\|30-100]` | GPU fan control with temp-based curve |
+| `vserve doctor` | Check system readiness |
+
+All commands support **fuzzy matching** — `vserve start qwen fp8` finds the right model.
+
+---
+
+## Prerequisites
+
+| Requirement | Check | Install |
+|------------|-------|---------|
+| NVIDIA GPU + drivers | `nvidia-smi` | [nvidia.com/drivers](https://www.nvidia.com/drivers) |
+| CUDA toolkit | `nvcc --version` | `sudo apt install nvidia-cuda-toolkit` |
+| vLLM 0.18+ | `vllm --version` | [docs.vllm.ai](https://docs.vllm.ai/en/latest/getting_started/installation.html) |
+| systemd | (most Linux servers) | See [troubleshooting](docs/troubleshooting.md) |
+| sudo access | for systemctl, fan control | |
+
+---
+
+## Configuration
+
+Auto-discovered on first run. Override at `~/.config/vserve/config.yaml`:
+
+```yaml
+vllm_root: /opt/vllm
+cuda_home: /usr/local/cuda
+service_name: vllm
+service_user: vllm
+port: 8888
+```
+
+---
+
+## Fan Control
+
+```bash
+vserve fan              # show status, interactive menu
+vserve fan auto         # temp-based curve with quiet hours
+vserve fan 80           # hold at 80% (persistent daemon)
+vserve fan off          # stop daemon, restore NVIDIA auto
+```
+
+The auto curve ramps with temperature and caps fan speed during quiet hours (configurable). Emergency override at 88C ignores quiet hours.
+
+---
+
+## Development
+
+```bash
+git clone https://github.com/Gavin-Qiao/vserve.git
+cd vserve
+uv sync --dev
+uv run pytest tests/              # 175 tests
+uv run ruff check src/ tests/     # lint
+uv run mypy src/vserve/           # type check
+```
+
+---
+
+## License
+
+[MIT](LICENSE)

vserve-0.1.0/README.md

@@ -0,0 +1,129 @@
+<div align="center">
+
+# vserve
+
+**A CLI for managing vLLM inference on GPU workstations.**
+
+Download models. Calculate limits. Serve with one command. Control fans.
+
+![Python 3.12+](https://img.shields.io/badge/python-3.12+-3776ab?style=flat-square&logo=python&logoColor=white)
+![vLLM 0.18+](https://img.shields.io/badge/vLLM-0.18+-ff6f00?style=flat-square)
+![Tests](https://img.shields.io/badge/tests-175%20passed-brightgreen?style=flat-square)
+![License](https://img.shields.io/badge/license-MIT-green?style=flat-square)
+
+</div>
+
+---
+
+## Install
+
+```bash
+uv tool install vserve
+```
+
+Or with pip:
+
+```bash
+pip install vserve
+```
+
+---
+
+## Quick Start
+
+```bash
+vserve init                  # scan GPU, vLLM, CUDA, systemd — write config
+vserve download              # search HuggingFace, pick variant, download
+vserve tune <model>          # calculate context/concurrency limits for your GPU
+vserve start <model>         # interactive config → serve via systemd
+```
+
+---
+
+## What It Does
+
+**vserve** manages the full lifecycle of serving LLMs with vLLM on a GPU workstation:
+
+- **Download** — search HuggingFace, see available weight variants (MXFP4, BF16, GGUF) with sizes, download only what you need
+- **Tune** — calculate exactly what context lengths and concurrency your GPU can handle, based on model architecture and available VRAM
+- **Start/Stop** — interactive config wizard, systemd service management, health check with timeout
+- **Fan control** — temperature-based curve daemon with quiet hours, or hold a fixed speed
+- **Multi-user** — file-based locking, terminal notifications when another user holds the GPU
+- **Doctor** — diagnose GPU, CUDA, vLLM, systemd issues with actionable fix suggestions
+
+---
+
+## Commands
+
+| Command | Description |
+|:--------|:------------|
+| `vserve` | Dashboard — GPU, models, status |
+| `vserve init` | Auto-discover vLLM and write config |
+| `vserve download [model]` | Search and download from HuggingFace with variant picker |
+| `vserve models [name]` | List models or show detail (fuzzy match) |
+| `vserve tune <model>` | Calculate context/concurrency limits for your GPU |
+| `vserve start [model]` | Configure and start serving |
+| `vserve stop` | Stop the vLLM service |
+| `vserve status` | Show current serving config |
+| `vserve fan [auto\|off\|30-100]` | GPU fan control with temp-based curve |
+| `vserve doctor` | Check system readiness |
+
+All commands support **fuzzy matching** — `vserve start qwen fp8` finds the right model.
+
+---
+
+## Prerequisites
+
+| Requirement | Check | Install |
+|------------|-------|---------|
+| NVIDIA GPU + drivers | `nvidia-smi` | [nvidia.com/drivers](https://www.nvidia.com/drivers) |
+| CUDA toolkit | `nvcc --version` | `sudo apt install nvidia-cuda-toolkit` |
+| vLLM 0.18+ | `vllm --version` | [docs.vllm.ai](https://docs.vllm.ai/en/latest/getting_started/installation.html) |
+| systemd | (most Linux servers) | See [troubleshooting](docs/troubleshooting.md) |
+| sudo access | for systemctl, fan control | |
+
+---
+
+## Configuration
+
+Auto-discovered on first run. Override at `~/.config/vserve/config.yaml`:
+
+```yaml
+vllm_root: /opt/vllm
+cuda_home: /usr/local/cuda
+service_name: vllm
+service_user: vllm
+port: 8888
+```
+
+---
+
+## Fan Control
+
+```bash
+vserve fan              # show status, interactive menu
+vserve fan auto         # temp-based curve with quiet hours
+vserve fan 80           # hold at 80% (persistent daemon)
+vserve fan off          # stop daemon, restore NVIDIA auto
+```
+
+The auto curve ramps with temperature and caps fan speed during quiet hours (configurable). Emergency override at 88C ignores quiet hours.
+
+---
+
+## Development
+
+```bash
+git clone https://github.com/Gavin-Qiao/vserve.git
+cd vserve
+uv sync --dev
+uv run pytest tests/              # 175 tests
+uv run ruff check src/ tests/     # lint
+uv run mypy src/vserve/           # type check
+```
+
+---
+
+## License
+
+[MIT](LICENSE)

vserve-0.1.0/docs/troubleshooting.md

@@ -0,0 +1,142 @@
+# Troubleshooting
+
+Hard-won lessons from running vLLM on NVIDIA workstation GPUs.
+
+## GPU Crashes (Xid Errors)
+
+### Xid 8 — GPU Stopped Processing
+
+**Symptom:** vLLM dies after hours of stable inference. Kernel log shows:
+```
+NVRM: krcWatchdog_IMPL: RC watchdog: GPU is probably locked!
+NVRM: Xid (PCI:0000:02:00): 8, pid=..., name=VLLM::EngineCor
+```
+
+**Root cause:** The GPU's recovery counter watchdog detected a hang. On Blackwell (SM120), this correlates with CUDA graphs under sustained FP8 load (see [vllm-project/vllm#35659](https://github.com/vllm-project/vllm/issues/35659)).
+
+**Recovery:**
+1. The GPU usually recovers automatically after the faulting process exits — no reset needed.
+2. `nvidia-smi --gpu-reset` is deprecated as of driver 570+.
+3. If the GPU is unresponsive, reload kernel modules: `sudo rmmod nvidia_uvm nvidia_drm nvidia_modeset nvidia && sudo modprobe nvidia`
+4. Last resort: reboot.
+
+**Prevention:**
+- Use the production driver branch (595.x), not the new-feature branch (590.x).
+- `vserve fan auto` keeps temps below throttle threshold.
+- systemd `Restart=always` with exponential backoff recovers automatically.
+- If crashes persist, try `--enforce-eager` (disables CUDA graphs, ~2.3x throughput cost).
+
+### vLLM exits cleanly but systemd doesn't restart
+
+**Cause:** vLLM's APIServer shuts down cleanly (exit 0) after an EngineCore crash. `Restart=on-failure` ignores exit 0.
+
+**Fix:** Use `Restart=always` in the systemd unit.
+
+## Fan Control
+
+### Why override NVIDIA's auto curve?
+
+The default auto curve on workstation GPUs (RTX PRO series) caps fan speed conservatively for noise. Under sustained 300W inference, this can allow temps to reach 85-90°C, causing:
+- Thermal throttling (reduced inference throughput)
+- Accelerated component aging (electromigration, capacitor degradation)
+- Increased risk of Xid errors
+
+### Coolbits setup
+
+Fan control via `nvidia-settings` requires Coolbits enabled in X11 config:
+
+```bash
+# /etc/X11/xorg.conf
+Section "Device"
+    Identifier     "GPU0"
+    Driver         "nvidia"
+    BusID          "PCI:2:0:0"    # check with: nvidia-smi --query-gpu=gpu_bus_id --format=csv,noheader
+    Option         "Coolbits" "4"  # bit 2 = manual fan control
+EndSection
+```
+
+On headless systems, `vserve fan` uses a temporary Xvfb virtual display — no persistent X server needed.
+
+### Hardware thermal failsafe
+
+Even if the fan daemon crashes and the fan is stuck at 30%, the GPU has hardware protection:
+- GPU Boost reduces clocks starting at ~83°C
+- Aggressive power limiting at ~90°C
+- Hardware shutdown at ~100-105°C (cannot be overridden by software)
+
+The `vserve fan` emergency override (100% fan at 88°C regardless of quiet hours) is software-level defense. The hardware failsafe is always present underneath.
+
+## JIT Compilation
+
+### FlashInfer JIT cache
+
+vLLM 0.18+ uses FlashInfer with just-in-time compiled CUDA kernels. First run for each model/KV-dtype combination triggers JIT compilation that takes 2-5 minutes.
+
+**Where the cache lives:** `$VLLM_ROOT/.cache/flashinfer/`
+
+**Problem:** If vLLM runs as a different user (e.g., systemd `User=vllm`), the JIT cache must be writable by that user. First-time startup will be slow.
+
+**The preheat solution:** `vserve tune` runs a preheat step that starts vLLM briefly as the service user to build the JIT cache before benchmarking. This uses `sudo -u <service_user>` with the correct `CUDA_HOME`, `PATH`, and `HF_HOME` environment.
+
+### torch.compile cache
+
+vLLM also uses `torch.compile` which has its own cache at `$VLLM_ROOT/.cache/vllm/torch_compile_cache/`. Same ownership considerations apply.
+
+### CUDA_HOME and PATH
+
+The JIT compiler needs:
+- `nvcc` accessible (CUDA toolkit bin directory in PATH)
+- `CUDA_HOME` set to the CUDA toolkit root
+- Matching CUDA version between the toolkit and the PyTorch build
+
+If JIT compilation fails with `nvcc not found` or architecture mismatches, check:
+```bash
+which nvcc
+nvcc --version
+python -c "import torch; print(torch.version.cuda)"
+```
+
+### ProtectSystem=strict breaks JIT
+
+Do NOT use `ProtectSystem=strict` in the vLLM systemd unit — it makes `/usr` read-only, preventing nvcc from writing temporary files during JIT compilation. The `gpu-fan.service` can use it (fan control doesn't need JIT), but the main `vllm.service` cannot.
+
+## Driver Management
+
+### Which driver branch to use
+
+| Branch | Status | Use |
+|--------|--------|-----|
+| 570.x | Old production | Pre-Blackwell only |
+| 575.x | Early Blackwell | Known issues, avoid |
+| 590.x | New-feature branch | Experimental, not for production |
+| 595.x | **Current production** | Recommended for Blackwell |
+
+Check your branch: `nvidia-smi` shows the driver version in the header.
+
+### Open vs. proprietary kernel modules
+
+Blackwell GPUs (RTX 50-series, RTX PRO 5000/6000) **require open kernel modules**. The proprietary modules do not support Blackwell at all. Always use packages with `-open` suffix (e.g., `nvidia-driver-595-server-open` or `nvidia-open` from the CUDA repo).
+
+### nvidia-settings version mismatch
+
+If `nvidia-settings --version` shows a different version than `nvidia-smi`, you have a partial driver upgrade. This can cause subtle issues. Fix by upgrading the full driver stack to match.
+
+## Headless Operation
+
+### Disabling the display manager
+
+For 24/7 inference, disable the display manager to free GPU memory and simplify driver reloads:
+
+```bash
+sudo systemctl disable gdm    # or lightdm/sddm
+sudo systemctl set-default multi-user.target
+```
+
+GUI is still available on demand: `sudo systemctl start gdm`
+
+### GPU memory with no display
+
+With no display manager: ~2 MiB GPU memory used (driver overhead only).
+With Xorg/GDM running: ~15 MiB (Xorg frame buffer).
+
+This is negligible for 48 GB, but matters for driver reload — fewer processes holding the GPU open means cleaner `rmmod`.

vserve-0.1.0/pyproject.toml

@@ -0,0 +1,47 @@
+[project]
+name = "vserve"
+version = "0.1.0"
+description = "CLI for managing vLLM inference on GPU workstations"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.12"
+authors = [{ name = "Mohan Qiao" }]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Environment :: GPU :: NVIDIA CUDA",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "typer>=0.15",
+    "rich>=14",
+    "huggingface-hub>=0.30",
+    "pyyaml>=6",
+    "nvidia-ml-py>=13.595.45",
+]
+
+[dependency-groups]
+dev = ["pytest>=8", "pytest-mock>=3", "ruff>=0.15", "mypy>=1.15", "types-PyYAML>=6", "types-requests>=2.33"]
+
+[project.urls]
+Homepage = "https://github.com/Gavin-Qiao/vserve"
+Repository = "https://github.com/Gavin-Qiao/vserve"
+Issues = "https://github.com/Gavin-Qiao/vserve/issues"
+
+[project.scripts]
+vserve = "vserve.cli:app"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/vserve"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

vserve-0.1.0/src/vserve/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"