vllm-router 0.1.9__tar.gz

vllm_router-0.1.9/Cargo.toml

@@ -0,0 +1,111 @@
+[package]
+name = "vllm_router_rs"
+version = "0.1.9"
+edition = "2021"
+
+[features]
+default = ["grpc-client"]
+grpc-client = []
+grpc-server = []
+
+[lib]
+name = "vllm_router_rs"
+# Pure Rust library: Just omit crate-type (defaults to rlib)
+# Python/C binding + Rust library: Use ["cdylib", "rlib"]
+crate-type = ["cdylib", "rlib"]
+
+[[bin]]
+name = "vllm-router"
+path = "src/main.rs"
+
+[dependencies]
+clap = { version = "4", features = ["derive"] }
+axum = { version = "0.8.4", features = ["macros", "ws", "tracing"] }
+tower = { version = "0.5", features = ["full"] }
+tower-http = { version = "0.6", features = ["trace", "compression-gzip", "cors", "timeout", "limit", "request-id", "util"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+bytes = "1.8.0"
+rand = "0.9.2"
+reqwest = { version = "0.12.8", features = ["stream", "blocking", "json"] }
+futures-util = "0.3"
+futures = "0.3"
+pyo3 = { version = "0.24", features = ["extension-module"] }
+dashmap = "6.1.0"
+http = "1.1.0"
+tokio = { version = "1.42.0", features = ["full"] }
+async-trait = "0.1"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "chrono"] }
+tracing-log = "0.2"
+tracing-appender = "0.2.3"
+chrono = "0.4"
+kube = { version = "1.1.0", features = ["runtime", "derive"] }
+k8s-openapi = { version = "0.25.0", features = ["v1_33"] }
+metrics = "0.24.2"
+metrics-exporter-prometheus = "0.17.0"
+uuid = { version = "1.10", features = ["v4", "serde"] }
+ulid = "1.2.1"
+parking_lot = "0.12.4"
+thiserror = "2.0.12"
+regex = "1.10"
+url = "2.5.4"
+tokio-stream = { version = "0.1", features = ["sync"] }
+anyhow = "1.0"
+tokenizers = { version = "0.22.2" }
+tiktoken-rs = { version = "0.7.0" }
+minijinja = { version = "2.0" }
+rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
+hf-hub = { version = "0.4.3", features = ["tokio"] }
+
+# gRPC and Protobuf dependencies
+tonic = { version = "0.12", features = ["tls", "gzip", "transport"] }
+prost = "0.13"
+prost-types = "0.13"
+deadpool = { version = "0.12", features = ["managed", "rt_tokio_1"] }
+backoff = { version = "0.4", features = ["tokio"] }
+strum = { version = "0.26", features = ["derive"] }
+once_cell = "1.21.3"
+zmq = "0.10.0"
+rmp-serde = "1.3"
+
+[build-dependencies]
+tonic-build = "0.12"
+prost-build = "0.13"
+
+[dev-dependencies]
+criterion = { version = "0.5", features = ["html_reports"] }
+tower = { version = "0.5", features = ["util"] }
+http-body-util = "0.1"
+portpicker = "0.1"
+tempfile = "3.8"
+lazy_static = "1.4"
+
+[[bench]]
+name = "request_processing"
+harness = false
+path = "benches/request_processing.rs"
+
+[[bench]]
+name = "tokenizer_benchmark"
+harness = false
+path = "benches/tokenizer_benchmark.rs"
+
+[profile.release]
+lto = "thin"
+codegen-units = 1
+
+[profile.dev]
+opt-level = 0
+debug = true
+split-debuginfo = "unpacked"
+incremental = true
+
+
+[profile.dev.build-override]
+opt-level = 3
+codegen-units = 1
+
+[profile.dev-opt]
+inherits = "dev"
+opt-level = 1

vllm_router-0.1.9/MANIFEST.in

@@ -0,0 +1,5 @@
+# Must include:
+include Cargo.toml        # Rust project configuration
+include build.rs          # Build script for protobuf generation
+recursive-include src *.rs  # Rust source files
+recursive-include src/proto *.proto  # Protobuf definitions

vllm_router-0.1.9/PKG-INFO

@@ -0,0 +1,266 @@
+Metadata-Version: 2.4
+Name: vllm-router
+Version: 0.1.9
+Summary: High-performance Rust-based load balancer for VLLM with multiple routing algorithms and prefill-decode disaggregation support
+Author-email: Byron Hsu <byronhsu1230@gmail.com>
+License: Apache-2.0
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: setproctitle
+Requires-Dist: aiohttp
+Requires-Dist: orjson
+Requires-Dist: uvicorn
+Requires-Dist: fastapi
+Requires-Dist: requests>=2.25.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
+
+# vLLM Router
+
+A high-performance and light-weight request forwarding system for vLLM large scale deployments, providing advanced load balancing methods and prefill/decode disaggregation support.
+
+### Key Features
+
+- **Core Architecture**: Request routing framework and async processing patterns
+- **Load Balancing**: Multiple algorithms (cache-aware, power of two, consistent hashing, random, round robin)
+- **Prefill-Decode Disaggregation**: Specialized routing for separated processing phases
+- **Service Discovery**: Kubernetes-native worker management and health monitoring
+- **Enterprise Features**: Circuit breakers, retry logic, metrics collection
+
+## Quick Start
+
+### Prerequisites
+
+**Rust and Cargo:**
+```bash
+# Install rustup (Rust installer and version manager)
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+
+# Follow the installation prompts, then reload your shell
+source $HOME/.cargo/env
+
+# Verify installation
+rustc --version
+cargo --version
+
+# Install protobuf compiler (on Ubuntu/Debian)
+sudo apt-get update
+sudo apt-get install -y protobuf-compiler libprotobuf-dev
+```
+
+**Python with pip installed**
+
+### Installation & Basic Usage
+
+#### Rust Binary
+```bash
+# Build Rust components
+cargo build --release
+```
+
+#### Python Package
+```bash
+pip install setuptools-rust wheel build
+python -m build
+pip install dist/*.whl
+
+# Rebuild & reinstall in one step during development
+python -m build && pip install --force-reinstall dist/*.whl
+```
+
+### Usage Examples
+
+#### Standard Data Parallelism Routing
+```bash
+# Launch router with data parallelism (8 replicas per worker URL)
+# When data-parallel-size > 1, the router automatically creates DP-aware workers
+./target/release/vllm-router \
+    --worker-urls http://worker1:8000 http://worker2:8000 \
+    --policy consistent_hash \
+    --intra-node-data-parallel-size 8
+
+# Alternative: using cargo run
+cargo run --release -- \
+    --worker-urls http://worker1:8000 http://worker2:8000 \
+    --policy consistent_hash \
+    --intra-node-data-parallel-size 8
+
+# Alternative: using python launcher
+vllm-router \
+  --worker-urls http://worker1:8000 http://worker2:8000 \
+    --policy consistent_hash \
+    --intra-node-data-parallel-size 8
+```
+
+#### Prefill-Decode Disaggregation
+```bash
+# When vLLM runs the NIXL connector, prefill/decode URLs are required.
+# See a working example in scripts/llama3.1/ folder.
+cargo run --release -- \
+    --policy consistent_hash \
+    --vllm-pd-disaggregation \
+    --prefill http://127.0.0.1:8081 \
+    --prefill http://127.0.0.1:8082 \
+    --decode http://127.0.0.1:8083 \
+    --decode http://127.0.0.1:8084 \
+    --decode http://127.0.0.1:8085 \
+    --decode http://127.0.0.1:8086 \
+    --host 127.0.0.1 \
+    --port 8090 \
+    --intra-node-data-parallel-size 1 \
+
+
+# When vLLM runs the NCCL connector, ZMQ based discovery is supported.
+# See a working example in scripts/install.sh
+cargo run --release -- \
+    --policy consistent_hash \
+    --vllm-pd-disaggregation \
+    --vllm-discovery-address 0.0.0.0:30001 \
+    --host 0.0.0.0 \
+    --port 10001 \
+    --prefill-policy consistent_hash \
+    --decode-policy consistent_hash
+```
+
+## Configuration
+
+### Metrics
+
+Prometheus metrics endpoint available at `127.0.0.1:29000` by default.
+
+```bash
+# Custom metrics configuration
+vllm-router \
+    --worker-urls http://localhost:8080 http://localhost:8081 \
+    --prometheus-host 0.0.0.0 \
+    --prometheus-port 9000
+```
+
+### Retries and Circuit Breakers
+
+#### Retry Configuration
+Retries are enabled by default with exponential backoff and jitter:
+
+```bash
+vllm-router \
+  --worker-urls http://localhost:8080 http://localhost:8081 \
+  --retry-max-retries 3 \
+  --retry-initial-backoff-ms 100 \
+  --retry-max-backoff-ms 10000 \
+  --retry-backoff-multiplier 2.0 \
+  --retry-jitter-factor 0.1
+```
+
+#### Circuit Breaker Configuration
+Circuit breakers protect workers and provide automatic recovery:
+
+```bash
+vllm-router \
+  --worker-urls http://localhost:8080 http://localhost:8081 \
+  --cb-failure-threshold 5 \
+  --cb-success-threshold 2 \
+  --cb-timeout-duration-secs 30 \
+  --cb-window-duration-secs 60
+```
+
+**Circuit Breaker State Machine:**
+- `Closed` → `Open` after N consecutive failures (failure-threshold)
+- `Open` → `HalfOpen` after timeout (timeout-duration-secs)
+- `HalfOpen` → `Closed` after M consecutive successes (success-threshold)
+
+**Retry Policy:** Retries on HTTP status codes 408/429/500/502/503/504, with backoff/jitter between attempts.
+
+### Request ID Tracking
+
+Track requests across distributed systems with configurable headers:
+
+```bash
+# Use custom request ID headers
+vllm-router \
+    --worker-urls http://localhost:8080 \
+    --request-id-headers x-trace-id x-request-id
+```
+
+**Default headers:** `x-request-id`, `x-correlation-id`, `x-trace-id`, `request-id`
+
+### Load Balancing Policies
+
+The router supports multiple load balancing policies:
+
+| Policy | Description | Session Affinity | Use Case |
+|--------|-------------|------------------|----------|
+| `round_robin` | Sequential distribution across workers | No | General purpose, even distribution |
+| `random` | Uniform random selection | No | Simple deployments |
+| `consistent_hash` | Routes same session/user to same worker | Yes | Multi-turn chat, KV cache reuse |
+| `power_of_two` | Picks least loaded of two random workers | No | Load-sensitive workloads |
+| `cache_aware` | Optimizes for prefix cache hits | Yes | Repeated prompts, few-shot |
+
+```bash
+# Example: Using consistent_hash with HTTP header for session affinity
+curl -X POST http://router:8000/v1/chat/completions \
+  -H "X-Session-ID: my-session-123" \
+  -H "Content-Type: application/json" \
+  -d '{"model": "llama-3", "messages": [{"role": "user", "content": "Hello!"}]}'
+```
+
+For detailed configuration options, hash key priorities, and usage examples, see [Load Balancing Documentation](docs/load_balancing/README.md).
+
+## Advanced Features
+
+### Kubernetes Service Discovery
+
+Automatic worker discovery and management in Kubernetes environments.
+
+#### Basic Service Discovery
+
+```bash
+vllm-router \
+    --service-discovery \
+    --selector app=vllm-worker role=inference \
+    --service-discovery-namespace default
+```
+
+### Command Line Arguments Reference
+
+#### Service Discovery
+- `--service-discovery`: Enable Kubernetes service discovery
+- `--service-discovery-port`: Port for worker URLs (default: 8000)
+- `--service-discovery-namespace`: Kubernetes namespace to watch
+- `--selector`: Label selectors for regular mode (format: `key1=value1 key2=value2`)
+
+## Development
+
+### Troubleshooting
+
+**VSCode Rust Analyzer Issues:**
+Set `rust-analyzer.linkedProjects` to the absolute path of `Cargo.toml`:
+
+```json
+{
+  "rust-analyzer.linkedProjects": ["/workspaces/vllm/vllm-router/Cargo.toml"]
+}
+```
+
+### CI/CD Pipeline
+
+The continuous integration pipeline includes comprehensive testing, benchmarking, and publishing:
+
+#### Build & Test
+1. **Build Wheels**: Uses `cibuildwheel` for manylinux x86_64 packages
+2. **Build Source Distribution**: Creates source distribution for pip fallback
+3. **Rust HTTP Server Benchmarking**: Performance testing of router overhead
+4. **Basic Inference Testing**: End-to-end validation through the router
+5. **PD Disaggregation Testing**: Benchmark and sanity checks for prefill-decode load balancing
+
+#### Publishing
+- **PyPI Publishing**: Wheels and source distributions published when version changes in `pyproject.toml`
+- **Container Images**: Docker images published using `/docker/Dockerfile.router`
+
+## Acknowledgement
+
+This project is a fork of [SGLang Model Gateway](https://github.com/sgl-project/sglang/tree/main/sgl-model-gateway), and we would like to explicitly acknowledge and thank the original authors for their work. At this stage, our fork includes only minimal changes to preserve the existing interface and ensure compatibility with vLLM. We anticipate further divergence as we pursue the roadmap we have in mind, which is the reason for creating the fork.

vllm_router-0.1.9/README.md

@@ -0,0 +1,244 @@
+# vLLM Router
+
+A high-performance and light-weight request forwarding system for vLLM large scale deployments, providing advanced load balancing methods and prefill/decode disaggregation support.
+
+### Key Features
+
+- **Core Architecture**: Request routing framework and async processing patterns
+- **Load Balancing**: Multiple algorithms (cache-aware, power of two, consistent hashing, random, round robin)
+- **Prefill-Decode Disaggregation**: Specialized routing for separated processing phases
+- **Service Discovery**: Kubernetes-native worker management and health monitoring
+- **Enterprise Features**: Circuit breakers, retry logic, metrics collection
+
+## Quick Start
+
+### Prerequisites
+
+**Rust and Cargo:**
+```bash
+# Install rustup (Rust installer and version manager)
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+
+# Follow the installation prompts, then reload your shell
+source $HOME/.cargo/env
+
+# Verify installation
+rustc --version
+cargo --version
+
+# Install protobuf compiler (on Ubuntu/Debian)
+sudo apt-get update
+sudo apt-get install -y protobuf-compiler libprotobuf-dev
+```
+
+**Python with pip installed**
+
+### Installation & Basic Usage
+
+#### Rust Binary
+```bash
+# Build Rust components
+cargo build --release
+```
+
+#### Python Package
+```bash
+pip install setuptools-rust wheel build
+python -m build
+pip install dist/*.whl
+
+# Rebuild & reinstall in one step during development
+python -m build && pip install --force-reinstall dist/*.whl
+```
+
+### Usage Examples
+
+#### Standard Data Parallelism Routing
+```bash
+# Launch router with data parallelism (8 replicas per worker URL)
+# When data-parallel-size > 1, the router automatically creates DP-aware workers
+./target/release/vllm-router \
+    --worker-urls http://worker1:8000 http://worker2:8000 \
+    --policy consistent_hash \
+    --intra-node-data-parallel-size 8
+
+# Alternative: using cargo run
+cargo run --release -- \
+    --worker-urls http://worker1:8000 http://worker2:8000 \
+    --policy consistent_hash \
+    --intra-node-data-parallel-size 8
+
+# Alternative: using python launcher
+vllm-router \
+  --worker-urls http://worker1:8000 http://worker2:8000 \
+    --policy consistent_hash \
+    --intra-node-data-parallel-size 8
+```
+
+#### Prefill-Decode Disaggregation
+```bash
+# When vLLM runs the NIXL connector, prefill/decode URLs are required.
+# See a working example in scripts/llama3.1/ folder.
+cargo run --release -- \
+    --policy consistent_hash \
+    --vllm-pd-disaggregation \
+    --prefill http://127.0.0.1:8081 \
+    --prefill http://127.0.0.1:8082 \
+    --decode http://127.0.0.1:8083 \
+    --decode http://127.0.0.1:8084 \
+    --decode http://127.0.0.1:8085 \
+    --decode http://127.0.0.1:8086 \
+    --host 127.0.0.1 \
+    --port 8090 \
+    --intra-node-data-parallel-size 1 \
+
+
+# When vLLM runs the NCCL connector, ZMQ based discovery is supported.
+# See a working example in scripts/install.sh
+cargo run --release -- \
+    --policy consistent_hash \
+    --vllm-pd-disaggregation \
+    --vllm-discovery-address 0.0.0.0:30001 \
+    --host 0.0.0.0 \
+    --port 10001 \
+    --prefill-policy consistent_hash \
+    --decode-policy consistent_hash
+```
+
+## Configuration
+
+### Metrics
+
+Prometheus metrics endpoint available at `127.0.0.1:29000` by default.
+
+```bash
+# Custom metrics configuration
+vllm-router \
+    --worker-urls http://localhost:8080 http://localhost:8081 \
+    --prometheus-host 0.0.0.0 \
+    --prometheus-port 9000
+```
+
+### Retries and Circuit Breakers
+
+#### Retry Configuration
+Retries are enabled by default with exponential backoff and jitter:
+
+```bash
+vllm-router \
+  --worker-urls http://localhost:8080 http://localhost:8081 \
+  --retry-max-retries 3 \
+  --retry-initial-backoff-ms 100 \
+  --retry-max-backoff-ms 10000 \
+  --retry-backoff-multiplier 2.0 \
+  --retry-jitter-factor 0.1
+```
+
+#### Circuit Breaker Configuration
+Circuit breakers protect workers and provide automatic recovery:
+
+```bash
+vllm-router \
+  --worker-urls http://localhost:8080 http://localhost:8081 \
+  --cb-failure-threshold 5 \
+  --cb-success-threshold 2 \
+  --cb-timeout-duration-secs 30 \
+  --cb-window-duration-secs 60
+```
+
+**Circuit Breaker State Machine:**
+- `Closed` → `Open` after N consecutive failures (failure-threshold)
+- `Open` → `HalfOpen` after timeout (timeout-duration-secs)
+- `HalfOpen` → `Closed` after M consecutive successes (success-threshold)
+
+**Retry Policy:** Retries on HTTP status codes 408/429/500/502/503/504, with backoff/jitter between attempts.
+
+### Request ID Tracking
+
+Track requests across distributed systems with configurable headers:
+
+```bash
+# Use custom request ID headers
+vllm-router \
+    --worker-urls http://localhost:8080 \
+    --request-id-headers x-trace-id x-request-id
+```
+
+**Default headers:** `x-request-id`, `x-correlation-id`, `x-trace-id`, `request-id`
+
+### Load Balancing Policies
+
+The router supports multiple load balancing policies:
+
+| Policy | Description | Session Affinity | Use Case |
+|--------|-------------|------------------|----------|
+| `round_robin` | Sequential distribution across workers | No | General purpose, even distribution |
+| `random` | Uniform random selection | No | Simple deployments |
+| `consistent_hash` | Routes same session/user to same worker | Yes | Multi-turn chat, KV cache reuse |
+| `power_of_two` | Picks least loaded of two random workers | No | Load-sensitive workloads |
+| `cache_aware` | Optimizes for prefix cache hits | Yes | Repeated prompts, few-shot |
+
+```bash
+# Example: Using consistent_hash with HTTP header for session affinity
+curl -X POST http://router:8000/v1/chat/completions \
+  -H "X-Session-ID: my-session-123" \
+  -H "Content-Type: application/json" \
+  -d '{"model": "llama-3", "messages": [{"role": "user", "content": "Hello!"}]}'
+```
+
+For detailed configuration options, hash key priorities, and usage examples, see [Load Balancing Documentation](docs/load_balancing/README.md).
+
+## Advanced Features
+
+### Kubernetes Service Discovery
+
+Automatic worker discovery and management in Kubernetes environments.
+
+#### Basic Service Discovery
+
+```bash
+vllm-router \
+    --service-discovery \
+    --selector app=vllm-worker role=inference \
+    --service-discovery-namespace default
+```
+
+### Command Line Arguments Reference
+
+#### Service Discovery
+- `--service-discovery`: Enable Kubernetes service discovery
+- `--service-discovery-port`: Port for worker URLs (default: 8000)
+- `--service-discovery-namespace`: Kubernetes namespace to watch
+- `--selector`: Label selectors for regular mode (format: `key1=value1 key2=value2`)
+
+## Development
+
+### Troubleshooting
+
+**VSCode Rust Analyzer Issues:**
+Set `rust-analyzer.linkedProjects` to the absolute path of `Cargo.toml`:
+
+```json
+{
+  "rust-analyzer.linkedProjects": ["/workspaces/vllm/vllm-router/Cargo.toml"]
+}
+```
+
+### CI/CD Pipeline
+
+The continuous integration pipeline includes comprehensive testing, benchmarking, and publishing:
+
+#### Build & Test
+1. **Build Wheels**: Uses `cibuildwheel` for manylinux x86_64 packages
+2. **Build Source Distribution**: Creates source distribution for pip fallback
+3. **Rust HTTP Server Benchmarking**: Performance testing of router overhead
+4. **Basic Inference Testing**: End-to-end validation through the router
+5. **PD Disaggregation Testing**: Benchmark and sanity checks for prefill-decode load balancing
+
+#### Publishing
+- **PyPI Publishing**: Wheels and source distributions published when version changes in `pyproject.toml`
+- **Container Images**: Docker images published using `/docker/Dockerfile.router`
+
+## Acknowledgement
+
+This project is a fork of [SGLang Model Gateway](https://github.com/sgl-project/sglang/tree/main/sgl-model-gateway), and we would like to explicitly acknowledge and thank the original authors for their work. At this stage, our fork includes only minimal changes to preserve the existing interface and ensure compatibility with vLLM. We anticipate further divergence as we pursue the roadmap we have in mind, which is the reason for creating the fork.

vllm_router-0.1.9/build.rs

@@ -0,0 +1,31 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Only regenerate if the proto file changes
+    println!("cargo:rerun-if-changed=src/proto/vllm_scheduler.proto");
+
+    // Configure protobuf compilation with custom settings
+    let config = prost_build::Config::new();
+
+    // Skip serde for types that use prost_types::Struct
+    // These cause conflicts and we don't need serde for all generated types
+
+    // Configure tonic-build for gRPC code generation
+    tonic_build::configure()
+        // Generate both client and server code
+        .build_server(true)
+        .build_client(true)
+        // Add a module-level attribute for documentation and clippy warnings
+        .server_mod_attribute(
+            "vllm.grpc.scheduler",
+            "#[allow(unused, clippy::mixed_attributes_style)]",
+        )
+        .client_mod_attribute(
+            "vllm.grpc.scheduler",
+            "#[allow(unused, clippy::mixed_attributes_style)]",
+        )
+        // Compile the proto file with the custom config
+        .compile_protos_with_config(config, &["src/proto/vllm_scheduler.proto"], &["src/proto"])?;
+
+    println!("cargo:warning=Protobuf compilation completed successfully");
+
+    Ok(())
+}

vllm_router-0.1.9/py_src/vllm_router/__init__.py

@@ -0,0 +1,9 @@
+from vllm_router.version import __version__
+
+try:
+    from vllm_router.router import Router
+
+    __all__ = ["__version__", "Router"]
+except ImportError:
+    # Router is not available if Rust extension is not built
+    __all__ = ["__version__"]