vllm-sr 0.1.0b2.dev20260115081807__tar.gz

vllm_sr-0.1.0b2.dev20260115081807/MANIFEST.in

@@ -0,0 +1,18 @@
+# Include documentation
+include README.md
+include QUICKSTART.md
+include PUBLISHING.md
+
+# Include templates (now in cli package)
+recursive-include cli/templates *.yaml
+recursive-include cli/templates *.json
+
+# Include requirements
+include requirements.txt
+
+# Exclude unnecessary files
+global-exclude __pycache__
+global-exclude *.py[co]
+global-exclude .DS_Store
+global-exclude *.egg-info
+

vllm_sr-0.1.0b2.dev20260115081807/PKG-INFO

@@ -0,0 +1,107 @@
+Metadata-Version: 2.4
+Name: vllm-sr
+Version: 0.1.0b2.dev20260115081807
+Summary: vLLM Semantic Router - Intelligent routing for Mixture-of-Models
+Author: vLLM-SR Team
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/vllm-project/vllm-semantic-router
+Project-URL: Documentation, https://github.com/vllm-project/vllm-semantic-router/blob/main/README.md
+Project-URL: Repository, https://github.com/vllm-project/vllm-semantic-router
+Project-URL: Issues, https://github.com/vllm-project/vllm-semantic-router/issues
+Keywords: vllm,semantic-router,llm,routing,caching
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: click>=8.1.7
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: jinja2>=3.1.4
+Requires-Dist: requests>=2.31.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: huggingface_hub[cli]>=0.20.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.4.1; extra == "dev"
+Requires-Dist: black>=22.0.0; extra == "dev"
+
+# vLLM Semantic Router
+
+Intelligent Router for Mixture-of-Models (MoM).
+
+GitHub: https://github.com/vllm-project/semantic-router
+
+## Quick Start
+
+### Installation
+
+```bash
+# Install from PyPI
+pip install vllm-sr
+
+# Or install from source (development)
+cd src/vllm-sr
+pip install -e .
+```
+
+### Usage
+
+```bash
+# Initialize vLLM Semantic Router Configuration
+vllm-sr init
+
+# Start the router (includes dashboard)
+vllm-sr serve
+
+# Open dashboard in browser
+vllm-sr dashboard
+
+# View logs
+vllm-sr logs router
+vllm-sr logs envoy
+vllm-sr logs dashboard
+
+# Check status
+vllm-sr status
+
+# Stop
+vllm-sr stop
+```
+
+## Features
+
+- **Router**: Intelligent request routing based on intent classification
+- **Envoy Proxy**: High-performance proxy with ext_proc integration
+- **Dashboard**: Web UI for monitoring and testing (http://localhost:8700)
+- **Metrics**: Prometheus metrics endpoint (http://localhost:9190/metrics)
+
+## Endpoints
+
+After running `vllm-sr serve`, the following endpoints are available:
+
+| Endpoint | Port | Description |
+|----------|------|-------------|
+| Dashboard | 8700 | Web UI for monitoring and Playground |
+| API | 8888* | Chat completions API (configurable in config.yaml) |
+| Metrics | 9190 | Prometheus metrics |
+| gRPC | 50051 | Router gRPC (internal) |
+
+*Default port, configurable via `listeners` in config.yaml
+
+## Configuration
+
+### File Descriptor Limits
+
+The CLI automatically sets file descriptor limits to 65,536 for Envoy proxy. To customize:
+
+```bash
+export VLLM_SR_NOFILE_LIMIT=100000  # Optional (min: 8192)
+vllm-sr serve
+```
+
+## License
+
+Apache 2.0

vllm_sr-0.1.0b2.dev20260115081807/README.md

@@ -0,0 +1,77 @@
+# vLLM Semantic Router
+
+Intelligent Router for Mixture-of-Models (MoM).
+
+GitHub: https://github.com/vllm-project/semantic-router
+
+## Quick Start
+
+### Installation
+
+```bash
+# Install from PyPI
+pip install vllm-sr
+
+# Or install from source (development)
+cd src/vllm-sr
+pip install -e .
+```
+
+### Usage
+
+```bash
+# Initialize vLLM Semantic Router Configuration
+vllm-sr init
+
+# Start the router (includes dashboard)
+vllm-sr serve
+
+# Open dashboard in browser
+vllm-sr dashboard
+
+# View logs
+vllm-sr logs router
+vllm-sr logs envoy
+vllm-sr logs dashboard
+
+# Check status
+vllm-sr status
+
+# Stop
+vllm-sr stop
+```
+
+## Features
+
+- **Router**: Intelligent request routing based on intent classification
+- **Envoy Proxy**: High-performance proxy with ext_proc integration
+- **Dashboard**: Web UI for monitoring and testing (http://localhost:8700)
+- **Metrics**: Prometheus metrics endpoint (http://localhost:9190/metrics)
+
+## Endpoints
+
+After running `vllm-sr serve`, the following endpoints are available:
+
+| Endpoint | Port | Description |
+|----------|------|-------------|
+| Dashboard | 8700 | Web UI for monitoring and Playground |
+| API | 8888* | Chat completions API (configurable in config.yaml) |
+| Metrics | 9190 | Prometheus metrics |
+| gRPC | 50051 | Router gRPC (internal) |
+
+*Default port, configurable via `listeners` in config.yaml
+
+## Configuration
+
+### File Descriptor Limits
+
+The CLI automatically sets file descriptor limits to 65,536 for Envoy proxy. To customize:
+
+```bash
+export VLLM_SR_NOFILE_LIMIT=100000  # Optional (min: 8192)
+vllm-sr serve
+```
+
+## License
+
+Apache 2.0

vllm_sr-0.1.0b2.dev20260115081807/cli/__init__.py

@@ -0,0 +1,3 @@
+"""vLLM Semantic Router CLI package."""
+
+__version__ = "0.1.0"

vllm_sr-0.1.0b2.dev20260115081807/cli/commands/__init__.py

@@ -0,0 +1,9 @@
+"""CLI commands for vLLM Semantic Router."""
+
+from .init import init_command
+from .config import config_command
+
+__all__ = [
+    "init_command",
+    "config_command",
+]

vllm_sr-0.1.0b2.dev20260115081807/cli/commands/config.py

@@ -0,0 +1,91 @@
+"""Config command implementation."""
+
+import sys
+import yaml
+from pathlib import Path
+
+from cli.parser import parse_user_config, ConfigParseError
+from cli.defaults import load_embedded_defaults, load_defaults
+from cli.merger import merge_configs
+from cli.validator import (
+    validate_user_config,
+    validate_merged_config,
+    print_validation_errors,
+)
+from cli.config_generator import generate_envoy_config_from_user_config
+from cli.utils import getLogger
+
+log = getLogger(__name__)
+
+
+def config_command(config_type: str, config_path: str = "config.yaml"):
+    """
+    Print generated configuration.
+
+    Args:
+        config_type: Type of config to print ('envoy' or 'router')
+        config_path: Path to user config.yaml (default: config.yaml)
+    """
+    if config_type not in ["envoy", "router"]:
+        log.error(f"Invalid config type: {config_type}")
+        log.error("Must be 'envoy' or 'router'")
+        sys.exit(1)
+
+    # Check if config file exists
+    if not Path(config_path).exists():
+        log.error(f"Config file not found: {config_path}")
+        log.error("Run 'vllm-sr init' to create a config file")
+        sys.exit(1)
+
+    # Parse user config
+    try:
+        user_config = parse_user_config(config_path)
+    except ConfigParseError as e:
+        log.error(f"Failed to parse configuration: {e}")
+        sys.exit(1)
+
+    # Validate user config
+    errors = validate_user_config(user_config)
+    if errors:
+        log.error("Configuration validation failed:")
+        print_validation_errors(errors)
+        sys.exit(1)
+
+    if config_type == "router":
+        # Generate router config (use local defaults if available)
+        defaults = load_defaults(".vllm-sr")
+        merged = merge_configs(user_config, defaults)
+
+        # Validate merged config
+        errors = validate_merged_config(merged)
+        if errors:
+            log.error("Merged configuration validation failed:")
+            print_validation_errors(errors)
+            sys.exit(1)
+
+        # Print router config as YAML
+        print(yaml.dump(merged, default_flow_style=False, sort_keys=False))
+
+    elif config_type == "envoy":
+        # Generate envoy config
+        try:
+            # Generate to a temporary string
+            import tempfile
+
+            with tempfile.NamedTemporaryFile(
+                mode="w", suffix=".yaml", delete=False
+            ) as f:
+                temp_path = f.name
+
+            generate_envoy_config_from_user_config(user_config, temp_path)
+
+            # Read and print
+            with open(temp_path, "r") as f:
+                print(f.read())
+
+            # Clean up
+            Path(temp_path).unlink()
+
+        except Exception as e:
+            log.error(f"Failed to generate Envoy config: {e}")
+            sys.exit(1)

vllm_sr-0.1.0b2.dev20260115081807/cli/commands/generate.py

@@ -0,0 +1,73 @@
+"""Generate command implementation."""
+
+import sys
+from pathlib import Path
+
+from cli.commands.serve import (
+    generate_router_config,
+    copy_defaults_reference,
+    ensure_output_directory,
+    DEFAULT_OUTPUT_DIR,
+)
+from cli.parser import parse_user_config, ConfigParseError
+from cli.config_generator import generate_envoy_config_from_user_config
+from cli.utils import getLogger
+
+log = getLogger(__name__)
+
+
+def generate_command(
+    config_path: str, output_dir: str = DEFAULT_OUTPUT_DIR, force: bool = False
+):
+    """
+    Generate configurations without starting services.
+
+    Args:
+        config_path: Path to user config.yaml
+        output_dir: Output directory for generated configs
+        force: Force overwrite existing files
+    """
+    log.info("=" * 60)
+    log.info("vLLM Semantic Router - Generate Configurations")
+    log.info("=" * 60)
+
+    # Ensure output directory exists
+    output_path = ensure_output_directory(output_dir)
+
+    # Parse user config (needed for Envoy generation)
+    try:
+        user_config = parse_user_config(config_path)
+    except ConfigParseError as e:
+        log.error(f"Failed to parse configuration: {e}")
+        sys.exit(1)
+
+    # Generate router config
+    router_config_path = generate_router_config(config_path, output_dir, force=force)
+
+    # Copy defaults for reference
+    defaults_path = copy_defaults_reference(output_dir)
+
+    # Generate Envoy config
+    envoy_config_path = None
+    try:
+        envoy_output = output_path / "envoy-config.yaml"
+        envoy_config_path = generate_envoy_config_from_user_config(
+            user_config, str(envoy_output)
+        )
+    except Exception as e:
+        log.warning(f"Failed to generate Envoy config: {e}")
+        log.warning("Continuing without Envoy config...")
+
+    log.info("=" * 60)
+    log.info("✓ Configurations generated successfully")
+    log.info(f"  Output directory: {output_path.absolute()}")
+    log.info(f"  Router config: {router_config_path.name}")
+    if envoy_config_path:
+        log.info(f"  Envoy config: {envoy_config_path.name}")
+    log.info(f"  Defaults reference: {defaults_path.name}")
+    log.info("=" * 60)
+    log.info("\nYou can now:")
+    log.info(f"  • Edit {router_config_path} to customize system settings")
+    if envoy_config_path:
+        log.info(f"  • Edit {envoy_config_path} to customize Envoy settings")
+    log.info(f"  • Start service with: vllm-sr serve {config_path}")

vllm_sr-0.1.0b2.dev20260115081807/cli/commands/init.py

@@ -0,0 +1,96 @@
+"""Init command implementation."""
+
+import os
+import shutil
+import sys
+from pathlib import Path
+from cli.utils import getLogger
+
+log = getLogger(__name__)
+
+
+def get_templates_dir():
+    """
+    Get the templates directory path.
+
+    Templates are now in cli/templates/ directory.
+    """
+    # Get cli package directory
+    cli_dir = Path(__file__).parent.parent
+    templates_dir = cli_dir / "templates"
+
+    return templates_dir if templates_dir.exists() else None
+
+
+def init_command(force: bool = False):
+    """
+    Initialize vLLM Semantic Router configuration.
+
+    Creates:
+    - config.yaml from config.template.yaml
+    - .vllm-sr/ directory with template files
+
+    Args:
+        force: Force overwrite existing files
+    """
+    log.info("=" * 60)
+    log.info("vLLM Semantic Router - Initialize Configuration")
+    log.info("=" * 60)
+
+    # Get current directory
+    current_dir = Path.cwd()
+    config_file = current_dir / "config.yaml"
+    vllm_sr_dir = current_dir / ".vllm-sr"
+
+    # Get templates directory
+    templates_dir = get_templates_dir()
+
+    if templates_dir is None or not templates_dir.exists():
+        log.error(f"Templates directory not found: {templates_dir}")
+        log.error("Please ensure vllm-sr is installed correctly.")
+        return False
+
+    # Copy config.template.yaml to config.yaml
+    template_config = templates_dir / "config.template.yaml"
+    if not template_config.exists():
+        log.error(f"Config template not found: {template_config}")
+        return False
+
+    if config_file.exists():
+        log.warning(f"config.yaml already exists, overwriting...")
+
+    shutil.copy2(template_config, config_file)
+    log.info(f"✓ Created config.yaml")
+
+    # Create .vllm-sr directory
+    if vllm_sr_dir.exists():
+        log.warning(f".vllm-sr/ directory already exists, overwriting...")
+        shutil.rmtree(vllm_sr_dir)
+
+    vllm_sr_dir.mkdir(exist_ok=True)
+    log.info(f"✓ Created .vllm-sr/ directory")
+
+    # Copy all template files to .vllm-sr/
+    copied_files = []
+    for template_file in templates_dir.iterdir():
+        if template_file.is_file() and template_file.name != "config.template.yaml":
+            dest_file = vllm_sr_dir / template_file.name
+            shutil.copy2(template_file, dest_file)
+            copied_files.append(template_file.name)
+            log.info(f"  • Copied {template_file.name}")
+
+    log.info("=" * 60)
+    log.info("✓ Initialization complete!")
+    log.info("")
+    log.info("Created files:")
+    log.info(f"  • config.yaml")
+    log.info(f"  • .vllm-sr/ ({len(copied_files)} files)")
+    log.info("")
+    log.info("Next steps:")
+    log.info("  1. Edit config.yaml to configure your setup")
+    log.info("     Or use the dashboard for visual configuration")
+    log.info("  2. Start service: vllm-sr serve")
+    log.info("  3. Open dashboard: http://localhost:8700")
+    log.info("=" * 60)
+
+    return True

vllm_sr-0.1.0b2.dev20260115081807/cli/commands/serve.py

@@ -0,0 +1,206 @@
+"""Serve command implementation."""
+
+import os
+import sys
+import yaml
+from pathlib import Path
+
+from cli.parser import parse_user_config, ConfigParseError
+from cli.defaults import load_embedded_defaults, get_defaults_yaml, load_defaults
+from cli.merger import merge_configs
+from cli.validator import (
+    validate_user_config,
+    validate_merged_config,
+    print_validation_errors,
+)
+from cli.config_generator import generate_envoy_config_from_user_config
+from cli.utils import getLogger
+
+log = getLogger(__name__)
+
+DEFAULT_OUTPUT_DIR = ".vllm-sr"
+
+
+def ensure_output_directory(output_dir: str) -> Path:
+    """
+    Ensure output directory exists.
+
+    Args:
+        output_dir: Output directory path
+
+    Returns:
+        Path: Output directory path object
+    """
+    output_path = Path(output_dir)
+    output_path.mkdir(parents=True, exist_ok=True)
+    return output_path
+
+
+def generate_router_config(
+    config_path: str, output_dir: str = DEFAULT_OUTPUT_DIR, force: bool = False
+) -> Path:
+    """
+    Generate router configuration from user config.
+
+    Args:
+        config_path: Path to user config.yaml
+        output_dir: Output directory
+        force: Force regeneration even if file exists
+
+    Returns:
+        Path: Path to generated router config
+    """
+    output_path = ensure_output_directory(output_dir)
+    router_config_path = output_path / "router-config.yaml"
+
+    # Check if router config already exists
+    if router_config_path.exists() and not force:
+        log.info(f"Using existing {router_config_path}")
+        log.info(f"  (Use --regenerate to recreate from {config_path})")
+        return router_config_path
+
+    log.info(f"Generating {router_config_path}...")
+
+    # Parse user config
+    try:
+        user_config = parse_user_config(config_path)
+    except ConfigParseError as e:
+        log.error(f"Failed to parse configuration: {e}")
+        sys.exit(1)
+
+    # Validate user config
+    errors = validate_user_config(user_config)
+    if errors:
+        print_validation_errors(errors)
+        sys.exit(1)
+
+    # Load defaults (prefer local router-defaults.yaml if it exists)
+    defaults = load_defaults(output_dir)
+
+    # Log which defaults were used
+    local_defaults_path = Path(output_dir) / "router-defaults.yaml"
+    if local_defaults_path.exists():
+        log.info(f"  Using local defaults: {local_defaults_path}")
+    else:
+        log.info(f"  Using embedded defaults")
+
+    # Merge configs
+    merged = merge_configs(user_config, defaults)
+
+    # Validate merged config
+    errors = validate_merged_config(merged)
+    if errors:
+        print_validation_errors(errors)
+        sys.exit(1)
+
+    # Write router config
+    with open(router_config_path, "w") as f:
+        yaml.dump(merged, f, default_flow_style=False, sort_keys=False)
+
+    log.info(f"✓ Generated {router_config_path}")
+
+    return router_config_path
+
+
+def copy_defaults_reference(output_dir: str) -> Path:
+    """
+    Copy embedded defaults to output directory for reference.
+
+    Will NOT overwrite existing router-defaults.yaml to preserve user modifications.
+
+    Args:
+        output_dir: Output directory
+
+    Returns:
+        Path: Path to defaults reference file
+    """
+    output_path = Path(output_dir)
+    defaults_path = output_path / "router-defaults.yaml"
+
+    if defaults_path.exists():
+        log.info(f"Using existing {defaults_path} (preserving user modifications)")
+        return defaults_path
+
+    log.info(f"Copying router-defaults.yaml (for reference)...")
+
+    with open(defaults_path, "w") as f:
+        f.write(get_defaults_yaml())
+
+    log.info(f"✓ Copied {defaults_path}")
+
+    return defaults_path
+
+
+def serve_command(
+    config_path: str,
+    output_dir: str = DEFAULT_OUTPUT_DIR,
+    regenerate: bool = False,
+    router_config: str = None,
+    envoy_config: str = None,
+):
+    """
+    Start vLLM Semantic Router service.
+
+    Args:
+        config_path: Path to user config.yaml
+        output_dir: Output directory for generated configs
+        regenerate: Force regenerate router config
+        router_config: Custom router config path (bypasses generation)
+        envoy_config: Custom Envoy config path (bypasses generation)
+    """
+    log.info("=" * 60)
+    log.info("vLLM Semantic Router - Starting Service")
+    log.info("=" * 60)
+
+    # Ensure output directory exists
+    output_path = ensure_output_directory(output_dir)
+
+    # Parse user config (needed for Envoy generation)
+    try:
+        user_config = parse_user_config(config_path)
+    except ConfigParseError as e:
+        log.error(f"Failed to parse configuration: {e}")
+        sys.exit(1)
+
+    # Generate or use existing router config
+    if router_config:
+        log.info(f"Using custom router config: {router_config}")
+        router_config_path = Path(router_config)
+    else:
+        router_config_path = generate_router_config(
+            config_path, output_dir, force=regenerate
+        )
+
+    # Copy defaults for reference
+    copy_defaults_reference(output_dir)
+
+    # Generate Envoy config
+    envoy_config_path = None
+    if envoy_config:
+        log.info(f"Using custom Envoy config: {envoy_config}")
+        envoy_config_path = Path(envoy_config)
+    else:
+        try:
+            envoy_output = output_path / "envoy-config.yaml"
+            envoy_config_path = generate_envoy_config_from_user_config(
+                user_config, str(envoy_output)
+            )
+        except Exception as e:
+            log.warning(f"Failed to generate Envoy config: {e}")
+            log.warning("Continuing without Envoy config...")
+
+    # TODO: Start services
+
+    log.info("=" * 60)
+    log.info("✓ Configuration generated successfully")
+    log.info(f"  Router config: {router_config_path}")
+    if envoy_config_path:
+        log.info(f"  Envoy config: {envoy_config_path}")
+    log.info(f"  Output directory: {output_dir}")
+    log.info("=" * 60)
+
+    # For now, just show what would happen
+    log.info("\nNext steps (to be implemented):")
+    log.info("  1. Start Envoy proxy")
+    log.info("  2. Start Router service")
+    log.info("  3. Wait for health checks")