dragon-cli 0.2.0__py3-none-any.whl

dragon_cli/recommendation.py

@@ -0,0 +1,295 @@
+"""Model recommendation engine for Dragon CLI.
+
+Analyzes hardware profile and recommends optimal models based on:
+- Available RAM and VRAM
+- CPU capabilities
+- User preferences (speed vs quality)
+"""
+
+from typing import Any, Dict, List, Optional, Tuple
+
+from .models.catalog import MODEL_CATALOG
+from .hardware import detect_hardware
+
+
+class ModelRecommender:
+    """Recommends LLM models based on hardware capabilities."""
+    
+    def __init__(self, hardware_profile: Optional[Dict[str, Any]] = None):
+        """Initialize recommender with optional hardware profile.
+        
+        Args:
+            hardware_profile: Hardware info dict from detect_hardware(),
+                             or None to auto-detect
+        """
+        if hardware_profile is None:
+            self.hardware = detect_hardware()
+        else:
+            self.hardware = hardware_profile
+    
+    def get_compatible_models(
+        self,
+        priority: str = "balanced",
+        max_context: Optional[int] = None,
+    ) -> List[Dict[str, Any]]:
+        """Get list of models compatible with current hardware.
+        
+        Args:
+            priority: One of "speed", "quality", "balanced", "memory"
+            max_context: Maximum context window size desired
+            
+        Returns:
+            List of compatible models with scores
+        """
+        ram_available = self.hardware.get("ram_available_gb", 8)
+        vram_available = self.hardware.get("gpu_vram_gb", 0)
+        gpu_available = self.hardware.get("gpu_available", False)
+        cpu_count = self.hardware.get("cpu_count", 4)
+        
+        compatible = []
+        
+        for model in MODEL_CATALOG:
+            ram_required = model.get("ram_min_gb", 0)
+            
+            # Basic compatibility check
+            if ram_required > ram_available * 1.2:  # Allow 20% overhead
+                continue
+            
+            # Calculate compatibility score
+            score = self._calculate_score(
+                model=model,
+                ram_available=ram_available,
+                vram_available=vram_available,
+                gpu_available=gpu_available,
+                cpu_count=cpu_count,
+                priority=priority,
+            )
+            
+            model_copy = model.copy()
+            model_copy["compatibility_score"] = score
+            model_copy["can_use_gpu"] = gpu_available and vram_available >= 4
+            model_copy["recommended"] = score >= 0.7
+            
+            compatible.append(model_copy)
+        
+        # Sort by score descending
+        compatible.sort(key=lambda x: x["compatibility_score"], reverse=True)
+        
+        return compatible
+    
+    def _calculate_score(
+        self,
+        model: Dict[str, Any],
+        ram_available: float,
+        vram_available: float,
+        gpu_available: bool,
+        cpu_count: int,
+        priority: str,
+    ) -> float:
+        """Calculate compatibility score for a model (0.0 to 1.0)."""
+        ram_required = model.get("ram_min_gb", 0)
+        
+        # Base score: how well RAM fits
+        ram_ratio = ram_available / max(ram_required, 1)
+        ram_score = min(1.0, ram_ratio / 1.5)  # Cap at 1.0
+        
+        # GPU bonus
+        gpu_score = 0.0
+        if gpu_available:
+            # Estimate VRAM needed (rough: 0.7 GB per billion params at Q4)
+            params_estimate = self._estimate_params_from_name(model.get("name", ""))
+            vram_needed = params_estimate * 0.7
+            if vram_available >= vram_needed:
+                gpu_score = 0.3  # Bonus for full GPU offload capability
+            elif vram_available >= vram_needed * 0.5:
+                gpu_score = 0.15  # Partial offload
+        
+        # CPU score (more cores = better for CPU inference)
+        cpu_score = min(1.0, cpu_count / 8) * 0.1
+        
+        # Priority adjustments
+        if priority == "speed":
+            # Favor smaller models
+            size_penalty = (ram_required / 16) * 0.2
+            total_score = ram_score + gpu_score + cpu_score - size_penalty
+        elif priority == "quality":
+            # Favor larger models
+            size_bonus = min(0.2, ram_required / 20)
+            total_score = ram_score + gpu_score + cpu_score + size_bonus
+        elif priority == "memory":
+            # Strongly favor smaller models
+            memory_efficiency = 1.0 - (ram_required / 20)
+            total_score = ram_score * 0.5 + memory_efficiency * 0.5 + gpu_score
+        else:  # balanced
+            total_score = ram_score + gpu_score + cpu_score
+        
+        return min(1.0, max(0.0, total_score))
+    
+    def _estimate_params_from_name(self, name: str) -> float:
+        """Estimate parameter count from model name (in billions)."""
+        import re
+        
+        # Look for patterns like "7B", "14B", "1.5B", etc.
+        matches = re.findall(r'(\d+\.?\d*)[Bb]', name)
+        if matches:
+            try:
+                return float(matches[0])
+            except ValueError:
+                pass
+        return 7.0  # Default assumption
+    
+    def recommend_best(
+        self,
+        priority: str = "balanced",
+    ) -> Optional[Dict[str, Any]]:
+        """Get the single best recommended model.
+        
+        Args:
+            priority: One of "speed", "quality", "balanced", "memory"
+            
+        Returns:
+            Best model dictionary or None if no compatible models
+        """
+        compatible = self.get_compatible_models(priority=priority)
+        if compatible:
+            return compatible[0]
+        return None
+    
+    def get_recommendation_reason(self, model: Dict[str, Any]) -> str:
+        """Get human-readable explanation for why a model was recommended."""
+        ram_available = self.hardware.get("ram_available_gb", 0)
+        ram_required = model.get("ram_min_gb", 0)
+        gpu_available = self.hardware.get("gpu_available", False)
+        vram = self.hardware.get("gpu_vram_gb", 0)
+        
+        reasons = []
+        
+        # RAM assessment
+        if ram_available >= ram_required * 1.5:
+            reasons.append(f"✓ Plenty of RAM ({ram_available:.1f} GB available, {ram_required} GB needed)")
+        elif ram_available >= ram_required:
+            reasons.append(f"✓ Sufficient RAM ({ram_available:.1f} GB available)")
+        else:
+            reasons.append(f"⚠ Limited RAM ({ram_available:.1f} GB available, {ram_required} GB recommended)")
+        
+        # GPU assessment
+        if gpu_available and vram >= 6:
+            reasons.append(f"✓ GPU acceleration available ({vram:.1f} GB VRAM)")
+        elif gpu_available:
+            reasons.append(f"○ GPU available but limited VRAM ({vram:.1f} GB)")
+        else:
+            reasons.append("○ CPU-only mode (no GPU detected)")
+        
+        # Model-specific notes
+        params = self._estimate_params_from_name(model.get("name", ""))
+        if params <= 3:
+            reasons.append("→ Fast inference, good for quick tasks")
+        elif params <= 8:
+            reasons.append("→ Good balance of speed and quality")
+        else:
+            reasons.append("→ Higher quality, slower inference")
+        
+        return "; ".join(reasons)
+    
+    def print_recommendations(
+        self,
+        top_n: int = 5,
+        priority: str = "balanced",
+    ) -> None:
+        """Print formatted recommendations to console."""
+        from .config import C_CYAN, C_RESET, C_GREEN, C_YELLOW, C_DIM, C_RED
+        
+        compatible = self.get_compatible_models(priority=priority)
+        
+        print(f"\n{C_CYAN}{'='*60}{C_RESET}")
+        print(f"{C_CYAN}MODEL RECOMMENDATIONS{C_RESET}")
+        print(f"{C_CYAN}{'='*60}{C_RESET}\n")
+        
+        # Show hardware summary
+        print(f"{C_DIM}Your Hardware:{C_RESET}")
+        print(f"  RAM: {self.hardware.get('ram_available_gb', 0):.1f} GB available")
+        if self.hardware.get('gpu_available'):
+            print(f"  GPU: {self.hardware.get('gpu_name', 'Unknown')} ({self.hardware.get('gpu_vram_gb', 0):.1f} GB VRAM)")
+        else:
+            print(f"  GPU: None detected (CPU-only)")
+        print()
+        
+        if not compatible:
+            print(f"{C_RED}No compatible models found for your hardware.{C_RESET}\n")
+            return
+        
+        print(f"{C_DIM}Priority: {priority.capitalize()}{C_RESET}\n")
+        print(f"{C_GREEN}Top Recommendations:{C_RESET}\n")
+        
+        for i, model in enumerate(compatible[:top_n], 1):
+            score = model.get("compatibility_score", 0)
+            recommended = model.get("recommended", False)
+            
+            badge = f"{C_GREEN}[RECOMMENDED]{C_RESET}" if recommended else f"{C_DIM}[Compatible]{C_RESET}"
+            
+            # Score indicator
+            if score >= 0.8:
+                score_str = f"{C_GREEN}██████████{C_RESET} {score:.0%}"
+            elif score >= 0.6:
+                score_str = f"{C_YELLOW}██████░░░░{C_RESET} {score:.0%}"
+            else:
+                score_str = f"{C_RED}████░░░░░░{C_RESET} {score:.0%}"
+            
+            print(f"  {i}. {model['name']}")
+            print(f"     {badge}")
+            print(f"     Match: {score_str}")
+            print(f"     Min RAM: {model.get('ram_min_gb', '?')} GB | Context: {model.get('ctx', '?')}")
+            
+            if i == 1:
+                reason = self.get_recommendation_reason(model)
+                print(f"     {C_DIM}Why: {reason}{C_RESET}")
+            print()
+
+
+def recommend_model_for_hardware(
+    priority: str = "balanced",
+    show_all: bool = False,
+) -> Optional[Dict[str, Any]]:
+    """Convenience function to get model recommendation.
+    
+    Args:
+        priority: One of "speed", "quality", "balanced", "memory"
+        show_all: If True, print all recommendations
+        
+    Returns:
+        Best recommended model or None
+    """
+    recommender = ModelRecommender()
+    
+    if show_all:
+        recommender.print_recommendations(priority=priority)
+    
+    return recommender.recommend_best(priority=priority)
+
+
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Get model recommendations for your hardware")
+    parser.add_argument(
+        "--priority",
+        choices=["speed", "quality", "balanced", "memory"],
+        default="balanced",
+        help="Optimization priority (default: balanced)"
+    )
+    parser.add_argument(
+        "--all",
+        action="store_true",
+        help="Show all compatible models"
+    )
+    
+    args = parser.parse_args()
+    
+    result = recommend_model_for_hardware(
+        priority=args.priority,
+        show_all=args.all
+    )
+    
+    if result and not args.all:
+        print(f"\n{result['name']}")
+        print(f"Min RAM: {result.get('ram_min_gb', '?')} GB")

dragon_cli/runtime.py

@@ -0,0 +1,252 @@
+"""Runtime configuration module for Dragon CLI.
+
+Decides execution parameters based on hardware and model:
+- CPU vs GPU vs hybrid execution
+- Context window size
+- Thread count
+- Batch size
+"""
+
+from typing import Any, Dict, Optional
+
+
+class RuntimeConfig:
+    """Configure runtime parameters for LLM inference."""
+    
+    def __init__(
+        self,
+        hardware_profile: Optional[Dict[str, Any]] = None,
+        model_info: Optional[Dict[str, Any]] = None,
+    ):
+        """Initialize runtime configuration.
+        
+        Args:
+            hardware_profile: Hardware info from detect_hardware()
+            model_info: Model metadata (ram_min_gb, ctx, etc.)
+        """
+        from .hardware import detect_hardware
+        
+        self.hardware = hardware_profile or detect_hardware()
+        self.model = model_info or {}
+    
+    def decide_execution_mode(self) -> str:
+        """Decide whether to use CPU, GPU, or hybrid execution.
+        
+        Returns:
+            One of: "gpu", "cpu", "hybrid"
+        """
+        gpu_available = self.hardware.get("gpu_available", False)
+        vram = self.hardware.get("gpu_vram_gb", 0)
+        ram = self.hardware.get("ram_available_gb", 8)
+        
+        # Estimate model VRAM requirement
+        params_billions = self._estimate_params()
+        vram_needed = params_billions * 0.7  # Rough estimate for Q4 quantization
+        
+        if gpu_available and vram >= vram_needed:
+            return "gpu"  # Full GPU offload possible
+        elif gpu_available and vram >= 4:
+            return "hybrid"  # Partial GPU offload
+        else:
+            return "cpu"  # CPU only
+    
+    def get_gpu_layers(self) -> int:
+        """Calculate optimal number of layers to offload to GPU.
+        
+        Returns:
+            Number of layers for n_gpu_layers parameter
+        """
+        execution_mode = self.decide_execution_mode()
+        
+        if execution_mode == "gpu":
+            # Offload all layers
+            return 999  # llama-cpp-python convention for "all layers"
+        elif execution_mode == "hybrid":
+            # Calculate partial offload
+            vram = self.hardware.get("gpu_vram_gb", 0)
+            params_billions = self._estimate_params()
+            
+            # Assume ~1GB per 1B params at Q4, adjust for layers
+            layers_per_gb = 2.5  # Approximate
+            max_layers = int(vram * layers_per_gb)
+            
+            # Typical models have 32-40 layers
+            total_layers = self._estimate_layer_count()
+            return min(max_layers, total_layers)
+        else:
+            return 0  # CPU only
+    
+    def get_thread_count(self) -> int:
+        """Calculate optimal thread count for CPU inference.
+        
+        Returns:
+            Number of threads for n_threads parameter
+        """
+        cpu_count = self.hardware.get("cpu_count", 4)
+        
+        # Leave some cores for system
+        # Physical cores usually better than logical for inference
+        physical_count = self.hardware.get("cpu_count_physical", cpu_count // 2)
+        
+        if physical_count > 0:
+            threads = max(1, physical_count - 1)
+        else:
+            threads = max(1, cpu_count - 2)
+        
+        # Cap at reasonable maximum
+        return min(threads, 16)
+    
+    def get_context_size(self) -> int:
+        """Determine optimal context window size.
+        
+        Returns:
+            Context size for n_ctx parameter
+        """
+        model_ctx = self.model.get("ctx", 4096)
+        ram = self.hardware.get("ram_available_gb", 8)
+        
+        # Context memory: roughly 2GB per 8K context for 7B model
+        # Scale by model size
+        params_billions = self._estimate_params()
+        memory_per_8k = params_billions / 7.0 * 2.0  # GB
+        
+        # Calculate max context we can afford
+        # Reserve 2GB for model weights and system
+        available_for_ctx = max(0, ram - 2 - (params_billions * 0.7))
+        
+        if available_for_ctx <= 0:
+            max_ctx = 2048
+        else:
+            max_ctx = int(available_for_ctx / memory_per_8k * 8192)
+            max_ctx = max(2048, min(max_ctx, 32768))  # Reasonable bounds
+        
+        # Use model's recommended context or calculated max, whichever is smaller
+        return min(model_ctx, max_ctx)
+    
+    def get_batch_size(self) -> int:
+        """Determine optimal batch size for inference.
+        
+        Returns:
+            Batch size for n_batch parameter
+        """
+        execution_mode = self.decide_execution_mode()
+        ram = self.hardware.get("ram_available_gb", 8)
+        
+        if execution_mode == "gpu":
+            # GPU can handle larger batches
+            return 512
+        elif ram >= 16:
+            return 256
+        elif ram >= 8:
+            return 128
+        else:
+            return 64
+    
+    def get_runtime_params(self) -> Dict[str, Any]:
+        """Get complete runtime parameters for llama-cpp-python.
+        
+        Returns:
+            Dictionary with all parameters for Llama() constructor
+        """
+        execution_mode = self.decide_execution_mode()
+        
+        params = {
+            "n_threads": self.get_thread_count(),
+            "n_ctx": self.get_context_size(),
+            "n_batch": self.get_batch_size(),
+            "n_gpu_layers": self.get_gpu_layers(),
+            "verbose": False,
+        }
+        
+        # Additional optimizations
+        if execution_mode == "gpu":
+            params["use_mmap"] = True
+            params["use_mlock"] = False
+        else:
+            params["use_mmap"] = True
+            params["use_mlock"] = False
+        
+        return params
+    
+    def _estimate_params(self) -> float:
+        """Estimate model parameters in billions from name."""
+        import re
+        
+        name = self.model.get("name", "")
+        matches = re.findall(r'(\d+\.?\d*)[Bb]', name)
+        if matches:
+            try:
+                return float(matches[0])
+            except ValueError:
+                pass
+        return 7.0  # Default
+    
+    def _estimate_layer_count(self) -> int:
+        """Estimate layer count from model name."""
+        name = self.model.get("name", "").lower()
+        
+        # Common patterns
+        if "llama" in name or "qwen" in name:
+            params = self._estimate_params()
+            if params <= 3:
+                return 28
+            elif params <= 8:
+                return 32
+            elif params <= 14:
+                return 40
+            else:
+                return 48
+        
+        return 32  # Default
+    
+    def print_config(self) -> None:
+        """Print runtime configuration summary."""
+        from .config import C_CYAN, C_RESET, C_GREEN, C_DIM
+        
+        params = self.get_runtime_params()
+        execution_mode = self.decide_execution_mode()
+        
+        print(f"\n{C_CYAN}{'='*50}{C_RESET}")
+        print(f"{C_CYAN}RUNTIME CONFIGURATION{C_RESET}")
+        print(f"{C_CYAN}{'='*50}{C_RESET}\n")
+        
+        print(f"{C_GREEN}Execution Mode:{C_RESET} {execution_mode.upper()}")
+        print(f"{C_GREEN}Threads:{C_RESET} {params['n_threads']}")
+        print(f"{C_GREEN}Context:{C_RESET} {params['n_ctx']} tokens")
+        print(f"{C_GREEN}Batch Size:{C_RESET} {params['n_batch']}")
+        print(f"{C_GREEN}GPU Layers:{C_RESET} {params['n_gpu_layers']}")
+        
+        if execution_mode != "cpu":
+            print(f"\n{C_DIM}→ GPU acceleration enabled{C_RESET}")
+        else:
+            print(f"\n{C_DIM}→ Running on CPU only{C_RESET}")
+        
+        print()
+
+
+def configure_runtime(
+    model_info: Optional[Dict[str, Any]] = None,
+    hardware_profile: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """Convenience function to get runtime configuration.
+    
+    Args:
+        model_info: Model metadata
+        hardware_profile: Hardware profile (auto-detected if None)
+        
+    Returns:
+        Runtime parameters dictionary for llama-cpp-python
+    """
+    config = RuntimeConfig(hardware_profile=hardware_profile, model_info=model_info)
+    return config.get_runtime_params()
+
+
+if __name__ == "__main__":
+    # Test runtime configuration
+    config = RuntimeConfig()
+    config.print_config()
+    
+    print("\nFull parameters:")
+    params = config.get_runtime_params()
+    for key, value in params.items():
+        print(f"  {key}: {value}")

dragon_cli/server.py

@@ -0,0 +1,144 @@
+"""Minimal OpenAI-compatible HTTP server exposing the loaded local model.
+
+No MCP tool-calling here by design — this is a small subset of the OpenAI
+chat completions API (/v1/models, /v1/chat/completions, streaming included),
+enough for any tool or SDK that already speaks that protocol to talk to the
+local model, e.g. by pointing an OpenAI client at http://127.0.0.1:8000/v1.
+Binds to localhost only unless --host is overridden explicitly.
+
+Built on FastAPI so interactive API docs come for free: Swagger UI at /docs,
+ReDoc at /redoc, raw schema at /openapi.json.
+"""
+
+import asyncio
+import json
+import threading
+from pathlib import Path
+
+import uvicorn
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.responses import HTMLResponse, Response, StreamingResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+from pydantic import BaseModel, Field
+from starlette.concurrency import run_in_threadpool
+
+from .config import C_CYAN, C_GREEN, C_RESET, C_YELLOW
+from .web import FAVICON_SVG
+
+BASE_DIR = Path(__file__).resolve().parent
+TEMPLATES = Jinja2Templates(directory=BASE_DIR / "templates")
+IMAGE_DIR = BASE_DIR.parents[2] / "image"
+
+# llama.cpp inference is not safe to run concurrently from multiple threads,
+# so every request (streaming or not) serializes on this lock.
+_llm_lock = asyncio.Lock()
+
+
+class ChatMessage(BaseModel):
+    role: str = Field(examples=["user"])
+    content: str = Field(examples=["Hello!"])
+
+
+class ChatCompletionRequest(BaseModel):
+    messages: list[ChatMessage] = Field(min_length=1)
+    model: str | None = Field(default=None, description="Ignored — only one model is loaded per server.")
+    temperature: float = 0.7
+    max_tokens: int = 1024
+    stream: bool = Field(default=False, description="If true, respond with server-sent events.")
+
+
+async def _stream_chat_completion(llm, kwargs):
+    async with _llm_lock:
+        queue: asyncio.Queue = asyncio.Queue()
+        loop = asyncio.get_running_loop()
+
+        def worker():
+            try:
+                for chunk in llm.create_chat_completion(stream=True, **kwargs):
+                    loop.call_soon_threadsafe(queue.put_nowait, chunk)
+            except Exception as exc:
+                loop.call_soon_threadsafe(queue.put_nowait, {"error": {"message": str(exc)}})
+            finally:
+                loop.call_soon_threadsafe(queue.put_nowait, None)
+
+        threading.Thread(target=worker, daemon=True).start()
+
+        while True:
+            chunk = await queue.get()
+            if chunk is None:
+                break
+            yield f"data: {json.dumps(chunk)}\n\n".encode()
+        yield b"data: [DONE]\n\n"
+
+
+def create_app(llm, model_name: str) -> FastAPI:
+    app = FastAPI(
+        title="Dragon CLI — local model server",
+        description="Minimal OpenAI-compatible API for the locally loaded GGUF model. "
+                     "No MCP tool-calling here — model only.",
+        version="1.0.0",
+    )
+    app.state.llm = llm
+    app.state.model_name = model_name
+
+    app.mount("/static", StaticFiles(directory=BASE_DIR / "static"), name="static")
+
+    @app.get("/", response_class=HTMLResponse, include_in_schema=False)
+    async def index(request: Request) -> HTMLResponse:
+        return TEMPLATES.TemplateResponse(request, "index.html", {
+            "model_name": model_name,
+            "base_url": str(request.base_url).rstrip("/"),
+        })
+
+    @app.get("/chat", response_class=HTMLResponse, include_in_schema=False)
+    async def chat(request: Request) -> HTMLResponse:
+        return TEMPLATES.TemplateResponse(request, "chat.html", {
+            "model_name": model_name,
+            "base_url": str(request.base_url).rstrip("/"),
+        })
+
+    @app.get("/favicon.ico", include_in_schema=False)
+    async def favicon() -> Response:
+        return Response(FAVICON_SVG, media_type="image/svg+xml")
+
+    @app.get("/v1/models", tags=["models"], summary="List the loaded model")
+    async def list_models() -> dict:
+        return {
+            "object": "list",
+            "data": [{"id": model_name, "object": "model", "owned_by": "dragon-cli"}],
+        }
+
+    @app.post("/v1/chat/completions", tags=["chat"], summary="Create a chat completion")
+    async def chat_completions(body: ChatCompletionRequest):
+        kwargs = {
+            "messages": [m.model_dump() for m in body.messages],
+            "temperature": body.temperature,
+            "max_tokens": body.max_tokens,
+        }
+
+        if not body.stream:
+            async with _llm_lock:
+                try:
+                    return await run_in_threadpool(llm.create_chat_completion, stream=False, **kwargs)
+                except Exception as exc:
+                    raise HTTPException(status_code=500, detail=str(exc))
+
+        return StreamingResponse(_stream_chat_completion(llm, kwargs), media_type="text/event-stream")
+
+    return app
+
+
+async def run_server(llm, model_name: str, host: str, port: int) -> None:
+    app = create_app(llm, model_name)
+    config = uvicorn.Config(app, host=host, port=port, log_level="warning")
+    server = uvicorn.Server(config)
+    print(f"{C_GREEN}[serve]{C_RESET} OpenAI-compatible endpoint at "
+          f"{C_CYAN}http://{host}:{port}/v1{C_RESET}")
+    print(f"{C_GREEN}[serve]{C_RESET} Interactive docs (Swagger UI): "
+          f"{C_CYAN}http://{host}:{port}/docs{C_RESET}")
+    print(f"{C_GREEN}[serve]{C_RESET} Chat UI: "
+          f"{C_CYAN}http://{host}:{port}/chat{C_RESET}")
+    backend = getattr(llm, "exec_mode", "CPU")
+    print(f"{C_YELLOW}[serve]{C_RESET} Model: {model_name} on {backend}. Press Ctrl+C to stop.")
+    await server.serve()