llmstack-cli 0.1.0__py3-none-any.whl

llmstack/core/hardware.py

@@ -0,0 +1,131 @@
+"""Detect GPU, CPU, and RAM available on the host machine."""
+
+from __future__ import annotations
+
+import platform
+import shutil
+import subprocess
+from dataclasses import dataclass
+from typing import Literal
+
+import psutil
+
+
+@dataclass(frozen=True)
+class HardwareProfile:
+    gpu_vendor: Literal["nvidia", "amd", "apple", "none"]
+    gpu_name: str | None
+    gpu_vram_mb: int
+    cpu_cores: int
+    ram_mb: int
+    os: Literal["linux", "darwin", "windows"]
+    docker_runtime: Literal["nvidia", "default"]
+
+
+def _detect_nvidia() -> tuple[str | None, int]:
+    """Return (gpu_name, vram_mb) via nvidia-smi, or (None, 0)."""
+    if not shutil.which("nvidia-smi"):
+        return None, 0
+    try:
+        out = subprocess.check_output(
+            ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"],
+            text=True,
+            timeout=5,
+        ).strip()
+        if not out:
+            return None, 0
+        # Take the first GPU
+        line = out.splitlines()[0]
+        name, vram = line.split(",", 1)
+        return name.strip(), int(float(vram.strip()))
+    except (subprocess.SubprocessError, ValueError):
+        return None, 0
+
+
+def _detect_apple() -> tuple[str | None, int]:
+    """Return (chip_name, unified_memory_mb) on macOS."""
+    if platform.system() != "Darwin":
+        return None, 0
+    try:
+        out = subprocess.check_output(
+            ["sysctl", "-n", "machdep.cpu.brand_string"], text=True, timeout=5
+        ).strip()
+        # On Apple Silicon, unified memory = total RAM
+        ram_bytes = psutil.virtual_memory().total
+        if "Apple" in out:
+            return out, int(ram_bytes / 1024 / 1024)
+        return None, 0
+    except (subprocess.SubprocessError, ValueError):
+        return None, 0
+
+
+def _check_nvidia_docker() -> bool:
+    """Check if nvidia-container-toolkit is available."""
+    if not shutil.which("nvidia-smi"):
+        return False
+    try:
+        subprocess.check_output(
+            ["docker", "info", "--format", "{{.Runtimes}}"],
+            text=True,
+            timeout=10,
+        )
+        # If nvidia runtime exists, docker info will mention it
+        out = subprocess.check_output(
+            ["docker", "info"], text=True, timeout=10
+        )
+        return "nvidia" in out.lower()
+    except (subprocess.SubprocessError, FileNotFoundError):
+        return False
+
+
+def detect_hardware() -> HardwareProfile:
+    """Detect hardware capabilities of the host machine."""
+    os_name: Literal["linux", "darwin", "windows"]
+    sys = platform.system()
+    if sys == "Linux":
+        os_name = "linux"
+    elif sys == "Darwin":
+        os_name = "darwin"
+    else:
+        os_name = "windows"
+
+    cpu_cores = psutil.cpu_count(logical=False) or psutil.cpu_count() or 1
+    ram_mb = int(psutil.virtual_memory().total / 1024 / 1024)
+
+    # Try NVIDIA first
+    gpu_name, gpu_vram = _detect_nvidia()
+    if gpu_name:
+        has_nvidia_docker = _check_nvidia_docker()
+        return HardwareProfile(
+            gpu_vendor="nvidia",
+            gpu_name=gpu_name,
+            gpu_vram_mb=gpu_vram,
+            cpu_cores=cpu_cores,
+            ram_mb=ram_mb,
+            os=os_name,
+            docker_runtime="nvidia" if has_nvidia_docker else "default",
+        )
+
+    # Try Apple Silicon
+    gpu_name, gpu_vram = _detect_apple()
+    if gpu_name:
+        return HardwareProfile(
+            gpu_vendor="apple",
+            gpu_name=gpu_name,
+            gpu_vram_mb=gpu_vram,
+            cpu_cores=cpu_cores,
+            ram_mb=ram_mb,
+            os=os_name,
+            docker_runtime="default",
+        )
+
+    # No GPU detected
+    return HardwareProfile(
+        gpu_vendor="none",
+        gpu_name=None,
+        gpu_vram_mb=0,
+        cpu_cores=cpu_cores,
+        ram_mb=ram_mb,
+        os=os_name,
+        docker_runtime="default",
+    )

llmstack/core/health.py

@@ -0,0 +1,23 @@
+"""Health check polling for services."""
+
+from __future__ import annotations
+
+import asyncio
+
+import httpx
+
+
+async def wait_healthy(url: str, timeout: int = 120, interval: float = 2.0) -> bool:
+    """Poll a health endpoint until it returns 200 or timeout expires."""
+    elapsed = 0.0
+    async with httpx.AsyncClient(timeout=5) as client:
+        while elapsed < timeout:
+            try:
+                resp = await client.get(url)
+                if resp.status_code == 200:
+                    return True
+            except (httpx.ConnectError, httpx.ReadTimeout, httpx.ConnectTimeout):
+                pass
+            await asyncio.sleep(interval)
+            elapsed += interval
+    return False

llmstack/core/resolver.py

@@ -0,0 +1,49 @@
+"""Resolve 'auto' config values into concrete choices based on hardware."""
+
+from __future__ import annotations
+
+from llmstack.config.schema import ModelSpec, EmbeddingSpec
+from llmstack.core.hardware import HardwareProfile
+
+
+def resolve_inference_backend(model: ModelSpec, hw: HardwareProfile) -> str:
+    """Pick the best inference backend for the detected hardware."""
+    if model.backend != "auto":
+        return model.backend
+
+    # vLLM requires NVIDIA GPU with sufficient VRAM
+    if hw.gpu_vendor == "nvidia" and hw.gpu_vram_mb >= 16_000:
+        return "vllm"
+
+    # Everything else: Ollama (supports CPU, Apple Silicon, smaller GPUs)
+    return "ollama"
+
+
+def resolve_embedding_backend(spec: EmbeddingSpec, hw: HardwareProfile) -> str:
+    """Pick the best embedding backend."""
+    if spec.backend != "auto":
+        return spec.backend
+
+    # TEI (Text Embeddings Inference) works well with GPU
+    if hw.gpu_vendor == "nvidia" and hw.gpu_vram_mb >= 4_000:
+        return "tei"
+
+    # Fallback: use Ollama for embeddings too (simpler, works everywhere)
+    return "ollama"
+
+
+def resolve_quantization(model: ModelSpec, hw: HardwareProfile) -> str | None:
+    """Auto-pick quantization based on available memory."""
+    if model.quantization is not None:
+        return model.quantization
+
+    # Only auto-quantize for very large models on limited hardware
+    model_lower = model.name.lower()
+    if "70b" in model_lower:
+        if hw.gpu_vram_mb < 48_000:
+            return "q4_k_m"
+    if "13b" in model_lower or "14b" in model_lower:
+        if hw.gpu_vram_mb < 16_000:
+            return "q4_k_m"
+
+    return None

llmstack/core/stack.py

@@ -0,0 +1,207 @@
+"""Stack orchestrator — manages the full lifecycle of an llmstack deployment."""
+
+from __future__ import annotations
+
+import secrets
+
+from rich.console import Console
+from rich.table import Table
+
+from llmstack.config.schema import StackConfig
+from llmstack.core.hardware import detect_hardware
+from llmstack.core.health import wait_healthy
+from llmstack.core.resolver import resolve_inference_backend, resolve_embedding_backend
+from llmstack.docker.manager import DockerManager
+from llmstack.services.base import ServiceBase, ServiceStatus, ServiceState
+from llmstack.services.inference.ollama import OllamaService
+from llmstack.services.inference.vllm import VllmService
+from llmstack.services.embeddings.tei import TEIService
+from llmstack.services.vectordb.qdrant import QdrantService
+from llmstack.services.cache.redis import RedisService
+from llmstack.services.gateway.service import GatewayService
+from llmstack.services.observe.prometheus import PrometheusService, GrafanaService
+
+console = Console()
+
+
+class Stack:
+    """Orchestrates boot and teardown of all services."""
+
+    def __init__(self, config: StackConfig):
+        self.config = config
+        self.hw = detect_hardware()
+        self.docker = DockerManager(network_name=config.docker.network)
+        self._services: list[ServiceBase] = []
+
+    def _build_services(self) -> list[ServiceBase]:
+        """Instantiate services in boot order:
+        vectordb -> cache -> inference -> embeddings
+        """
+        services: list[ServiceBase] = []
+
+        # 1. Vector DB
+        services.append(QdrantService(self.config.services.vectors))
+
+        # 2. Cache
+        services.append(RedisService(self.config.services.cache))
+
+        # 3. Inference
+        inference_backend = resolve_inference_backend(self.config.models.chat, self.hw)
+        if inference_backend == "vllm":
+            services.append(VllmService(self.config.models.chat, self.hw))
+        else:
+            services.append(OllamaService(self.config.models.chat, self.hw))
+
+        # 4. Embeddings
+        embed_backend = resolve_embedding_backend(self.config.models.embeddings, self.hw)
+        if embed_backend == "tei":
+            services.append(TEIService(self.config.models.embeddings, self.hw))
+        # If embed_backend == "ollama", we reuse the Ollama container (no extra service)
+
+        # 5. Gateway
+        inference_url = self._resolve_inference_url(services, backend=inference_backend)
+        embeddings_url = self._resolve_embeddings_url(services, embed_backend)
+        qdrant_url = f"http://llmstack-qdrant:{self.config.services.vectors.port}"
+        redis_url = f"redis://llmstack-redis:{self.config.services.cache.port}"
+
+        services.append(GatewayService(
+            config=self.config.gateway,
+            inference_url=inference_url,
+            embeddings_url=embeddings_url,
+            qdrant_url=qdrant_url,
+            redis_url=redis_url,
+        ))
+
+        # 6. Observability (optional)
+        if self.config.observe.metrics:
+            services.append(PrometheusService(self.config.observe))
+            services.append(GrafanaService(self.config.observe))
+
+        return services
+
+    def _resolve_inference_url(self, services: list[ServiceBase], backend: str) -> str:
+        for svc in services:
+            if svc.category == "inference":
+                return svc.openai_base_url() or ""
+        return ""
+
+    def _resolve_embeddings_url(self, services: list[ServiceBase], backend: str) -> str:
+        for svc in services:
+            if svc.category == "embeddings":
+                return svc.openai_base_url() or ""
+        # Fallback to inference (Ollama can do embeddings)
+        for svc in services:
+            if svc.category == "inference":
+                return svc.openai_base_url() or ""
+        return ""
+
+    async def up(self) -> None:
+        """Boot all services in order with health checks."""
+        self._services = self._build_services()
+        self.docker.ensure_network()
+
+        # Generate API key if needed
+        if self.config.gateway.auth == "api_key" and not self.config.gateway.api_keys:
+            key = f"sk-llmstack-{secrets.token_urlsafe(24)}"
+            self.config.gateway.api_keys = [key]
+            console.print(f"\n[bold green]Generated API key:[/] {key}\n")
+
+        for svc in self._services:
+            console.print(f"  [cyan]Starting {svc.name}...[/]", end="")
+            self.docker.run_service(svc)
+
+            # Health check (skip Redis — no HTTP health endpoint)
+            if svc.category != "cache":
+                healthy = await wait_healthy(svc.health_url(), timeout=180)
+                if not healthy:
+                    console.print(" [red]FAILED[/]")
+                    raise RuntimeError(f"Service {svc.name} failed to start")
+
+            console.print(" [green]ready[/]")
+
+            # Post-start hook (e.g., pull model)
+            if svc.category == "inference":
+                model_name = self.config.models.chat.name
+                console.print(f"  [cyan]Pulling model {model_name}...[/]", end="")
+            await svc.post_start()
+            if svc.category == "inference":
+                console.print(" [green]done[/]")
+
+        # Print summary
+        self._print_summary()
+
+    def down(self, remove_volumes: bool = False) -> list[str]:
+        """Stop all services in reverse order."""
+        return self.docker.stop_all(remove_volumes=remove_volumes)
+
+    def status(self) -> list[ServiceStatus]:
+        """Get status of all managed services."""
+        containers = self.docker.list_services()
+        result = []
+        for info in containers:
+            state = ServiceState.RUNNING if info["status"] == "running" else ServiceState.STOPPED
+            ports = info.get("ports", {})
+            port = None
+            if ports:
+                first = list(ports.values())[0]
+                if first and isinstance(first, list) and first:
+                    port = first[0].get("HostPort")
+
+            result.append(ServiceStatus(
+                name=info["name"],
+                state=state,
+                port=int(port) if port else None,
+                container_id=info["container_id"],
+            ))
+        return result
+
+    def _get_inference_url(self) -> str:
+        """Get the OpenAI base URL for the inference service."""
+        for svc in self._services:
+            if svc.category == "inference":
+                return svc.openai_base_url() or ""
+        return ""
+
+    def _get_embeddings_url(self) -> str:
+        """Get the embeddings URL."""
+        for svc in self._services:
+            if svc.category == "embeddings":
+                return svc.openai_base_url() or ""
+        # Fallback: use Ollama for embeddings
+        for svc in self._services:
+            if svc.category == "inference" and isinstance(svc, OllamaService):
+                return svc.openai_base_url() or ""
+        return ""
+
+    def _print_summary(self) -> None:
+        """Print a summary table of running services."""
+        table = Table(title="LLMStack Services", show_header=True)
+        table.add_column("Service", style="cyan")
+        table.add_column("Category")
+        table.add_column("Status", style="green")
+        table.add_column("URL")
+
+        for svc in self._services:
+            url = svc.health_url()
+            # Clean up URL for display
+            for suffix in ["/healthz", "/health", "/api/tags"]:
+                url = url.replace(suffix, "")
+            table.add_row(svc.name, svc.category, "running", url)
+
+        console.print()
+        console.print(table)
+
+        # Print usage hint
+        inference_svc = next((s for s in self._services if s.category == "inference"), None)
+        if inference_svc:
+            base = inference_svc.health_url()
+            for suffix in ["/healthz", "/health", "/api/tags"]:
+                base = base.replace(suffix, "")
+            model = self.config.models.chat.name
+            console.print("\n[bold]Try it:[/]")
+            console.print(
+                f"  curl {base}/v1/chat/completions \\\n"
+                f"    -H 'Content-Type: application/json' \\\n"
+                f"    -d '{{\"model\":\"{model}\",\"messages\":[{{\"role\":\"user\",\"content\":\"Hello!\"}}]}}'"
+            )
+        console.print()

llmstack/docker/manager.py

@@ -0,0 +1,134 @@
+"""Docker container lifecycle management."""
+
+from __future__ import annotations
+
+from typing import Iterator
+
+import docker
+from docker.errors import NotFound, APIError
+from docker.models.containers import Container
+
+from llmstack.services.base import ServiceBase
+
+
+class DockerManager:
+    """Wraps Docker SDK to manage llmstack containers."""
+
+    LABEL_MANAGED = "llmstack.managed"
+    LABEL_SERVICE = "llmstack.service"
+
+    def __init__(self, network_name: str = "llmstack_net"):
+        try:
+            self.client = docker.from_env()
+            self.client.ping()
+        except docker.errors.DockerException as exc:
+            raise SystemExit(
+                "Cannot connect to Docker daemon. Is Docker running?\n"
+                "Install: https://docs.docker.com/get-docker/"
+            ) from exc
+        self.network_name = network_name
+
+    def ensure_network(self) -> None:
+        """Create the bridge network if it doesn't exist."""
+        try:
+            self.client.networks.get(self.network_name)
+        except NotFound:
+            self.client.networks.create(self.network_name, driver="bridge")
+
+    def run_service(self, service: ServiceBase) -> Container:
+        """Start a container for a service. Removes any existing container with the same name."""
+        spec = service.container_spec()
+        name = spec.pop("name", f"llmstack-{service.name}")
+
+        # Remove existing container if present
+        try:
+            existing = self.client.containers.get(name)
+            existing.stop(timeout=10)
+            existing.remove(force=True)
+        except NotFound:
+            pass
+
+        labels = {
+            self.LABEL_MANAGED: "true",
+            self.LABEL_SERVICE: service.name,
+        }
+
+        container = self.client.containers.run(
+            detach=True,
+            name=name,
+            network=self.network_name,
+            labels=labels,
+            **spec,
+        )
+        return container
+
+    def stop_service(self, service_name: str) -> None:
+        """Stop and remove a container by service name."""
+        for container in self._managed_containers():
+            if container.labels.get(self.LABEL_SERVICE) == service_name:
+                container.stop(timeout=10)
+                container.remove(force=True)
+                return
+
+    def stop_all(self, remove_volumes: bool = False) -> list[str]:
+        """Stop and remove all llmstack containers. Returns names of stopped containers."""
+        stopped = []
+        for container in self._managed_containers():
+            name = container.name
+            container.stop(timeout=10)
+            container.remove(force=True)
+            stopped.append(name)
+
+        if remove_volumes:
+            for vol in self.client.volumes.list():
+                if vol.name.startswith("llmstack_"):
+                    try:
+                        vol.remove(force=True)
+                    except APIError:
+                        pass
+
+        # Remove network
+        try:
+            net = self.client.networks.get(self.network_name)
+            net.remove()
+        except (NotFound, APIError):
+            pass
+
+        return stopped
+
+    def get_container(self, service_name: str) -> Container | None:
+        """Find a running container by service name."""
+        for container in self._managed_containers():
+            if container.labels.get(self.LABEL_SERVICE) == service_name:
+                return container
+        return None
+
+    def stream_logs(self, service_name: str, follow: bool = True, tail: int = 50) -> Iterator[str]:
+        """Yield decoded log lines from a service container."""
+        container = self.get_container(service_name)
+        if container is None:
+            raise ValueError(f"No running container for service '{service_name}'")
+
+        for chunk in container.logs(stream=True, follow=follow, tail=tail):
+            yield chunk.decode("utf-8", errors="replace")
+
+    def list_services(self) -> list[dict]:
+        """Return info about all managed containers."""
+        result = []
+        for container in self._managed_containers():
+            container.reload()
+            result.append({
+                "name": container.labels.get(self.LABEL_SERVICE, "unknown"),
+                "container_name": container.name,
+                "container_id": container.short_id,
+                "status": container.status,
+                "ports": container.ports,
+            })
+        return result
+
+    def _managed_containers(self) -> list[Container]:
+        """List all containers with the llmstack.managed label."""
+        return self.client.containers.list(
+            all=True,
+            filters={"label": f"{self.LABEL_MANAGED}=true"},
+        )

llmstack/gateway/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN pip install --no-cache-dir \
+    fastapi>=0.115 \
+    uvicorn[standard]>=0.30 \
+    httpx>=0.27 \
+    starlette>=0.40
+
+COPY . /app/llmstack/gateway/
+ENV PYTHONPATH=/app
+
+EXPOSE 8000
+
+CMD ["uvicorn", "llmstack.gateway.main:app", "--host", "0.0.0.0", "--port", "8000"]

llmstack/gateway/main.py

@@ -0,0 +1,52 @@
+"""LLMStack Gateway — OpenAI-compatible API gateway."""
+
+from __future__ import annotations
+
+import os
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from llmstack.gateway.routes.chat import router as chat_router
+from llmstack.gateway.routes.embeddings import router as embeddings_router
+from llmstack.gateway.routes.models import router as models_router
+from llmstack.gateway.routes.health import router as health_router
+from llmstack.gateway.middleware.auth import AuthMiddleware
+from llmstack.gateway.middleware.metrics import MetricsMiddleware
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(
+        title="LLMStack Gateway",
+        description="OpenAI-compatible API gateway for LLMStack",
+        version="0.1.0",
+    )
+
+    # CORS
+    cors_origins = os.getenv("LLMSTACK_CORS_ORIGINS", "*").split(",")
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=cors_origins,
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Auth
+    api_keys = os.getenv("LLMSTACK_API_KEYS", "")
+    if api_keys:
+        app.add_middleware(AuthMiddleware, api_keys=api_keys.split(","))
+
+    # Metrics
+    app.add_middleware(MetricsMiddleware)
+
+    # Routes
+    app.include_router(chat_router, prefix="/v1")
+    app.include_router(embeddings_router, prefix="/v1")
+    app.include_router(models_router, prefix="/v1")
+    app.include_router(health_router)
+
+    return app
+
+
+app = create_app()

llmstack/gateway/middleware/auth.py

@@ -0,0 +1,32 @@
+"""API key authentication middleware."""
+
+from __future__ import annotations
+
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+
+
+class AuthMiddleware(BaseHTTPMiddleware):
+    def __init__(self, app, api_keys: list[str]):
+        super().__init__(app)
+        self.api_keys = set(k.strip() for k in api_keys if k.strip())
+
+    async def dispatch(self, request: Request, call_next):
+        # Skip auth for health checks and docs
+        if request.url.path in ("/healthz", "/metrics", "/docs", "/openapi.json"):
+            return await call_next(request)
+
+        if not self.api_keys:
+            return await call_next(request)
+
+        auth = request.headers.get("Authorization", "")
+        if auth.startswith("Bearer "):
+            token = auth[7:]
+            if token in self.api_keys:
+                return await call_next(request)
+
+        return JSONResponse(
+            status_code=401,
+            content={"error": {"message": "Invalid API key", "type": "auth_error"}},
+        )