llmstack-cli 0.1.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

llmstack/__init__.py

@@ -1,3 +1,3 @@
 """LLMStack — One command. Full LLM stack. Zero config."""
 
-__version__ = "0.1.0"
+__version__ = "0.3.0"

llmstack/cli/app.py

@@ -80,6 +80,24 @@ def logs(
     _logs(service=service, follow=follow, tail=tail)
 
 
+@app.command()
+def chat(
+    model: str = typer.Option(None, "--model", "-m", help="Model name to chat with"),
+) -> None:
+    """Interactive chat with your local LLM."""
+    from llmstack.cli.commands.chat import chat as _chat
+    _chat(model=model)
+
+
+@app.command()
+def export(
+    output: str = typer.Option("docker-compose.yml", "--output", "-o", help="Output file path"),
+) -> None:
+    """Export llmstack.yaml as a standalone docker-compose.yml."""
+    from llmstack.cli.commands.export import export as _export
+    _export(output=output)
+
+
 @app.command()
 def doctor() -> None:
     """Check system requirements and diagnose issues."""

llmstack/cli/commands/chat.py

@@ -0,0 +1,104 @@
+"""llmstack chat — interactive terminal chat with your local LLM."""
+
+from __future__ import annotations
+
+import json
+import sys
+
+import httpx
+
+from llmstack.cli.console import console
+from llmstack.config.loader import load_config
+
+
+def chat(model: str | None = None) -> None:
+    """Start an interactive chat session with the running LLM."""
+    config = load_config()
+    base_url = f"http://localhost:{config.gateway.port}"
+    model_name = model or config.models.chat.name
+
+    # Get API key
+    api_key = config.gateway.api_keys[0] if config.gateway.api_keys else ""
+    headers = {"Content-Type": "application/json"}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    # Verify gateway is reachable
+    try:
+        resp = httpx.get(f"{base_url}/healthz", headers=headers, timeout=5)
+        if resp.status_code != 200:
+            console.print("[error]Gateway is not healthy. Run 'llmstack up' first.[/]")
+            sys.exit(1)
+    except httpx.ConnectError:
+        console.print("[error]Cannot connect to gateway. Run 'llmstack up' first.[/]")
+        sys.exit(1)
+
+    console.print(f"\n[bold]LLMStack Chat[/] — model: [cyan]{model_name}[/]")
+    console.print("[dim]Type 'exit' or Ctrl+C to quit. '/clear' to reset conversation.[/]\n")
+
+    messages: list[dict[str, str]] = []
+
+    while True:
+        try:
+            user_input = console.input("[bold green]You:[/] ")
+        except (KeyboardInterrupt, EOFError):
+            console.print("\n[dim]Goodbye![/]")
+            break
+
+        text = user_input.strip()
+        if not text:
+            continue
+        if text.lower() in ("exit", "quit"):
+            console.print("[dim]Goodbye![/]")
+            break
+        if text == "/clear":
+            messages.clear()
+            console.print("[dim]Conversation cleared.[/]\n")
+            continue
+
+        messages.append({"role": "user", "content": text})
+
+        # Stream response
+        console.print("[bold cyan]Assistant:[/] ", end="")
+        assistant_text = ""
+
+        try:
+            with httpx.stream(
+                "POST",
+                f"{base_url}/v1/chat/completions",
+                headers=headers,
+                json={"model": model_name, "messages": messages, "stream": True},
+                timeout=httpx.Timeout(300, connect=10),
+            ) as resp:
+                if resp.status_code != 200:
+                    console.print(f"[error]Error {resp.status_code}[/]")
+                    messages.pop()
+                    continue
+
+                for line in resp.iter_lines():
+                    if not line.startswith("data: "):
+                        continue
+                    data = line[6:]
+                    if data.strip() == "[DONE]":
+                        break
+                    try:
+                        chunk = json.loads(data)
+                        delta = chunk["choices"][0].get("delta", {})
+                        token = delta.get("content", "")
+                        if token:
+                            print(token, end="", flush=True)
+                            assistant_text += token
+                    except (json.JSONDecodeError, KeyError, IndexError):
+                        continue
+
+        except httpx.ConnectError:
+            console.print("[error]Connection lost.[/]")
+            messages.pop()
+            continue
+        except KeyboardInterrupt:
+            pass
+
+        print()  # newline after streamed response
+        if assistant_text:
+            messages.append({"role": "assistant", "content": assistant_text})
+        print()

llmstack/cli/commands/export.py

@@ -0,0 +1,190 @@
+"""llmstack export — generate a standalone docker-compose.yml from llmstack.yaml."""
+
+from __future__ import annotations
+
+import yaml
+
+from llmstack.cli.console import console
+from llmstack.config.loader import load_config
+from llmstack.config.schema import StackConfig
+from llmstack.core.hardware import detect_hardware
+from llmstack.core.resolver import resolve_inference_backend, resolve_embedding_backend
+
+
+def _build_compose(config: StackConfig) -> dict:
+    """Convert StackConfig into a docker-compose.yml dict."""
+    hw = detect_hardware()
+    inference_backend = resolve_inference_backend(config.models.chat, hw)
+    embed_backend = resolve_embedding_backend(config.models.embeddings, hw)
+
+    services: dict = {}
+    volumes: dict = {}
+    network_name = config.docker.network
+
+    # 1. Qdrant
+    services["qdrant"] = {
+        "image": "qdrant/qdrant:latest",
+        "container_name": "llmstack-qdrant",
+        "ports": [f"{config.services.vectors.port}:6333", f"{config.services.vectors.port + 1}:6334"],
+        "volumes": ["qdrant_data:/qdrant/storage"],
+        "restart": "unless-stopped",
+        "networks": [network_name],
+    }
+    volumes["qdrant_data"] = None
+
+    # 2. Redis
+    services["redis"] = {
+        "image": "redis:7-alpine",
+        "container_name": "llmstack-redis",
+        "ports": [f"{config.services.cache.port}:6379"],
+        "command": f"redis-server --maxmemory {config.services.cache.max_memory} --maxmemory-policy allkeys-lru",
+        "restart": "unless-stopped",
+        "networks": [network_name],
+    }
+
+    # 3. Inference
+    if inference_backend == "vllm":
+        model = config.models.chat
+        cmd = f"--model {model.name} --host 0.0.0.0 --port 8000 --max-model-len {model.context_length}"
+        if model.quantization:
+            cmd += f" --quantization {model.quantization}"
+        services["vllm"] = {
+            "image": "vllm/vllm-openai:latest",
+            "container_name": "llmstack-vllm",
+            "ports": ["8001:8000"],
+            "command": cmd,
+            "volumes": ["vllm_cache:/root/.cache/huggingface"],
+            "shm_size": "4g",
+            "deploy": {"resources": {"reservations": {"devices": [{"driver": "nvidia", "count": "all", "capabilities": ["gpu"]}]}}},
+            "restart": "unless-stopped",
+            "networks": [network_name],
+        }
+        volumes["vllm_cache"] = None
+        inference_url = "http://vllm:8000/v1"
+    else:
+        svc: dict = {
+            "image": "ollama/ollama:latest",
+            "container_name": "llmstack-ollama",
+            "ports": ["11434:11434"],
+            "volumes": ["ollama_data:/root/.ollama"],
+            "restart": "unless-stopped",
+            "networks": [network_name],
+        }
+        if hw.gpu_vendor == "nvidia":
+            svc["deploy"] = {"resources": {"reservations": {"devices": [{"driver": "nvidia", "count": "all", "capabilities": ["gpu"]}]}}}
+        services["ollama"] = svc
+        volumes["ollama_data"] = None
+        inference_url = "http://ollama:11434/v1"
+
+    # 4. Embeddings
+    embeddings_url = inference_url  # fallback: use Ollama for embeddings
+    if embed_backend == "tei":
+        tei_image = "ghcr.io/huggingface/text-embeddings-inference:cpu-latest"
+        tei_svc: dict = {
+            "image": tei_image,
+            "container_name": "llmstack-tei",
+            "ports": ["8002:80"],
+            "command": f"--model-id {config.models.embeddings.name} --port 80",
+            "volumes": ["tei_cache:/data"],
+            "restart": "unless-stopped",
+            "networks": [network_name],
+        }
+        if hw.gpu_vendor == "nvidia":
+            tei_svc["image"] = "ghcr.io/huggingface/text-embeddings-inference:latest"
+            tei_svc["deploy"] = {"resources": {"reservations": {"devices": [{"driver": "nvidia", "count": "all", "capabilities": ["gpu"]}]}}}
+        services["tei"] = tei_svc
+        volumes["tei_cache"] = None
+        embeddings_url = "http://tei:80/v1"
+
+    # 5. Gateway
+    qdrant_url = f"http://qdrant:{config.services.vectors.port}"
+    redis_url = f"redis://redis:{config.services.cache.port}"
+    api_keys = ",".join(config.gateway.api_keys) if config.gateway.api_keys else ""
+
+    services["gateway"] = {
+        "build": {
+            "context": ".",
+            "dockerfile": "src/llmstack/gateway/Dockerfile",
+        },
+        "container_name": "llmstack-gateway",
+        "ports": [f"{config.gateway.port}:8000"],
+        "environment": {
+            "LLMSTACK_INFERENCE_URL": inference_url,
+            "LLMSTACK_EMBEDDINGS_URL": embeddings_url,
+            "LLMSTACK_QDRANT_URL": qdrant_url,
+            "LLMSTACK_REDIS_URL": redis_url,
+            "LLMSTACK_API_KEYS": api_keys,
+            "LLMSTACK_CORS_ORIGINS": ",".join(config.gateway.cors),
+            "LLMSTACK_REQUEST_TIMEOUT": str(config.gateway.request_timeout),
+            "LLMSTACK_RATE_LIMIT": config.gateway.rate_limit,
+        },
+        "depends_on": ["qdrant", "redis"],
+        "restart": "unless-stopped",
+        "networks": [network_name],
+    }
+    # Add inference dependency
+    if inference_backend == "vllm":
+        services["gateway"]["depends_on"].append("vllm")
+    else:
+        services["gateway"]["depends_on"].append("ollama")
+    if embed_backend == "tei":
+        services["gateway"]["depends_on"].append("tei")
+
+    # 6. Observability
+    if config.observe.metrics:
+        services["prometheus"] = {
+            "image": "prom/prometheus:latest",
+            "container_name": "llmstack-prometheus",
+            "ports": ["9090:9090"],
+            "command": [
+                "--config.file=/etc/prometheus/prometheus.yml",
+                f"--storage.tsdb.retention.time={config.observe.retention}",
+                "--web.enable-lifecycle",
+            ],
+            "volumes": [
+                "./prometheus.yml:/etc/prometheus/prometheus.yml:ro",
+                "prometheus_data:/prometheus",
+            ],
+            "restart": "unless-stopped",
+            "networks": [network_name],
+        }
+        volumes["prometheus_data"] = None
+
+        services["grafana"] = {
+            "image": "grafana/grafana:latest",
+            "container_name": "llmstack-grafana",
+            "ports": [f"{config.observe.dashboard_port}:3000"],
+            "environment": {
+                "GF_SECURITY_ADMIN_USER": "admin",
+                "GF_SECURITY_ADMIN_PASSWORD": "llmstack",
+                "GF_AUTH_ANONYMOUS_ENABLED": "true",
+                "GF_AUTH_ANONYMOUS_ORG_ROLE": "Viewer",
+            },
+            "restart": "unless-stopped",
+            "networks": [network_name],
+        }
+
+    compose = {
+        "version": "3.8",
+        "services": services,
+        "volumes": volumes,
+        "networks": {network_name: {"driver": "bridge"}},
+    }
+
+    return compose
+
+
+def export(output: str = "docker-compose.yml") -> None:
+    """Generate a docker-compose.yml from the current llmstack.yaml."""
+    config = load_config()
+    compose = _build_compose(config)
+
+    # Clean up None volume values for yaml output
+    compose["volumes"] = {k: {} for k in compose["volumes"]}
+
+    with open(output, "w") as f:
+        yaml.dump(compose, f, default_flow_style=False, sort_keys=False)
+
+    service_count = len(compose["services"])
+    console.print(f"\n[success]Exported {service_count} services to {output}[/]")
+    console.print(f"Run with: [bold]docker compose -f {output} up -d[/]\n")

llmstack/core/health.py

@@ -16,7 +16,7 @@ async def wait_healthy(url: str, timeout: int = 120, interval: float = 2.0) -> b
                 resp = await client.get(url)
                 if resp.status_code == 200:
                     return True
-            except (httpx.ConnectError, httpx.ReadTimeout, httpx.ConnectTimeout):
+            except httpx.HTTPError:
                 pass
             await asyncio.sleep(interval)
             elapsed += interval

llmstack/core/stack.py

@@ -7,6 +7,7 @@ import secrets
 from rich.console import Console
 from rich.table import Table
 
+from llmstack.config.loader import save_config
 from llmstack.config.schema import StackConfig
 from llmstack.core.hardware import detect_hardware
 from llmstack.core.health import wait_healthy
@@ -100,11 +101,21 @@ class Stack:
         self._services = self._build_services()
         self.docker.ensure_network()
 
+        # Build local images if needed
+        for svc in self._services:
+            if hasattr(svc, "build_info") and svc.build_info():
+                info = svc.build_info()
+                console.print(f"  [cyan]Building {svc.name} image...[/]", end="")
+                self.docker.build_image(**info)
+                console.print(" [green]done[/]")
+
         # Generate API key if needed
         if self.config.gateway.auth == "api_key" and not self.config.gateway.api_keys:
             key = f"sk-llmstack-{secrets.token_urlsafe(24)}"
             self.config.gateway.api_keys = [key]
-            console.print(f"\n[bold green]Generated API key:[/] {key}\n")
+            save_config(self.config)
+            console.print(f"\n[bold green]Generated API key:[/] {key}")
+            console.print("[dim]Saved to llmstack.yaml[/]\n")
 
         for svc in self._services:
             console.print(f"  [cyan]Starting {svc.name}...[/]", end="")

llmstack/docker/manager.py

@@ -35,6 +35,11 @@ class DockerManager:
         except NotFound:
             self.client.networks.create(self.network_name, driver="bridge")
 
+    def build_image(self, path: str, dockerfile: str, tag: str) -> str:
+        """Build a Docker image from a local Dockerfile. Returns the image tag."""
+        self.client.images.build(path=path, dockerfile=dockerfile, tag=tag, rm=True)
+        return tag
+
     def run_service(self, service: ServiceBase) -> Container:
         """Start a container for a service. Removes any existing container with the same name."""
         spec = service.container_spec()
@@ -53,13 +58,24 @@ class DockerManager:
             self.LABEL_SERVICE: service.name,
         }
 
-        container = self.client.containers.run(
-            detach=True,
-            name=name,
-            network=self.network_name,
-            labels=labels,
-            **spec,
-        )
+        try:
+            container = self.client.containers.run(
+                detach=True,
+                name=name,
+                network=self.network_name,
+                labels=labels,
+                **spec,
+            )
+        except APIError as exc:
+            msg = str(exc)
+            if "port is already allocated" in msg or "address already in use" in msg:
+                ports = spec.get("ports", {})
+                port = next(iter(ports.values()), "unknown") if ports else "unknown"
+                raise SystemExit(
+                    f"Port {port} is already in use. "
+                    f"Stop the conflicting service or change the port in llmstack.yaml."
+                ) from exc
+            raise
         return container
 
     def stop_service(self, service_name: str) -> None:

llmstack/gateway/Dockerfile

@@ -6,9 +6,13 @@ RUN pip install --no-cache-dir \
     fastapi>=0.115 \
     uvicorn[standard]>=0.30 \
     httpx>=0.27 \
-    starlette>=0.40
+    starlette>=0.40 \
+    redis>=5.0 \
+    pydantic>=2.0
+
+COPY src/llmstack/__init__.py /app/llmstack/__init__.py
+COPY src/llmstack/gateway/ /app/llmstack/gateway/
 
-COPY . /app/llmstack/gateway/
 ENV PYTHONPATH=/app
 
 EXPOSE 8000

llmstack/gateway/cache.py

@@ -0,0 +1,187 @@
+"""Semantic response cache backed by Redis.
+
+Caches LLM completions keyed by a hash of (model + messages).
+Supports TTL-based expiration and cache hit/miss metrics.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import time
+from dataclasses import dataclass, field
+
+import redis.asyncio as aioredis
+
+REDIS_URL = os.getenv("LLMSTACK_REDIS_URL", "")
+CACHE_TTL = int(os.getenv("LLMSTACK_CACHE_TTL", "3600"))  # 1 hour default
+CACHE_ENABLED = os.getenv("LLMSTACK_CACHE_ENABLED", "true").lower() == "true"
+
+# Prefix to avoid key collisions
+_KEY_PREFIX = "llmstack:cache:"
+_STATS_KEY = "llmstack:cache:stats"
+
+
+@dataclass
+class CacheStats:
+    hits: int = 0
+    misses: int = 0
+    evictions: int = 0
+    avg_hit_latency_ms: float = 0.0
+    _hit_latencies: list[float] = field(default_factory=list, repr=False)
+
+    def record_hit(self, latency_ms: float) -> None:
+        self.hits += 1
+        self._hit_latencies.append(latency_ms)
+        self.avg_hit_latency_ms = sum(self._hit_latencies) / len(self._hit_latencies)
+
+    def record_miss(self) -> None:
+        self.misses += 1
+
+    @property
+    def hit_rate(self) -> float:
+        total = self.hits + self.misses
+        return self.hits / total if total > 0 else 0.0
+
+    def to_dict(self) -> dict:
+        return {
+            "hits": self.hits,
+            "misses": self.misses,
+            "hit_rate": round(self.hit_rate, 4),
+            "avg_hit_latency_ms": round(self.avg_hit_latency_ms, 2),
+        }
+
+
+class ResponseCache:
+    """Redis-backed LLM response cache with semantic key hashing."""
+
+    def __init__(self, redis_url: str = "", ttl: int = CACHE_TTL):
+        self._url = redis_url or REDIS_URL
+        self._ttl = ttl
+        self._redis: aioredis.Redis | None = None
+        self._stats = CacheStats()
+        self._connected = False
+
+    async def connect(self) -> None:
+        """Lazily connect to Redis."""
+        if self._connected or not self._url:
+            return
+        try:
+            self._redis = aioredis.from_url(
+                self._url,
+                decode_responses=True,
+                socket_connect_timeout=3,
+                socket_timeout=5,
+                retry_on_timeout=True,
+            )
+            await self._redis.ping()
+            self._connected = True
+        except Exception:
+            self._redis = None
+            self._connected = False
+
+    async def close(self) -> None:
+        if self._redis:
+            await self._redis.aclose()
+            self._connected = False
+
+    @staticmethod
+    def _build_cache_key(model: str, messages: list[dict], temperature: float = 1.0) -> str:
+        """Deterministic hash of the request for cache lookup.
+
+        Only caches when temperature <= 0.1 (near-deterministic outputs).
+        For higher temperatures, returns empty string (skip cache).
+        """
+        if temperature > 0.1:
+            return ""
+
+        # Normalize messages to a stable representation
+        normalized = []
+        for msg in messages:
+            normalized.append({
+                "role": msg.get("role", ""),
+                "content": msg.get("content", ""),
+            })
+
+        payload = json.dumps({"model": model, "messages": normalized}, sort_keys=True)
+        digest = hashlib.sha256(payload.encode()).hexdigest()
+        return f"{_KEY_PREFIX}{digest}"
+
+    async def get(self, model: str, messages: list[dict], temperature: float = 1.0) -> dict | None:
+        """Look up a cached response. Returns None on miss."""
+        if not self._connected or not CACHE_ENABLED:
+            return None
+
+        key = self._build_cache_key(model, messages, temperature)
+        if not key:
+            self._stats.record_miss()
+            return None
+
+        start = time.monotonic()
+        try:
+            raw = await self._redis.get(key)  # type: ignore[union-attr]
+            if raw is None:
+                self._stats.record_miss()
+                return None
+
+            latency_ms = (time.monotonic() - start) * 1000
+            self._stats.record_hit(latency_ms)
+
+            cached = json.loads(raw)
+            # Mark response as cached
+            cached["_cached"] = True
+            cached["_cache_age_s"] = int(time.time() - cached.get("_cached_at", 0))
+            return cached
+        except Exception:
+            self._stats.record_miss()
+            return None
+
+    async def put(self, model: str, messages: list[dict], response: dict,
+                  temperature: float = 1.0) -> None:
+        """Store a response in the cache."""
+        if not self._connected or not CACHE_ENABLED:
+            return
+
+        key = self._build_cache_key(model, messages, temperature)
+        if not key:
+            return
+
+        try:
+            response["_cached_at"] = int(time.time())
+            raw = json.dumps(response)
+            await self._redis.set(key, raw, ex=self._ttl)  # type: ignore[union-attr]
+        except Exception:
+            pass  # Cache write failure is non-fatal
+
+    async def invalidate(self, pattern: str = "*") -> int:
+        """Delete cache entries matching a pattern. Returns count deleted."""
+        if not self._connected:
+            return 0
+        try:
+            keys = []
+            async for key in self._redis.scan_iter(f"{_KEY_PREFIX}{pattern}"):  # type: ignore[union-attr]
+                keys.append(key)
+            if keys:
+                return await self._redis.delete(*keys)  # type: ignore[union-attr]
+            return 0
+        except Exception:
+            return 0
+
+    @property
+    def stats(self) -> CacheStats:
+        return self._stats
+
+
+# Module-level singleton
+_cache: ResponseCache | None = None
+
+
+async def get_cache() -> ResponseCache:
+    """Get or create the global cache instance."""
+    global _cache
+    if _cache is None:
+        _cache = ResponseCache()
+    if not _cache._connected:
+        await _cache.connect()
+    return _cache