llamawatch 0.6.0__py3-none-any.whl

llamawatch/__init__.py

@@ -0,0 +1,2 @@
+"""llamawatch — self-hosted ops dashboard for local LLMs."""
+__version__ = "0.6.0"

llamawatch/__main__.py

@@ -0,0 +1,73 @@
+"""llamawatch CLI entry point."""
+
+import argparse
+import os
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="llamawatch", description="Self-hosted ops dashboard for local LLMs")
+    subparsers = parser.add_subparsers(dest="command")
+
+    init_parser = subparsers.add_parser("init", help="Auto-detect backends and write config")
+    init_parser.add_argument("--guided", action="store_true", help="Interactive setup wizard")
+
+    parser.add_argument("--port", type=int, help="Override port")
+    parser.add_argument("--host", type=str, help="Override host")
+    parser.add_argument("--no-auth", action="store_true", help="Disable auth for this session")
+    parser.add_argument("--sample", type=str, metavar="FILE",
+                        help="Serve a fixed JSON snapshot (demo/tests); config is read from the file's folder")
+
+    args = parser.parse_args()
+
+    if args.command == "init":
+        from .auto_detect import run_init
+        run_init(guided=args.guided)
+        return
+
+    cli_overrides = {}
+    if args.port:
+        cli_overrides["port"] = args.port
+    if args.host:
+        cli_overrides["host"] = args.host
+    if args.no_auth:
+        cli_overrides["auth_enabled"] = False
+
+    from .config import load_config
+    config_dir = None
+    if args.sample:
+        import os
+        from pathlib import Path
+        from .sample import ENV
+        sample_path = Path(args.sample).resolve()
+        os.environ[ENV] = str(sample_path)
+        config_dir = sample_path.parent
+    config = load_config(cli_overrides, config_dir=config_dir)
+
+    host = config.get("host", "127.0.0.1")
+    port = config.get("port", 8400)
+
+    # Secure-by-default warning: exposing to the network with no password means
+    # only the localhost-gated dangerous actions are protected; reads are open.
+    loopback = host in ("127.0.0.1", "::1", "localhost")
+    if not loopback and not config.get("auth_enabled"):
+        print(
+            "\n  ⚠  llamawatch is bound to a NETWORK address (" + str(host) + ") with NO password.\n"
+            "     Anyone on your network can view the dashboard. Shell/Docker actions stay\n"
+            "     localhost-only, but you should set a password: Settings → General → Authentication.\n",
+            file=sys.stderr,
+        )
+
+    import uvicorn
+    # Give open streams (SSE, websockets, terminals) five seconds to finish,
+    # then leave hard: a leftover worker thread must not hang the stop until
+    # systemd aborts the process and writes a core dump.
+    uvicorn.run("llamawatch.server:app", host=host, port=port, log_level="info",
+                timeout_graceful_shutdown=5)
+    sys.stdout.flush()
+    sys.stderr.flush()
+    os._exit(0)
+
+
+if __name__ == "__main__":
+    main()

llamawatch/adapters/__init__.py

@@ -0,0 +1,108 @@
+"""Backend adapter registry for LlamaWatch.
+
+Provides AdapterRegistry factory that creates the right adapter
+based on backend type configuration.
+"""
+
+from .base import BackendAdapter
+from .llamacpp import LlamaCppAdapter
+from .ollama import OllamaAdapter
+from .openai_compat import OpenAICompatAdapter
+
+_ADAPTER_TYPES: dict[str, type[BackendAdapter]] = {
+    "llamacpp": LlamaCppAdapter,
+    "ollama": OllamaAdapter,
+    "openai": OpenAICompatAdapter,
+}
+
+
+def create_adapter(backend_config: dict, top_config: dict | None = None) -> BackendAdapter:
+    """Module-level factory: create a single adapter from a backend config dict.
+
+    Parameters
+    ----------
+    backend_config:
+        A single backend entry (type, url, name, …).
+    top_config:
+        Optional top-level config used to inherit ``model_names``.
+
+    Returns
+    -------
+    BackendAdapter
+        Concrete adapter instance (LlamaCppAdapter, OllamaAdapter, …).
+    """
+    top_config = top_config or {}
+    backend_type = backend_config.get("type", "openai")
+    url = backend_config.get("url", "")
+
+    adapter_cls = _ADAPTER_TYPES.get(backend_type, OpenAICompatAdapter)
+
+    merged_config = {**backend_config}
+    if "model_names" not in merged_config and "model_names" in top_config:
+        merged_config["model_names"] = top_config["model_names"]
+
+    return adapter_cls(url=url, config=merged_config)
+
+
+class AdapterRegistry:
+    """Factory that creates and manages backend adapters from config."""
+
+    def __init__(self, config: dict) -> None:
+        self._adapters: list[BackendAdapter] = []
+        self._by_name: dict[str, BackendAdapter] = {}
+        self._top_config = config
+        self._load(config)
+
+    # ------------------------------------------------------------------
+    # Public dict-style view (name → adapter) for hot-reload consumers
+    # ------------------------------------------------------------------
+
+    @property
+    def adapters(self) -> dict[str, BackendAdapter]:
+        """Dict mapping backend name → adapter (live view, not a copy)."""
+        return self._by_name
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    def _load(self, config: dict) -> None:
+        """Populate internal lists from *config* without clearing first."""
+        backends = config.get("backends", [])
+        for backend_cfg in backends:
+            name = backend_cfg.get("name", backend_cfg.get("url", ""))
+            adapter = create_adapter(backend_cfg, top_config=config)
+            self._adapters.append(adapter)
+            self._by_name[name] = adapter
+
+    # ------------------------------------------------------------------
+    # Hot-reload
+    # ------------------------------------------------------------------
+
+    def rebuild(self, config: dict) -> None:
+        """Replace all adapters in-place from a new config dict.
+
+        Existing adapter objects are discarded and recreated from *config*.
+        All references to ``registry.adapters`` (the dict) will see the
+        updated entries because we mutate the same dict object.
+        """
+        self._adapters.clear()
+        self._by_name.clear()
+        self._top_config = config
+        self._load(config)
+
+    # ------------------------------------------------------------------
+    # Public accessors (unchanged API)
+    # ------------------------------------------------------------------
+
+    def get_primary(self) -> BackendAdapter | None:
+        """Return the first adapter, or None if no backends configured."""
+        return self._adapters[0] if self._adapters else None
+
+    def get_all(self) -> list[BackendAdapter]:
+        """Return all adapters."""
+        return list(self._adapters)
+
+    def get_by_name(self, name: str) -> BackendAdapter | None:
+        """Return adapter matching the given name, or None."""
+        return self._by_name.get(name)

llamawatch/adapters/base.py

@@ -0,0 +1,43 @@
+"""Backend adapter base class for LlamaWatch."""
+
+from abc import ABC, abstractmethod
+from typing import AsyncGenerator
+
+
+class BackendAdapter(ABC):
+    """Abstract base for all LLM backend adapters."""
+
+    def __init__(self, url: str, config: dict) -> None:
+        self.url: str = url.rstrip("/")
+        self.config: dict = config
+        self._model_names: dict[str, str] = config.get("model_names", {})
+        self.timeout: float = config.get("timeout", 3.0)
+
+    @abstractmethod
+    def health(self) -> str:
+        """Return 'healthy', 'unreachable', or 'error'."""
+
+    @abstractmethod
+    def model_name(self) -> str:
+        """Return the currently loaded model ID."""
+
+    @abstractmethod
+    def is_generating(self) -> bool:
+        """Return True if active inference is happening."""
+
+    @abstractmethod
+    def models_available(self) -> list[dict]:
+        """Return list of available models."""
+
+    @abstractmethod
+    def chat_completions_url(self) -> str:
+        """Return the chat completions endpoint URL."""
+
+    def model_friendly_name(self) -> str:
+        """Map model ID to user-configured display name."""
+        name = self.model_name()
+        return self._model_names.get(name, name)
+
+    def swap_status(self) -> dict | None:
+        """Return swap status info, or None if not swapping. Override in subclasses."""
+        return None

llamawatch/adapters/llamacpp.py

@@ -0,0 +1,148 @@
+"""llama.cpp / swap-proxy backend adapter for LlamaWatch."""
+
+import json
+import time
+import urllib.request
+from pathlib import Path
+
+from .base import BackendAdapter
+
+SWAP_LOCK = Path("/tmp/model_swap_in_progress")
+
+# Swap step timing thresholds (seconds elapsed -> step number + description)
+# Based on observed model-swap timings
+_SWAP_STEPS = [
+    (5, 1, "Unloading old model"),
+    (15, 2, "Waiting for unload"),
+    (25, 3, "GTT memory release (10s)"),
+    (35, 4, "Flushing RAM cache"),
+    (55, 5, "Warming up subsystems"),
+    (999, 6, "Loading new model"),
+]
+
+
+def _http_get(url: str, timeout: float = 3.0) -> str | None:
+    """GET a URL and return the response body, or None on failure."""
+    try:
+        req = urllib.request.Request(url)
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.read().decode("utf-8")
+    except Exception:
+        return None
+
+
+def _http_get_json(url: str, timeout: float = 3.0) -> dict | list | None:
+    """GET a URL and parse as JSON, or None on failure."""
+    body = _http_get(url, timeout)
+    if body is None:
+        return None
+    try:
+        return json.loads(body)
+    except Exception:
+        return None
+
+
+class LlamaCppAdapter(BackendAdapter):
+    """Adapter for llama.cpp servers, with optional swap-proxy support."""
+
+    def __init__(self, url: str, config: dict) -> None:
+        super().__init__(url, config)
+        self.is_swap_proxy: bool = config.get("swap_proxy", False)
+        self.direct_port: int = config.get("direct_port", 8080)
+
+    def health(self) -> str:
+        """Check /health endpoint."""
+        body = _http_get(f"{self.url}/health", timeout=3.0)
+        if body is None:
+            return "unreachable"
+        # llama.cpp /health returns JSON with "status" field
+        try:
+            data = json.loads(body)
+            status = data.get("status", "")
+            if status == "ok":
+                return "healthy"
+            if status in ("no slot available", "loading model"):
+                return "healthy"
+            return "error"
+        except (json.JSONDecodeError, AttributeError):
+            # Non-JSON 200 response — treat as healthy
+            return "healthy"
+
+    def model_name(self) -> str:
+        """Get model ID from /v1/models endpoint."""
+        data = _http_get_json(f"{self.url}/v1/models", timeout=3.0)
+        if data and "data" in data and len(data["data"]) > 0:
+            return data["data"][0].get("id", "unknown")
+        return "unknown"
+
+    def is_generating(self) -> bool:
+        """Check if any llama-server slot is actively processing.
+
+        NOTE: Queries localhost:{direct_port}/slots directly. For remote
+        proxy setups where llama-server is not on localhost, returns False.
+        """
+        model_id = self.model_name()
+        if model_id != "unknown":
+            url = f"http://localhost:{self.direct_port}/slots?model={model_id}"
+        else:
+            url = f"http://localhost:{self.direct_port}/slots"
+        data = _http_get_json(url, timeout=2.0)
+        if data and isinstance(data, list):
+            for slot in data:
+                if slot.get("is_processing", False):
+                    return True
+        return False
+
+    def models_available(self) -> list[dict]:
+        """Get available models from /v1/models."""
+        data = _http_get_json(f"{self.url}/v1/models", timeout=3.0)
+        if data and "data" in data:
+            return data["data"]
+        return []
+
+    def chat_completions_url(self) -> str:
+        """Return the chat completions endpoint URL."""
+        return f"{self.url}/v1/chat/completions"
+
+    def swap_status(self) -> dict | None:
+        """Check if a model swap is in progress by reading the lock file."""
+        if not self.is_swap_proxy:
+            return None
+        try:
+            if not SWAP_LOCK.exists():
+                return None
+
+            content = SWAP_LOCK.read_text().strip()
+            lines = content.split("\n")
+            if not lines:
+                return None
+
+            timestamp = float(lines[0].strip())
+            elapsed = time.time() - timestamp
+
+            swap_from: str | None = None
+            swap_to: str | None = None
+            if len(lines) > 1 and "->" in lines[1]:
+                parts = lines[1].split("->", 1)
+                if len(parts) == 2:
+                    swap_from = parts[0].strip()
+                    swap_to = parts[1].strip()
+
+            # Determine step from elapsed time
+            step = 6
+            step_desc = "Loading new model"
+            for threshold, step_num, desc in _SWAP_STEPS:
+                if elapsed < threshold:
+                    step = step_num
+                    step_desc = desc
+                    break
+
+            return {
+                "swap_from": swap_from,
+                "swap_to": swap_to,
+                "swap_step": step,
+                "swap_step_desc": step_desc,
+                "swap_elapsed": round(elapsed),
+            }
+        except Exception:
+            return None

llamawatch/adapters/ollama.py

@@ -0,0 +1,43 @@
+"""Ollama backend adapter for LlamaWatch."""
+
+from .base import BackendAdapter
+
+# Reuse HTTP helpers from llamacpp module (stdlib only, no extra deps)
+from .llamacpp import _http_get, _http_get_json
+
+
+class OllamaAdapter(BackendAdapter):
+    """Adapter for Ollama API servers."""
+
+    def health(self) -> str:
+        """Check Ollama health via root endpoint."""
+        body = _http_get(f"{self.url}/", timeout=3.0)
+        if body is None:
+            return "unreachable"
+        return "healthy"
+
+    def model_name(self) -> str:
+        """Get the currently running model from /api/ps."""
+        data = _http_get_json(f"{self.url}/api/ps", timeout=3.0)
+        if data and "models" in data and len(data["models"]) > 0:
+            return data["models"][0].get("name", "unknown")
+        return "unknown"
+
+    def is_generating(self) -> bool:
+        """Ollama does not expose per-request generation status.
+
+        Always returns False. The generating indicator will not work for
+        Ollama backends — this is a known limitation of the Ollama API.
+        """
+        return False
+
+    def models_available(self) -> list[dict]:
+        """Get available models from /api/tags."""
+        data = _http_get_json(f"{self.url}/api/tags", timeout=3.0)
+        if data and "models" in data:
+            return data["models"]
+        return []
+
+    def chat_completions_url(self) -> str:
+        """Return the Ollama chat completions endpoint (OpenAI-compatible)."""
+        return f"{self.url}/v1/chat/completions"

llamawatch/adapters/openai_compat.py

@@ -0,0 +1,39 @@
+"""Generic OpenAI-compatible backend adapter for LlamaWatch."""
+
+from .base import BackendAdapter
+from .llamacpp import _http_get, _http_get_json
+
+
+class OpenAICompatAdapter(BackendAdapter):
+    """Adapter for any OpenAI-compatible API server."""
+
+    def health(self) -> str:
+        """Check health by querying /v1/models."""
+        data = _http_get_json(f"{self.url}/v1/models", timeout=3.0)
+        if data is None:
+            return "unreachable"
+        if "data" in data:
+            return "healthy"
+        return "error"
+
+    def model_name(self) -> str:
+        """Get the first model from /v1/models."""
+        data = _http_get_json(f"{self.url}/v1/models", timeout=3.0)
+        if data and "data" in data and len(data["data"]) > 0:
+            return data["data"][0].get("id", "unknown")
+        return "unknown"
+
+    def is_generating(self) -> bool:
+        """Generic OpenAI servers don't expose generation status."""
+        return False
+
+    def models_available(self) -> list[dict]:
+        """Get available models from /v1/models."""
+        data = _http_get_json(f"{self.url}/v1/models", timeout=3.0)
+        if data and "data" in data:
+            return data["data"]
+        return []
+
+    def chat_completions_url(self) -> str:
+        """Return the chat completions endpoint URL."""
+        return f"{self.url}/v1/chat/completions"