llama-tui 0.1.0__tar.gz

llama_tui-0.1.0/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.3
+Name: llama-tui
+Version: 0.1.0
+Summary: Lightweight TUI manager for llama.cpp models on Raspberry Pi
+Requires-Dist: click-extra>=7.12
+Requires-Python: >=3.14
+Description-Content-Type: text/markdown
+
+src/
+└── llama_tui/
+    ├── __init__.py        # main() entry for the console script
+    ├── cli.py             # Click-extra CLI group & commands
+    ├── config.py          # Slotted dataclasses (server, system, RPC)
+    ├── models.py          # Model catalog (slotted, verified)
+    ├── download.py        # hf download wrapper
+    ├── server.py          # llama-server lifecycle manager
+    ├── swap.py            # swap, governor, hugepages helpers
+    └── rpc.py             # o7p3 RPC connectivity check

llama_tui-0.1.0/README.md

@@ -0,0 +1,10 @@
+src/
+└── llama_tui/
+    ├── __init__.py        # main() entry for the console script
+    ├── cli.py             # Click-extra CLI group & commands
+    ├── config.py          # Slotted dataclasses (server, system, RPC)
+    ├── models.py          # Model catalog (slotted, verified)
+    ├── download.py        # hf download wrapper
+    ├── server.py          # llama-server lifecycle manager
+    ├── swap.py            # swap, governor, hugepages helpers
+    └── rpc.py             # o7p3 RPC connectivity check

llama_tui-0.1.0/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "llama-tui"
+version = "0.1.0"
+description = "Lightweight TUI manager for llama.cpp models on Raspberry Pi"
+readme = "README.md"
+requires-python = ">=3.14"
+dependencies = [
+    "click-extra>=7.12",
+]
+
+[project.scripts]
+llama-tui = "llama_tui:main"
+
+[build-system]
+requires = ["uv_build>=0.11.16,<0.12.0"]
+build-backend = "uv_build"

llama_tui-0.1.0/src/llama_tui/__init__.py

@@ -0,0 +1,3 @@
+"""Llama TUI — lightweight LLM manager for Raspberry Pi clusters."""
+
+from llama_tui.cli import main

llama_tui-0.1.0/src/llama_tui/cli.py

@@ -0,0 +1,159 @@
+import click_extra as click
+import sys
+
+from .config import Config, DEFAULT_CONFIG
+from .models import registry, ModelRole, MODEL_CATALOG
+from .download import DownloadManager
+from .server import ServerManager
+from .swap import swap_off, swap_on, swap_status, set_governor, enable_hugepages
+from .rpc import rpc_status
+
+
+@click.group()
+@click.version_option(package_name="llama-tui")
+def main() -> None:
+    """🦙 Llama TUI — Lightweight model manager for Raspberry Pi LLM clusters."""
+
+
+@main.command()
+@click.option("--force", is_flag=True, help="Force re-download even if cached")
+@click.option("--workers", default=4, help="Parallel download workers")
+@click.option("--role", "roles", multiple=True,
+              type=click.Choice([r.name.lower() for r in ModelRole], case_sensitive=False),
+              help="Filter by model role (repeatable)")
+def download(force: bool, workers: int, roles: tuple[str, ...]) -> None:
+    """Download all models into the HuggingFace cache."""
+    click.echo("📥 Downloading models...")
+    mgr = DownloadManager(force=force, max_workers=workers)
+    role_set = [ModelRole[r.upper()] for r in roles] if roles else None
+    results = mgr.download_all(roles=role_set)
+
+    ok = sum(1 for r in results if r.success)
+    fail = len(results) - ok
+    if fail == 0:
+        click.secho(f"✅ All {ok} models downloaded successfully.", fg="green")
+    else:
+        click.secho(f"⚠️  {ok} ok, {fail} failed.", fg="yellow")
+
+
+@main.command()
+@click.option("--role", "role_name", default="chat",
+              type=click.Choice([r.name.lower() for r in ModelRole], case_sensitive=False),
+              help="Which model role to serve")
+@click.option("--port", type=int, help="Override default port")
+@click.option("--ctx-size", type=int, default=8192, help="Context window size")
+@click.option("--rpc/--no-rpc", default=False, help="Enable RPC offload to o7p3")
+def serve(role_name: str, port: int | None, ctx_size: int, rpc: bool) -> None:
+    """Start llama-server for a specific model."""
+    role = ModelRole[role_name.upper()]
+    entry = registry.get(role)
+    if entry is None:
+        click.secho(f"❌ No model found for role: {role_name}", fg="red")
+        sys.exit(1)
+
+    cfg = Config()
+    cfg.server.model_ref = f"{entry.repo}:{entry.quant}"
+    cfg.server.port = port or entry.port or 8383
+    cfg.server.ctx_size = ctx_size
+    cfg.rpc.enabled = rpc
+
+    click.echo(f"🚀 Starting {entry.name} on port {cfg.server.port}...")
+
+    click.echo("🔧 System prep: swap off, performance governor, hugepages...")
+    swap_off()
+    set_governor("performance")
+    enable_hugepages("madvise")
+
+    mgr = ServerManager(
+        config=cfg.server,
+        sys_cfg=cfg.system,
+        rpc_cfg=cfg.rpc,
+    )
+    if not mgr.start():
+        click.secho("❌ Failed to start server.", fg="red")
+        sys.exit(1)
+
+    click.echo("⏳ Waiting for server to become healthy...")
+    if mgr.wait_ready():
+        click.secho(f"✅ Server ready at http://127.0.0.1:{cfg.server.port}", fg="green")
+    else:
+        click.secho("❌ Server did not become healthy.", fg="red")
+        mgr.stop()
+        sys.exit(1)
+
+
+@main.command()
+def swap() -> None:
+    """Show swap status."""
+    click.echo(f"Swap: {swap_status()}")
+
+@main.command()
+def prep() -> None:
+    """Apply all system optimisations for inference."""
+    click.echo("🔧 Applying system optimisations...")
+    if swap_off():
+        click.secho("  ✅ Swap disabled", fg="green")
+    else:
+        click.secho("  ❌ Swap disable failed", fg="red")
+    if set_governor("performance"):
+        click.secho("  ✅ CPU governor: performance", fg="green")
+    else:
+        click.secho("  ❌ Governor set failed", fg="red")
+    if enable_hugepages("madvise"):
+        click.secho("  ✅ Transparent hugepages: madvise", fg="green")
+    else:
+        click.secho("  ❌ Hugepages set failed", fg="red")
+    click.secho("✅ System ready for inference.", fg="green")
+
+
+@main.command()
+def restore() -> None:
+    """Restore system defaults after inference."""
+    click.echo("🔧 Restoring system defaults...")
+    swap_on()
+    set_governor("schedutil")
+    enable_hugepages("always")
+    click.secho("✅ System restored.", fg="green")
+
+
+@main.command()
+def models() -> None:
+    """List all models in the catalog."""
+    from rich.table import Table
+    from rich.console import Console
+
+    console = Console()
+    table = Table(title="🦙 Model Catalog")
+    table.add_column("Role", style="cyan")
+    table.add_column("Name", style="green")
+    table.add_column("Quant", style="yellow")
+    table.add_column("Size", style="magenta")
+    table.add_column("Host", style="blue")
+
+    for m in MODEL_CATALOG:
+        size_str = f"{m.size_mb} MB" if m.size_mb > 0 else "—"
+        mm_str = "📷" if m.mmproj else ""
+        note = f" [{m.note}]" if m.note else ""
+        table.add_row(
+            m.role.name, f"{m.name} {mm_str}{note}",
+            m.quant, size_str, m.host,
+        )
+    console.print(table)
+
+
+@main.command()
+def rpc() -> None:
+    """Check RPC connectivity to o7p3."""
+    click.echo(rpc_status())
+
+
+@main.command()
+@click.pass_context
+def status(ctx: click.Context) -> None:
+    """Show full system + model status."""
+    click.echo("🦙 Llama TUI Status")
+    click.echo("=" * 40)
+    click.echo(f"Swap: {swap_status()}")
+    click.echo(rpc_status())
+    click.echo(f"\nModels: {len(MODEL_CATALOG)} in catalog")
+    ctx.invoke(models)

llama_tui-0.1.0/src/llama_tui/config.py

@@ -0,0 +1,44 @@
+from dataclasses import dataclass, field
+
+@dataclass(slots=True)
+class ServerConfig:
+    model_ref: str = ""
+    port: int = 8383
+    host: str = "0.0.0.0"
+    ngl: int = 0
+    flash_attn: bool = True
+    ctx_size: int = 8192
+    threads: int = 3
+    threads_batch: int = 3
+    parallel: int = 2
+    cache_type_k: str = "q8_0"
+    cache_type_v: str = "q8_0"
+    mlock: bool = True
+    cache_reuse: int = 256
+    rope_scaling: str = "linear"
+    rope_scale: float = 8.0
+    priority: int = 2
+
+@dataclass(slots=True)
+class SystemConfig:
+    manage_swap: bool = True
+    cpu_governor: str = "performance"
+    transparent_hugepages: str = "madvise"
+    vulkan_disabled: bool = True
+    opencl_disabled: bool = True
+    blis_single_thread: bool = True
+
+@dataclass(slots=True)
+class RPCConfig:
+    enabled: bool = False
+    host: str = "o7p3.stalk-symmetric.ts.net"
+    port: int = 50052
+    draft_model: str = "MaziyarPanahi/gemma-3-1b-it-GGUF:Q4_K_M"
+
+@dataclass(slots=True)
+class Config:
+    server: ServerConfig = field(default_factory=ServerConfig)
+    system: SystemConfig = field(default_factory=SystemConfig)
+    rpc: RPCConfig = field(default_factory=RPCConfig)
+
+DEFAULT_CONFIG = Config()

llama_tui-0.1.0/src/llama_tui/download.py

@@ -0,0 +1,63 @@
+import subprocess
+import sys
+from dataclasses import dataclass, field
+
+from .models import MODEL_CATALOG, ModelEntry
+
+@dataclass(slots=True)
+class DownloadResult:
+    model: str
+    success: bool
+    message: str = ""
+
+@dataclass(slots=True)
+class DownloadManager:
+    force: bool = False
+    max_workers: int = 4
+    results: list[DownloadResult] = field(default_factory=list)
+
+    def download(self, entry: ModelEntry) -> DownloadResult:
+        repo_quant = f"{entry.repo}:{entry.quant}"
+        cmd = [
+            "hf", "download", entry.repo,
+            "--include", f"*{entry.quant}.gguf",
+            "--max-workers", str(self.max_workers),
+        ]
+        if self.force:
+            cmd.append("--force-download")
+
+        # Multimodal projector: separate call to avoid glob vs explicit conflict
+        if entry.mmproj:
+            proj_cmd = [
+                "hf", "download", entry.repo,
+                "--include", "mmproj-*f16.gguf",
+                "--max-workers", str(self.max_workers),
+            ]
+            try:
+                subprocess.run(proj_cmd, check=True, capture_output=True, text=True, timeout=300)
+            except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+                return DownloadResult(model=repo_quant, success=False,
+                                      message=f"Projector download failed: {e}")
+
+        try:
+            result = subprocess.run(cmd, check=True, capture_output=True, text=True, timeout=600)
+            return DownloadResult(model=repo_quant, success=True, message=result.stdout.strip())
+        except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
+            return DownloadResult(model=repo_quant, success=False,
+                                  message=getattr(e, 'stderr', str(e)))
+
+    def download_all(self, *, roles=None) -> list[DownloadResult]:
+        self.results.clear()
+        entries = MODEL_CATALOG
+        if roles:
+            entries = [e for e in MODEL_CATALOG if e.role in roles]
+        for entry in entries:
+            res = self.download(entry)
+            self.results.append(res)
+            status = "✅" if res.success else "❌"
+            print(f"  {status} {res.model}: {res.message[:120]}")
+        return self.results
+
+    @property
+    def all_ok(self) -> bool:
+        return all(r.success for r in self.results) if self.results else False

llama_tui-0.1.0/src/llama_tui/models.py

@@ -0,0 +1,52 @@
+from dataclasses import dataclass, field
+from enum import Enum, auto
+
+class ModelRole(Enum):
+    CHAT = auto()
+    VISION = auto()
+    CODE = auto()
+    SECURITY = auto()
+    STEM = auto()
+    SPEC_DRAFT = auto()
+    EMBEDDED = auto()
+
+@dataclass(slots=True)
+class ModelEntry:
+    role: ModelRole
+    name: str
+    repo: str          # HuggingFace user/repo
+    quant: str         # e.g. Q4_K_M
+    host: str          # "o7p5" or "o7p3"
+    port: int = 0
+    size_mb: int = 0
+    mmproj: bool = False
+    needs_mmproj: bool = False
+    note: str = ""
+
+# Verified catalog — May 2026
+MODEL_CATALOG: list[ModelEntry] = [
+    ModelEntry(ModelRole.CHAT, "Llama 3.2 3B", "unsloth/Llama-3.2-3B-Instruct-GGUF", "Q4_K_M", "o7p5", 8383, 1900),
+    ModelEntry(ModelRole.VISION, "MiniCPM-V 4.6", "openbmb/MiniCPM-V-4.6-gguf", "Q4_K_M", "o7p5", 8384, 1600, mmproj=True, needs_mmproj=True),
+    ModelEntry(ModelRole.CODE, "Agent.Nano.Coder 2B", "WithinUsAI/Agent.Nano.Coder-2B-gguf", "Q4_K_M", "o7p5", 8385, 1200),
+    ModelEntry(ModelRole.SECURITY, "Qwen3 4B SafeRL", "ShahzebKhoso/Qwen3-4B-SafeRL-GGUF", "Q4_K_M", "o7p5", 8386, 2500),
+    ModelEntry(ModelRole.STEM, "LFM2.5 1.2B", "bartowski/LiquidAI_LFM2.5-1.2B-Instruct-GGUF", "Q8_0", "o7p5", 8387, 1250),
+    ModelEntry(ModelRole.SPEC_DRAFT, "Qwen2.5 Coder 0.5B", "featherless-ai-quants/unsloth-Qwen2.5-Coder-0.5B-GGUF", "Q4_K_M", "o7p5", 0, 398),
+    ModelEntry(ModelRole.EMBEDDED, "Gemma 3 1B", "MaziyarPanahi/gemma-3-1b-it-GGUF", "Q4_K_M", "o7p3", 50052, 800),
+]
+
+@dataclass(slots=True)
+class ModelRegistry:
+    _by_role: dict[ModelRole, ModelEntry] = field(default_factory=dict)
+    _by_name: dict[str, ModelEntry] = field(default_factory=dict)
+
+    def __post_init__(self):
+        for m in MODEL_CATALOG:
+            self._by_role[m.role] = m
+            self._by_name[m.name] = m
+
+    def get(self, key: ModelRole | str) -> ModelEntry | None:
+        if isinstance(key, ModelRole):
+            return self._by_role.get(key)
+        return self._by_name.get(key)
+
+registry = ModelRegistry()

llama_tui-0.1.0/src/llama_tui/rpc.py

@@ -0,0 +1,14 @@
+import socket
+
+def check_rpc(host: str = "o7p3.stalk-symmetric.ts.net", port: int = 50052) -> bool:
+    try:
+        with socket.create_connection((host, port), timeout=5):
+            return True
+    except (socket.timeout, ConnectionRefusedError, OSError):
+        return False
+
+def rpc_status(host: str = "o7p3.stalk-symmetric.ts.net", port: int = 50052) -> str:
+    reachable = check_rpc(host, port)
+    if reachable:
+        return f"✅ RPC server reachable at {host}:{port}"
+    return f"❌ RPC server unreachable at {host}:{port}"

llama_tui-0.1.0/src/llama_tui/server.py

@@ -0,0 +1,108 @@
+import os
+import signal
+import subprocess
+import sys
+import time
+import urllib.request
+import urllib.error
+from dataclasses import dataclass, field
+
+from .config import ServerConfig, SystemConfig, RPCConfig, DEFAULT_CONFIG
+
+@dataclass(slots=True)
+class ServerManager:
+    config: ServerConfig = field(default_factory=lambda: DEFAULT_CONFIG.server)
+    sys_cfg: SystemConfig = field(default_factory=lambda: DEFAULT_CONFIG.system)
+    rpc_cfg: RPCConfig = field(default_factory=lambda: DEFAULT_CONFIG.rpc)
+    _process: subprocess.Popen | None = None
+
+    def _build_env(self) -> dict[str, str]:
+        env = os.environ.copy()
+        if self.sys_cfg.vulkan_disabled:
+            env["GGML_VULKAN"] = "0"
+        if self.sys_cfg.opencl_disabled:
+            env["GGML_OPENCL"] = "0"
+        if self.sys_cfg.blis_single_thread:
+            env["BLIS_NUM_THREADS"] = "1"
+            env["OMP_NUM_THREADS"] = "1"
+            env["GOMP_SPINCOUNT"] = "0"
+        env["LLAMA_ARG_THREADS"] = str(self.config.threads)
+        env["LLAMA_ARG_THREADS_BATCH"] = str(self.config.threads_batch)
+        return env
+
+    def _build_cmd(self) -> list[str]:
+        cmd = [
+            "taskset", "-c", f"0-{self.config.threads - 1}",
+            "llama-server",
+            "-hf", self.config.model_ref,
+            "-ngl", str(self.config.ngl),
+            "-fa", "on" if self.config.flash_attn else "off",
+            "-ctk", self.config.cache_type_k,
+            "-ctv", self.config.cache_type_v,
+            "-c", str(self.config.ctx_size),
+            "-t", str(self.config.threads),
+            "-tb", str(self.config.threads_batch),
+            "-np", str(self.config.parallel),
+            "--prio", str(self.config.priority),
+            "--cache-reuse", str(self.config.cache_reuse),
+            "--port", str(self.config.port),
+            "--host", self.config.host,
+            "--rope-scaling", self.config.rope_scaling,
+            "--rope-scale", str(self.config.rope_scale),
+            "--spec-type", "ngram-mod",
+            "--spec-ngram-mod-n-min", "48",
+            "--spec-ngram-mod-n-max", "64",
+            "--spec-ngram-mod-n-match", "24",
+            "--spec-draft-n-max", "3",
+        ]
+        if self.config.mlock:
+            cmd.append("--mlock")
+        if self.rpc_cfg.enabled:
+            cmd.extend(["--rpc", f"{self.rpc_cfg.host}:{self.rpc_cfg.port}"])
+        return cmd
+
+    def start(self) -> bool:
+        if self._process and self._process.poll() is None:
+            return True
+        cmd = self._build_cmd()
+        env = self._build_env()
+        try:
+            self._process = subprocess.Popen(
+                cmd, env=env,
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                start_new_session=True,
+            )
+            return True
+        except FileNotFoundError:
+            print("❌ llama-server not found in PATH", file=sys.stderr)
+            return False
+
+    def stop(self, timeout: int = 10) -> bool:
+        if self._process is None or self._process.poll() is not None:
+            return True
+        self._process.terminate()
+        try:
+            self._process.wait(timeout=timeout)
+            return True
+        except subprocess.TimeoutExpired:
+            self._process.kill()
+            self._process.wait()
+            return False
+
+    @property
+    def is_running(self) -> bool:
+        return self._process is not None and self._process.poll() is None
+
+    def wait_ready(self, timeout: int = 300) -> bool:
+        deadline = time.monotonic() + timeout
+        url = f"http://127.0.0.1:{self.config.port}/health"
+        while time.monotonic() < deadline:
+            if not self.is_running:
+                return False
+            try:
+                with urllib.request.urlopen(url, timeout=3) as r:
+                    if r.status == 200:
+                        return True
+            except (urllib.error.URLError, OSError):
+                time.sleep(1)
+        return False

llama_tui-0.1.0/src/llama_tui/swap.py

@@ -0,0 +1,47 @@
+import subprocess
+import sys
+
+def swap_off() -> bool:
+    try:
+        subprocess.run(["sudo", "swapoff", "-a"], check=True, capture_output=True, text=True)
+        return True
+    except subprocess.CalledProcessError as e:
+        print(f"⚠️  swapoff failed: {e.stderr.strip()}", file=sys.stderr)
+        return False
+
+def swap_on() -> bool:
+    try:
+        subprocess.run(["sudo", "swapon", "-a"], check=True, capture_output=True, text=True)
+        return True
+    except subprocess.CalledProcessError as e:
+        print(f"⚠️  swapon failed: {e.stderr.strip()}", file=sys.stderr)
+        return False
+
+def swap_status() -> str:
+    try:
+        r = subprocess.run(["swapon", "--show"], capture_output=True, text=True)
+        return r.stdout.strip() or "Swap is off"
+    except Exception:
+        return "Unknown"
+
+def set_governor(governor: str = "performance") -> bool:
+    try:
+        subprocess.run(
+            ["sudo", "sh", "-c",
+             f"echo {governor} | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor"],
+            check=True, capture_output=True, text=True,
+        )
+        return True
+    except subprocess.CalledProcessError:
+        return False
+
+def enable_hugepages(mode: str = "madvise") -> bool:
+    try:
+        subprocess.run(
+            ["sudo", "sh", "-c",
+             f"echo {mode} | tee /sys/kernel/mm/transparent_hugepage/enabled"],
+            check=True, capture_output=True, text=True,
+        )
+        return True
+    except subprocess.CalledProcessError:
+        return False