polyserve 0.1.0__py3-none-any.whl

polyserve/__init__.py

@@ -0,0 +1,6 @@
+"""PolyServe: an autotuner for LLM serving.
+
+probe -> select backends -> prepare model -> plan memory -> calibrate -> cache -> serve
+"""
+
+__version__ = "0.1.0"

polyserve/backends/__init__.py

@@ -0,0 +1,36 @@
+"""Backend registry. New hardware = new class here, no core changes."""
+
+from __future__ import annotations
+
+from typing import Dict
+
+from polyserve.backends.base import Backend, BaseBackend, LaunchSpec, LlmtraceHooks, Process, free_port
+
+
+def registry() -> Dict[str, BaseBackend]:
+    from polyserve.backends.llamacpp import LlamaCppCpuBackend, LlamaCppCudaBackend
+    from polyserve.backends.sglang import SglangBackend
+    from polyserve.backends.vllm import VllmBackend
+    from polyserve.backends.vllm_cpu import VllmCpuBackend
+
+    backends = [VllmBackend(), SglangBackend(), LlamaCppCudaBackend(), LlamaCppCpuBackend(), VllmCpuBackend()]
+    return {b.name: b for b in backends}
+
+
+def get_backend(name: str) -> BaseBackend:
+    reg = registry()
+    if name not in reg:
+        raise KeyError(f"unknown backend {name!r}; known: {sorted(reg)}")
+    return reg[name]
+
+
+__all__ = [
+    "Backend",
+    "BaseBackend",
+    "LaunchSpec",
+    "LlmtraceHooks",
+    "Process",
+    "free_port",
+    "registry",
+    "get_backend",
+]

polyserve/backends/base.py

@@ -0,0 +1,314 @@
+"""Backend interface (spec section 3) plus the subprocess wrapper every backend uses."""
+
+from __future__ import annotations
+
+import logging
+import os
+import signal
+import socket
+import subprocess
+import sys
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Dict, List, Optional, Protocol, Tuple, runtime_checkable
+
+import httpx
+
+from polyserve.memory import MemoryModel
+from polyserve.models import Config, HardwareDescriptor, ModelSpec, PreparedModel
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class LlmtraceHooks:
+    """How the calibration driver should talk to (and measure) this backend."""
+
+    completions_path: str = "/v1/completions"
+    health_path: str = "/health"
+    models_path: str = "/v1/models"
+    stream_usage: bool = True  # backend reports usage in the final stream chunk when asked
+    model_name: Optional[str] = None  # value to put in the "model" field of requests
+    tokenizer_id: Optional[str] = None  # HF id whose tokenizer counts tokens when usage is absent
+    gpu_ids: List[int] = field(default_factory=list)  # NVML indices llmtrace should sample
+    process_memory: bool = False  # sample RSS of the backend process (CPU backends)
+
+
+@dataclass
+class LaunchSpec:
+    args: List[str]
+    env: Dict[str, str] = field(default_factory=dict)
+    cwd: Optional[str] = None
+
+
+class Process:
+    """A launched backend server: subprocess + readiness probe + logs."""
+
+    def __init__(self, spec: LaunchSpec, port: int, health_url: str, log_path: Optional[Path] = None):
+        self.spec = spec
+        self.port = port
+        self.health_url = health_url
+        self.log_path = log_path
+        self._proc: Optional[subprocess.Popen] = None
+        self._log_fh = None
+        self.started_at: Optional[float] = None
+        self.ready_at: Optional[float] = None
+
+    @property
+    def pid(self) -> Optional[int]:
+        return self._proc.pid if self._proc else None
+
+    def start(self) -> "Process":
+        env = dict(os.environ)
+        env.update(self.spec.env)
+        if self.log_path:
+            self.log_path.parent.mkdir(parents=True, exist_ok=True)
+            self._log_fh = open(self.log_path, "ab")
+            stdout = self._log_fh
+        else:
+            stdout = subprocess.DEVNULL
+        logger.info("launch: %s", " ".join(self.spec.args))
+        kwargs = {}
+        if sys.platform != "win32":
+            kwargs["start_new_session"] = True  # own process group so we can kill children
+        self._proc = subprocess.Popen(
+            self.spec.args,
+            env=env,
+            cwd=self.spec.cwd,
+            stdout=stdout,
+            stderr=subprocess.STDOUT,
+            stdin=subprocess.DEVNULL,
+            **kwargs,
+        )
+        self.started_at = time.monotonic()
+        return self
+
+    def alive(self) -> bool:
+        return self._proc is not None and self._proc.poll() is None
+
+    def returncode(self) -> Optional[int]:
+        return self._proc.poll() if self._proc else None
+
+    def wait_ready(self, timeout: float = 600.0, poll: float = 1.0) -> bool:
+        deadline = time.monotonic() + timeout
+        with httpx.Client(timeout=5.0) as client:
+            while time.monotonic() < deadline:
+                if not self.alive():
+                    logger.error("backend exited during startup (rc=%s); see %s", self.returncode(), self.log_path)
+                    return False
+                try:
+                    r = client.get(self.health_url)
+                    if r.status_code < 500:
+                        self.ready_at = time.monotonic()
+                        return True
+                except httpx.HTTPError:
+                    pass
+                time.sleep(poll)
+        logger.error("backend not ready after %.0fs; see %s", timeout, self.log_path)
+        return False
+
+    def stop(self, grace: float = 15.0) -> None:
+        if self._proc is None:
+            return
+        if self.alive():
+            try:
+                if sys.platform != "win32":
+                    os.killpg(self._proc.pid, signal.SIGTERM)  # own session: group id == server pid
+                else:
+                    self._proc.terminate()
+            except Exception:
+                pass
+            try:
+                self._proc.wait(timeout=grace)
+            except subprocess.TimeoutExpired:
+                try:
+                    if sys.platform != "win32":
+                        os.killpg(self._proc.pid, signal.SIGKILL)
+                    else:
+                        self._proc.kill()
+                except Exception:
+                    pass
+                self._proc.wait(timeout=10)
+        if sys.platform != "win32":
+            # The server can be gone while its children are not: a vLLM API server that died leaves
+            # VLLM::EngineCore holding the GPU, and every later trial then sees less memory. The
+            # children share the server's process group, so reap whatever is left of it.
+            try:
+                os.killpg(self._proc.pid, signal.SIGKILL)
+            except OSError:
+                pass
+        if self._log_fh:
+            self._log_fh.close()
+            self._log_fh = None
+
+    def tail_log(self, n: int = 40) -> str:
+        if not self.log_path or not self.log_path.exists():
+            return ""
+        try:
+            lines = self.log_path.read_text(errors="replace").splitlines()
+            return "\n".join(lines[-n:])
+        except OSError:
+            return ""
+
+
+CTX_GRID = (2048, 4096, 8192, 16384, 32768, 65536, 131072)
+CTX_GRID_WIDTH = 3  # sizes tried per calibration: the smallest that fits the workload and the next two
+
+
+def ctx_grid(max_pos: int, min_ctx: int = 0) -> List[int]:
+    """Context lengths to try: the three smallest standard sizes that fit the model and hold the workload.
+
+    Returns [] if the model cannot hold `min_ctx` at all.
+    """
+    if min_ctx > max_pos:
+        return []
+    grid = [c for c in CTX_GRID if min_ctx <= c <= max_pos]
+    if not grid:
+        # Workload needs more than the largest standard size that fits: use the model max.
+        grid = [max_pos]
+    return grid[:CTX_GRID_WIDTH]
+
+
+def render_extra(extra: Dict[str, object], skip: Tuple[str, ...] = ()) -> List[str]:
+    """Extra launch flags. `True` renders as a bare flag; `False` and None are dropped."""
+    out: List[str] = []
+    for k, v in extra.items():
+        if k in skip or v is None or v is False:
+            continue
+        flag = f"--{k.replace('_', '-')}"
+        out += [flag] if v is True else [flag, str(v)]
+    return out
+
+
+def free_port(preferred: Optional[int] = None) -> int:
+    if preferred:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            try:
+                s.bind(("127.0.0.1", preferred))
+                return preferred
+            except OSError:
+                pass
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+@runtime_checkable
+class Backend(Protocol):
+    name: str
+
+    def available(self, hw: HardwareDescriptor) -> bool: ...
+
+    def supports(self, hw: HardwareDescriptor, model: ModelSpec) -> bool: ...
+
+    def prepare(self, model: ModelSpec, hw: HardwareDescriptor, quants: Optional[List[str]] = None) -> PreparedModel: ...
+
+    def memory_model(self, hw: HardwareDescriptor) -> MemoryModel: ...
+
+    def estimate_memory(self, cfg: Config, model: PreparedModel, hw: HardwareDescriptor) -> int: ...
+
+    def candidate_configs(self, hw: HardwareDescriptor, model: PreparedModel, min_ctx: int = 0) -> List[Config]: ...
+
+    def launch_spec(self, cfg: Config, model: PreparedModel, port: int) -> LaunchSpec: ...
+
+    def launch(self, cfg: Config, model: PreparedModel, port: int, log_path: Optional[Path] = None) -> Process: ...
+
+    def workload_hooks(self, hw: HardwareDescriptor, model: PreparedModel) -> LlmtraceHooks: ...
+
+    def default_config(self, hw: HardwareDescriptor, model: PreparedModel, min_ctx: int = 0) -> Config: ...
+
+    def version(self, hw: HardwareDescriptor) -> Optional[str]: ...
+
+
+class BaseBackend:
+    """Shared plumbing. Concrete backends override the abstract-ish methods."""
+
+    name: str = "base"
+    runtime_workspace_bytes: int = 1024 * 1024 * 1024
+
+    def available(self, hw: HardwareDescriptor) -> bool:
+        return hw.backend_available(self.name)
+
+    def version(self, hw: HardwareDescriptor) -> Optional[str]:
+        return hw.backend_version(self.name)
+
+    def memory_model(self, hw: HardwareDescriptor) -> MemoryModel:
+        raise NotImplementedError
+
+    def calibrated_memory(self, hw: HardwareDescriptor, kv_tokens_fn, device: str) -> MemoryModel:
+        """MemoryModel using this machine's fitted workspace/margin when `polyserve memory-report --apply` ran."""
+        from polyserve.hardware import hardware_hash
+        from polyserve.memcal import margin_override, workspace_override
+        from polyserve.memory import DEFAULT_MARGIN_FRACTION
+
+        hh = hardware_hash(hw)
+        ws = workspace_override(hh, self.name)
+        mf = margin_override(hh, self.name)
+        return MemoryModel(
+            runtime_workspace=ws if ws is not None else self.runtime_workspace_bytes,
+            kv_tokens_fn=kv_tokens_fn,
+            device=device,
+            margin_fraction=mf if mf is not None else DEFAULT_MARGIN_FRACTION,
+            calibrated=ws is not None or mf is not None,
+        )
+
+    def estimate_memory(self, cfg: Config, model: PreparedModel, hw: HardwareDescriptor) -> int:
+        from polyserve.memory import estimate
+
+        return estimate(hw, model, cfg, self.memory_model(hw)).total
+
+    def materialize(self, model: PreparedModel, quants: List[str]) -> PreparedModel:
+        """Download / convert weights for the quants the planner kept. Default: nothing to do."""
+        return model
+
+    def prefill_variants(self, cfg: Config) -> List[Config]:
+        """Configs differing from `cfg` only in the prefill knob. Default: this backend has none."""
+        return []
+
+    def disagg_launch_spec(self, cfg: Config, model: PreparedModel, port: int, role: str,
+                           kv_transfer_config: dict, gpu_index: int, side_channel_port: int) -> LaunchSpec:
+        """Launch one engine of a disaggregated prefill/decode pair."""
+        raise NotImplementedError(f"{self.name} does not support disaggregated prefill/decode")
+
+    # ---- optional search dimensions (default: this backend offers none)
+
+    supports_tp: bool = False
+
+    def supported_quants(self, hw: HardwareDescriptor) -> List[str]:
+        """Every weight precision this backend could run on `hw` (what --quant filters)."""
+        return []
+
+    def kv_dtypes(self, hw: HardwareDescriptor) -> List[str]:
+        """Quantized KV-cache types worth trying on `hw`."""
+        return []
+
+    def batch_ladder(self) -> Tuple[int, ...]:
+        """Batch sizes, smallest first, that a smaller KV cache may let the search step up to."""
+        return ()
+
+    def prefix_variants(self, cfg: Config) -> List[Config]:
+        """Prefix-cache settings to try when the workload's prompts share a prefix."""
+        return []
+
+    def spec_variants(self, cfg: Config, model: PreparedModel) -> List[Config]:
+        """Speculative-decoding settings to try."""
+        return []
+
+    def replica_launch_spec(self, cfg: Config, model: PreparedModel, port: int, gpu_index: int) -> LaunchSpec:
+        """One full engine pinned to one GPU, for the replicas layout."""
+        spec = self.launch_spec(cfg, model, port)
+        spec.env = {**spec.env, "CUDA_VISIBLE_DEVICES": str(gpu_index)}
+        return spec
+
+    def launch_spec(self, cfg: Config, model: PreparedModel, port: int) -> LaunchSpec:
+        raise NotImplementedError
+
+    health_path: str = "/health"
+
+    def workload_hooks(self, hw: HardwareDescriptor, model: PreparedModel) -> LlmtraceHooks:
+        return LlmtraceHooks(gpu_ids=[hw.gpu.index] if hw.gpu else [])
+
+    def launch(self, cfg: Config, model: PreparedModel, port: int, log_path: Optional[Path] = None) -> Process:
+        spec = self.launch_spec(cfg, model, port)
+        return Process(spec, port, f"http://127.0.0.1:{port}{self.health_path}", log_path=log_path).start()

polyserve/backends/llamacpp.py

@@ -0,0 +1,298 @@
+"""llama.cpp backend: `llama-server`, CUDA (any NVIDIA GPU, incl. Pascal/Volta) and CPU."""
+
+from __future__ import annotations
+
+import functools
+import logging
+import os
+import subprocess
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+from polyserve import speculative
+from polyserve.backends.base import BaseBackend, LaunchSpec, LlmtraceHooks, ctx_grid, render_extra
+from polyserve.gguf import (
+    GGUF_QUANTS,
+    GGUFCandidate,
+    convert_and_quantize,
+    download_gguf,
+    estimate_gguf_bytes,
+    search_hub_gguf,
+)
+from polyserve.hardware import llama_server_binary
+from polyserve.hfconfig import load_arch
+from polyserve.memory import MemoryModel
+from polyserve.models import Config, HardwareDescriptor, MiB, ModelSpec, PreparedModel
+
+
+@functools.lru_cache(maxsize=8)
+def server_help(binary: str) -> str:
+    """`llama-server --help`, to follow flag renames across llama.cpp versions. Empty if it cannot run."""
+    try:
+        out = subprocess.run([binary, "--help"], capture_output=True, text=True, timeout=60)
+        return out.stdout + out.stderr
+    except Exception:
+        return ""
+
+
+def _modern_spec(binary: str) -> bool:
+    return "--spec-type" in server_help(binary)
+
+logger = logging.getLogger(__name__)
+
+
+class LlamaCppBackend(BaseBackend):
+    """Shared implementation; the CUDA and CPU variants differ in offload and workspace."""
+
+    name = "llamacpp"
+    cuda = False
+    runtime_workspace_bytes = 768 * MiB  # compute buffers (scale with n_batch) + CUDA context
+
+    def supports(self, hw: HardwareDescriptor, model: ModelSpec) -> bool:
+        if self.cuda and (hw.gpu is None or hw.gpu.vendor != "nvidia"):
+            return False
+        return llama_server_binary() is not None
+
+    # ---- prepare: resolve (no download); materialize: download/convert only what the planner kept
+
+    def prepare(self, model: ModelSpec, hw: HardwareDescriptor, quants: Optional[List[str]] = None) -> PreparedModel:
+        arch = load_arch(model)
+        quants = list(quants or GGUF_QUANTS)
+        prepared = PreparedModel(spec=model, backend=self.name, arch=arch)
+        if os.path.isfile(model.hf_id) and model.hf_id.lower().endswith(".gguf"):
+            # Local GGUF file given directly.
+            q = _quant_from_filename(model.hf_id) or "Q4_K_M"
+            prepared.gguf_paths[q] = model.hf_id
+            prepared.weights_bytes[q] = os.path.getsize(model.hf_id)
+            return prepared
+        try:
+            found: Dict[str, GGUFCandidate] = search_hub_gguf(model, quants)
+        except Exception as exc:
+            logger.warning("GGUF hub search failed (%s); will fall back to conversion", exc)
+            found = {}
+        params = arch.num_params or 0
+        for q in quants:
+            cand = found.get(q)
+            if cand is not None:
+                prepared.weights_bytes[q] = cand.size_bytes or estimate_gguf_bytes(params, q)
+                prepared.gguf_paths[q] = f"hf://{cand.repo_id}/{cand.filename}"
+            else:
+                prepared.weights_bytes[q] = estimate_gguf_bytes(params, q)
+                prepared.gguf_paths[q] = f"convert://{q}"
+        # A small same-family draft model for speculative decoding, when the Hub has a GGUF of it.
+        draft = speculative.draft_for(model.hf_id)
+        if draft:
+            try:
+                d = search_hub_gguf(ModelSpec(hf_id=draft), ["Q8_0"]).get("Q8_0")
+                if d is not None:
+                    prepared.draft_paths[draft] = f"hf://{d.repo_id}/{d.filename}"
+            except Exception as exc:
+                logger.debug("no draft GGUF for %s: %s", draft, exc)
+        return prepared
+
+    def materialize(self, model: PreparedModel, quants: List[str]) -> PreparedModel:
+        need_convert: List[str] = []
+        for q in quants:
+            ref = model.gguf_paths.get(q)
+            if ref is None:
+                continue
+            if ref.startswith("hf://"):
+                repo, _, fname = ref[len("hf://"):].rpartition("/")
+                path = download_gguf(GGUFCandidate(repo_id=repo, filename=fname, quant=q))
+                model.gguf_paths[q] = str(path)
+                model.weights_bytes[q] = path.stat().st_size
+            elif ref.startswith("convert://"):
+                need_convert.append(q)
+        if need_convert:
+            produced = convert_and_quantize(model.spec, need_convert)
+            for q, path in produced.items():
+                model.gguf_paths[q] = str(path)
+                model.weights_bytes[q] = Path(path).stat().st_size
+            for q in need_convert:
+                if q not in produced:
+                    model.gguf_paths.pop(q, None)
+                    model.weights_bytes.pop(q, None)
+        for draft, ref in list(model.draft_paths.items()):
+            if ref.startswith("hf://"):
+                repo, _, fname = ref[len("hf://"):].rpartition("/")
+                try:
+                    model.draft_paths[draft] = str(download_gguf(GGUFCandidate(repo_id=repo, filename=fname,
+                                                                               quant="Q8_0")))
+                except Exception as exc:
+                    logger.warning("draft model %s unavailable: %s", draft, exc)
+                    model.draft_paths.pop(draft, None)
+        return model
+
+    # ---- memory: llama-server allocates the full KV for -c up front (per-slot ctx x n_parallel)
+
+    def memory_model(self, hw: HardwareDescriptor) -> MemoryModel:
+        return self.calibrated_memory(hw, lambda cfg: cfg.ctx * cfg.batch, "gpu" if self.cuda else "cpu")
+
+    # ---- configs
+
+    def supported_quants(self, hw: HardwareDescriptor) -> List[str]:
+        return list(GGUF_QUANTS)
+
+    def kv_dtypes(self, hw: HardwareDescriptor) -> List[str]:
+        return ["q8_0", "q4_0"]
+
+    def batch_ladder(self) -> Tuple[int, ...]:
+        return (1, 4, 8, 16)
+
+    def prefix_variants(self, cfg: Config) -> List[Config]:
+        # Reuse cached prompt chunks by KV shifting, and share one KV buffer across slots so a common
+        # prefix computed by one slot serves every slot.
+        options = [{"cache_reuse": 256}, {"kv_unified": True}]
+        return [cfg.model_copy(update={"extra": {**cfg.extra, **o}}) for o in options
+                if any(cfg.extra.get(k) != v for k, v in o.items())]
+
+    def spec_variants(self, cfg: Config, model: PreparedModel) -> List[Config]:
+        specs: List[str] = []
+        binary = llama_server_binary()
+        if binary and _modern_spec(binary):  # built-in n-gram lookup: no second model needed
+            specs.append(speculative.ngram(speculative.NGRAM_TOKENS_LLAMACPP))
+        draft = speculative.draft_for(model.spec.hf_id)
+        path = model.draft_paths.get(draft or "")
+        if draft and path and not path.startswith(("hf://", "convert://")):
+            specs.append(speculative.draft(draft, speculative.DRAFT_TOKENS_LLAMACPP))
+        return [cfg.model_copy(update={"spec_decode": s}) for s in specs if s != cfg.spec_decode]
+
+    def _quants(self, model: PreparedModel) -> List[str]:
+        return [q for q in GGUF_QUANTS if q in model.weights_bytes] or list(model.weights_bytes)
+
+    def candidate_configs(self, hw: HardwareDescriptor, model: PreparedModel, min_ctx: int = 0) -> List[Config]:
+        layers = model.arch.num_layers
+        ctxs = ctx_grid(model.arch.max_position_embeddings, min_ctx)
+        if self.cuda:
+            ngls = sorted({layers + 1, (3 * layers) // 4, layers // 2}, reverse=True)  # +1 = output layer too
+            n_batches = [512]
+        else:
+            ngls = [0]
+            n_batches = [512, 2048]
+        out: List[Config] = []
+        for quant in self._quants(model):
+            for ngl in ngls:
+                for ctx in ctxs:
+                    for np_ in (1, 4, 8):
+                        for nb in n_batches:
+                            out.append(
+                                Config(
+                                    backend=self.name, quant=quant, ctx=ctx, batch=np_, n_gpu_layers=ngl, n_batch=nb
+                                )
+                            )
+        return out
+
+    def default_config(self, hw: HardwareDescriptor, model: PreparedModel, min_ctx: int = 0) -> Config:
+        quants = self._quants(model)
+        return Config(
+            backend=self.name,
+            quant="Q4_K_M" if "Q4_K_M" in quants else quants[0],
+            ctx=min(max(4096, min_ctx), model.arch.max_position_embeddings),
+            batch=4,
+            n_gpu_layers=(model.arch.num_layers + 1) if self.cuda else 0,
+            n_batch=512,
+        )
+
+    # ---- launch
+
+    def _spec_args(self, spec: str, model: PreparedModel, binary: str) -> List[str]:
+        """Speculative-decoding flags. llama.cpp (2026) moved to `--spec-type` and renamed --draft-max;
+        older builds only know the draft-model flags."""
+        kind, draft, k = speculative.parse(spec)
+        modern = _modern_spec(binary)
+        if kind == "ngram":
+            if not modern:
+                raise RuntimeError("this llama-server has no built-in n-gram speculative decoding")
+            return ["--spec-type", "ngram-mod", "--spec-ngram-mod-n-max", str(k)]
+        if kind != "draft":
+            raise RuntimeError(f"llama.cpp has no {kind} speculative decoding")
+        draft_path = model.draft_paths.get(draft or "")
+        if not draft_path or draft_path.startswith(("hf://", "convert://")):
+            raise RuntimeError(f"draft model for {spec} not materialized")
+        out = ["-md", draft_path]
+        out += ["--spec-type", "draft-simple", "--spec-draft-n-max", str(k)] if modern else ["--draft-max", str(k)]
+        if self.cuda:
+            out += ["-ngld", "999"]
+        return out
+
+    def launch_spec(self, cfg: Config, model: PreparedModel, port: int) -> LaunchSpec:
+        binary = llama_server_binary()
+        if binary is None:
+            raise RuntimeError("llama-server not found on PATH (set $LLAMA_SERVER)")
+        path = model.gguf_paths.get(cfg.quant)
+        if not path or path.startswith(("hf://", "convert://")):
+            raise RuntimeError(f"GGUF for {cfg.quant} not materialized: {path}")
+        n_parallel = max(1, cfg.batch)
+        args = [
+            binary,
+            "-m", path,
+            "--host", "127.0.0.1",
+            "--port", str(port),
+            "-c", str(cfg.ctx * n_parallel),  # -c is total context, split across slots
+            "-np", str(n_parallel),
+            "-b", str(max(cfg.n_batch or 512, cfg.prefill_budget or 0)),  # logical batch must cover -ub
+            "-ngl", str(cfg.n_gpu_layers if cfg.n_gpu_layers is not None else (999 if self.cuda else 0)),
+            "--alias", model.spec.hf_id,
+        ]
+        if cfg.prefill_budget is not None:
+            args += ["-ub", str(cfg.prefill_budget)]
+        threads = cfg.extra.get("threads")
+        if threads:
+            args += ["-t", str(threads)]
+        if self.cuda or cfg.kv_dtype != "auto":
+            args += ["-fa", "on"]  # a quantized V cache needs flash attention, on CPU too
+        if cfg.kv_dtype != "auto":
+            args += ["-ctk", cfg.kv_dtype, "-ctv", cfg.kv_dtype]
+        if cfg.spec_decode:
+            args += self._spec_args(cfg.spec_decode, model, binary)
+        args += render_extra(cfg.extra, skip=("threads",))
+        env = {}
+        if not self.cuda:
+            env["CUDA_VISIBLE_DEVICES"] = ""
+        return LaunchSpec(args=args, env=env)
+
+    # Physical prompt-processing batch (-ub) to try around the engine default of 512.
+    UBATCHES = (256, 1024, 2048)
+
+    def prefill_variants(self, cfg: Config) -> List[Config]:
+        return [cfg.model_copy(update={"prefill_budget": ub, "n_batch": max(cfg.n_batch or 512, ub)})
+                for ub in self.UBATCHES if ub != cfg.prefill_budget]
+
+    def workload_hooks(self, hw: HardwareDescriptor, model: PreparedModel) -> LlmtraceHooks:
+        return LlmtraceHooks(
+            health_path="/health",
+            model_name=model.spec.hf_id,
+            tokenizer_id=model.spec.hf_id,
+            gpu_ids=[hw.gpu.index] if (self.cuda and hw.gpu) else [],
+            process_memory=not self.cuda,
+        )
+
+
+class LlamaCppCudaBackend(LlamaCppBackend):
+    name = "llamacpp-cuda"
+    cuda = True
+
+
+class LlamaCppCpuBackend(LlamaCppBackend):
+    name = "llamacpp-cpu"
+    cuda = False
+    runtime_workspace_bytes = 512 * MiB
+
+    def candidate_configs(self, hw: HardwareDescriptor, model: PreparedModel, min_ctx: int = 0) -> List[Config]:
+        cfgs = super().candidate_configs(hw, model, min_ctx)
+        for c in cfgs:
+            c.extra["threads"] = hw.cpu.physical_cores
+        return cfgs
+
+    def default_config(self, hw: HardwareDescriptor, model: PreparedModel, min_ctx: int = 0) -> Config:
+        c = super().default_config(hw, model, min_ctx)
+        c.extra["threads"] = hw.cpu.physical_cores
+        return c
+
+
+def _quant_from_filename(path: str) -> Optional[str]:
+    base = os.path.basename(path).upper()
+    for q in GGUF_QUANTS + ("F16", "BF16", "F32"):
+        if q in base:
+            return q
+    return None