flyteplugins-llamacpp 2.7.0__py3-none-any.whl

flyteplugins/llamacpp/__init__.py

@@ -0,0 +1,4 @@
+__all__ = ["DEFAULT_LLAMA_CPP_IMAGE", "LlamaCppAppEnvironment", "build_llama_cpp_image"]
+
+from flyteplugins.llamacpp._app_environment import LlamaCppAppEnvironment
+from flyteplugins.llamacpp._image import DEFAULT_LLAMA_CPP_IMAGE, build_llama_cpp_image

flyteplugins/llamacpp/_app_environment.py

@@ -0,0 +1,269 @@
+from __future__ import annotations
+
+import shlex
+from dataclasses import dataclass, field, replace
+from typing import Any, Literal, Optional, Union
+
+import flyte.app
+import rich.repr
+from flyte import Environment, Image, Resources, SecretRequest
+from flyte.app import ArtifactValue, Parameter, RunOutput
+from flyte.app._types import Port
+from flyte.models import SerializationContext
+
+from flyteplugins.llamacpp._image import DEFAULT_LLAMA_CPP_IMAGE
+
+
+def _shell_safe(args: list[str]) -> list[str]:
+    """Quote args that have to survive a trip through a shell.
+
+    `fserve` runs the app with `Popen(" ".join(args), shell=True)` (flyte/_bin/serve.py), so
+    any token carrying spaces or quotes -- a `--chat-template` blob, say -- reaches the server
+    mangled unless it is quoted here. `shlex.quote` is the identity function for ordinary
+    tokens, so this is a no-op for everything else.
+
+    Tokens starting with `$` are left alone: `fserve` expands those against the container
+    environment *before* joining, and quoting would turn the marker into a literal.
+    """
+    return [arg if arg.startswith("$") else shlex.quote(arg) for arg in args]
+
+
+@rich.repr.auto
+@dataclass(kw_only=True, repr=True)
+class LlamaCppAppEnvironment(flyte.app.AppEnvironment):
+    """
+    App environment backed by llama.cpp (llama-server) for serving GGUF models.
+
+    This environment serves an OpenAI-compatible endpoint (under `/v1`) plus the llama.cpp
+    Web UI, with the specified GGUF model and configuration. llama.cpp shines where vLLM and
+    SGLang don't fit: quantized GGUF weights, partial CPU offload of models larger than VRAM,
+    and CPU-only serving.
+
+    Args:
+        name: The name of the application.
+        port: Port the application listens on. Defaults to 8080.
+        requests: Compute resource requests for application.
+        secrets: Secrets that are requested for application.
+        limits: Compute resource limits for application.
+        env_vars: Environment variables to set for the application.
+        scaling: Scaling configuration for the app environment.
+        domain: Domain to use for the app.
+        cluster_pool: The target cluster_pool where the app should be deployed.
+        requires_auth: Whether the public URL requires authentication.
+        type: Type of app.
+        extra_args: Extra args to pass to `llama-server`, e.g. `"--ctx-size 32768 --jinja"`.
+            Run `llama-server --help` or see
+            https://github.com/ggml-org/llama.cpp/tree/master/tools/server for details.
+        model_path: Remote path to the GGUF weights -- a directory containing `.gguf` file(s)
+            or a direct path to one (e.g. s3://bucket/path/to/model), or a
+            `RunOutput`/`ArtifactValue` resolved at deploy time. The weights are downloaded
+            into the container and the served `.gguf` is located at startup (for sharded
+            models, the `-00001-of-` shard is picked; llama-server finds the rest).
+        model_hf_path: Hugging Face GGUF repo, optionally with a quant tag (e.g.
+            `ggml-org/gemma-3-4b-it-GGUF:Q4_K_M`). Passed to llama-server as `--hf-repo`,
+            which downloads the weights at startup.
+        model_id: Model id exposed by the server (llama-server's `--alias`).
+        draft_model_path: Remote path to the draft model GGUF used for speculative decoding,
+            or a `RunOutput`/`ArtifactValue` resolved at deploy time. Downloaded alongside the
+            target model and passed as `--model-draft`. Tune the speculation via `extra_args`
+            (`--draft-max`, `--draft-min`, `--gpu-layers-draft`, ...).
+        draft_model_hf_path: Hugging Face GGUF repo for the draft model, as an alternative to
+            `draft_model_path`. Passed as `--hf-repo-draft`.
+    """
+
+    port: int | Port = 8080
+    type: str = "llama.cpp"
+    extra_args: str | list[str] = ""
+    model_path: str | RunOutput | ArtifactValue = ""
+    model_hf_path: str = ""
+    model_id: str = ""
+    draft_model_path: str | RunOutput | ArtifactValue = ""
+    draft_model_hf_path: str = ""
+    image: str | Image | Literal["auto"] = DEFAULT_LLAMA_CPP_IMAGE
+    # Under /tmp, and that is not cosmetic: ``fserve`` materializes each mounted Parameter
+    # through ``_ensure_dest_writable``, which needs the *image's* user to be able to create the
+    # parent directory. The released Flyte base image runs non-root, so a mount at the
+    # filesystem root -- or under /root -- fails with "Permission denied" before the engine ever
+    # starts. /tmp is writable for any user and lives on the same overlay filesystem the weights
+    # are already budgeted against by ``disk=``.
+    _model_mount_path: str = field(default="/tmp/flyte/model", init=False)
+    _draft_model_mount_path: str = field(default="/tmp/flyte/draft-model", init=False)
+
+    def __post_init__(self):
+        if self.env_vars is None:
+            self.env_vars = {}
+
+        if self._server is not None:
+            raise ValueError("server function cannot be set for LlamaCppAppEnvironment")
+
+        if self._on_startup is not None:
+            raise ValueError("on_startup function cannot be set for LlamaCppAppEnvironment")
+
+        if self._on_shutdown is not None:
+            raise ValueError("on_shutdown function cannot be set for LlamaCppAppEnvironment")
+
+        if self.model_id == "":
+            raise ValueError("model_id must be defined")
+
+        if self.model_path == "" and self.model_hf_path == "":
+            raise ValueError("model_path or model_hf_path must be defined")
+        if self.model_path != "" and self.model_hf_path != "":
+            raise ValueError("model_path and model_hf_path cannot be set at the same time")
+
+        if self.draft_model_path != "" and self.draft_model_hf_path != "":
+            raise ValueError("draft_model_path and draft_model_hf_path cannot be set at the same time")
+
+        if self.args:
+            raise ValueError("args cannot be set for LlamaCppAppEnvironment. Use `extra_args` to add extra arguments.")
+
+        if isinstance(self.extra_args, str):
+            extra_args = shlex.split(self.extra_args)
+        else:
+            extra_args = list(self.extra_args)
+
+        # The GGUF filename inside a mounted directory is unknown at deploy time, so mounted
+        # weights go through the `llama-cpp-fserve` shim, which resolves `--model-dir` /
+        # `--draft-model-dir` to concrete .gguf paths and execs llama-server.
+        if self.model_path:
+            model_args = ["--model-dir", self._model_mount_path]
+        else:
+            model_args = ["--hf-repo", self.model_hf_path]
+
+        draft_args: list[str] = []
+        if self.draft_model_path:
+            draft_args = ["--draft-model-dir", self._draft_model_mount_path]
+        elif self.draft_model_hf_path:
+            draft_args = ["--hf-repo-draft", self.draft_model_hf_path]
+
+        # llama-server binds 127.0.0.1 by default, which is unreachable from outside the
+        # container.
+        host_args = [] if "--host" in extra_args else ["--host", "0.0.0.0"]
+
+        self.args = _shell_safe(
+            [
+                "llama-cpp-fserve",
+                *model_args,
+                "--alias",
+                self.model_id,
+                *host_args,
+                "--port",
+                str(self.get_port().port),
+                *draft_args,
+                *extra_args,
+            ]
+        )
+
+        if self.parameters:
+            raise ValueError("parameters cannot be set for LlamaCppAppEnvironment")
+
+        parameters: list[Parameter] = []
+        if self.model_path:
+            parameters.append(
+                Parameter(
+                    name="model_path",
+                    value=self.model_path,
+                    download=True,
+                    mount=self._model_mount_path,
+                )
+            )
+        if self.draft_model_path:
+            parameters.append(
+                Parameter(
+                    name="draft_model_path",
+                    value=self.draft_model_path,
+                    download=True,
+                    mount=self._draft_model_mount_path,
+                )
+            )
+        if parameters:
+            self.parameters = parameters
+
+        self.links = [flyte.app.Link(path="/", title="llama.cpp Web UI", is_relative=True), *self.links]
+
+        if self.image is None or self.image == "auto":
+            self.image = DEFAULT_LLAMA_CPP_IMAGE
+
+        super().__post_init__()
+
+    def container_args(self, serialization_context: SerializationContext) -> list[str]:
+        """Return the container arguments for llama.cpp."""
+        if isinstance(self.args, str):
+            return shlex.split(self.args)
+        return self.args or []
+
+    def clone_with(
+        self,
+        name: str,
+        image: Optional[Union[str, Image, Literal["auto"]]] = None,
+        resources: Optional[Resources] = None,
+        env_vars: Optional[dict[str, str]] = None,
+        secrets: Optional[SecretRequest] = None,
+        depends_on: Optional[list[Environment]] = None,
+        description: Optional[str] = None,
+        interruptible: Optional[bool] = None,
+        **kwargs: Any,
+    ) -> LlamaCppAppEnvironment:
+        port = kwargs.pop("port", None)
+        extra_args = kwargs.pop("extra_args", None)
+        if "model_path" in kwargs:
+            set_model_path = True
+            model_path = kwargs.pop("model_path", "") or ""
+        else:
+            set_model_path = False
+            model_path = self.model_path
+        if "model_hf_path" in kwargs:
+            set_model_hf_path = True
+            model_hf_path = kwargs.pop("model_hf_path", "") or ""
+        else:
+            set_model_hf_path = False
+            model_hf_path = self.model_hf_path
+        if "draft_model_path" in kwargs:
+            set_draft_model_path = True
+            draft_model_path = kwargs.pop("draft_model_path", "") or ""
+        else:
+            set_draft_model_path = False
+            draft_model_path = self.draft_model_path
+        if "draft_model_hf_path" in kwargs:
+            set_draft_model_hf_path = True
+            draft_model_hf_path = kwargs.pop("draft_model_hf_path", "") or ""
+        else:
+            set_draft_model_hf_path = False
+            draft_model_hf_path = self.draft_model_hf_path
+        model_id = kwargs.pop("model_id", None)
+
+        if kwargs:
+            raise TypeError(f"Unexpected keyword arguments: {list(kwargs.keys())}")
+
+        kwargs = self._get_kwargs()
+        kwargs["name"] = name
+        kwargs["args"] = None
+        kwargs["parameters"] = None
+        if image is not None:
+            kwargs["image"] = image
+        if resources is not None:
+            kwargs["resources"] = resources
+        if env_vars is not None:
+            kwargs["env_vars"] = env_vars
+        if secrets is not None:
+            kwargs["secrets"] = secrets
+        if depends_on is not None:
+            kwargs["depends_on"] = depends_on
+        if description is not None:
+            kwargs["description"] = description
+        if interruptible is not None:
+            kwargs["interruptible"] = interruptible
+        if port is not None:
+            kwargs["port"] = port
+        if extra_args is not None:
+            kwargs["extra_args"] = extra_args
+        if set_model_path:
+            kwargs["model_path"] = model_path
+        if set_model_hf_path:
+            kwargs["model_hf_path"] = model_hf_path
+        if set_draft_model_path:
+            kwargs["draft_model_path"] = draft_model_path
+        if set_draft_model_hf_path:
+            kwargs["draft_model_hf_path"] = draft_model_hf_path
+        if model_id is not None:
+            kwargs["model_id"] = model_id
+        return replace(self, **kwargs)

flyteplugins/llamacpp/_constants.py

@@ -0,0 +1,24 @@
+"""Constants shared by the llama.cpp image build and server shim."""
+
+LLAMA_CPP_REPO = "https://github.com/ggml-org/llama.cpp"
+LLAMA_CPP_INSTALL_DIR = "/opt/llama.cpp"
+LLAMA_SERVER_BINARY = f"{LLAMA_CPP_INSTALL_DIR}/build/bin/llama-server"
+
+CUDA_HOME = "/usr/local/cuda-12.8"
+CUDA_TOOLKIT_PACKAGE = "cuda-toolkit-12-8"
+# CUDA stubs let the linker resolve libcuda.so on GPU-less build machines; the real
+# driver library is injected by the container runtime on the serving node.
+CUDA_STUB_LIB = f"{CUDA_HOME}/lib64/stubs"
+
+# Default target arch: 89 = Ada Lovelace (L4/L40S). Image builds run on CPU-only
+# nodes, so CMAKE_CUDA_ARCHITECTURES=native is out. Pass a ";"-separated list to
+# ``build_llama_cpp_image(cuda_arch=...)`` to produce a fat binary, e.g. "80;86;89;90"
+# to also cover A100 (80), A10 (86), and H100 (90).
+DEFAULT_CUDA_ARCH = "89"
+
+# Node is required to build llama-server's embedded Web UI from source. The UI is a
+# Vite 7 / SvelteKit app (needs Node >=22.12); without Node, cmake's llama-ui-assets
+# target skips the npm build and falls back to a prebuilt UI download that is
+# version-mismatched against embed.cpp and fails the build.
+NODE_VERSION = "22.12.0"
+NODE_HOME = "/opt/node"

flyteplugins/llamacpp/_image.py

@@ -0,0 +1,153 @@
+"""Container image with llama.cpp built from source (CUDA-enabled by default)."""
+
+from __future__ import annotations
+
+import base64
+
+import flyte
+
+from flyteplugins.llamacpp._constants import (
+    CUDA_HOME,
+    CUDA_STUB_LIB,
+    CUDA_TOOLKIT_PACKAGE,
+    DEFAULT_CUDA_ARCH,
+    LLAMA_CPP_INSTALL_DIR,
+    LLAMA_CPP_REPO,
+    NODE_HOME,
+    NODE_VERSION,
+)
+
+_NODE_TARBALL = f"node-v{NODE_VERSION}-linux-x64"
+
+
+def _node_install_commands() -> list[str]:
+    return [
+        f"wget -q https://nodejs.org/dist/v{NODE_VERSION}/{_NODE_TARBALL}.tar.xz -O /tmp/node.tar.xz",
+        f"mkdir -p {NODE_HOME} && tar -xJf /tmp/node.tar.xz -C {NODE_HOME} --strip-components=1 && rm /tmp/node.tar.xz",
+    ]
+
+
+def _run_script(script: str) -> str:
+    """Serialize a shell script for an image-builder RUN step without quoting hazards.
+
+    The remote image builder mangles embedded quoting: double quotes inside a command
+    are dropped, so a fat-binary `-DCMAKE_CUDA_ARCHITECTURES="80;86;89"` reaches `sh`
+    as `80;86;89` and the `;` splits it into separate commands ("sh: 1: 86: not").
+    Base64-encoding the whole script and decoding it into `sh` at build time keeps
+    every character intact regardless of how the builder serializes RUN commands.
+    """
+    encoded = base64.b64encode(script.encode()).decode()
+    return f"echo {encoded} | base64 -d | sh"
+
+
+def _clone_commands(repo: str, ref: str | None) -> list[str]:
+    commands = [f"git clone {repo} {LLAMA_CPP_INSTALL_DIR}"]
+    if ref is not None:
+        commands.append(f"cd {LLAMA_CPP_INSTALL_DIR} && git checkout {ref}")
+    return commands
+
+
+def build_llama_cpp_image(
+    *,
+    name: str = "llama-cpp-app-image",
+    cuda: bool = True,
+    cuda_arch: str = DEFAULT_CUDA_ARCH,
+    repo: str = LLAMA_CPP_REPO,
+    ref: str | None = None,
+) -> flyte.Image:
+    """Build a Debian image with llama-server compiled from source.
+
+    Args:
+        name: Name of the image.
+        cuda: Build with CUDA support (GGML_CUDA=ON). Set to False for a CPU-only image.
+        cuda_arch: Target CUDA architecture(s) for the kernel build, as a ";"-separated
+            list of compute capabilities (e.g. "89" for L4/L40S, "80;86;89;90" for a fat
+            binary that also covers A100/A10/H100). Ignored when `cuda=False`.
+        repo: Git repository to build llama.cpp from.
+        ref: Git ref (tag, branch, or commit) to check out. None builds the default
+            branch tip; pin a release tag (e.g. "b6148") for reproducible builds.
+    """
+    image = flyte.Image.from_debian_base(name=name).with_apt_packages(
+        "git",
+        "build-essential",
+        "cmake",
+        "pkg-config",
+        "wget",
+        "curl",
+        "ca-certificates",
+        "libnuma-dev",
+        "libssl-dev",
+        "pciutils",
+        "libcurl4-openssl-dev",
+        "xz-utils",
+    )
+
+    if cuda:
+        image = image.with_commands(
+            [
+                (
+                    "wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/"
+                    "cuda-keyring_1.1-1_all.deb"
+                ),
+                "dpkg -i cuda-keyring_1.1-1_all.deb",
+                "apt-get update",
+                f"apt-get install -y {CUDA_TOOLKIT_PACKAGE}",
+            ]
+        )
+        cmake_configure = (
+            f"LIBRARY_PATH={CUDA_STUB_LIB}:$LIBRARY_PATH "
+            f"cmake -S {LLAMA_CPP_INSTALL_DIR} -B {LLAMA_CPP_INSTALL_DIR}/build "
+            "-DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=ON "
+            "-DCMAKE_BUILD_TYPE=Release "
+            f'-DCMAKE_CUDA_ARCHITECTURES="{cuda_arch}" '
+            f'-DCMAKE_EXE_LINKER_FLAGS="-Wl,-rpath-link,{CUDA_STUB_LIB}" '
+            f'-DCMAKE_SHARED_LINKER_FLAGS="-Wl,-rpath-link,{CUDA_STUB_LIB}"'
+        )
+        cmake_build_prefix = f"LIBRARY_PATH={CUDA_STUB_LIB}:$LIBRARY_PATH "
+        stub_commands = [f"ln -sf {CUDA_STUB_LIB}/libcuda.so {CUDA_STUB_LIB}/libcuda.so.1"]
+    else:
+        cmake_configure = (
+            f"cmake -S {LLAMA_CPP_INSTALL_DIR} -B {LLAMA_CPP_INSTALL_DIR}/build "
+            "-DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release"
+        )
+        cmake_build_prefix = ""
+        stub_commands = []
+
+    image = image.with_commands(
+        _node_install_commands()
+        + _clone_commands(repo, ref)
+        + stub_commands
+        + [
+            # The cmake configure carries a ";"-separated CMAKE_CUDA_ARCHITECTURES;
+            # ship it via _run_script so the ";" and quotes survive the builder.
+            _run_script(cmake_configure),
+            # Build llama-server. The llama-ui-assets target runs an npm build of
+            # tools/ui to embed the Web UI; put node/npm on PATH. The UI's .npmrc
+            # sets engine-strict=true, which makes a transitive dep's Node bound a
+            # fatal EBADENGINE; override it so install proceeds (Node here satisfies
+            # the UI's real Vite requirement).
+            _run_script(
+                f"PATH={NODE_HOME}/bin:$PATH npm_config_engine_strict=false "
+                f"{cmake_build_prefix}"
+                f"cmake --build {LLAMA_CPP_INSTALL_DIR}/build --config Release "
+                "-j $(nproc) --target llama-server"
+            ),
+        ]
+    )
+
+    env_vars = {
+        "PATH": f"{LLAMA_CPP_INSTALL_DIR}/build/bin:{CUDA_HOME}/bin:$PATH"
+        if cuda
+        else (f"{LLAMA_CPP_INSTALL_DIR}/build/bin:$PATH"),
+        # Under /tmp so `-hf`-style downloads work with the non-root user the
+        # released Flyte base image runs as.
+        "LLAMA_CACHE": "/tmp/llama.cpp/cache",
+    }
+    if cuda:
+        env_vars["CUDA_HOME"] = CUDA_HOME
+
+    # The plugin itself provides the `llama-cpp-fserve` entrypoint (and pulls in flyte).
+    return image.with_env_vars(env_vars).with_pip_packages("flyteplugins-llamacpp", pre=True)
+
+
+DEFAULT_LLAMA_CPP_IMAGE = build_llama_cpp_image()

flyteplugins/llamacpp/_server.py

@@ -0,0 +1,73 @@
+"""`llama-cpp-fserve`: resolve mounted GGUF weights, then exec llama-server.
+
+The app environment mounts model weights as a *directory* (the GGUF filename inside a
+`RunOutput`/blob-store directory is unknown at deploy time), but llama-server takes a
+path to a concrete `.gguf` file. This shim bridges the two: it rewrites
+
+- `--model-dir <dir>` -> `--model <resolved .gguf>`
+- `--draft-model-dir <dir>` -> `--model-draft <resolved .gguf>`
+
+leaving every other argument untouched, and then replaces itself with llama-server.
+"""
+
+from __future__ import annotations
+
+import glob
+import logging
+import os
+import shutil
+import sys
+from pathlib import Path
+
+from flyteplugins.llamacpp._constants import LLAMA_SERVER_BINARY
+
+logger = logging.getLogger(__name__)
+
+_DIR_FLAG_TO_MODEL_FLAG = {
+    "--model-dir": "--model",
+    "--draft-model-dir": "--model-draft",
+}
+
+
+def find_gguf(path: str) -> str:
+    """Resolve the GGUF file to serve from a mounted file or directory.
+
+    For sharded models only the first shard is passed to llama-server (it discovers
+    the rest itself), so `*-00001-of-*.gguf` wins over other matches.
+    """
+    if os.path.isfile(path):
+        return path
+    matches = sorted(glob.glob(os.path.join(path, "**", "*.gguf"), recursive=True))
+    if not matches:
+        raise FileNotFoundError(f"No .gguf files found under {path!r}")
+    first_shards = [m for m in matches if "-00001-of-" in Path(m).name]
+    return first_shards[0] if first_shards else matches[0]
+
+
+def build_command(argv: list[str]) -> list[str]:
+    """Build the llama-server argv, resolving `--model-dir`/`--draft-model-dir`."""
+    server = shutil.which("llama-server") or LLAMA_SERVER_BINARY
+    cmd = [server]
+    i = 0
+    while i < len(argv):
+        arg = argv[i]
+        model_flag = _DIR_FLAG_TO_MODEL_FLAG.get(arg)
+        if model_flag is not None:
+            if i + 1 >= len(argv):
+                raise ValueError(f"{arg} requires a value")
+            cmd.extend([model_flag, find_gguf(argv[i + 1])])
+            i += 2
+        else:
+            cmd.append(arg)
+            i += 1
+    return cmd
+
+
+def main() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    )
+    cmd = build_command(sys.argv[1:])
+    logger.info("Starting llama-server: %s", " ".join(cmd))
+    os.execv(cmd[0], cmd)

flyteplugins_llamacpp-2.7.0.dist-info/METADATA

@@ -0,0 +1,128 @@
+Metadata-Version: 2.4
+Name: flyteplugins-llamacpp
+Version: 2.7.0
+Summary: llama.cpp plugin for flyte
+Author-email: Niels Bantilan <cosmicbboy@users.noreply.github.com>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: flyte>=2.0.0b43
+
+# Union llama.cpp Plugin
+
+Serve GGUF models with [llama.cpp](https://github.com/ggml-org/llama.cpp)'s `llama-server` behind Flyte Apps.
+
+This plugin provides the `LlamaCppAppEnvironment` class for deploying quantized (GGUF) LLMs
+with an OpenAI-compatible API (under `/v1`) and the built-in llama.cpp Web UI. llama.cpp shines
+where vLLM and SGLang don't fit: quantized GGUF weights, partial CPU offload of models larger
+than VRAM, and CPU-only serving.
+
+## Installation
+
+```bash
+pip install --pre flyteplugins-llamacpp
+```
+
+## Usage
+
+```python
+import flyte
+import flyte.app
+from flyteplugins.llamacpp import LlamaCppAppEnvironment
+
+llama_app = LlamaCppAppEnvironment(
+    name="my-llm-app",
+    # A directory (or direct path) of GGUF weights in object storage...
+    model_path="s3://your-bucket/models/your-model-gguf",
+    model_id="your-model-id",
+    resources=flyte.Resources(cpu="4", memory="32Gi", gpu="L40s:1", disk="100Gi"),
+    scaling=flyte.app.Scaling(replicas=(0, 1), scaledown_after=300),
+)
+
+if __name__ == "__main__":
+    flyte.init_from_config()
+    app = flyte.serve(llama_app)
+    print(f"Deployed llama.cpp app: {app.url}")
+```
+
+`model_path` accepts a remote directory or file path, a `RunOutput` (e.g. from a prefetch task
+that downloaded the GGUF), or an `ArtifactValue`. The weights are downloaded into the container
+and the served `.gguf` is located at startup; for sharded models the `-00001-of-` shard is
+selected and llama-server discovers the rest.
+
+Alternatively, point directly at a Hugging Face GGUF repo (with an optional quant tag) and let
+llama-server download it at startup:
+
+```python
+llama_app = LlamaCppAppEnvironment(
+    name="gemma-app",
+    model_hf_path="ggml-org/gemma-3-4b-it-GGUF:Q4_K_M",
+    model_id="gemma-3-4b-it",
+    resources=flyte.Resources(cpu="4", memory="16Gi", gpu="L4:1", disk="50Gi"),
+)
+```
+
+## The default image
+
+llama.cpp ships no GPU pip wheel, so the default image compiles `llama-server` from source with
+CUDA enabled (plus the embedded Web UI). The default targets compute capability 8.9 (L4/L40S);
+use `build_llama_cpp_image` to target other GPUs, pin a llama.cpp release for reproducible
+builds, or build a CPU-only image:
+
+```python
+from flyteplugins.llamacpp import LlamaCppAppEnvironment, build_llama_cpp_image
+
+llama_app = LlamaCppAppEnvironment(
+    name="my-llm-app",
+    image=build_llama_cpp_image(
+        cuda_arch="80;86;89;90",  # fat binary: A100, A10, L4/L40S, H100
+        ref="b6148",              # pin a llama.cpp release tag
+    ),
+    ...
+)
+```
+
+`build_llama_cpp_image(cuda=False)` produces a CPU-only image for serving small quantized
+models without a GPU.
+
+## Speculative decoding
+
+Point `draft_model_path` (object storage, `RunOutput`, or `ArtifactValue`) or
+`draft_model_hf_path` at a small draft GGUF and it is passed to llama-server as
+`--model-draft` / `--hf-repo-draft`. Tune the speculation via `extra_args`:
+
+```python
+llama_app = LlamaCppAppEnvironment(
+    name="qwen3-spec",
+    model_path="s3://your-bucket/models/qwen3-32b-gguf",
+    model_id="qwen3-32b",
+    draft_model_hf_path="ggml-org/Qwen3-0.6B-GGUF:Q8_0",
+    extra_args="--draft-max 16 --draft-min 1 --gpu-layers-draft 99",
+    resources=flyte.Resources(cpu="8", memory="64Gi", gpu="L40s:1", disk="120Gi"),
+)
+```
+
+## Extra arguments
+
+`extra_args` is appended to `llama-server`, as either a string or a list:
+
+```python
+llama_app = LlamaCppAppEnvironment(
+    name="my-llm-app",
+    model_path="s3://your-bucket/models/your-model-gguf",
+    model_id="your-model-id",
+    extra_args="--ctx-size 32768 --parallel 4 --jinja",
+)
+```
+
+Useful flags: `--ctx-size` (context length), `--parallel` (concurrent request slots),
+`--jinja` (enable the model's chat template, needed for tool calling), `--n-gpu-layers`
+(limit GPU offload for models larger than VRAM; recent llama.cpp offloads everything by
+default), `--cache-type-k/--cache-type-v` (quantized KV cache), `--flash-attn`.
+
+Arguments are quoted before they reach the server, so values containing spaces or JSON survive
+intact. Arguments of the form `$MY_VAR` are left unquoted so that Flyte still expands them from
+the app's environment.
+
+Run `llama-server --help` or see the
+[llama-server docs](https://github.com/ggml-org/llama.cpp/tree/master/tools/server)
+for all options.

flyteplugins_llamacpp-2.7.0.dist-info/RECORD

@@ -0,0 +1,10 @@
+flyteplugins/llamacpp/__init__.py,sha256=zVnwPFuug96hLpbR0VsgkdBtDUTyJV8FAAdg3ke_KGE,252
+flyteplugins/llamacpp/_app_environment.py,sha256=NdvoJ3UNipZ2HMIsd_r1KwUwO4xL0iiqPGb4mALhp1k,11723
+flyteplugins/llamacpp/_constants.py,sha256=W_EfK5ITz8-7Sq56MAAA1l6MXH1agk4YO_LIRzN6fUY,1205
+flyteplugins/llamacpp/_image.py,sha256=fDmug_s_QBAJ03gWS1BNyUx1Okm3CUjYg_lUW3Lxrg0,5838
+flyteplugins/llamacpp/_server.py,sha256=c3qI3zrTr_qnoc9njMCZoZJkYYfAdd2wbezH1tBeF0c,2377
+flyteplugins_llamacpp-2.7.0.dist-info/METADATA,sha256=VlALvkkc_GCNFwuQKzWDA0gni9tvBdx6KnpR9yCFSOs,4548
+flyteplugins_llamacpp-2.7.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+flyteplugins_llamacpp-2.7.0.dist-info/entry_points.txt,sha256=40N9KHFJ9tN29aXctZzU1VBcCBPAuI4B25xJ6SgFFko,72
+flyteplugins_llamacpp-2.7.0.dist-info/top_level.txt,sha256=cgd779rPu9EsvdtuYgUxNHHgElaQvPn74KhB5XSeMBE,13
+flyteplugins_llamacpp-2.7.0.dist-info/RECORD,,

flyteplugins_llamacpp-2.7.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

flyteplugins_llamacpp-2.7.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+llama-cpp-fserve = flyteplugins.llamacpp._server:main

flyteplugins_llamacpp-2.7.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+flyteplugins