zeroquantz 0.1.0__py3-none-any.whl

zeroquantz/quantization/gguf_pipeline.py

@@ -0,0 +1,210 @@
+"""Production HF → quantized-GGUF pipeline — pip-only, no C/C++ toolchain.
+
+Every stage runs in an isolated ``tools/gguf`` env (built once) and hard-fails on
+error, so the output is a *proven* llama.cpp-loadable GGUF:
+
+    convert   HF safetensors -> f16 GGUF   via llama.cpp's convert_hf_to_gguf.py
+              (fetched as source, no build; all architectures + tokenizers)
+    quantize  f16 GGUF -> quantized GGUF   via llama-cpp-python (llama_model_quantize)
+    verify    load-test                    via llama-cpp-python (Llama load) — the gate
+
+llama-cpp-python ships prebuilt CPU wheels, so the whole pipeline installs with pip
+and needs no compiler. (GPU serving can use its CUDA wheel index; quantization is
+CPU work regardless.) ``plan()`` previews the commands without running.
+"""
+
+from __future__ import annotations
+
+import contextlib
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from zeroquantz.core.exceptions import QuantizationError
+from zeroquantz.core.subenv import EnvSpec, SubEnvManager
+from zeroquantz.quantization.native import ensure_llamacpp_src, llamacpp_src
+from zeroquantz.utils.logging import get_logger
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+log = get_logger(__name__)
+
+# Quantize a GGUF via llama-cpp-python's binding (handles top-level or low-level layout).
+_QUANTIZE_SCRIPT = '''
+import ctypes, sys
+import llama_cpp
+low = getattr(llama_cpp, "llama_cpp", llama_cpp)
+inp, out, ftype = sys.argv[1], sys.argv[2], sys.argv[3]
+const = "LLAMA_FTYPE_MOSTLY_" + ftype
+if not hasattr(low, const):
+    sys.stderr.write("unsupported type: %s\\n" % ftype); sys.exit(2)
+params = low.llama_model_quantize_default_params()
+params.ftype = getattr(low, const)
+rc = low.llama_model_quantize(inp.encode("utf-8"), out.encode("utf-8"), ctypes.byref(params))
+sys.exit(0 if rc == 0 else 1)
+'''
+
+# Load-test gate: if llama.cpp can construct the model, it is compatible.
+_VERIFY_SCRIPT = '''
+import sys
+from llama_cpp import Llama
+try:
+    Llama(model_path=sys.argv[1], n_ctx=64, n_gpu_layers=0, verbose=False)
+    print("OK: loaded", sys.argv[1]); sys.exit(0)
+except Exception as e:
+    sys.stderr.write("FAIL: %s\\n" % e); sys.exit(1)
+'''
+
+
+@dataclass
+class PipelineStage:
+    name: str
+    ready: bool
+    command: list[str]
+    hint: str = ""
+
+
+@dataclass
+class GgufArtifacts:
+    out_dir: Path
+    f16_gguf: Path
+    quantized_gguf: Path
+    quant_type: str
+
+
+class GgufPipeline:
+    """Orchestrates convert → quantize → verify into a proven GGUF (pip-only)."""
+
+    # One isolated env holds the converter deps + llama-cpp-python. Pinned to 3.12
+    # so prebuilt wheels are available (no compiler). Built on first use.
+    TOOLS_ENV = EnvSpec(
+        kind="tools",
+        name="gguf",
+        pip=("llama-cpp-python", "gguf", "sentencepiece", "protobuf",
+             "transformers", "safetensors", "numpy", "torch"),
+        python_version="3.12",
+    )
+
+    def __init__(self, subenv: SubEnvManager | None = None) -> None:
+        self._envs = subenv or SubEnvManager()
+
+    # ---- paths / setup ------------------------------------------------------
+
+    def _paths(self, quant: str, out_dir: str | Path) -> tuple[Path, Path, Path]:
+        out = Path(out_dir)
+        return out, out / "model.f16.gguf", out / f"model.{quant}.gguf"
+
+    def _env_dir(self) -> Path:
+        return self._envs.env_dir(self.TOOLS_ENV)
+
+    def _tools_python(self) -> Path:
+        return self._envs.venv_python(self._env_dir())
+
+    def _write_scripts(self) -> tuple[Path, Path]:
+        env_dir = self._env_dir()
+        q = env_dir / "zq_quantize.py"
+        v = env_dir / "zq_verify.py"
+        q.write_text(_QUANTIZE_SCRIPT, encoding="utf-8")
+        v.write_text(_VERIFY_SCRIPT, encoding="utf-8")
+        return q, v
+
+    # ---- preview ------------------------------------------------------------
+
+    def plan(self, model_dir: str, quant: str, out_dir: str | Path) -> list[PipelineStage]:
+        _out, f16, quantized = self._paths(quant, out_dir)
+        src = llamacpp_src()
+        py = self._tools_python()
+        env_ready = self._envs.is_ready(self.TOOLS_ENV)
+        script = str((src / "convert_hf_to_gguf.py") if src else "convert_hf_to_gguf.py")
+        env_dir = self._env_dir()
+        return [
+            PipelineStage(
+                "convert",
+                ready=bool(src) and env_ready,
+                command=[str(py), script, model_dir, "--outfile", str(f16), "--outtype", "f16"],
+                hint="fetched via shallow git clone; tools/gguf env built on first run",
+            ),
+            PipelineStage(
+                "quantize",
+                ready=env_ready,
+                command=[str(py), str(env_dir / "zq_quantize.py"), str(f16), str(quantized), quant],
+                hint="llama-cpp-python (prebuilt CPU wheel) in the tools/gguf env",
+            ),
+            PipelineStage(
+                "verify",
+                ready=env_ready,
+                command=[str(py), str(env_dir / "zq_verify.py"), str(quantized)],
+                hint="llama-cpp-python load-test",
+            ),
+        ]
+
+    # ---- building blocks (also usable standalone) ---------------------------
+
+    def ensure_env(self, *, progress: Callable[[str, float], None] | None = None) -> Path:
+        py = self._envs.ensure(self.TOOLS_ENV, progress=progress)
+        self._write_scripts()
+        return py
+
+    def quantize(
+        self, in_gguf: str, out_gguf: str, quant: str,
+        *, progress: Callable[[str, float], None] | None = None,
+    ) -> Path:
+        py = self.ensure_env(progress=progress)
+        script = self._env_dir() / "zq_quantize.py"
+        if progress:
+            progress(f"quantize -> {quant}", 0.6)
+        self._envs.run([str(py), str(script), str(in_gguf), str(out_gguf), quant])
+        out = Path(out_gguf)
+        if not out.exists():
+            raise QuantizationError("quantize produced no output GGUF.", detail=str(out))
+        return out
+
+    def verify(self, gguf: str, *, progress: Callable[[str, float], None] | None = None) -> None:
+        py = self.ensure_env(progress=progress)
+        script = self._env_dir() / "zq_verify.py"
+        if progress:
+            progress("verify (llama.cpp load-test)", 0.9)
+        self._envs.run([str(py), str(script), str(gguf)])  # raises on nonzero exit
+
+    def serve_command(self, gguf: str, port: int) -> list[str]:
+        return [str(self._tools_python()), "-m", "llama_cpp.server",
+                "--model", str(gguf), "--host", "0.0.0.0", "--port", str(port)]
+
+    # ---- full pipeline ------------------------------------------------------
+
+    def run(
+        self,
+        model_dir: str,
+        quant: str,
+        out_dir: str | Path,
+        *,
+        progress: Callable[[str, float], None] | None = None,
+        keep_f16: bool = False,
+    ) -> GgufArtifacts:
+        out, f16, quantized = self._paths(quant, out_dir)
+        out.mkdir(parents=True, exist_ok=True)
+
+        def step(msg: str, frac: float) -> None:
+            log.info("gguf-pipeline: %s", msg)
+            if progress:
+                progress(msg, frac)
+
+        step("preparing tools/gguf env (first run installs the pip deps)", 0.05)
+        py = self.ensure_env(progress=progress)
+        src = ensure_llamacpp_src(progress=progress)
+
+        step("convert: HF -> f16 GGUF", 0.2)
+        self._envs.run([str(py), str(src / "convert_hf_to_gguf.py"), model_dir,
+                        "--outfile", str(f16), "--outtype", "f16"])
+        if not f16.exists():
+            raise QuantizationError("convert produced no f16 GGUF.", detail=str(f16))
+
+        self.quantize(str(f16), str(quantized), quant, progress=progress)
+        self.verify(str(quantized), progress=progress)
+
+        if not keep_f16:
+            with contextlib.suppress(OSError):
+                f16.unlink()
+        step("done", 1.0)
+        return GgufArtifacts(out_dir=out, f16_gguf=f16, quantized_gguf=quantized, quant_type=quant)

zeroquantz/quantization/isolated.py

@@ -0,0 +1,248 @@
+"""Run quantization toolchains in isolated sub-environments.
+
+Many quantizers pin mutually-incompatible torch/transformers versions (e.g.
+GPTQModel wants torch≥2.8 while the archived AutoGPTQ froze at 2.2.1), so they
+cannot share one environment. Each toolchain gets a dedicated venv (built and
+cached by :class:`~zeroquantz.core.subenv.SubEnvManager` under
+``~/.zeroquantz/envs/quant/<family>/``); this module installs just that toolchain
+and runs a generated quantization script as a subprocess — passing config via a
+JSON file and reading results back from a ``manifest.json`` (rather than parsing
+tqdm-polluted stdout).
+
+The heavy work (multi-GB installs + quantization) happens in the child process.
+``plan_only=True`` returns the exact env + commands + script without executing,
+so a run can be inspected first.
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from zeroquantz.core.exceptions import QuantizationError
+from zeroquantz.core.subenv import EnvSpec, SubEnvManager, torch_index_for
+from zeroquantz.utils.logging import get_logger
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from zeroquantz.quantization.catalog import QuantFormat
+
+log = get_logger(__name__)
+
+
+# ---- per-toolchain quantization scripts ------------------------------------
+# Each reads {"model", "output", "bits", "group_size", "calibration"} from a JSON
+# config (argv --config) and writes {"output_model_dir": ...} to --manifest.
+
+_MANIFEST_TAIL = '''
+import json, sys
+from pathlib import Path
+_manifest = sys.argv[sys.argv.index("--manifest") + 1]
+Path(_manifest).write_text(json.dumps({"output_model_dir": cfg["output"], "status": "ok"}))
+print("ZEROQUANTZ_DONE", cfg["output"])
+'''
+
+# NOTE: recipes target each library's current documented API. They are best-effort
+# and may need updates as libraries evolve; run with plan_only to inspect first.
+_RECIPES: dict[str, str] = {
+    "awq": '''
+import json, sys
+from pathlib import Path
+cfg = json.loads(Path(sys.argv[sys.argv.index("--config") + 1]).read_text())
+from awq import AutoAWQForCausalLM
+from transformers import AutoTokenizer
+model = AutoAWQForCausalLM.from_pretrained(cfg["model"], safetensors=True)
+tok = AutoTokenizer.from_pretrained(cfg["model"], trust_remote_code=True)
+model.quantize(tok, quant_config={
+    "w_bit": cfg.get("bits", 4), "q_group_size": cfg.get("group_size", 128),
+    "zero_point": True, "version": "GEMM",
+})
+model.save_quantized(cfg["output"]); tok.save_pretrained(cfg["output"])
+''' + _MANIFEST_TAIL,
+
+    "gptq": '''
+import json, sys
+from pathlib import Path
+cfg = json.loads(Path(sys.argv[sys.argv.index("--config") + 1]).read_text())
+from gptqmodel import GPTQModel, QuantizeConfig
+qc = QuantizeConfig(bits=cfg.get("bits", 4), group_size=cfg.get("group_size", 128))
+model = GPTQModel.load(cfg["model"], qc)
+model.quantize(cfg["calibration"], batch_size=1)
+model.save(cfg["output"])
+''' + _MANIFEST_TAIL,
+
+    "autoround": '''
+import json, sys
+from pathlib import Path
+cfg = json.loads(Path(sys.argv[sys.argv.index("--config") + 1]).read_text())
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from auto_round import AutoRound
+model = AutoModelForCausalLM.from_pretrained(cfg["model"], torch_dtype="auto", device_map="auto")
+tok = AutoTokenizer.from_pretrained(cfg["model"])
+ar = AutoRound(model, tok, bits=cfg.get("bits", 4), group_size=cfg.get("group_size", 128), nsamples=128)
+ar.quantize_and_save(cfg["output"], format="auto_gptq")
+''' + _MANIFEST_TAIL,
+
+    "hqq": '''
+import json, sys
+from pathlib import Path
+cfg = json.loads(Path(sys.argv[sys.argv.index("--config") + 1]).read_text())
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from hqq.models.hf.base import AutoHQQHFModel
+from hqq.core.quantize import BaseQuantizeConfig
+model = AutoModelForCausalLM.from_pretrained(cfg["model"], torch_dtype=torch.float16)
+tok = AutoTokenizer.from_pretrained(cfg["model"])
+qcfg = BaseQuantizeConfig(nbits=cfg.get("bits", 4), group_size=cfg.get("group_size", 64))
+AutoHQQHFModel.quantize_model(model, quant_config=qcfg, compute_dtype=torch.float16, device="cuda")
+AutoHQQHFModel.save_quantized(model, cfg["output"]); tok.save_pretrained(cfg["output"])
+''' + _MANIFEST_TAIL,
+}
+
+
+@dataclass
+class IsolatedPlan:
+    """A previewable plan for an isolated quantization run."""
+
+    family: str
+    env_dir: Path
+    python: Path
+    pip_specs: tuple[str, ...]
+    torch_index: str | None
+    script_path: Path
+    commands: list[str] = field(default_factory=list)
+
+
+@dataclass
+class IsolatedResult:
+    family: str
+    output_dir: str
+    env_dir: Path
+    elapsed_s: float = 0.0
+
+
+class IsolatedEnvRunner:
+    """Create and drive per-toolchain isolated environments (kind='quant')."""
+
+    def __init__(self, root: Path | None = None) -> None:
+        self._envs = SubEnvManager(root=root)
+
+    # ---- spec / helpers -----------------------------------------------------
+
+    def _spec(self, fmt: QuantFormat, cuda_version: str | None) -> EnvSpec:
+        return EnvSpec(
+            kind="quant", name=fmt.family, pip=tuple(fmt.pip),
+            torch_index=torch_index_for(cuda_version),
+        )
+
+    def supports(self, family: str) -> bool:
+        return family in _RECIPES
+
+    @staticmethod
+    def venv_python(env_dir: Path) -> Path:
+        return SubEnvManager.venv_python(env_dir)
+
+    # ---- planning -----------------------------------------------------------
+
+    def plan(self, fmt: QuantFormat, *, cuda_version: str | None = None) -> IsolatedPlan:
+        if not self.supports(fmt.family):
+            raise QuantizationError(
+                f"No isolated quantization recipe for '{fmt.family}'.",
+                detail="ZeroQuantz can plan/estimate this format but not run it yet.",
+                suggestions=["/recommend", "pick an in-env or supported isolated format"],
+            )
+        spec = self._spec(fmt, cuda_version)
+        base = self._envs.plan(spec)
+        commands = [*base.commands,
+                    f"{base.python} quantize.py --config run_cfg.json --manifest manifest.json"]
+        return IsolatedPlan(
+            family=fmt.family, env_dir=base.env_dir, python=base.python, pip_specs=fmt.pip,
+            torch_index=spec.torch_index, script_path=base.env_dir / "quantize.py",
+            commands=commands,
+        )
+
+    # ---- environment build --------------------------------------------------
+
+    def ensure_env(
+        self,
+        fmt: QuantFormat,
+        *,
+        cuda_version: str | None = None,
+        progress: Callable[[str, float], None] | None = None,
+    ) -> Path:
+        return self._envs.ensure(self._spec(fmt, cuda_version), progress=progress)
+
+    # ---- execution ----------------------------------------------------------
+
+    def quantize(
+        self,
+        fmt: QuantFormat,
+        model_ref: str,
+        output_dir: str,
+        *,
+        bits: int | None = None,
+        group_size: int = 128,
+        calibration: list[str] | None = None,
+        cuda_version: str | None = None,
+        plan_only: bool = False,
+        progress: Callable[[str, float], None] | None = None,
+    ) -> IsolatedResult | IsolatedPlan:
+        if not self.supports(fmt.family):
+            raise QuantizationError(
+                f"No isolated quantization recipe for '{fmt.family}'.",
+                detail=f"'{fmt.label}' is produced by {fmt.produced_by}; "
+                "ZeroQuantz catalogs/estimates it but does not run it automatically yet.",
+            )
+        if plan_only:
+            return self.plan(fmt, cuda_version=cuda_version)
+
+        import time
+
+        spec = self._spec(fmt, cuda_version)
+        py = self.ensure_env(fmt, cuda_version=cuda_version, progress=progress)
+        env_dir = self._envs.env_dir(spec)
+        script_path = env_dir / "quantize.py"
+        script_path.write_text(_RECIPES[fmt.family], encoding="utf-8")
+        config = {
+            "model": model_ref,
+            "output": str(Path(output_dir).resolve()),
+            "bits": bits or max(2, round(fmt.bits_per_weight)),
+            "group_size": group_size,
+            "calibration": calibration or _default_calibration(),
+        }
+        cfg_path = env_dir / "run_cfg.json"
+        cfg_path.write_text(json.dumps(config), encoding="utf-8")
+        manifest_path = env_dir / "manifest.json"
+        manifest_path.unlink(missing_ok=True)
+
+        if progress:
+            progress(f"quantizing with {fmt.family}", 0.6)
+        started = time.perf_counter()
+        self._envs.run(
+            [str(py), "-u", str(script_path), "--config", str(cfg_path),
+             "--manifest", str(manifest_path)],
+            progress=progress,
+        )
+        elapsed = time.perf_counter() - started
+
+        if not manifest_path.exists():
+            raise QuantizationError(
+                f"Isolated {fmt.family} run finished without producing a manifest.",
+                detail="The quantization script did not complete; check the logs.",
+            )
+        manifest = json.loads(manifest_path.read_text())
+        if progress:
+            progress("done", 1.0)
+        return IsolatedResult(
+            family=fmt.family, output_dir=manifest["output_model_dir"], env_dir=env_dir,
+            elapsed_s=elapsed,
+        )
+
+
+def _default_calibration() -> list[str]:
+    from zeroquantz.profiling.calibration import default_calibration
+
+    return default_calibration().prompts

zeroquantz/quantization/memory.py

@@ -0,0 +1,133 @@
+"""Heuristic memory estimation shared by backends and the candidate generator.
+
+Everything here is an *estimate* derived from parameter counts and architecture
+dimensions — never a measurement. The candidate generator labels it as such, and
+the benchmark subsystem is what produces measured numbers.
+
+Model of VRAM at inference time:
+
+    vram = quantized_weights + cuda_context + kv_cache + activations
+
+* ``quantized_weights`` — a two-bucket model: parameters a weight-only backend
+  actually quantizes (linear layers) at the target precision, plus the
+  parameters it keeps at compute precision (embeddings, and the untied LM head).
+* ``cuda_context`` — a fixed allocator/context overhead.
+* ``kv_cache`` — grows with sequence length and (grouped) KV heads.
+* ``activations`` — a modest term proportional to hidden size and sequence.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from zeroquantz.utils import units
+
+if TYPE_CHECKING:
+    from zeroquantz.models.metadata import ModelProfile
+    from zeroquantz.quantization.config import QuantizationConfig
+
+# Fixed CUDA context / allocator overhead, in GiB. Realistic for a single device.
+CUDA_CONTEXT_GIB = 0.6
+# Activation memory is roughly proportional to hidden*seq*batch; this multiplies it.
+ACTIVATION_FACTOR = 2.0
+# Default profiling shape used for the estimate.
+DEFAULT_SEQ_LEN = 2048
+DEFAULT_BATCH = 1
+
+
+def estimate_weight_bytes(model: ModelProfile, config: QuantizationConfig) -> float:
+    """Estimated bytes of *weights* after applying ``config`` to ``model``."""
+    base_bytes = units.bytes_per_param(config.compute_dtype)
+    non_quant = model.non_quantizable_params * base_bytes
+    q_params = model.quantizable_params
+
+    if config.method == "mixed":
+        low = config.extra.get("low_precision", "int4")
+        high = config.extra.get("high_precision", "int8")
+        high_fraction = float(config.extra.get("high_fraction", 0.25))
+        per_param = high_fraction * units.bytes_per_param(high) + (
+            1 - high_fraction
+        ) * units.bytes_per_param(low)
+        quant = q_params * per_param
+    else:
+        quant = q_params * units.bytes_per_param(config.weight_precision)
+
+    return non_quant + quant
+
+
+def estimate_kv_cache_bytes(
+    model: ModelProfile,
+    *,
+    seq_len: int = DEFAULT_SEQ_LEN,
+    batch: int = DEFAULT_BATCH,
+    cache_dtype: str = "float16",
+) -> float:
+    """KV-cache bytes for a given sequence length and batch."""
+    layers = model.num_layers
+    kv_heads = model.num_key_value_heads or model.num_attention_heads
+    head_dim = model.head_dim
+    if not (layers and kv_heads and head_dim):
+        return 0.0
+    per_token = 2 * layers * kv_heads * head_dim * units.bytes_per_param(cache_dtype)
+    return per_token * seq_len * batch
+
+
+def estimate_activation_bytes(
+    model: ModelProfile,
+    *,
+    seq_len: int = DEFAULT_SEQ_LEN,
+    batch: int = DEFAULT_BATCH,
+    compute_dtype: str = "bfloat16",
+) -> float:
+    """A coarse activation-memory estimate."""
+    hidden = model.hidden_size
+    if not hidden:
+        return 0.0
+    return hidden * seq_len * batch * units.bytes_per_param(compute_dtype) * ACTIVATION_FACTOR
+
+
+def estimate_runtime_overhead_bytes(
+    model: ModelProfile,
+    *,
+    seq_len: int = DEFAULT_SEQ_LEN,
+    batch: int = DEFAULT_BATCH,
+    compute_dtype: str = "bfloat16",
+) -> float:
+    context = units.gb_to_bytes(CUDA_CONTEXT_GIB)
+    kv = estimate_kv_cache_bytes(model, seq_len=seq_len, batch=batch)
+    act = estimate_activation_bytes(
+        model, seq_len=seq_len, batch=batch, compute_dtype=compute_dtype
+    )
+    return context + kv + act
+
+
+def estimate_vram_bytes(
+    model: ModelProfile,
+    config: QuantizationConfig,
+    *,
+    seq_len: int = DEFAULT_SEQ_LEN,
+    batch: int = DEFAULT_BATCH,
+) -> float:
+    """Total estimated VRAM to *run* ``model`` under ``config``."""
+    weights = estimate_weight_bytes(model, config)
+    overhead = estimate_runtime_overhead_bytes(
+        model, seq_len=seq_len, batch=batch, compute_dtype=config.compute_dtype
+    )
+    return weights + overhead
+
+
+def estimate_weight_gb(model: ModelProfile, config: QuantizationConfig) -> float:
+    return round(units.bytes_to_gb(estimate_weight_bytes(model, config)), 2)
+
+
+def estimate_vram_gb(
+    model: ModelProfile,
+    config: QuantizationConfig,
+    *,
+    seq_len: int = DEFAULT_SEQ_LEN,
+    batch: int = DEFAULT_BATCH,
+) -> float:
+    return round(
+        units.bytes_to_gb(estimate_vram_bytes(model, config, seq_len=seq_len, batch=batch)),
+        2,
+    )

zeroquantz/quantization/native.py

@@ -0,0 +1,91 @@
+"""Locate ZeroQuantz's optional Rust binary and the llama.cpp converter source.
+
+The GGUF pipeline is otherwise pip-only (llama-cpp-python); the only native binary
+is the experimental Rust converter, and llama.cpp is needed only as *source* for its
+``convert_hf_to_gguf.py`` script (no compilation).
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from zeroquantz.core.exceptions import ZeroQuantzError
+from zeroquantz.utils.paths import paths
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+_LLAMACPP_REPO = "https://github.com/ggml-org/llama.cpp.git"
+
+
+def repo_root() -> Path:
+    """The repository root (parent of the installed ``zeroquantz`` package)."""
+    import zeroquantz
+
+    return Path(zeroquantz.__file__).resolve().parents[1]
+
+
+def find_native_binary(name: str) -> Path | None:
+    """Return the path to a built Rust binary (on PATH or ./target/release), or None."""
+    on_path = shutil.which(name)
+    if on_path:
+        return Path(on_path)
+    base = repo_root() / "target" / "release"
+    for exe in (f"{name}.exe", name):
+        if (base / exe).exists():
+            return base / exe
+    return None
+
+
+def llamacpp_src() -> Path | None:
+    """Locate a llama.cpp source tree containing ``convert_hf_to_gguf.py``.
+
+    Prefers ``$ZEROQUANTZ_LLAMACPP_SRC``, then the shallow clone ZeroQuantz manages at
+    ``~/.zeroquantz/llama.cpp``.
+    """
+    import os
+
+    env = os.environ.get("ZEROQUANTZ_LLAMACPP_SRC")
+    if env and (Path(env) / "convert_hf_to_gguf.py").exists():
+        return Path(env)
+    managed = paths().home / "llama.cpp"
+    if (managed / "convert_hf_to_gguf.py").exists():
+        return managed
+    return None
+
+
+def ensure_llamacpp_src(progress: Callable[[str, float], None] | None = None) -> Path:
+    """Return a llama.cpp source tree, shallow-cloning it if needed (no build).
+
+    Only the Python converter (``convert_hf_to_gguf.py``) is used from it, so this is
+    a source checkout, not a compile — needs ``git`` but no C/C++ toolchain.
+    """
+    existing = llamacpp_src()
+    if existing is not None:
+        return existing
+    if shutil.which("git") is None:
+        raise ZeroQuantzError(
+            "git is required to fetch llama.cpp's converter script.",
+            suggestions=[
+                "install git, or",
+                "set ZEROQUANTZ_LLAMACPP_SRC=/path/to/a/llama.cpp/checkout",
+            ],
+        )
+    dest = paths().ensure().home / "llama.cpp"
+    if progress:
+        progress("fetching llama.cpp converter (shallow clone, no build)", 0.02)
+    try:
+        subprocess.run(
+            ["git", "clone", "--depth", "1", _LLAMACPP_REPO, str(dest)],
+            check=True, capture_output=True, text=True,
+        )
+    except subprocess.CalledProcessError as exc:
+        raise ZeroQuantzError(
+            "Failed to clone llama.cpp.", detail=(exc.stderr or "").strip()[:500]
+        ) from exc
+    if not (dest / "convert_hf_to_gguf.py").exists():
+        raise ZeroQuantzError(f"cloned llama.cpp but convert_hf_to_gguf.py is missing in {dest}.")
+    return dest