quantfit 0.1.0__py3-none-any.whl

quantfit/__init__.py

@@ -0,0 +1,6 @@
+"""quantfit — quantize an LLM if it fits your GPU."""
+from quantfit.gpufit import FitReport, check_fit
+from quantfit.spec import DEFAULT_SPEC, QuantSpec
+
+__version__ = "0.1.0"
+__all__ = ["FitReport", "check_fit", "QuantSpec", "DEFAULT_SPEC", "__version__"]

quantfit/backends/__init__.py

@@ -0,0 +1 @@
+"""Quantization backends. Each turns a (method, scheme) into a saved artifact."""

quantfit/backends/compressed_tensors.py

@@ -0,0 +1,104 @@
+"""compressed-tensors backend (llm-compressor): the method × scheme matrix.
+
+awq / gptq / autoround / smoothquant calibrate; fp8 / rtn do not. All emit
+compressed-tensors (vLLM-loadable). For the calibrated algorithms the only
+cross-method difference is the algorithm itself — same calibration, same format
+— so the methods are comparable, not confounded.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from quantfit.spec import QuantSpec
+
+_TARGETS = ["Linear"]
+_IGNORE = ["lm_head"]
+_SMOOTHING_STRENGTH = 0.8  # SmoothQuant migration strength (standard default)
+
+
+def build_recipe(method: str, scheme: str):
+    """Construct the llm-compressor recipe (modifier or modifier list) for a method."""
+    from llmcompressor.modifiers.awq import AWQModifier
+    from llmcompressor.modifiers.quantization import GPTQModifier, QuantizationModifier
+    from llmcompressor.modifiers.smoothquant import SmoothQuantModifier
+
+    common = dict(targets=_TARGETS, ignore=_IGNORE)
+    if method == "awq":
+        return AWQModifier(scheme=scheme, **common)
+    if method == "gptq":
+        return GPTQModifier(scheme=scheme, **common)
+    if method == "smoothquant":
+        return [
+            SmoothQuantModifier(smoothing_strength=_SMOOTHING_STRENGTH),
+            GPTQModifier(scheme=scheme, **common),
+        ]
+    if method in ("fp8", "rtn"):
+        return QuantizationModifier(scheme=scheme, **common)
+    raise ValueError(f"no compressed-tensors recipe for method {method!r}")
+
+
+def calib_dataset(spec: QuantSpec, tokenizer, token: str | None = None):
+    """Packed fixed-length calibration: concatenate text, chunk into seq-len blocks.
+
+    Uniform-length sequences are required by AutoRound (it stacks samples and
+    rejects ragged lengths) and are the standard GPTQ/AWQ calibration form, so one
+    packed dataset serves every calibrated method. Deterministic under the spec.
+    """
+    from datasets import Dataset, load_dataset
+
+    ds = load_dataset(
+        spec.calib_dataset, spec.calib_config, split=spec.calib_split, token=token
+    )
+    ds = ds.filter(lambda ex: ex["text"] is not None and ex["text"].strip() != "")
+    ds = ds.shuffle(seed=spec.seed)
+
+    needed = spec.calib_samples * spec.calib_seqlen
+    buf: list[int] = []
+    for ex in ds:
+        buf.extend(tokenizer(ex["text"]).input_ids)
+        if len(buf) >= needed:
+            break
+    blocks = [
+        buf[i : i + spec.calib_seqlen]
+        for i in range(0, needed, spec.calib_seqlen)
+    ]
+    return Dataset.from_dict(
+        {"input_ids": blocks, "attention_mask": [[1] * len(b) for b in blocks]}
+    )
+
+
+def quantize_ct(
+    model_id: str,
+    method: str,
+    scheme: str,
+    out_dir: str,
+    spec: QuantSpec,
+    needs_calibration: bool,
+    token: str | None = None,
+    offload: bool = False,
+) -> Path:
+    """Run llm-compressor oneshot for `method`/`scheme` into `out_dir`."""
+    from llmcompressor import oneshot
+    from transformers import AutoTokenizer
+
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    tokenizer = AutoTokenizer.from_pretrained(model_id, token=token)
+
+    kwargs: dict = dict(
+        model=model_id,
+        tokenizer=tokenizer,
+        recipe=build_recipe(method, scheme),
+        output_dir=str(out),
+    )
+    if offload:
+        # Quantize layer-by-layer with the model held on CPU -> fits any size.
+        kwargs["sequential_offload_device"] = "cpu"
+    if needs_calibration:
+        kwargs.update(
+            dataset=calib_dataset(spec, tokenizer, token=token),
+            num_calibration_samples=spec.calib_samples,
+            max_seq_length=spec.calib_seqlen,
+        )
+    oneshot(**kwargs)
+    return out

quantfit/backends/gguf.py

@@ -0,0 +1,121 @@
+"""GGUF backend (llama.cpp).
+
+Produces GGUF k-quants (Q4_K_M, etc.) for the Ollama / llama.cpp / LM Studio
+world. Quantization is CPU-only. Two tools are provisioned into a cache on first
+use (or located via QUANTFIT_LLAMACPP pointing at a llama.cpp checkout):
+  - the prebuilt `llama-quantize` binary (from the pinned llama.cpp release zip)
+  - the repo's `convert_hf_to_gguf.py` (HF safetensors -> GGUF f16); it imports a
+    sibling `conversion` package, so a shallow clone of the repo is required.
+"""
+from __future__ import annotations
+
+import os
+import platform
+import subprocess
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+
+LLAMACPP_TAG = "b9817"  # pinned release; binary + convert script must match
+GGUF_TYPES = (
+    "Q2_K", "Q3_K_S", "Q3_K_M", "Q4_K_S", "Q4_K_M", "Q5_K_M", "Q6_K", "Q8_0", "IQ4_XS",
+)
+
+_REPO = "https://github.com/ggml-org/llama.cpp"
+_RELEASES = "https://github.com/ggml-org/llama.cpp/releases/download"
+
+
+def _cache_dir() -> Path:
+    root = os.environ.get("QUANTFIT_CACHE")
+    d = Path(root) if root else Path.home() / ".cache" / "quantfit"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def _exe_name() -> str:
+    return "llama-quantize.exe" if platform.system() == "Windows" else "llama-quantize"
+
+
+def _binary_asset() -> str:
+    sysname = platform.system()
+    if sysname == "Windows":
+        return f"llama-{LLAMACPP_TAG}-bin-win-cpu-x64.zip"
+    if sysname == "Linux":
+        return f"llama-{LLAMACPP_TAG}-bin-ubuntu-x64.zip"
+    raise RuntimeError(
+        f"no prebuilt llama.cpp binary wired for {sysname}; install llama.cpp and "
+        f"set QUANTFIT_LLAMACPP to its directory"
+    )
+
+
+def _first_match(root: Path, name: str) -> Path | None:
+    if (root / name).exists():
+        return root / name
+    return next(iter(root.rglob(name)), None)
+
+
+def llama_quantize_bin() -> Path:
+    """Locate (env) or download+extract the llama-quantize binary."""
+    exe = _exe_name()
+    env = os.environ.get("QUANTFIT_LLAMACPP")
+    if env and (hit := _first_match(Path(env), exe)):
+        return hit
+
+    bindir = _cache_dir() / f"llamacpp-bin-{LLAMACPP_TAG}"
+    if hit := _first_match(bindir, exe):
+        return hit
+
+    asset = _binary_asset()
+    zip_path = _cache_dir() / asset
+    if not zip_path.exists():
+        urllib.request.urlretrieve(f"{_RELEASES}/{LLAMACPP_TAG}/{asset}", zip_path)
+    bindir.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(zip_path) as z:
+        z.extractall(bindir)
+    if hit := _first_match(bindir, exe):
+        return hit
+    raise RuntimeError(f"{exe} not found inside {asset}")
+
+
+def convert_script() -> Path:
+    """Locate (env) or shallow-clone the repo's convert_hf_to_gguf.py."""
+    name = "convert_hf_to_gguf.py"
+    env = os.environ.get("QUANTFIT_LLAMACPP")
+    if env and (Path(env) / name).exists():
+        return Path(env) / name
+
+    repo = _cache_dir() / f"llama.cpp-{LLAMACPP_TAG}"
+    if (repo / name).exists():
+        return repo / name
+    subprocess.run(
+        ["git", "clone", "--depth", "1", "--branch", LLAMACPP_TAG, _REPO, str(repo)],
+        check=True,
+    )
+    if not (repo / name).exists():
+        raise RuntimeError(f"{name} missing after cloning {_REPO}@{LLAMACPP_TAG}")
+    return repo / name
+
+
+def quantize_gguf(model_id: str, qtype: str, out_dir: str, token: str | None = None) -> Path:
+    """HF model -> GGUF f16 -> quantized GGUF (CPU-only)."""
+    from huggingface_hub import snapshot_download
+
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+
+    quant_bin = llama_quantize_bin()
+    convert = convert_script()
+    model_dir = snapshot_download(model_id, token=token)
+
+    f16 = out / "model.f16.gguf"
+    final = out / f"model.{qtype}.gguf"
+    env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
+
+    subprocess.run(
+        [sys.executable, str(convert), model_dir, "--outtype", "f16", "--outfile", str(f16)],
+        check=True, env=env,
+    )
+    subprocess.run([str(quant_bin), str(f16), str(final), qtype], check=True, env=env)
+    f16.unlink(missing_ok=True)  # drop the large f16 intermediate
+    return out

quantfit/cli.py

@@ -0,0 +1,103 @@
+"""quantfit CLI — `check`, `list`, `quantize`."""
+from __future__ import annotations
+
+import argparse
+import sys
+
+from quantfit.registry import METHODS
+
+
+def _force_utf8_stdio() -> None:
+    # llm-compressor / gptqmodel loggers emit unicode; a Windows cp1252 console
+    # otherwise crashes mid-run with UnicodeEncodeError.
+    for stream in (sys.stdout, sys.stderr):
+        try:
+            stream.reconfigure(encoding="utf-8")  # type: ignore[union-attr]
+        except (AttributeError, ValueError):
+            pass
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="quantfit", description="Quantize an LLM if it fits your GPU."
+    )
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    pc = sub.add_parser("check", help="will this model fit your GPU?")
+    pc.add_argument("--model", required=True, help="HF model id")
+
+    sub.add_parser("list", help="list supported methods + schemes")
+
+    pv = sub.add_parser("verify", help="smoke-load a quantized artifact + generate")
+    pv.add_argument("--model", required=True, help="path to a quantized output dir or .gguf")
+
+    pvs = sub.add_parser("verify-safety", help="refusal preservation: fp16 baseline vs quantized")
+    pvs.add_argument("--fp16", required=True, help="HF id of the fp16 baseline")
+    pvs.add_argument("--quant", required=True, help="path to the quantized artifact")
+    pvs.add_argument("--max-new-tokens", type=int, default=64)
+
+    pq = sub.add_parser("quantize", help="quantize a model")
+    pq.add_argument("--model", required=True, help="HF model id (the FP16 base)")
+    pq.add_argument("--method", required=True, choices=tuple(METHODS))
+    pq.add_argument("--scheme", default=None, help="override the method's default scheme")
+    pq.add_argument("--out", required=True, help="output directory")
+    pq.add_argument("--push", default=None, help="HF repo id to upload the result to")
+    pq.add_argument("--private", action="store_true", help="push as a private repo")
+    pq.add_argument("--offload", action="store_true", help="quantize on CPU (fits any size, slower)")
+    pq.add_argument("--no-check", action="store_true", help="skip the GPU pre-flight")
+    return p
+
+
+def main(argv: list[str] | None = None) -> int:
+    _force_utf8_stdio()
+    args = _build_parser().parse_args(argv)
+
+    if args.cmd == "check":
+        from quantfit.fit import plan
+
+        cap = plan(args.model)
+        print(cap.reason())
+        return 0 if cap.fits else 2
+
+    if args.cmd == "list":
+        from quantfit.registry import catalog
+
+        print(catalog())
+        return 0
+
+    if args.cmd == "verify":
+        from quantfit.verify import verify
+
+        ok, msg = verify(args.model)
+        print(("PASS: " if ok else "FAIL: ") + msg)
+        return 0 if ok else 2
+
+    if args.cmd == "verify-safety":
+        from quantfit.safety.verify import verify_safety
+
+        tax = verify_safety(args.fp16, args.quant, max_new_tokens=args.max_new_tokens)
+        print(tax.summary())  # aggregates only — never echoes raw probe prompts/completions
+        return 0 if tax.clean else 2
+
+    if args.cmd == "quantize":
+        from quantfit.quantize import CannotQuantize, push, quantize
+        from quantfit.registry import UnsupportedCombo
+
+        try:
+            out = quantize(
+                args.model, args.method, args.out,
+                scheme=args.scheme, run_check=not args.no_check, offload=args.offload,
+            )
+        except (CannotQuantize, UnsupportedCombo) as exc:
+            print(exc)
+            return 2
+        print(f"quantized -> {out}")
+        if args.push:
+            print(f"pushed -> {push(str(out), args.push, private=args.private)}")
+        return 0
+
+    return 1  # unreachable: subparser is required
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

quantfit/fit.py

@@ -0,0 +1,130 @@
+"""Capacity decision: in-GPU / CPU-offload / refuse.
+
+Three resources gate a quantization job:
+  - disk : weights must be downloaded (unless cached) + the output written. Needed
+           in BOTH gpu and offload modes, so it's a precondition.
+  - VRAM : enough -> quantize in-GPU (fast).
+  - RAM  : not enough VRAM but enough RAM -> hold the model on CPU and stream
+           layers to the GPU (offload; slower; fits any size).
+Refuse only when none of the above can be satisfied, and always name the actual
+limiting resource.
+"""
+from __future__ import annotations
+
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+
+import psutil
+
+from quantfit.gpufit import (
+    CALIB_OVERHEAD_FACTOR,
+    HEADROOM_BYTES,
+    estimate_fp16_bytes,
+    gpu_free_bytes,
+)
+
+# Offload holds the model in CPU RAM and moves one layer at a time to the GPU.
+OFFLOAD_RAM_FACTOR = 1.15
+# Quantized output is smaller than fp16; reserve this fraction (covers 8-bit, the
+# largest common output; 4-bit is ~half this).
+OUTPUT_DISK_FACTOR = 0.6
+_GIB = 1024**3
+
+MODE_GPU = "gpu"
+MODE_OFFLOAD = "offload"
+MODE_REFUSE = "refuse"
+
+LIMIT_NONE = ""
+LIMIT_DISK = "disk"
+LIMIT_MACHINE = "machine"
+
+
+@dataclass(frozen=True)
+class CapacityPlan:
+    model_id: str
+    fp16_bytes: int
+    gpu_free: int
+    ram_available: int
+    disk_free: int
+    disk_need: int
+    mode: str
+    limit: str
+
+    @property
+    def fits(self) -> bool:
+        return self.mode != MODE_REFUSE
+
+    @property
+    def offload(self) -> bool:
+        return self.mode == MODE_OFFLOAD
+
+    def reason(self) -> str:
+        g = lambda b: f"{b / _GIB:.1f}"
+        if self.mode == MODE_GPU:
+            return (
+                f"OK (in-GPU): {self.model_id} ~{g(self.fp16_bytes)} GB, "
+                f"{g(self.gpu_free)} GB VRAM free."
+            )
+        if self.mode == MODE_OFFLOAD:
+            return (
+                f"OK (offload): {self.model_id} ~{g(self.fp16_bytes)} GB won't fit "
+                f"{g(self.gpu_free)} GB VRAM — quantizing via CPU "
+                f"({g(self.ram_available)} GB RAM). Slower."
+            )
+        if self.limit == LIMIT_DISK:
+            return (
+                f"CAN'T QUANTIZE: {self.model_id} needs ~{g(self.disk_need)} GB free "
+                f"disk (download + output) but only {g(self.disk_free)} GB is free."
+            )
+        return (
+            f"CAN'T QUANTIZE: {self.model_id} ~{g(self.fp16_bytes)} GB needs more than "
+            f"{g(self.gpu_free)} GB VRAM and {g(self.ram_available)} GB RAM. "
+            f"Use a bigger machine."
+        )
+
+
+def _existing_parent(path: str) -> str:
+    p = Path(path).resolve()
+    while not p.exists():
+        p = p.parent
+    return str(p)
+
+
+def _cached_weight_bytes(model_id: str) -> int:
+    """Bytes of this model's safetensors already in the HF cache (0 if absent)."""
+    from huggingface_hub.constants import HF_HUB_CACHE
+
+    snap = Path(HF_HUB_CACHE) / ("models--" + model_id.replace("/", "--")) / "snapshots"
+    if not snap.exists():
+        return 0
+    total = 0
+    for f in snap.rglob("*.safetensors"):
+        try:
+            total += f.stat().st_size
+        except OSError:
+            pass
+    return total
+
+
+def plan(model_id: str, out_dir: str = ".", token: str | None = None) -> CapacityPlan:
+    """Decide gpu / offload / refuse for quantizing `model_id`."""
+    fp16 = estimate_fp16_bytes(model_id, token=token)
+    gpu = gpu_free_bytes()
+    ram = int(psutil.virtual_memory().available)
+    disk = shutil.disk_usage(_existing_parent(out_dir)).free
+
+    download_need = max(0, fp16 - _cached_weight_bytes(model_id))
+    disk_need = download_need + int(fp16 * OUTPUT_DISK_FACTOR)
+    gpu_need = int(fp16 * CALIB_OVERHEAD_FACTOR) + HEADROOM_BYTES
+    ram_need = int(fp16 * OFFLOAD_RAM_FACTOR) + HEADROOM_BYTES
+
+    if disk < disk_need:
+        mode, limit = MODE_REFUSE, LIMIT_DISK
+    elif gpu_need <= gpu:
+        mode, limit = MODE_GPU, LIMIT_NONE
+    elif ram_need <= ram:
+        mode, limit = MODE_OFFLOAD, LIMIT_NONE
+    else:
+        mode, limit = MODE_REFUSE, LIMIT_MACHINE
+    return CapacityPlan(model_id, fp16, gpu, ram, disk, disk_need, mode, limit)

quantfit/gpufit.py

@@ -0,0 +1,97 @@
+"""GPU capacity pre-flight.
+
+The whole point of the tool: decide whether a model can be quantized in-GPU on
+*this* machine BEFORE downloading weights or starting a multi-minute job. The
+estimate is the FP16 footprint of the released weights (read from the Hub file
+metadata, no download) times a calibration-overhead factor, plus fixed headroom,
+compared against free VRAM. Errs toward refusal — a clear "can't quantize" beats
+an OOM crash 20 minutes in.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from huggingface_hub import HfApi
+
+# In-GPU PTQ (GPTQ/AWQ) holds the FP16 model plus per-layer Hessian/activation
+# buffers; observed peak runs ~1.25x the released FP16 size. Headroom covers the
+# CUDA context, calibration-batch activations, and allocator fragmentation.
+CALIB_OVERHEAD_FACTOR = 1.25
+HEADROOM_BYTES = 2 * 1024**3
+_GIB = 1024**3
+# Prefer safetensors; fall back to .bin only if no safetensors shards exist
+# (summing both would double-count repos that ship both formats).
+_WEIGHT_SUFFIXES = (".safetensors", ".bin")
+
+
+@dataclass(frozen=True)
+class FitReport:
+    model_id: str
+    fp16_bytes: int
+    required_bytes: int
+    free_bytes: int
+    fits: bool
+
+    @property
+    def fp16_gib(self) -> float:
+        return self.fp16_bytes / _GIB
+
+    @property
+    def required_gib(self) -> float:
+        return self.required_bytes / _GIB
+
+    @property
+    def free_gib(self) -> float:
+        return self.free_bytes / _GIB
+
+    def reason(self) -> str:
+        if self.fits:
+            return (
+                f"OK: {self.model_id} is ~{self.fp16_gib:.1f} GB FP16, needs "
+                f"~{self.required_gib:.1f} GB to quantize, {self.free_gib:.1f} GB free."
+            )
+        return (
+            f"CAN'T QUANTIZE: {self.model_id} needs ~{self.required_gib:.1f} GB "
+            f"in-GPU but only {self.free_gib:.1f} GB is free. Use a bigger GPU "
+            f"or a smaller model."
+        )
+
+
+def estimate_fp16_bytes(model_id: str, token: str | None = None) -> int:
+    """Sum the released weight-file sizes from Hub metadata (no weight download)."""
+    info = HfApi().model_info(model_id, files_metadata=True, token=token)
+    by_suffix: dict[str, int] = {}
+    for f in info.siblings:
+        for suffix in _WEIGHT_SUFFIXES:
+            if f.rfilename.endswith(suffix) and f.size:
+                by_suffix[suffix] = by_suffix.get(suffix, 0) + f.size
+    for suffix in _WEIGHT_SUFFIXES:
+        if by_suffix.get(suffix):
+            return by_suffix[suffix]
+    raise ValueError(
+        f"{model_id}: no weight-file sizes found via Hub metadata; cannot "
+        "estimate footprint (model may be gated without access, or unavailable)."
+    )
+
+
+def gpu_free_bytes() -> int:
+    """Free VRAM on the current default CUDA device, in bytes."""
+    import torch
+
+    if not torch.cuda.is_available():
+        raise RuntimeError("no CUDA GPU visible; quantfit needs a GPU to quantize.")
+    free, _total = torch.cuda.mem_get_info()
+    return int(free)
+
+
+def check_fit(
+    model_id: str,
+    token: str | None = None,
+    overhead: float = CALIB_OVERHEAD_FACTOR,
+    headroom_bytes: int = HEADROOM_BYTES,
+) -> FitReport:
+    """Estimate footprint vs. free VRAM and return a fit verdict."""
+    fp16 = estimate_fp16_bytes(model_id, token=token)
+    required = int(fp16 * overhead) + headroom_bytes
+    free = gpu_free_bytes()
+    return FitReport(model_id, fp16, required, free, fits=required <= free)

quantfit/quantize.py

@@ -0,0 +1,98 @@
+"""Dispatcher: validate the request, GPU pre-flight, route to a backend, card it."""
+from __future__ import annotations
+
+from pathlib import Path
+
+from quantfit.fit import MODE_REFUSE, plan
+from quantfit.registry import BACKEND_CT, BACKEND_GGUF, resolve
+from quantfit.spec import DEFAULT_SPEC, QuantSpec
+
+
+class CannotQuantize(RuntimeError):
+    """Raised when the model won't fit the GPU (pre-flight refusal)."""
+
+
+def quantize(
+    model_id: str,
+    method: str,
+    out_dir: str,
+    scheme: str | None = None,
+    spec: QuantSpec = DEFAULT_SPEC,
+    token: str | None = None,
+    run_check: bool = True,
+    offload: bool = False,
+) -> Path:
+    """Quantize `model_id` with `method` (+ optional `scheme`) into `out_dir`."""
+    m, resolved_scheme = resolve(method, scheme)
+
+    # The 3-tier capacity plan applies to GPU quantization (compressed-tensors).
+    # GGUF quantization is CPU-only, so it skips the VRAM/offload plan.
+    if run_check and m.backend == BACKEND_CT:
+        cap = plan(model_id, out_dir, token=token)
+        if cap.mode == MODE_REFUSE:
+            raise CannotQuantize(cap.reason())
+        if cap.offload:
+            offload = True
+
+    if m.backend == BACKEND_CT:
+        from quantfit.backends.compressed_tensors import quantize_ct
+
+        out = quantize_ct(
+            model_id, m.name, resolved_scheme, out_dir, spec,
+            m.needs_calibration, token=token, offload=offload,
+        )
+    elif m.backend == BACKEND_GGUF:
+        from quantfit.backends.gguf import quantize_gguf
+
+        out = quantize_gguf(model_id, resolved_scheme, out_dir, token=token)
+    else:
+        raise NotImplementedError(f"backend {m.backend!r} is not wired yet")
+
+    _write_card(Path(out), model_id, m.name, resolved_scheme, spec)
+    return Path(out)
+
+
+def _write_card(out: Path, model_id: str, method: str, scheme: str, spec: QuantSpec) -> None:
+    if method == "gguf":
+        card = f"""---
+base_model: {model_id}
+tags: [quantized, gguf, {scheme.lower()}, llama.cpp, quantfit]
+---
+
+# {out.name}
+
+GGUF {scheme} quantization of `{model_id}`, produced with
+[quantfit](https://github.com/Sahil170595/quantfit).
+
+Loads in llama.cpp / Ollama / LM Studio. k-quant, no calibration (no imatrix).
+"""
+    else:
+        card = f"""---
+base_model: {model_id}
+tags: [quantized, {method}, {scheme.lower()}, compressed-tensors, quantfit]
+---
+
+# {out.name}
+
+{method.upper()} quantization ({scheme}) of `{model_id}`, produced with
+[quantfit](https://github.com/Sahil170595/quantfit).
+
+## Provenance
+- method: {method}, scheme: {scheme}, group_size {spec.group_size}
+- calibration: {spec.calib_dataset}/{spec.calib_config} [{spec.calib_split}], \
+{spec.calib_samples} samples, seq-len {spec.calib_seqlen}, seed {spec.seed}
+- spec fingerprint: `{spec.fingerprint()}`
+
+Loads in vLLM via the compressed-tensors backend.
+"""
+    (out / "README.md").write_text(card, encoding="utf-8")
+
+
+def push(out_dir: str, repo_id: str, token: str | None = None, private: bool = False) -> str:
+    """Upload a quantized output folder to the Hub."""
+    from huggingface_hub import HfApi
+
+    api = HfApi(token=token)
+    api.create_repo(repo_id, exist_ok=True, private=private, repo_type="model")
+    api.upload_folder(folder_path=str(out_dir), repo_id=repo_id, repo_type="model")
+    return f"https://huggingface.co/{repo_id}"