quantfit 0.1.0__tar.gz

quantfit-0.1.0/LICENSE

@@ -0,0 +1,154 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or Derivative
+          Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law, shall any Contributor be liable
+      to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2026 Sahil Kadadekar
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

quantfit-0.1.0/PKG-INFO

@@ -0,0 +1,121 @@
+Metadata-Version: 2.4
+Name: quantfit
+Version: 0.1.0
+Summary: Quantize an LLM and check it still refuses what it should — a GPU-aware quantization CLI with a built-in safety-tax check.
+Author: Sahil Kadadekar
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/Sahil170595/quantfit
+Project-URL: Repository, https://github.com/Sahil170595/quantfit
+Keywords: llm,quantization,awq,gptq,gguf,fp8,gpu,safety
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.4
+Requires-Dist: transformers>=4.49
+Requires-Dist: huggingface_hub>=0.25
+Requires-Dist: datasets>=3.0
+Requires-Dist: accelerate>=1.0
+Requires-Dist: llmcompressor>=0.5
+Requires-Dist: gptqmodel>=2.0
+Requires-Dist: psutil>=5.9
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Provides-Extra: gguf
+Requires-Dist: gguf>=0.10; extra == "gguf"
+Dynamic: license-file
+
+# quantfit
+
+**Quantize an LLM — and check it still refuses what it should.**
+
+Quantization makes a model cheaper to serve. It can also quietly strip safety
+behavior: a 4-bit model that answers prompts the fp16 model refused is a regression
+you will not see in a perplexity number. `quantfit` quantizes across the SOTA method
+matrix, is honest about whether a model fits your GPU, and — uniquely — measures the
+**safety tax** of the quantization it just performed.
+
+```bash
+pip install quantfit
+
+quantfit check        --model Qwen/Qwen2.5-7B-Instruct                 # will it fit? (no download)
+quantfit quantize     --model Qwen/Qwen2.5-1.5B-Instruct --method awq --out ./out
+quantfit verify-safety --fp16 Qwen/Qwen2.5-1.5B-Instruct --quant ./out  # did quantization break refusals?
+```
+
+## The safety check — what nothing else does
+
+`verify-safety` generates from both the fp16 baseline and the quantized model over a
+curated probe set, judges each response refusal/compliance with a local classifier,
+and reports the tax as a **vector**, the way it actually matters:
+
+```
+safety-tax over 40 probes (REGRESSION):
+  refusal-robustness (expected-unsafe n=12): fp16 refused 12 -> quant 12 | 0 harmful-compliance regressions
+  over-refusal       (expected-safe   n=28): fp16 refused 18 -> quant 18 | 2 new false refusals
+  by zone: borderline[10->10/16] clear_safe[8->8/12] clear_unsafe[12->12/12]
+```
+
+Two axes, not one number:
+- **refusal-robustness** — on prompts that *should* be refused, did the quant start
+  complying? (the dangerous direction)
+- **over-refusal** — on prompts that *should* be answered, did the quant start
+  refusing? (the usability direction)
+
+A scalar refusal-delta can read 0 while both axes move in opposite directions; the
+vector + per-zone breakdown catches it. Local judge, curated public probes, no
+external API and no raw harmful corpora — so the check is distributable.
+
+## GPU-aware quantization
+
+**3-tier capacity.** `check` reads HF metadata (no download) to estimate the footprint:
+fits VRAM → quantize in-GPU; too big for VRAM but fits RAM+disk → **CPU offload** (a
+27B can quantize on a 12 GB GPU); won't fit even offloaded → refuse, naming the real
+limit. No OOM 20 minutes into a job.
+
+**Method × scheme matrix** (one llm-compressor backend, vLLM-loadable):
+
+| method | what | default scheme |
+|---|---|---|
+| `awq` | activation-aware weight quant (best 4-bit quality) | W4A16_ASYM |
+| `gptq` | Hessian/OBQ weight quant | W4A16 |
+| `smoothquant` | activation smoothing + W8A8 | W8A8 |
+| `fp8` | FP8 E4M3 dynamic, no calibration | FP8_DYNAMIC |
+| `rtn` | round-to-nearest baseline | W4A16 |
+
+Schemes (`--scheme`): `W4A16`, `W4A16_ASYM`, `W8A16`, `W8A8`, `INT8`, `W4A8`,
+`FP8_DYNAMIC`, `NVFP4`, `MXFP4`. Defaults are the validated paths; FP4 schemes need
+Blackwell to *serve* (quantfit can still produce them anywhere).
+
+**GGUF** (`--method gguf`) for Ollama / llama.cpp: `Q2_K`..`Q8_0` + IQ-quants.
+Auto-provisions the prebuilt `llama-quantize` binary + convert script (override with
+`QUANTFIT_LLAMACPP`).
+
+One frozen packed calibration (wikitext-103, 128 samples, seq-len 2048, seed 42,
+group-size 128) is shared across the calibrated methods, so they are comparable.
+
+## What it is — and isn't
+
+- It **quantizes** (wrapping llm-compressor + llama.cpp) and **checks safety
+  preservation**. Both are real and validated end-to-end.
+- It does **not** auto-select the config for you yet — you pick `--method`. Automatic
+  config selection is a real capability, but it is published research
+  ([AMQ](https://arxiv.org/abs/2509.12019),
+  [KL-Lens](https://arxiv.org/abs/2604.13440)); a routing layer that *implements* it is
+  planned, not claimed here.
+
+## Docker
+
+`Dockerfile` builds an isolated CUDA image. For GGUF in Docker, the official
+`ghcr.io/ggml-org/llama.cpp:full` image carries the convert + quantize tooling.
+
+## License
+
+Apache-2.0.

quantfit-0.1.0/README.md

@@ -0,0 +1,87 @@
+# quantfit
+
+**Quantize an LLM — and check it still refuses what it should.**
+
+Quantization makes a model cheaper to serve. It can also quietly strip safety
+behavior: a 4-bit model that answers prompts the fp16 model refused is a regression
+you will not see in a perplexity number. `quantfit` quantizes across the SOTA method
+matrix, is honest about whether a model fits your GPU, and — uniquely — measures the
+**safety tax** of the quantization it just performed.
+
+```bash
+pip install quantfit
+
+quantfit check        --model Qwen/Qwen2.5-7B-Instruct                 # will it fit? (no download)
+quantfit quantize     --model Qwen/Qwen2.5-1.5B-Instruct --method awq --out ./out
+quantfit verify-safety --fp16 Qwen/Qwen2.5-1.5B-Instruct --quant ./out  # did quantization break refusals?
+```
+
+## The safety check — what nothing else does
+
+`verify-safety` generates from both the fp16 baseline and the quantized model over a
+curated probe set, judges each response refusal/compliance with a local classifier,
+and reports the tax as a **vector**, the way it actually matters:
+
+```
+safety-tax over 40 probes (REGRESSION):
+  refusal-robustness (expected-unsafe n=12): fp16 refused 12 -> quant 12 | 0 harmful-compliance regressions
+  over-refusal       (expected-safe   n=28): fp16 refused 18 -> quant 18 | 2 new false refusals
+  by zone: borderline[10->10/16] clear_safe[8->8/12] clear_unsafe[12->12/12]
+```
+
+Two axes, not one number:
+- **refusal-robustness** — on prompts that *should* be refused, did the quant start
+  complying? (the dangerous direction)
+- **over-refusal** — on prompts that *should* be answered, did the quant start
+  refusing? (the usability direction)
+
+A scalar refusal-delta can read 0 while both axes move in opposite directions; the
+vector + per-zone breakdown catches it. Local judge, curated public probes, no
+external API and no raw harmful corpora — so the check is distributable.
+
+## GPU-aware quantization
+
+**3-tier capacity.** `check` reads HF metadata (no download) to estimate the footprint:
+fits VRAM → quantize in-GPU; too big for VRAM but fits RAM+disk → **CPU offload** (a
+27B can quantize on a 12 GB GPU); won't fit even offloaded → refuse, naming the real
+limit. No OOM 20 minutes into a job.
+
+**Method × scheme matrix** (one llm-compressor backend, vLLM-loadable):
+
+| method | what | default scheme |
+|---|---|---|
+| `awq` | activation-aware weight quant (best 4-bit quality) | W4A16_ASYM |
+| `gptq` | Hessian/OBQ weight quant | W4A16 |
+| `smoothquant` | activation smoothing + W8A8 | W8A8 |
+| `fp8` | FP8 E4M3 dynamic, no calibration | FP8_DYNAMIC |
+| `rtn` | round-to-nearest baseline | W4A16 |
+
+Schemes (`--scheme`): `W4A16`, `W4A16_ASYM`, `W8A16`, `W8A8`, `INT8`, `W4A8`,
+`FP8_DYNAMIC`, `NVFP4`, `MXFP4`. Defaults are the validated paths; FP4 schemes need
+Blackwell to *serve* (quantfit can still produce them anywhere).
+
+**GGUF** (`--method gguf`) for Ollama / llama.cpp: `Q2_K`..`Q8_0` + IQ-quants.
+Auto-provisions the prebuilt `llama-quantize` binary + convert script (override with
+`QUANTFIT_LLAMACPP`).
+
+One frozen packed calibration (wikitext-103, 128 samples, seq-len 2048, seed 42,
+group-size 128) is shared across the calibrated methods, so they are comparable.
+
+## What it is — and isn't
+
+- It **quantizes** (wrapping llm-compressor + llama.cpp) and **checks safety
+  preservation**. Both are real and validated end-to-end.
+- It does **not** auto-select the config for you yet — you pick `--method`. Automatic
+  config selection is a real capability, but it is published research
+  ([AMQ](https://arxiv.org/abs/2509.12019),
+  [KL-Lens](https://arxiv.org/abs/2604.13440)); a routing layer that *implements* it is
+  planned, not claimed here.
+
+## Docker
+
+`Dockerfile` builds an isolated CUDA image. For GGUF in Docker, the official
+`ghcr.io/ggml-org/llama.cpp:full` image carries the convert + quantize tooling.
+
+## License
+
+Apache-2.0.

quantfit-0.1.0/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "quantfit"
+version = "0.1.0"
+description = "Quantize an LLM and check it still refuses what it should — a GPU-aware quantization CLI with a built-in safety-tax check."
+readme = "README.md"
+requires-python = ">=3.10"
+license = "Apache-2.0"
+authors = [{ name = "Sahil Kadadekar" }]
+keywords = ["llm", "quantization", "awq", "gptq", "gguf", "fp8", "gpu", "safety"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Intended Audience :: Developers",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+  "torch>=2.4",
+  "transformers>=4.49",
+  "huggingface_hub>=0.25",
+  "datasets>=3.0",
+  "accelerate>=1.0",
+  "llmcompressor>=0.5",
+  "gptqmodel>=2.0",
+  "psutil>=5.9",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0"]
+gguf = ["gguf>=0.10"]  # for the GGUF backend's convert step
+
+[project.urls]
+Homepage = "https://github.com/Sahil170595/quantfit"
+Repository = "https://github.com/Sahil170595/quantfit"
+
+[project.scripts]
+quantfit = "quantfit.cli:main"
+
+[tool.setuptools.packages.find]
+include = ["quantfit*"]
+
+[tool.pytest.ini_options]
+# Make `pytest tests/` import the package without an editable install (keeps CI light).
+pythonpath = ["."]

quantfit-0.1.0/quantfit/__init__.py

@@ -0,0 +1,6 @@
+"""quantfit — quantize an LLM if it fits your GPU."""
+from quantfit.gpufit import FitReport, check_fit
+from quantfit.spec import DEFAULT_SPEC, QuantSpec
+
+__version__ = "0.1.0"
+__all__ = ["FitReport", "check_fit", "QuantSpec", "DEFAULT_SPEC", "__version__"]

quantfit-0.1.0/quantfit/backends/__init__.py

@@ -0,0 +1 @@
+"""Quantization backends. Each turns a (method, scheme) into a saved artifact."""

quantfit-0.1.0/quantfit/backends/compressed_tensors.py

@@ -0,0 +1,104 @@
+"""compressed-tensors backend (llm-compressor): the method × scheme matrix.
+
+awq / gptq / autoround / smoothquant calibrate; fp8 / rtn do not. All emit
+compressed-tensors (vLLM-loadable). For the calibrated algorithms the only
+cross-method difference is the algorithm itself — same calibration, same format
+— so the methods are comparable, not confounded.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from quantfit.spec import QuantSpec
+
+_TARGETS = ["Linear"]
+_IGNORE = ["lm_head"]
+_SMOOTHING_STRENGTH = 0.8  # SmoothQuant migration strength (standard default)
+
+
+def build_recipe(method: str, scheme: str):
+    """Construct the llm-compressor recipe (modifier or modifier list) for a method."""
+    from llmcompressor.modifiers.awq import AWQModifier
+    from llmcompressor.modifiers.quantization import GPTQModifier, QuantizationModifier
+    from llmcompressor.modifiers.smoothquant import SmoothQuantModifier
+
+    common = dict(targets=_TARGETS, ignore=_IGNORE)
+    if method == "awq":
+        return AWQModifier(scheme=scheme, **common)
+    if method == "gptq":
+        return GPTQModifier(scheme=scheme, **common)
+    if method == "smoothquant":
+        return [
+            SmoothQuantModifier(smoothing_strength=_SMOOTHING_STRENGTH),
+            GPTQModifier(scheme=scheme, **common),
+        ]
+    if method in ("fp8", "rtn"):
+        return QuantizationModifier(scheme=scheme, **common)
+    raise ValueError(f"no compressed-tensors recipe for method {method!r}")
+
+
+def calib_dataset(spec: QuantSpec, tokenizer, token: str | None = None):
+    """Packed fixed-length calibration: concatenate text, chunk into seq-len blocks.
+
+    Uniform-length sequences are required by AutoRound (it stacks samples and
+    rejects ragged lengths) and are the standard GPTQ/AWQ calibration form, so one
+    packed dataset serves every calibrated method. Deterministic under the spec.
+    """
+    from datasets import Dataset, load_dataset
+
+    ds = load_dataset(
+        spec.calib_dataset, spec.calib_config, split=spec.calib_split, token=token
+    )
+    ds = ds.filter(lambda ex: ex["text"] is not None and ex["text"].strip() != "")
+    ds = ds.shuffle(seed=spec.seed)
+
+    needed = spec.calib_samples * spec.calib_seqlen
+    buf: list[int] = []
+    for ex in ds:
+        buf.extend(tokenizer(ex["text"]).input_ids)
+        if len(buf) >= needed:
+            break
+    blocks = [
+        buf[i : i + spec.calib_seqlen]
+        for i in range(0, needed, spec.calib_seqlen)
+    ]
+    return Dataset.from_dict(
+        {"input_ids": blocks, "attention_mask": [[1] * len(b) for b in blocks]}
+    )
+
+
+def quantize_ct(
+    model_id: str,
+    method: str,
+    scheme: str,
+    out_dir: str,
+    spec: QuantSpec,
+    needs_calibration: bool,
+    token: str | None = None,
+    offload: bool = False,
+) -> Path:
+    """Run llm-compressor oneshot for `method`/`scheme` into `out_dir`."""
+    from llmcompressor import oneshot
+    from transformers import AutoTokenizer
+
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    tokenizer = AutoTokenizer.from_pretrained(model_id, token=token)
+
+    kwargs: dict = dict(
+        model=model_id,
+        tokenizer=tokenizer,
+        recipe=build_recipe(method, scheme),
+        output_dir=str(out),
+    )
+    if offload:
+        # Quantize layer-by-layer with the model held on CPU -> fits any size.
+        kwargs["sequential_offload_device"] = "cpu"
+    if needs_calibration:
+        kwargs.update(
+            dataset=calib_dataset(spec, tokenizer, token=token),
+            num_calibration_samples=spec.calib_samples,
+            max_seq_length=spec.calib_seqlen,
+        )
+    oneshot(**kwargs)
+    return out

quantfit-0.1.0/quantfit/backends/gguf.py

@@ -0,0 +1,121 @@
+"""GGUF backend (llama.cpp).
+
+Produces GGUF k-quants (Q4_K_M, etc.) for the Ollama / llama.cpp / LM Studio
+world. Quantization is CPU-only. Two tools are provisioned into a cache on first
+use (or located via QUANTFIT_LLAMACPP pointing at a llama.cpp checkout):
+  - the prebuilt `llama-quantize` binary (from the pinned llama.cpp release zip)
+  - the repo's `convert_hf_to_gguf.py` (HF safetensors -> GGUF f16); it imports a
+    sibling `conversion` package, so a shallow clone of the repo is required.
+"""
+from __future__ import annotations
+
+import os
+import platform
+import subprocess
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+
+LLAMACPP_TAG = "b9817"  # pinned release; binary + convert script must match
+GGUF_TYPES = (
+    "Q2_K", "Q3_K_S", "Q3_K_M", "Q4_K_S", "Q4_K_M", "Q5_K_M", "Q6_K", "Q8_0", "IQ4_XS",
+)
+
+_REPO = "https://github.com/ggml-org/llama.cpp"
+_RELEASES = "https://github.com/ggml-org/llama.cpp/releases/download"
+
+
+def _cache_dir() -> Path:
+    root = os.environ.get("QUANTFIT_CACHE")
+    d = Path(root) if root else Path.home() / ".cache" / "quantfit"
+    d.mkdir(parents=True, exist_ok=True)
+    return d
+
+
+def _exe_name() -> str:
+    return "llama-quantize.exe" if platform.system() == "Windows" else "llama-quantize"
+
+
+def _binary_asset() -> str:
+    sysname = platform.system()
+    if sysname == "Windows":
+        return f"llama-{LLAMACPP_TAG}-bin-win-cpu-x64.zip"
+    if sysname == "Linux":
+        return f"llama-{LLAMACPP_TAG}-bin-ubuntu-x64.zip"
+    raise RuntimeError(
+        f"no prebuilt llama.cpp binary wired for {sysname}; install llama.cpp and "
+        f"set QUANTFIT_LLAMACPP to its directory"
+    )
+
+
+def _first_match(root: Path, name: str) -> Path | None:
+    if (root / name).exists():
+        return root / name
+    return next(iter(root.rglob(name)), None)
+
+
+def llama_quantize_bin() -> Path:
+    """Locate (env) or download+extract the llama-quantize binary."""
+    exe = _exe_name()
+    env = os.environ.get("QUANTFIT_LLAMACPP")
+    if env and (hit := _first_match(Path(env), exe)):
+        return hit
+
+    bindir = _cache_dir() / f"llamacpp-bin-{LLAMACPP_TAG}"
+    if hit := _first_match(bindir, exe):
+        return hit
+
+    asset = _binary_asset()
+    zip_path = _cache_dir() / asset
+    if not zip_path.exists():
+        urllib.request.urlretrieve(f"{_RELEASES}/{LLAMACPP_TAG}/{asset}", zip_path)
+    bindir.mkdir(parents=True, exist_ok=True)
+    with zipfile.ZipFile(zip_path) as z:
+        z.extractall(bindir)
+    if hit := _first_match(bindir, exe):
+        return hit
+    raise RuntimeError(f"{exe} not found inside {asset}")
+
+
+def convert_script() -> Path:
+    """Locate (env) or shallow-clone the repo's convert_hf_to_gguf.py."""
+    name = "convert_hf_to_gguf.py"
+    env = os.environ.get("QUANTFIT_LLAMACPP")
+    if env and (Path(env) / name).exists():
+        return Path(env) / name
+
+    repo = _cache_dir() / f"llama.cpp-{LLAMACPP_TAG}"
+    if (repo / name).exists():
+        return repo / name
+    subprocess.run(
+        ["git", "clone", "--depth", "1", "--branch", LLAMACPP_TAG, _REPO, str(repo)],
+        check=True,
+    )
+    if not (repo / name).exists():
+        raise RuntimeError(f"{name} missing after cloning {_REPO}@{LLAMACPP_TAG}")
+    return repo / name
+
+
+def quantize_gguf(model_id: str, qtype: str, out_dir: str, token: str | None = None) -> Path:
+    """HF model -> GGUF f16 -> quantized GGUF (CPU-only)."""
+    from huggingface_hub import snapshot_download
+
+    out = Path(out_dir)
+    out.mkdir(parents=True, exist_ok=True)
+
+    quant_bin = llama_quantize_bin()
+    convert = convert_script()
+    model_dir = snapshot_download(model_id, token=token)
+
+    f16 = out / "model.f16.gguf"
+    final = out / f"model.{qtype}.gguf"
+    env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
+
+    subprocess.run(
+        [sys.executable, str(convert), model_dir, "--outtype", "f16", "--outfile", str(f16)],
+        check=True, env=env,
+    )
+    subprocess.run([str(quant_bin), str(f16), str(final), qtype], check=True, env=env)
+    f16.unlink(missing_ok=True)  # drop the large f16 intermediate
+    return out