ternary-quant 0.1.6__tar.gz

ternary_quant-0.1.6/LICENSE

@@ -0,0 +1,184 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and
+configuration files.
+
+"Object" form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object
+code, generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form,
+made available under the License, as indicated by a copyright notice that is
+included in or attached to the work (an example is provided in the Appendix
+below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original
+version of the Work and any modifications or additions to that Work or
+Derivative Works thereof, that is intentionally submitted to Licensor for
+inclusion in the Work by the copyright owner or by an individual or Legal
+Entity authorized to submit on behalf of the copyright owner. For the purposes
+of this definition, "submitted" means any form of electronic, verbal, or
+written communication sent to the Licensor or its representatives, including
+but not limited to communication on electronic mailing lists, source code
+control systems, and issue tracking systems that are managed by, or on behalf
+of, the Licensor for the purpose of discussing and improving the Work, but
+excluding communication that is conspicuously marked or otherwise designated in
+writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of this
+License, each Contributor hereby grants to You a perpetual, worldwide,
+non-exclusive, no-charge, royalty-free, irrevocable copyright license to
+reproduce, prepare Derivative Works of, publicly display, publicly perform,
+sublicense, and distribute the Work and such Derivative Works in Source or
+Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of this
+License, each Contributor hereby grants to You a perpetual, worldwide,
+non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this
+section) patent license to make, have made, use, offer to sell, sell, import,
+and otherwise transfer the Work, where such license applies only to those
+patent claims licensable by such Contributor that are necessarily infringed by
+their Contribution(s) alone or by combination of their Contribution(s) with the
+Work to which such Contribution(s) was submitted. If You institute patent
+litigation against any entity (including a cross-claim or counterclaim in a
+lawsuit) alleging that the Work or a Contribution incorporated within the Work
+constitutes direct or contributory patent infringement, then any patent
+licenses granted to You under this License for that Work shall terminate as of
+the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the Work or
+Derivative Works thereof in any medium, with or without modifications, and in
+Source or Object form, provided that You meet the following conditions:
+
+(a) You must give any other recipients of the Work or Derivative Works a copy
+of this License; and
+
+(b) You must cause any modified files to carry prominent notices stating that
+You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works that You
+distribute, all copyright, patent, trademark, and attribution notices from the
+Source form of the Work, excluding those notices that do not pertain to any
+part of the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its distribution, then
+any Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License.
+
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a
+whole, provided Your use, reproduction, and distribution of the Work otherwise
+complies with the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise, any
+Contribution intentionally submitted for inclusion in the Work by You to the
+Licensor shall be under the terms and conditions of this License, without any
+additional terms or conditions. Notwithstanding the above, nothing herein shall
+supersede or modify the terms of any separate license agreement you may have
+executed with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade names,
+trademarks, service marks, or product names of the Licensor, except as required
+for reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or agreed to in
+writing, Licensor provides the Work (and each Contributor provides its
+Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied, including, without limitation, any warranties
+or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE. You are solely responsible for determining the
+appropriateness of using or redistributing the Work and assume any risks
+associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory, whether in
+tort (including negligence), contract, or otherwise, unless required by
+applicable law (such as deliberate and grossly negligent acts) or agreed to in
+writing, shall any Contributor be liable to You for damages, including any
+direct, indirect, special, incidental, or consequential damages of any
+character arising as a result of this License or out of the use or inability to
+use the Work (including but not limited to damages for loss of goodwill, work
+stoppage, computer failure or malfunction, or any and all other commercial
+damages or losses), even if such Contributor has been advised of the
+possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing the Work or
+Derivative Works thereof, You may choose to offer, and charge a fee for,
+acceptance of support, warranty, indemnity, or other liability obligations
+and/or rights consistent with this License. However, in accepting such
+obligations, You may act only on Your own behalf and on Your sole
+responsibility, not on behalf of any other Contributor, and only if You agree
+to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own
+identifying information. (Don't include the brackets!) The text should be
+enclosed in the appropriate comment syntax for the file format. We also
+recommend that a file or class name and description of purpose be included on
+the same "printed page" as the copyright notice for easier identification
+within third-party archives.
+
+Copyright 2026 Asad Ismail
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

ternary_quant-0.1.6/PKG-INFO

@@ -0,0 +1,214 @@
+Metadata-Version: 2.4
+Name: ternary-quant
+Version: 0.1.6
+Summary: Post-training ternary quantization for HuggingFace generative models
+Author: Asad Ismail
+License: Apache-2.0
+Project-URL: Repository, https://github.com/Asad-Ismail/ternary-quant
+Project-URL: Issues, https://github.com/Asad-Ismail/ternary-quant/issues
+Keywords: quantization,ternary,llm,vlm,post-training-quantization,generative-models
+Classifier: Development Status :: 3 - Alpha
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.1.0
+Requires-Dist: transformers>=4.46.0
+Requires-Dist: accelerate>=0.28.0
+Requires-Dist: datasets>=2.18.0
+Requires-Dist: safetensors>=0.4.0
+Requires-Dist: tqdm
+Requires-Dist: numpy
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Provides-Extra: paper
+Requires-Dist: hqq>=0.2.8.post1; extra == "paper"
+Requires-Dist: matplotlib>=3.7.0; extra == "paper"
+Provides-Extra: multimodal
+Requires-Dist: pillow; extra == "multimodal"
+Requires-Dist: num2words; extra == "multimodal"
+Requires-Dist: torchvision; extra == "multimodal"
+Provides-Extra: eval
+Requires-Dist: lm-eval>=0.4.0; extra == "eval"
+Provides-Extra: serving
+Requires-Dist: vllm>=0.4.0; extra == "serving"
+Dynamic: license-file
+
+<div align="center">
+
+```
+ _____     ___                _   
+|_   _|__ / _ \ _  _ __ _ _ _| |_ 
+  | ||___| (_) | || / _` | ' \  _|
+  |_|     \__\_\\_,_\__,_|_||_\__|
+```
+
+**Component-first post-training ternary quantization for HuggingFace models.
+HF-native PTQ for VLMs, seq2seq, and audio — with packed runtime modes for selected 7B-class models.**
+
+[![CI](https://github.com/Asad-Ismail/ternary-quant/actions/workflows/ci.yml/badge.svg)](https://github.com/Asad-Ismail/ternary-quant/actions/workflows/ci.yml)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
+[![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/)
+[![PyPI version](https://img.shields.io/pypi/v/ternary-quant.svg)](https://pypi.org/project/ternary-quant/)
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Asad-Ismail/ternary-quant/blob/main/notebooks/quickstart.ipynb)
+[![Demo](https://img.shields.io/badge/🤗%20Space-Live%20Demo-blue)](https://huggingface.co/spaces/AsadIsmail/ternary-quant-demo)
+
+**[Try the live demo](https://huggingface.co/spaces/AsadIsmail/ternary-quant-demo)**  &nbsp;|&nbsp;  Pre-quantized: [Qwen3-1.7B](https://huggingface.co/AsadIsmail/Qwen3-1.7B-ternary) &middot; [Qwen2-VL-2B](https://huggingface.co/AsadIsmail/Qwen2-VL-2B-ternary) &middot; [Gemma4-E2B](https://huggingface.co/AsadIsmail/Gemma4-E2B-ternary)
+
+</div>
+
+GGUF/llama.cpp is the strongest default for local deployment of supported models,
+and current llama.cpp builds support selected multimodal models through
+`libmtmd` (including image input and experimental audio input). `ternary-quant`
+targets a different gap: HuggingFace/Python-native, component-first
+post-training ternary quantization of existing FP16 checkpoints. Decoder-LLM
+benchmarks retain 95–99% of FP16 quality (97.7% mean across seven models);
+multimodal/audio quality is model-card and validation-suite dependent.
+
+```
+Qwen2.5-7B:   15.2 GB FP16  →  9.6 GB triton_memory  ✓ fits 12 GB GPU (RTX 4070/3080)
+Phi-3.5-mini:  7.6 GB FP16  →  4.6 GB triton_memory  ✓ fits  6 GB GPU (RTX 3060/GTX 1060)
+Mistral-7B:   14.0 GB FP16  →  5.0 GB stored           (98.7% ARC quality, stays in Python/HF)
+FLAN-T5-small:154 MB FP16  →  125 MB stored            (5.1× decoder component)
+```
+
+---
+
+## Quickstart
+
+```bash
+pip install ternary-quant
+# or with uv
+uv pip install ternary-quant
+
+# Vision-Language Model
+ternary-quant quantize-broad Qwen/Qwen2-VL-2B-Instruct \
+  --output ./qwen2vl-quant \
+  --components text_backbone vision_backbone \
+  --scheme tritplane3
+
+# Causal LM
+ternary-quant quantize-broad Qwen/Qwen3-4B \
+  --output ./qwen3-4b-quant --components text_backbone --scheme tritplane3
+
+# Seq2seq / Audio
+ternary-quant quantize-broad openai/whisper-small \
+  --output ./whisper-quant --components decoder
+
+# Generate
+ternary-quant generate ./qwen2vl-quant --prompt "Describe this image" --runtime-mode cached
+```
+
+Or use a pre-quantized model directly:
+
+```python
+from ternary_quant.inference import load_ternary_model, generate_text
+
+model, tokenizer = load_ternary_model("AsadIsmail/Qwen3-1.7B-ternary", device="cuda", runtime_mode="cached")
+print(generate_text(model, tokenizer, "What is ternary quantization?", max_new_tokens=100))
+```
+
+---
+
+## What it does
+
+`ternary-quant` is **post-training quantization (PTQ)** — it takes supported
+existing FP16 HuggingFace checkpoints and pushes selected transformer weights
+toward {-1, 0, +1} (ternary). No retraining required.
+
+**PTQ ternary vs trained ternary (BitNet):** BitNet-style models are trained
+from scratch to operate with ternary weights, which is different from converting
+an existing FP16 checkpoint after training. `ternary-quant` does PTQ: take an
+existing FP16 model (Qwen, Mistral, Whisper, VLMs) and quantize selected
+components without retraining. GGUF/llama.cpp supports trained ternary formats
+for compatible models, but not this repo's HF-native component-first PTQ
+workflow for arbitrary checkpoints.
+
+---
+
+## Key results
+
+**95–99% decoder-LLM quality retention** across 7 models (0.5B-7B), measured with lm-eval 0-shot (ARC, HellaSwag, WinoGrande):
+
+| Model | Avg retain | Compression |
+|---|---|---|
+| Phi-2 (2.7B) | **99.2%** | 1.8x |
+| Qwen2.5-0.5B | **99.1%** | 1.8x |
+| Mistral-7B | **98.7%** | 2.8x |
+| Qwen3-4B | **98.1%** | 1.3x |
+| Qwen3-1.7B | **97.5%** | 2.7x |
+
+Full results: [docs/KNOWN_GOOD_MODELS.md](docs/KNOWN_GOOD_MODELS.md)
+
+Validated paths cover decoder LLMs, selected VLMs (Qwen2-VL, Gemma-4,
+SmolVLM), seq2seq (FLAN-T5), and audio (Whisper).
+
+---
+
+## How it compares
+
+| Tool | LLMs | VLMs | Seq2seq / Audio | PTQ ternary | Component selection | Stays in Python/HF |
+|---|---|---|---|---|---|---|
+| GGUF / llama.cpp | ✓✓✓ | ✓ (selected models via libmtmd) | partial / experimental | trained only | partial | ✗ |
+| bitsandbytes | ✓✓ | partial | ✗ | ✗ | ✗ | ✓ |
+| GPTQ / AWQ | ✓✓ | partial | ✗ | ✗ | ✗ | ✓ |
+| PT²-LLM / PTQTP | ✓✓ | ✗ | ✗ | ✓ | ✗ | ✗ |
+| **ternary-quant** | **✓✓** | **✓✓** | **✓✓** | **✓** | **✓** | **✓** |
+
+**Choose the right tool:**
+- Standard decoder LLM, want max speed locally → **GGUF** (llama.cpp is faster on CPU, better ecosystem)
+- PTQ ternary on decoder LLMs, care about peak quality → **PT²-LLM** (ICLR 2026, strongest ternary PTQ results on decoder LLMs)
+- Need HF-native component-first ternary PTQ for a VLM, seq2seq, audio, or custom model → **ternary-quant**
+- 4-bit LLM with best quality-per-bit → **HQQ-4** (outperforms ternary-quant on standard LLMs)
+
+---
+
+## Runtime modes
+
+| Mode | Hardware | Weights | Speed | Use case |
+|---|---|---|---|---|
+| `cached` | CPU / GPU / MPS | FP16 in memory | **fastest** | Default — dequantizes once at load |
+| `triton_memory` | NVIDIA GPU | **packed 2-bit** | ~3-6x slower | Fit oversized models on small GPUs |
+| `metal` | Apple Silicon | **packed 2-bit** | ~3-4x slower | Fit oversized models on Mac |
+
+`triton_memory` and `metal` keep weights as packed 2-bit ternary codes throughout inference — no FP16 weight matrix is ever materialized. This trades speed for memory, enabling models that don't fit otherwise.
+
+> **`metal` mode requires torch ≥ 2.7** (for `torch.mps.compile_shader`). On older torch it degrades to `cached`. Apple-Silicon MPS generation also needs torch ≥ 2.7 to avoid a GQA matmul crash present in 2.6.
+
+| Model | FP16 VRAM | triton_memory | Fits on |
+|---|---|---|---|
+| Phi-3.5-mini (3.8B) | 7.6 GB | **4.6 GB** | RTX 3060 6GB |
+| Qwen2.5-7B (7.6B) | 15.2 GB | **9.6 GB** | RTX 4070 12GB |
+
+| Model | Cached (Mac) | Metal (Mac) | Memory saved |
+|---|---|---|---|
+| Qwen3-1.7B | 10.4 GB / 46 tok/s | 4.6 GB / 12 tok/s | 2.3x |
+| Qwen2-VL-2B | 16.8 GB / 52 tok/s | 7.4 GB / 13 tok/s | 2.3x |
+
+Full details: [docs/RUNTIME_MODES.md](docs/RUNTIME_MODES.md)
+
+---
+
+## Learn more
+
+- [Known good models](docs/KNOWN_GOOD_MODELS.md) — measured results and commands for validated architecture families
+- [Runtime modes](docs/RUNTIME_MODES.md) — triton_memory, metal, CPU inference details
+- [Research track](docs/RESEARCH_ADVANCED_TERNARY.md) — RAST + CCRE for true 1.58-bit compression
+- [Examples](examples/README.md) — text, seq2seq, VLM walkthroughs
+- [Paper](paper/paper.tex) — full experimental results and method description (LaTeX)
+- [Release notes](releases/v0.1.6.md) — what changed in this version
+
+---
+
+## Honest positioning
+
+- **HQQ-4 beats ternary-quant on quality-per-bit** for standard LLMs — use it when quality is the priority
+- **PT²-LLM and PTQTP** achieve strong PTQ ternary results on decoder-only LLMs — ternary-quant's algorithmic contribution on that axis is parallel, not superior
+- The broad path is a ternary PTQ research/runtime path, not a general replacement for strong INT4 deployments
+- The research track achieves true near-ternary weights with a real quality cost — this is the active research frontier, not a production-ready path
+- GGUF is better for local deployment of supported standard LLMs and selected multimodal models; ternary-quant's value is component-first ternary PTQ in the HuggingFace ecosystem

ternary_quant-0.1.6/README.md

@@ -0,0 +1,173 @@
+<div align="center">
+
+```
+ _____     ___                _   
+|_   _|__ / _ \ _  _ __ _ _ _| |_ 
+  | ||___| (_) | || / _` | ' \  _|
+  |_|     \__\_\\_,_\__,_|_||_\__|
+```
+
+**Component-first post-training ternary quantization for HuggingFace models.
+HF-native PTQ for VLMs, seq2seq, and audio — with packed runtime modes for selected 7B-class models.**
+
+[![CI](https://github.com/Asad-Ismail/ternary-quant/actions/workflows/ci.yml/badge.svg)](https://github.com/Asad-Ismail/ternary-quant/actions/workflows/ci.yml)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE)
+[![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/)
+[![PyPI version](https://img.shields.io/pypi/v/ternary-quant.svg)](https://pypi.org/project/ternary-quant/)
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Asad-Ismail/ternary-quant/blob/main/notebooks/quickstart.ipynb)
+[![Demo](https://img.shields.io/badge/🤗%20Space-Live%20Demo-blue)](https://huggingface.co/spaces/AsadIsmail/ternary-quant-demo)
+
+**[Try the live demo](https://huggingface.co/spaces/AsadIsmail/ternary-quant-demo)**  &nbsp;|&nbsp;  Pre-quantized: [Qwen3-1.7B](https://huggingface.co/AsadIsmail/Qwen3-1.7B-ternary) &middot; [Qwen2-VL-2B](https://huggingface.co/AsadIsmail/Qwen2-VL-2B-ternary) &middot; [Gemma4-E2B](https://huggingface.co/AsadIsmail/Gemma4-E2B-ternary)
+
+</div>
+
+GGUF/llama.cpp is the strongest default for local deployment of supported models,
+and current llama.cpp builds support selected multimodal models through
+`libmtmd` (including image input and experimental audio input). `ternary-quant`
+targets a different gap: HuggingFace/Python-native, component-first
+post-training ternary quantization of existing FP16 checkpoints. Decoder-LLM
+benchmarks retain 95–99% of FP16 quality (97.7% mean across seven models);
+multimodal/audio quality is model-card and validation-suite dependent.
+
+```
+Qwen2.5-7B:   15.2 GB FP16  →  9.6 GB triton_memory  ✓ fits 12 GB GPU (RTX 4070/3080)
+Phi-3.5-mini:  7.6 GB FP16  →  4.6 GB triton_memory  ✓ fits  6 GB GPU (RTX 3060/GTX 1060)
+Mistral-7B:   14.0 GB FP16  →  5.0 GB stored           (98.7% ARC quality, stays in Python/HF)
+FLAN-T5-small:154 MB FP16  →  125 MB stored            (5.1× decoder component)
+```
+
+---
+
+## Quickstart
+
+```bash
+pip install ternary-quant
+# or with uv
+uv pip install ternary-quant
+
+# Vision-Language Model
+ternary-quant quantize-broad Qwen/Qwen2-VL-2B-Instruct \
+  --output ./qwen2vl-quant \
+  --components text_backbone vision_backbone \
+  --scheme tritplane3
+
+# Causal LM
+ternary-quant quantize-broad Qwen/Qwen3-4B \
+  --output ./qwen3-4b-quant --components text_backbone --scheme tritplane3
+
+# Seq2seq / Audio
+ternary-quant quantize-broad openai/whisper-small \
+  --output ./whisper-quant --components decoder
+
+# Generate
+ternary-quant generate ./qwen2vl-quant --prompt "Describe this image" --runtime-mode cached
+```
+
+Or use a pre-quantized model directly:
+
+```python
+from ternary_quant.inference import load_ternary_model, generate_text
+
+model, tokenizer = load_ternary_model("AsadIsmail/Qwen3-1.7B-ternary", device="cuda", runtime_mode="cached")
+print(generate_text(model, tokenizer, "What is ternary quantization?", max_new_tokens=100))
+```
+
+---
+
+## What it does
+
+`ternary-quant` is **post-training quantization (PTQ)** — it takes supported
+existing FP16 HuggingFace checkpoints and pushes selected transformer weights
+toward {-1, 0, +1} (ternary). No retraining required.
+
+**PTQ ternary vs trained ternary (BitNet):** BitNet-style models are trained
+from scratch to operate with ternary weights, which is different from converting
+an existing FP16 checkpoint after training. `ternary-quant` does PTQ: take an
+existing FP16 model (Qwen, Mistral, Whisper, VLMs) and quantize selected
+components without retraining. GGUF/llama.cpp supports trained ternary formats
+for compatible models, but not this repo's HF-native component-first PTQ
+workflow for arbitrary checkpoints.
+
+---
+
+## Key results
+
+**95–99% decoder-LLM quality retention** across 7 models (0.5B-7B), measured with lm-eval 0-shot (ARC, HellaSwag, WinoGrande):
+
+| Model | Avg retain | Compression |
+|---|---|---|
+| Phi-2 (2.7B) | **99.2%** | 1.8x |
+| Qwen2.5-0.5B | **99.1%** | 1.8x |
+| Mistral-7B | **98.7%** | 2.8x |
+| Qwen3-4B | **98.1%** | 1.3x |
+| Qwen3-1.7B | **97.5%** | 2.7x |
+
+Full results: [docs/KNOWN_GOOD_MODELS.md](docs/KNOWN_GOOD_MODELS.md)
+
+Validated paths cover decoder LLMs, selected VLMs (Qwen2-VL, Gemma-4,
+SmolVLM), seq2seq (FLAN-T5), and audio (Whisper).
+
+---
+
+## How it compares
+
+| Tool | LLMs | VLMs | Seq2seq / Audio | PTQ ternary | Component selection | Stays in Python/HF |
+|---|---|---|---|---|---|---|
+| GGUF / llama.cpp | ✓✓✓ | ✓ (selected models via libmtmd) | partial / experimental | trained only | partial | ✗ |
+| bitsandbytes | ✓✓ | partial | ✗ | ✗ | ✗ | ✓ |
+| GPTQ / AWQ | ✓✓ | partial | ✗ | ✗ | ✗ | ✓ |
+| PT²-LLM / PTQTP | ✓✓ | ✗ | ✗ | ✓ | ✗ | ✗ |
+| **ternary-quant** | **✓✓** | **✓✓** | **✓✓** | **✓** | **✓** | **✓** |
+
+**Choose the right tool:**
+- Standard decoder LLM, want max speed locally → **GGUF** (llama.cpp is faster on CPU, better ecosystem)
+- PTQ ternary on decoder LLMs, care about peak quality → **PT²-LLM** (ICLR 2026, strongest ternary PTQ results on decoder LLMs)
+- Need HF-native component-first ternary PTQ for a VLM, seq2seq, audio, or custom model → **ternary-quant**
+- 4-bit LLM with best quality-per-bit → **HQQ-4** (outperforms ternary-quant on standard LLMs)
+
+---
+
+## Runtime modes
+
+| Mode | Hardware | Weights | Speed | Use case |
+|---|---|---|---|---|
+| `cached` | CPU / GPU / MPS | FP16 in memory | **fastest** | Default — dequantizes once at load |
+| `triton_memory` | NVIDIA GPU | **packed 2-bit** | ~3-6x slower | Fit oversized models on small GPUs |
+| `metal` | Apple Silicon | **packed 2-bit** | ~3-4x slower | Fit oversized models on Mac |
+
+`triton_memory` and `metal` keep weights as packed 2-bit ternary codes throughout inference — no FP16 weight matrix is ever materialized. This trades speed for memory, enabling models that don't fit otherwise.
+
+> **`metal` mode requires torch ≥ 2.7** (for `torch.mps.compile_shader`). On older torch it degrades to `cached`. Apple-Silicon MPS generation also needs torch ≥ 2.7 to avoid a GQA matmul crash present in 2.6.
+
+| Model | FP16 VRAM | triton_memory | Fits on |
+|---|---|---|---|
+| Phi-3.5-mini (3.8B) | 7.6 GB | **4.6 GB** | RTX 3060 6GB |
+| Qwen2.5-7B (7.6B) | 15.2 GB | **9.6 GB** | RTX 4070 12GB |
+
+| Model | Cached (Mac) | Metal (Mac) | Memory saved |
+|---|---|---|---|
+| Qwen3-1.7B | 10.4 GB / 46 tok/s | 4.6 GB / 12 tok/s | 2.3x |
+| Qwen2-VL-2B | 16.8 GB / 52 tok/s | 7.4 GB / 13 tok/s | 2.3x |
+
+Full details: [docs/RUNTIME_MODES.md](docs/RUNTIME_MODES.md)
+
+---
+
+## Learn more
+
+- [Known good models](docs/KNOWN_GOOD_MODELS.md) — measured results and commands for validated architecture families
+- [Runtime modes](docs/RUNTIME_MODES.md) — triton_memory, metal, CPU inference details
+- [Research track](docs/RESEARCH_ADVANCED_TERNARY.md) — RAST + CCRE for true 1.58-bit compression
+- [Examples](examples/README.md) — text, seq2seq, VLM walkthroughs
+- [Paper](paper/paper.tex) — full experimental results and method description (LaTeX)
+- [Release notes](releases/v0.1.6.md) — what changed in this version
+
+---
+
+## Honest positioning
+
+- **HQQ-4 beats ternary-quant on quality-per-bit** for standard LLMs — use it when quality is the priority
+- **PT²-LLM and PTQTP** achieve strong PTQ ternary results on decoder-only LLMs — ternary-quant's algorithmic contribution on that axis is parallel, not superior
+- The broad path is a ternary PTQ research/runtime path, not a general replacement for strong INT4 deployments
+- The research track achieves true near-ternary weights with a real quality cost — this is the active research frontier, not a production-ready path
+- GGUF is better for local deployment of supported standard LLMs and selected multimodal models; ternary-quant's value is component-first ternary PTQ in the HuggingFace ecosystem

ternary_quant-0.1.6/pyproject.toml

@@ -0,0 +1,56 @@
+[build-system]
+requires = ["setuptools>=68.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ternary-quant"
+version = "0.1.6"
+description = "Post-training ternary quantization for HuggingFace generative models"
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "Apache-2.0"}
+authors = [{name = "Asad Ismail"}]
+keywords = ["quantization", "ternary", "llm", "vlm", "post-training-quantization", "generative-models"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "torch>=2.1.0",
+    "transformers>=4.46.0",
+    "accelerate>=0.28.0",
+    "datasets>=2.18.0",
+    "safetensors>=0.4.0",
+    "tqdm",
+    "numpy",
+]
+
+[project.urls]
+Repository = "https://github.com/Asad-Ismail/ternary-quant"
+Issues = "https://github.com/Asad-Ismail/ternary-quant/issues"
+
+[project.optional-dependencies]
+dev = ["pytest", "ruff"]
+paper = ["hqq>=0.2.8.post1", "matplotlib>=3.7.0"]
+multimodal = ["pillow", "num2words", "torchvision"]
+eval = ["lm-eval>=0.4.0"]
+serving = ["vllm>=0.4.0"]
+
+[project.scripts]
+ternary-quant = "ternary_quant.cli:main"
+
+[tool.setuptools]
+license-files = ["LICENSE"]
+
+[tool.setuptools.packages.find]
+include = ["ternary_quant*"]
+
+[tool.ruff]
+line-length = 100
+
+[tool.ruff.lint]
+select = ["F"]  # pyflakes: unused imports, undefined names, unused variables

ternary_quant-0.1.6/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+