quf 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. quf-0.1.0/LICENSE +21 -0
  2. quf-0.1.0/MANIFEST.in +2 -0
  3. quf-0.1.0/PKG-INFO +177 -0
  4. quf-0.1.0/README.md +137 -0
  5. quf-0.1.0/docs/BACKENDS.md +20 -0
  6. quf-0.1.0/docs/ENV.md +38 -0
  7. quf-0.1.0/docs/FORMAT.md +173 -0
  8. quf-0.1.0/docs/RESULTS.md +77 -0
  9. quf-0.1.0/pyproject.toml +69 -0
  10. quf-0.1.0/quf/__init__.py +28 -0
  11. quf-0.1.0/quf/bench/__init__.py +1 -0
  12. quf-0.1.0/quf/bench/lm_eval_adapter.py +203 -0
  13. quf-0.1.0/quf/bench/suite.py +306 -0
  14. quf-0.1.0/quf/cli.py +235 -0
  15. quf-0.1.0/quf/convert/__init__.py +1 -0
  16. quf-0.1.0/quf/convert/converter.py +280 -0
  17. quf-0.1.0/quf/convert/gptq.py +109 -0
  18. quf-0.1.0/quf/device.py +41 -0
  19. quf-0.1.0/quf/format/__init__.py +1 -0
  20. quf-0.1.0/quf/format/constants.py +24 -0
  21. quf-0.1.0/quf/format/errors.py +39 -0
  22. quf-0.1.0/quf/format/reader.py +136 -0
  23. quf-0.1.0/quf/format/schema.py +112 -0
  24. quf-0.1.0/quf/format/writer.py +106 -0
  25. quf-0.1.0/quf/quant/__init__.py +1 -0
  26. quf-0.1.0/quf/quant/dequant.py +46 -0
  27. quf-0.1.0/quf/quant/gptq.py +384 -0
  28. quf-0.1.0/quf/quant/pack.py +41 -0
  29. quf-0.1.0/quf/quant/quantize.py +106 -0
  30. quf-0.1.0/quf/runtime/__init__.py +1 -0
  31. quf-0.1.0/quf/runtime/backends/__init__.py +1 -0
  32. quf-0.1.0/quf/runtime/backends/base.py +22 -0
  33. quf-0.1.0/quf/runtime/backends/cpu.py +31 -0
  34. quf-0.1.0/quf/runtime/backends/marlin.py +159 -0
  35. quf-0.1.0/quf/runtime/backends/torch_fused.py +161 -0
  36. quf-0.1.0/quf/runtime/backends/torch_gpu.py +42 -0
  37. quf-0.1.0/quf/runtime/errors.py +20 -0
  38. quf-0.1.0/quf/runtime/generate.py +113 -0
  39. quf-0.1.0/quf/runtime/linear.py +33 -0
  40. quf-0.1.0/quf/runtime/model.py +316 -0
  41. quf-0.1.0/quf/runtime/selector.py +98 -0
  42. quf-0.1.0/quf/tokenization/__init__.py +6 -0
  43. quf-0.1.0/quf/tokenization/calibration.py +54 -0
  44. quf-0.1.0/quf/tokenization/wrapper.py +83 -0
  45. quf-0.1.0/quf.egg-info/PKG-INFO +177 -0
  46. quf-0.1.0/quf.egg-info/SOURCES.txt +82 -0
  47. quf-0.1.0/quf.egg-info/dependency_links.txt +1 -0
  48. quf-0.1.0/quf.egg-info/entry_points.txt +2 -0
  49. quf-0.1.0/quf.egg-info/requires.txt +18 -0
  50. quf-0.1.0/quf.egg-info/top_level.txt +1 -0
  51. quf-0.1.0/setup.cfg +4 -0
  52. quf-0.1.0/tests/test_backend_equivalence.py +69 -0
  53. quf-0.1.0/tests/test_bench_json.py +45 -0
  54. quf-0.1.0/tests/test_bench_suite.py +147 -0
  55. quf-0.1.0/tests/test_calibration.py +53 -0
  56. quf-0.1.0/tests/test_cli.py +63 -0
  57. quf-0.1.0/tests/test_cli_info.py +66 -0
  58. quf-0.1.0/tests/test_context_length.py +56 -0
  59. quf-0.1.0/tests/test_converter_preservation.py +53 -0
  60. quf-0.1.0/tests/test_converter_smoke.py +82 -0
  61. quf-0.1.0/tests/test_converter_unit.py +183 -0
  62. quf-0.1.0/tests/test_device.py +89 -0
  63. quf-0.1.0/tests/test_format_corruption.py +175 -0
  64. quf-0.1.0/tests/test_format_layout.py +112 -0
  65. quf-0.1.0/tests/test_format_roundtrip.py +81 -0
  66. quf-0.1.0/tests/test_fused_backend.py +80 -0
  67. quf-0.1.0/tests/test_gptq_error.py +54 -0
  68. quf-0.1.0/tests/test_gptq_layout.py +67 -0
  69. quf-0.1.0/tests/test_gptq_unit.py +182 -0
  70. quf-0.1.0/tests/test_gpu.py +141 -0
  71. quf-0.1.0/tests/test_kv_cache.py +47 -0
  72. quf-0.1.0/tests/test_model.py +85 -0
  73. quf-0.1.0/tests/test_pack.py +52 -0
  74. quf-0.1.0/tests/test_pyproject_metadata.py +36 -0
  75. quf-0.1.0/tests/test_quant_error.py +67 -0
  76. quf-0.1.0/tests/test_quant_group_size.py +30 -0
  77. quf-0.1.0/tests/test_quant_shape.py +57 -0
  78. quf-0.1.0/tests/test_runtime_unit.py +146 -0
  79. quf-0.1.0/tests/test_sampling.py +59 -0
  80. quf-0.1.0/tests/test_selector.py +68 -0
  81. quf-0.1.0/tests/test_selector_cli.py +121 -0
  82. quf-0.1.0/tests/test_selector_extended.py +67 -0
  83. quf-0.1.0/tests/test_tokenizer_fallback.py +75 -0
  84. quf-0.1.0/tests/test_tokenizer_roundtrip.py +49 -0
quf-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 vtrsharvin-sj
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
quf-0.1.0/MANIFEST.in ADDED
@@ -0,0 +1,2 @@
1
+ include README.md
2
+ recursive-include docs *.md
quf-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,177 @@
1
+ Metadata-Version: 2.4
2
+ Name: quf
3
+ Version: 0.1.0
4
+ Summary: QUF: Quantized Unified Format for 4-bit quantized decoder-only LLMs
5
+ Author: vtrsharvin-sj
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/vtrsharvin-sj/Quantized-Model-Format
8
+ Project-URL: Repository, https://github.com/vtrsharvin-sj/Quantized-Model-Format
9
+ Project-URL: Issues, https://github.com/vtrsharvin-sj/Quantized-Model-Format/issues
10
+ Keywords: quantization,llm,gptq,4-bit,pytorch
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Operating System :: Microsoft :: Windows
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Requires-Dist: numpy>=1.24
25
+ Requires-Dist: torch>=2.7
26
+ Provides-Extra: cuda
27
+ Provides-Extra: convert
28
+ Requires-Dist: torch>=2.7; extra == "convert"
29
+ Requires-Dist: transformers; extra == "convert"
30
+ Requires-Dist: datasets; extra == "convert"
31
+ Requires-Dist: accelerate; extra == "convert"
32
+ Requires-Dist: gigatoken; extra == "convert"
33
+ Requires-Dist: tokenizers; extra == "convert"
34
+ Requires-Dist: safetensors; extra == "convert"
35
+ Requires-Dist: huggingface_hub; extra == "convert"
36
+ Provides-Extra: test
37
+ Requires-Dist: pytest>=7; extra == "test"
38
+ Requires-Dist: hypothesis>=6; extra == "test"
39
+ Dynamic: license-file
40
+
41
+ # QUF: Quantized Unified Format
42
+
43
+ QUF is a single-file container (`.quf`) for a 4-bit quantized decoder-only LLM. One file holds the model config, tokenizer files, quantization scheme and all tensors. This package provides a reference writer and reader, a native GPTQ converter, a runtime with CPU and CUDA backends, and a benchmark suite.
44
+
45
+ QUF runs natively on Windows (and any platform with PyTorch). No WSL or Linux environment is needed.
46
+
47
+ Supported architectures: `Qwen2ForCausalLM` (first), with `Qwen3ForCausalLM` addable through the architecture allow-list. Weights are 4-bit, group size 128, packed 8 per `int32`, with fp16 scales. Embeddings and `lm_head` stay fp16.
48
+
49
+ ## Quickstart
50
+
51
+ ```
52
+ pip install quf
53
+ quf convert Qwen/Qwen2.5-0.5B out.quf
54
+ quf run out.quf --prompt "The capital of France is" --max-new-tokens 32 --temperature 0
55
+ quf bench out.quf --out results.json
56
+ ```
57
+
58
+ `quf convert` needs the convert extra: `pip install "quf[convert]"`.
59
+
60
+ ## Status: what is and is not verified
61
+
62
+ | Component | Status |
63
+ | --- | --- |
64
+ | Format writer/reader, quantizer, packer, CPU backend | Implemented. Covered by the CPU test suite. |
65
+ | Native GPTQ converter (`quf convert`) | Verified on native Windows with Qwen/Qwen2.5-0.5B ([docs/RESULTS.md](docs/RESULTS.md)). |
66
+ | `torch_gpu` backend on RTX 5080 Laptop (compute capability 12.0) | Verified: `quf run` and `quf bench` work on CUDA with a real converted Qwen2.5-0.5B. |
67
+ | `torch_fused` backend | Verified, but slower than `torch_gpu` on this GPU (uses less memory). |
68
+ | gigatoken on Windows | Verified: gigatoken 0.11.0 installs and runs, 36.8x faster than HF `tokenizers` on WikiText-2 with identical ids. |
69
+ | Perplexity target (3 percent) | **Measured and missed** on Qwen2.5-0.5B: +8.47% and +7.41% over fp16. |
70
+ | Marlin backend | Optional, never selected automatically, not verified on this hardware. |
71
+
72
+ Details are in [docs/ENV.md](docs/ENV.md) and [docs/RESULTS.md](docs/RESULTS.md).
73
+
74
+ ## Installation
75
+
76
+ Requires Python 3.10 or newer.
77
+
78
+ ```
79
+ pip install quf # core: format, quantizer, runtime (torch is a base dependency)
80
+ pip install "quf[cuda]" # same as core; kept so the extra name resolves
81
+ pip install "quf[convert]" # adds transformers, datasets, accelerate, gigatoken, tokenizers,
82
+ # safetensors, huggingface_hub for `quf convert`
83
+ pip install "quf[test]" # pytest and hypothesis
84
+ ```
85
+
86
+ From a source checkout, use `pip install .` and the same extras.
87
+
88
+ The `quf` console script is installed with the package. Run `quf --help` to list subcommands: `info`, `convert`, `run`, `bench`.
89
+
90
+ For a CUDA build of PyTorch on an RTX 50-series GPU, install a CUDA 12.8 wheel (the version recorded in ENV.md is `2.11.0+cu128`). With no GPU, everything works on CPU.
91
+
92
+ ## Convert
93
+
94
+ `quf convert` loads a Hugging Face Qwen checkpoint with transformers and quantizes it with QUF's own GPTQ implementation (4 bit, group size 128). It runs on native Windows, Linux and macOS.
95
+
96
+ ```
97
+ quf convert Qwen/Qwen2.5-0.5B out.quf
98
+ quf convert ./local-model out.quf --dataset calib.txt --samples 128 --seq-len 512 --seed 0
99
+ ```
100
+
101
+ Options: `--dataset` (text file, wikitext config or HF dataset id; default `wikitext-2-raw-v1`), `--samples` (128), `--seq-len` (512), `--seed` (0), `--overwrite`. The command prints the output size and compression ratio. It refuses to replace an existing output unless `--overwrite` is given, rejects unsupported architectures before heavy work, and deletes a partial file if conversion fails.
102
+
103
+ Inspect the result:
104
+
105
+ ```
106
+ quf info out.quf
107
+ ```
108
+
109
+ This prints the model config, quantization scheme, tensor count and file size. A corrupt file exits with status 1 and an error message.
110
+
111
+ ## Run
112
+
113
+ ```
114
+ quf run out.quf --prompt "Hello" --max-new-tokens 64 --temperature 0
115
+ ```
116
+
117
+ Options: `--max-new-tokens` (64), `--temperature` (0 means greedy), `--top-k`, `--top-p`, `--seed` (0), `--no-stream`, `--device {auto,cpu,cuda}` (auto), `--backend {auto,cached,fused,marlin}` (auto), `--prefer-fused`.
118
+
119
+ Backends:
120
+
121
+ - `cpu`: used when no NVIDIA GPU is present.
122
+ - `torch_gpu` (`--backend cached`): dequantizes weights to fp16 at load. Default on GPU when the weights fit in free GPU memory.
123
+ - `torch_fused` (`--backend fused`): keeps packed int4 on the GPU. Chosen automatically when the cached weights do not fit, or with `--prefer-fused`.
124
+ - `marlin`: optional, only with `--backend marlin`, never automatic. Unverified on this hardware.
125
+
126
+ The choice and its reason are logged to stderr. `--device cuda` without an NVIDIA GPU exits with status 1. A prompt that exceeds the context length and GPU out-of-memory also exit with status 1.
127
+
128
+ ## Benchmark
129
+
130
+ ```
131
+ quf bench out.quf --out results.json
132
+ quf bench out.quf --out results.json --fp16-ref Qwen/Qwen2.5-0.5B --gguf model-q4.gguf --device auto
133
+ ```
134
+
135
+ The results JSON records WikiText perplexity, file size, tokens per second, backend, device name and software versions. With `--fp16-ref`, the relative perplexity increase over fp16 is printed next to the documented target of at most 3 percent. The target is reported, not enforced, and was missed on Qwen2.5-0.5B (see below). `--gguf` adds a GGUF Q4 comparison. An lm-evaluation-harness adapter is in `quf/bench/lm_eval_adapter.py`.
136
+
137
+ ## Measured results
138
+
139
+ Qwen/Qwen2.5-0.5B, converted natively on Windows (4 bit, group size 128), RTX 5080 Laptop GPU, PyTorch 2.11.0+cu128, WikiText-2 raw. Full details and raw JSON: [docs/RESULTS.md](docs/RESULTS.md).
140
+
141
+ | Metric | Value |
142
+ | --- | --- |
143
+ | fp16 perplexity | 13.07 |
144
+ | QUF perplexity (32 x 512 calibration) | 14.18 (+8.47%) |
145
+ | QUF perplexity (128 x 1024 calibration) | 14.04 (+7.41%) |
146
+ | 3% target | **Missed** (+7 to +8%) |
147
+ | File size | 469,483,072 bytes, about 2.1x smaller than fp16 weights (about 988 MB). Embeddings and `lm_head` stay fp16, which limits the ratio on a small model. |
148
+ | Decode, CPU | 9.6 tokens/s |
149
+ | Decode, RTX 5080 `torch_gpu` (default) | 20.7 tokens/s |
150
+ | Decode, RTX 5080 `torch_fused` | 9.1 tokens/s (slower, less memory) |
151
+ | gigatoken vs HF tokenizers | 0.23 s vs 8.44 s (36.8x, identical ids) |
152
+
153
+ The cause of the perplexity gap was not investigated. Qwen2.5-1.5B was not run. Speeds are single measurements and the decode loop is not optimized.
154
+
155
+ ## Documentation
156
+
157
+ - [docs/FORMAT.md](docs/FORMAT.md): byte layout, header fields, tensor naming, dtypes and the canonical Packed_Int4 order.
158
+ - [docs/ENV.md](docs/ENV.md): environment results for native Windows and the backend selection rules.
159
+ - [docs/RESULTS.md](docs/RESULTS.md): measured accuracy, size and speed.
160
+
161
+ README.md, docs/ENV.md and docs/FORMAT.md ship with the package: they are in the source distribution and are installed to `share/doc/quf` by the wheel.
162
+
163
+ ## Tests
164
+
165
+ ```
166
+ pytest -m "not gpu and not convert" # CPU suite, no NVIDIA GPU needed
167
+ pytest -m gpu # needs an NVIDIA GPU
168
+ pytest -m convert # needs the convert extra and downloads a model
169
+ ```
170
+
171
+ Tests marked `gpu` are skipped automatically when no NVIDIA GPU is available.
172
+
173
+ ## Links and license
174
+
175
+ Source and issues: https://github.com/vtrsharvin-sj/Quantized-Model-Format
176
+
177
+ Released under the MIT license. See [LICENSE](LICENSE).
quf-0.1.0/README.md ADDED
@@ -0,0 +1,137 @@
1
+ # QUF: Quantized Unified Format
2
+
3
+ QUF is a single-file container (`.quf`) for a 4-bit quantized decoder-only LLM. One file holds the model config, tokenizer files, quantization scheme and all tensors. This package provides a reference writer and reader, a native GPTQ converter, a runtime with CPU and CUDA backends, and a benchmark suite.
4
+
5
+ QUF runs natively on Windows (and any platform with PyTorch). No WSL or Linux environment is needed.
6
+
7
+ Supported architectures: `Qwen2ForCausalLM` (first), with `Qwen3ForCausalLM` addable through the architecture allow-list. Weights are 4-bit, group size 128, packed 8 per `int32`, with fp16 scales. Embeddings and `lm_head` stay fp16.
8
+
9
+ ## Quickstart
10
+
11
+ ```
12
+ pip install quf
13
+ quf convert Qwen/Qwen2.5-0.5B out.quf
14
+ quf run out.quf --prompt "The capital of France is" --max-new-tokens 32 --temperature 0
15
+ quf bench out.quf --out results.json
16
+ ```
17
+
18
+ `quf convert` needs the convert extra: `pip install "quf[convert]"`.
19
+
20
+ ## Status: what is and is not verified
21
+
22
+ | Component | Status |
23
+ | --- | --- |
24
+ | Format writer/reader, quantizer, packer, CPU backend | Implemented. Covered by the CPU test suite. |
25
+ | Native GPTQ converter (`quf convert`) | Verified on native Windows with Qwen/Qwen2.5-0.5B ([docs/RESULTS.md](docs/RESULTS.md)). |
26
+ | `torch_gpu` backend on RTX 5080 Laptop (compute capability 12.0) | Verified: `quf run` and `quf bench` work on CUDA with a real converted Qwen2.5-0.5B. |
27
+ | `torch_fused` backend | Verified, but slower than `torch_gpu` on this GPU (uses less memory). |
28
+ | gigatoken on Windows | Verified: gigatoken 0.11.0 installs and runs, 36.8x faster than HF `tokenizers` on WikiText-2 with identical ids. |
29
+ | Perplexity target (3 percent) | **Measured and missed** on Qwen2.5-0.5B: +8.47% and +7.41% over fp16. |
30
+ | Marlin backend | Optional, never selected automatically, not verified on this hardware. |
31
+
32
+ Details are in [docs/ENV.md](docs/ENV.md) and [docs/RESULTS.md](docs/RESULTS.md).
33
+
34
+ ## Installation
35
+
36
+ Requires Python 3.10 or newer.
37
+
38
+ ```
39
+ pip install quf # core: format, quantizer, runtime (torch is a base dependency)
40
+ pip install "quf[cuda]" # same as core; kept so the extra name resolves
41
+ pip install "quf[convert]" # adds transformers, datasets, accelerate, gigatoken, tokenizers,
42
+ # safetensors, huggingface_hub for `quf convert`
43
+ pip install "quf[test]" # pytest and hypothesis
44
+ ```
45
+
46
+ From a source checkout, use `pip install .` and the same extras.
47
+
48
+ The `quf` console script is installed with the package. Run `quf --help` to list subcommands: `info`, `convert`, `run`, `bench`.
49
+
50
+ For a CUDA build of PyTorch on an RTX 50-series GPU, install a CUDA 12.8 wheel (the version recorded in ENV.md is `2.11.0+cu128`). With no GPU, everything works on CPU.
51
+
52
+ ## Convert
53
+
54
+ `quf convert` loads a Hugging Face Qwen checkpoint with transformers and quantizes it with QUF's own GPTQ implementation (4 bit, group size 128). It runs on native Windows, Linux and macOS.
55
+
56
+ ```
57
+ quf convert Qwen/Qwen2.5-0.5B out.quf
58
+ quf convert ./local-model out.quf --dataset calib.txt --samples 128 --seq-len 512 --seed 0
59
+ ```
60
+
61
+ Options: `--dataset` (text file, wikitext config or HF dataset id; default `wikitext-2-raw-v1`), `--samples` (128), `--seq-len` (512), `--seed` (0), `--overwrite`. The command prints the output size and compression ratio. It refuses to replace an existing output unless `--overwrite` is given, rejects unsupported architectures before heavy work, and deletes a partial file if conversion fails.
62
+
63
+ Inspect the result:
64
+
65
+ ```
66
+ quf info out.quf
67
+ ```
68
+
69
+ This prints the model config, quantization scheme, tensor count and file size. A corrupt file exits with status 1 and an error message.
70
+
71
+ ## Run
72
+
73
+ ```
74
+ quf run out.quf --prompt "Hello" --max-new-tokens 64 --temperature 0
75
+ ```
76
+
77
+ Options: `--max-new-tokens` (64), `--temperature` (0 means greedy), `--top-k`, `--top-p`, `--seed` (0), `--no-stream`, `--device {auto,cpu,cuda}` (auto), `--backend {auto,cached,fused,marlin}` (auto), `--prefer-fused`.
78
+
79
+ Backends:
80
+
81
+ - `cpu`: used when no NVIDIA GPU is present.
82
+ - `torch_gpu` (`--backend cached`): dequantizes weights to fp16 at load. Default on GPU when the weights fit in free GPU memory.
83
+ - `torch_fused` (`--backend fused`): keeps packed int4 on the GPU. Chosen automatically when the cached weights do not fit, or with `--prefer-fused`.
84
+ - `marlin`: optional, only with `--backend marlin`, never automatic. Unverified on this hardware.
85
+
86
+ The choice and its reason are logged to stderr. `--device cuda` without an NVIDIA GPU exits with status 1. A prompt that exceeds the context length and GPU out-of-memory also exit with status 1.
87
+
88
+ ## Benchmark
89
+
90
+ ```
91
+ quf bench out.quf --out results.json
92
+ quf bench out.quf --out results.json --fp16-ref Qwen/Qwen2.5-0.5B --gguf model-q4.gguf --device auto
93
+ ```
94
+
95
+ The results JSON records WikiText perplexity, file size, tokens per second, backend, device name and software versions. With `--fp16-ref`, the relative perplexity increase over fp16 is printed next to the documented target of at most 3 percent. The target is reported, not enforced, and was missed on Qwen2.5-0.5B (see below). `--gguf` adds a GGUF Q4 comparison. An lm-evaluation-harness adapter is in `quf/bench/lm_eval_adapter.py`.
96
+
97
+ ## Measured results
98
+
99
+ Qwen/Qwen2.5-0.5B, converted natively on Windows (4 bit, group size 128), RTX 5080 Laptop GPU, PyTorch 2.11.0+cu128, WikiText-2 raw. Full details and raw JSON: [docs/RESULTS.md](docs/RESULTS.md).
100
+
101
+ | Metric | Value |
102
+ | --- | --- |
103
+ | fp16 perplexity | 13.07 |
104
+ | QUF perplexity (32 x 512 calibration) | 14.18 (+8.47%) |
105
+ | QUF perplexity (128 x 1024 calibration) | 14.04 (+7.41%) |
106
+ | 3% target | **Missed** (+7 to +8%) |
107
+ | File size | 469,483,072 bytes, about 2.1x smaller than fp16 weights (about 988 MB). Embeddings and `lm_head` stay fp16, which limits the ratio on a small model. |
108
+ | Decode, CPU | 9.6 tokens/s |
109
+ | Decode, RTX 5080 `torch_gpu` (default) | 20.7 tokens/s |
110
+ | Decode, RTX 5080 `torch_fused` | 9.1 tokens/s (slower, less memory) |
111
+ | gigatoken vs HF tokenizers | 0.23 s vs 8.44 s (36.8x, identical ids) |
112
+
113
+ The cause of the perplexity gap was not investigated. Qwen2.5-1.5B was not run. Speeds are single measurements and the decode loop is not optimized.
114
+
115
+ ## Documentation
116
+
117
+ - [docs/FORMAT.md](docs/FORMAT.md): byte layout, header fields, tensor naming, dtypes and the canonical Packed_Int4 order.
118
+ - [docs/ENV.md](docs/ENV.md): environment results for native Windows and the backend selection rules.
119
+ - [docs/RESULTS.md](docs/RESULTS.md): measured accuracy, size and speed.
120
+
121
+ README.md, docs/ENV.md and docs/FORMAT.md ship with the package: they are in the source distribution and are installed to `share/doc/quf` by the wheel.
122
+
123
+ ## Tests
124
+
125
+ ```
126
+ pytest -m "not gpu and not convert" # CPU suite, no NVIDIA GPU needed
127
+ pytest -m gpu # needs an NVIDIA GPU
128
+ pytest -m convert # needs the convert extra and downloads a model
129
+ ```
130
+
131
+ Tests marked `gpu` are skipped automatically when no NVIDIA GPU is available.
132
+
133
+ ## Links and license
134
+
135
+ Source and issues: https://github.com/vtrsharvin-sj/Quantized-Model-Format
136
+
137
+ Released under the MIT license. See [LICENSE](LICENSE).
@@ -0,0 +1,20 @@
1
+ # Backend measurements (task 12.5)
2
+
3
+ Model: Qwen2.5-0.5B converted with `quf convert` (469,483,072 bytes). GPU: NVIDIA GeForce RTX 5080 Laptop (sm_120, 16 GiB). torch 2.11.0+cu128, Windows native. Triton is not installed, so `torch_fused` ran in eager mode (no `torch.compile`).
4
+
5
+ Command: `python scripts/bench_backends.py model.quf --tokens 64 --out docs/bench_backends.json`
6
+ Greedy decode of 64 tokens after a 17-token prompt, batch 1, after an 8-token warm-up. Each backend runs in its own process. Raw numbers are in `docs/bench_backends.json`.
7
+
8
+ | Backend | Decode tok/s | Prefill | Load | GPU memory after load | GPU peak |
9
+ |---|---|---|---|---|---|
10
+ | cpu | 9.6 | 140 ms | 2.4 s | none (process RSS 2104 MiB) | none |
11
+ | torch_gpu (cached fp16) | 20.7 | 38 ms | 1.0 s | 958 MiB | 1489 MiB |
12
+ | torch_fused (eager) | 9.1 | 96 ms | 0.7 s | 446 MiB | 977 MiB |
13
+
14
+ Findings:
15
+ - The fused backend uses about 2.1x less weight memory than the cached backend but decodes about 2.3x slower, since it unpacks and dequantizes on every call. It is also slower than the CPU run by a small margin.
16
+ - Fused and cached backends produce identical greedy tokens for the first 16 outputs.
17
+ - All backends are slow in absolute terms (batch 1, Python-level layer loop). The numbers are measured, not tuned.
18
+ - Selector decision: the fused backend is not faster, so `torch_gpu` (cached) stays the default. `torch_fused` is chosen automatically only when the cached fp16 weights do not fit in free GPU memory (not the case here: about 1.5 GiB needed against 16 GiB) or with `--backend fused`.
19
+ - Re-measure with Triton installed and `compile=True` before changing this decision.
20
+ - Property 10 and Property 20 GPU tests (torch_gpu and fused variants, excluding Marlin) pass on this machine. Without Triton the compiled variant falls back to eager or is skipped.
quf-0.1.0/docs/ENV.md ADDED
@@ -0,0 +1,38 @@
1
+ # Environment results
2
+
3
+ Produced with `python scripts/env_spike.py` (Requirement 2) and the real-model runs in [RESULTS.md](RESULTS.md). Statuses: **verified** (ran and passed), **failed** (ran, failure recorded), **not verified** (not run, reason given).
4
+
5
+ Reference hardware: RTX 5080 Laptop GPU, compute capability 12.0 (sm_120), 16303 MiB, driver 621.14.
6
+
7
+ ## Windows (native)
8
+
9
+ QUF runs natively on Windows. No WSL or Linux environment is used.
10
+
11
+ | Item | Result |
12
+ | --- | --- |
13
+ | OS / Python | Windows 11 (10.0.26300), Python 3.13.0rc2 |
14
+ | CUDA toolkit (`nvcc`) | Not installed. Not needed for PyTorch wheels. |
15
+ | PyTorch build | 2.11.0+cu128 (CUDA 12.8 runtime). Arch list includes `sm_120`. |
16
+ | transformers / datasets / tokenizers | 5.23.0 / 5.2.0 / 0.23.3 |
17
+ | gigatoken | 0.11.0, installed and working |
18
+
19
+ ### PyTorch CUDA kernels on compute capability 12.0
20
+ **Verified: pass.** An fp16 256x256 matmul executed on the GPU and returned a finite result. `quf run` and `quf bench` also ran on CUDA with a converted Qwen2.5-0.5B.
21
+
22
+ ### gigatoken on Windows
23
+ **Verified: pass.** gigatoken 0.11.0 installs with pip and runs on native Windows (`gt.Tokenizer(hf_tokenizer)`, `encode(text)`). On the WikiText-2 train split it produced the same token ids as Hugging Face `tokenizers` and was 36.8x faster in one measurement (0.23 s vs 8.44 s). See [RESULTS.md](RESULTS.md). The Hugging Face `tokenizers` fallback remains available when gigatoken is not installed.
24
+
25
+ ### Marlin on compute capability 12.0
26
+ **Not verified.** No Marlin kernel has run on this device. Marlin is an optional backend, is never selected automatically, and is not needed for conversion or normal use.
27
+
28
+ ## Backend designation
29
+
30
+ **`torch_gpu` is the GPU backend for this device.** It was measured at 20.71 tokens/s on Qwen2.5-0.5B, against 9.08 for `torch_fused` and 9.61 on CPU (see [RESULTS.md](RESULTS.md)).
31
+
32
+ ## Backend selection
33
+
34
+ `quf run` and `quf bench` choose the backend with `--device {auto,cpu,cuda}` and `--backend {auto,cached,fused,marlin}`.
35
+
36
+ - `auto` selects CPU when no NVIDIA GPU is present. With a GPU it selects the cached backend (`torch_gpu`, fp16 weights dequantized at load) when those weights fit in free GPU memory. It selects the fused backend (`torch_fused`, packed int4 kept on the GPU) when the cached weights do not fit, or when `--prefer-fused` is given. If the fused self-test or load fails, the error is logged and the cached backend is used.
37
+ - `--backend cached` and `--backend fused` force that GPU backend.
38
+ - `--backend marlin` forces the optional Marlin backend. It needs an extra package that is not part of any install extra. Without it, the command exits with status 1 and an error naming the missing package.
@@ -0,0 +1,173 @@
1
+ # QUF v1 file format
2
+
3
+ QUF (Quantized Unified Format) is a single-file container for a 4-bit quantized decoder-only LLM. One file holds the model configuration, tokenizer information, the quantization scheme, and every tensor.
4
+
5
+ This document matches the reference writer and reader in `quf/format/`. All multi-byte integers are little-endian.
6
+
7
+ ## 1. Byte layout
8
+
9
+ ```
10
+ offset size field
11
+ 0 4 magic: the ASCII bytes "QUF1"
12
+ 4 4 version: uint32 (currently 1)
13
+ 8 8 header_len: uint64, byte length of the JSON header
14
+ 16 N header: UTF-8 JSON, N = header_len
15
+ 16+N pad zero bytes up to the next multiple of 64
16
+ ... tensor data region
17
+ ```
18
+
19
+ The fixed prefix (magic, version, header_len) is 16 bytes. The JSON header follows immediately.
20
+
21
+ Tensor data starts at the first multiple of 64 at or after `16 + header_len`. Each tensor is stored at its `offset` from the Tensor_Table. Rules:
22
+
23
+ - `offset` is an absolute file offset, counted from byte 0 of the file.
24
+ - Every `offset` is a multiple of 64.
25
+ - Gaps between tensors, and after the last tensor, are zero padding up to the next multiple of 64.
26
+ - A tensor occupies `prod(shape) * element_size` bytes, stored C-contiguous (row-major) with no per-tensor framing. The size is computed from `dtype` and `shape` and is not stored.
27
+ - Tensors are written in Tensor_Table order, so offsets increase.
28
+
29
+ Because a tensor's `offset` is a multiple of 64 and elements are at most 4 bytes, tensors can be memory-mapped and viewed in place without copying.
30
+
31
+ ## 2. Header JSON
32
+
33
+ The header is a single JSON object. The writer emits it compactly (no extra whitespace); readers must accept any valid JSON.
34
+
35
+ ```json
36
+ {
37
+ "model_config": {
38
+ "architecture": "Qwen2ForCausalLM",
39
+ "hidden_size": 896,
40
+ "num_hidden_layers": 24,
41
+ "num_attention_heads": 14,
42
+ "num_key_value_heads": 2,
43
+ "intermediate_size": 4864,
44
+ "vocab_size": 151936,
45
+ "max_position_embeddings": 32768,
46
+ "rms_norm_eps": 1e-6,
47
+ "rope_theta": 1000000.0,
48
+ "tie_word_embeddings": true
49
+ },
50
+ "tokenizer": {
51
+ "model_id": "Qwen/Qwen2.5-0.5B",
52
+ "eos_token_id": 151643,
53
+ "bos_token_id": null,
54
+ "files": { "tokenizer.json": "<embedded string or tensor name>" }
55
+ },
56
+ "quantization": {
57
+ "method": "gptq",
58
+ "bits": 4,
59
+ "group_size": 128,
60
+ "symmetric": false,
61
+ "pack_dtype": "int32",
62
+ "pack_order": "lsb_first"
63
+ },
64
+ "tensors": [
65
+ { "name": "model.layers.0.self_attn.q_proj.qweight", "dtype": "int32",
66
+ "shape": [112, 896], "offset": 4096 }
67
+ ]
68
+ }
69
+ ```
70
+
71
+ ### 2.1 Top-level fields
72
+
73
+ All four fields are required. A missing field is a header schema error.
74
+
75
+ | Field | Type | Meaning |
76
+ |---|---|---|
77
+ | `model_config` | object | Architecture and dimensions needed to build the decoder (see the example). |
78
+ | `tokenizer` | object | Tokenizer information: `model_id`, `eos_token_id`, `bos_token_id`, and `files`. |
79
+ | `quantization` | object | The quantization scheme. |
80
+ | `tensors` | list | The Tensor_Table, one entry per tensor. |
81
+
82
+ `model_config`, `tokenizer`, and `quantization` must be JSON objects. `tensors` must be a list.
83
+
84
+ ### 2.2 `quantization`
85
+
86
+ | Field | Value in v1 | Meaning |
87
+ |---|---|---|
88
+ | `group_size` | `128` | Required. The writer rejects any other value. |
89
+ | `bits` | `4` | Bits per weight. |
90
+ | `symmetric` | `true` / `false` | When `false`, each quantized layer has a `.qzeros` tensor. Treated as `false` if absent. |
91
+ | `method` | e.g. `"gptq"` | How the weights were produced. Informational. |
92
+ | `pack_dtype` | `"int32"` | Container type of Packed_Int4 values. |
93
+ | `pack_order` | `"lsb_first"` | The canonical order in section 4. |
94
+
95
+ ### 2.3 Tensor_Table entries
96
+
97
+ Each entry has these required fields:
98
+
99
+ | Field | Type | Meaning |
100
+ |---|---|---|
101
+ | `name` | string | Unique tensor name. |
102
+ | `dtype` | string | One of the codes in section 3. |
103
+ | `shape` | list of non-negative integers | Dimensions, outermost first. |
104
+ | `offset` | non-negative integer | Absolute file offset, a multiple of 64. |
105
+
106
+ Tensor names must be unique within a file.
107
+
108
+ ## 3. Data types
109
+
110
+ | Code | Element | Size |
111
+ |---|---|---|
112
+ | `fp16` | IEEE 754 half, little-endian | 2 bytes |
113
+ | `fp32` | IEEE 754 single, little-endian | 4 bytes |
114
+ | `int32` | signed 32-bit integer, little-endian | 4 bytes |
115
+
116
+ An unknown dtype code is a header schema error.
117
+
118
+ ## 4. Canonical Packed_Int4 order
119
+
120
+ Eight 4-bit values (each 0 to 15) are packed into one `int32`. Value `i` of a group of eight occupies bits `4*(i % 8)` through `4*(i % 8) + 3`, so value 0 is in the least significant nibble:
121
+
122
+ ```
123
+ packed = v0 | (v1 << 4) | (v2 << 8) | (v3 << 12)
124
+ | (v4 << 16) | (v5 << 20) | (v6 << 24) | (v7 << 28)
125
+ ```
126
+
127
+ Consecutive values are consecutive elements along the packed dimension. Runtimes that need another layout (for example Marlin) repack in memory at load time. The file is never rewritten.
128
+
129
+ ## 5. Tensor naming and layout
130
+
131
+ Names follow Hugging Face Qwen2 conventions, for example `model.layers.0.self_attn.q_proj`.
132
+
133
+ For each quantized linear layer `<name>` with weight shape `[out, in]`, where `in % 128 == 0` and `out % 8 == 0`:
134
+
135
+ | Tensor | dtype | shape | Notes |
136
+ |---|---|---|---|
137
+ | `<name>.qweight` | `int32` | `[in/8, out]` | Packed_Int4 along the input dimension, in the order of section 4. |
138
+ | `<name>.scales` | `fp16` | `[in/128, out]` | One scale per group of 128 input elements. |
139
+ | `<name>.qzeros` | `int32` | `[in/128, out/8]` | Packed_Int4 zero points along the output dimension. Present only when `symmetric` is `false`. |
140
+
141
+ Dequantization of element `(o, i)` with group `g = i // 128`:
142
+
143
+ ```
144
+ w[o, i] = (q[o, i] - zero[g, o]) * scales[g, o]
145
+ ```
146
+
147
+ where `q` is the unpacked 4-bit value and `zero` is the unpacked `qzeros` value, or the constant `8` when the scheme is symmetric. The result is fp16 with the original shape `[out, in]`.
148
+
149
+ Tensors that are not quantized linear weights are stored unquantized:
150
+
151
+ - `model.embed_tokens.weight` and `lm_head.weight` (when not tied) must be `fp16`. The writer rejects any other dtype for names ending in `embed_tokens.weight` or `lm_head.weight`.
152
+ - Normalization weights and biases are stored as `fp16`.
153
+
154
+ The writer requires every `.qweight` to have an `fp16` `.scales` tensor, and a `.qzeros` tensor when the scheme is asymmetric.
155
+
156
+ ## 6. Reader validation
157
+
158
+ A reader opens the file read-only and validates in this order, stopping at the first failure:
159
+
160
+ 1. **Magic.** The first 4 bytes must be `QUF1`. Otherwise: invalid magic error, reporting the bytes found.
161
+ 2. **Version.** A version greater than the supported version (1) is an unsupported version error that reports the file's version and the supported version. A file too short to hold the version field is a truncated header error.
162
+ 3. **Header length.** The file must hold the 16-byte prefix, and `16 + header_len` must not exceed the file size. Otherwise: truncated header error.
163
+ 4. **JSON parse.** The header bytes must be valid UTF-8 JSON. Otherwise: header schema error.
164
+ 5. **Required fields.** All top-level and per-tensor required fields must be present and well typed. Otherwise: header schema error naming the field.
165
+ 6. **Tensor bounds.** For each tensor, `offset` must be a multiple of 64 and `offset + nbytes` must not exceed the file size. Otherwise: tensor bounds error naming the tensor. Duplicate names are a header schema error.
166
+
167
+ All of these errors derive from `QUFFormatError`. Reads return memory-mapped, zero-copy views of the tensor data.
168
+
169
+ ## 7. Compatibility
170
+
171
+ - A reader must reject any file whose version is higher than it supports.
172
+ - Version 1 fixes `group_size` at 128 and 4-bit weights.
173
+ - Unknown extra keys in header objects are ignored by the reference reader, which leaves room for additive changes.
@@ -0,0 +1,77 @@
1
+ # Measured results (Qwen/Qwen2.5-0.5B, native Windows)
2
+
3
+ All numbers come from real runs on this machine. Raw data: `bench_qwen2.5-0.5b.json`, `bench_qwen2.5-0.5b-128x1024.json`, `real_model_results_qwen2.5-0.5b*.json`, `bench_backends.json` (all in this directory). Produced by `scripts/verify_real_model.py` and `scripts/bench_backends.py`.
4
+
5
+ ## Environment
6
+
7
+ | Item | Value |
8
+ | --- | --- |
9
+ | OS | Windows (native, no WSL) |
10
+ | GPU | NVIDIA GeForce RTX 5080 Laptop GPU, compute capability 12.0 |
11
+ | Python | 3.13.0rc2 |
12
+ | PyTorch | 2.11.0+cu128 (CUDA 12.8) |
13
+ | transformers / datasets / tokenizers | 5.23.0 / 5.2.0 / 0.23.3 |
14
+ | gigatoken | 0.11.0 |
15
+ | quf | 0.1.0 |
16
+
17
+ ## Accuracy and size (WikiText-2 raw, `quf bench --fp16-ref Qwen/Qwen2.5-0.5B --device cuda`)
18
+
19
+ Two conversions were made with the native GPTQ quantizer (4 bit, group size 128), differing only in calibration (WikiText-2 train).
20
+
21
+ | Calibration | fp16 perplexity | QUF perplexity | Relative increase | 3% target |
22
+ | --- | --- | --- | --- | --- |
23
+ | 32 samples x 512 tokens | 13.0703 | 14.1767 | +8.47% | **Missed** |
24
+ | 128 samples x 1024 tokens | 13.0703 | 14.0384 | +7.41% | **Missed** |
25
+
26
+ The documented target of at most 3 percent perplexity increase was **not met**. More calibration data lowered the gap (8.5% to 7.4%) but not near 3%. I did not investigate the cause or tune the quantizer (for example act-order, damping, or symmetric vs asymmetric), so I cannot say whether the gap is inherent to 4-bit g128 on a 0.5B model or due to the implementation. Small models are known to lose more under 4-bit quantization, but that is not tested here.
27
+
28
+ | Item | Value |
29
+ | --- | --- |
30
+ | QUF file size | 469,483,072 bytes (447.7 MiB) for both conversions |
31
+ | fp16 weights (494,032,768 params x 2 bytes, computed) | about 988,065,536 bytes, so QUF is about 2.1x smaller |
32
+
33
+ The file is only about 2.1x smaller than fp16 because the embedding and `lm_head` stay fp16 by design, and they are a large share of this small model.
34
+
35
+ ## Speed
36
+
37
+ `tokens_per_second` in the bench JSON (RTX 5080 Laptop, `torch_gpu` backend): 16.68 (32x512 file) and 14.64 (128x1024 file). These are from the bench run and vary between runs. The decode loop is not optimized.
38
+
39
+ Backend comparison (`scripts/bench_backends.py`, 64 new tokens, 17-token prompt, greedy):
40
+
41
+ | Backend | Device | Decode tokens/s | Peak GPU memory |
42
+ | --- | --- | --- | --- |
43
+ | cpu | CPU | 9.61 | n/a (RSS 2.2 GB) |
44
+ | torch_gpu (cached fp16 weights) | RTX 5080 Laptop | 20.71 | 1.56 GB |
45
+ | torch_fused | RTX 5080 Laptop | 9.08 | 1.02 GB |
46
+
47
+ All three backends produced the same greedy tokens. The fused backend was slower than the cached backend, so the selector keeps `torch_gpu` as the default and chooses it with the reason "cached fp16 weights fit and the fused backend is not faster". Fused uses less GPU memory.
48
+
49
+ ## Generation sample (`quf run`, prompt `The capital of France is`, `--temperature 0 --seed 1234 --max-new-tokens 32`)
50
+
51
+ Both CPU and CUDA exited 0 and printed identical text. From the 128x1024 file:
52
+
53
+ ```
54
+ Paris. The capital of the United States is Washington, D.C. The capital of the United Kingdom is London. The capital of the United States is Washington,
55
+ ```
56
+
57
+ Wall time including model load: CPU 10.5 s, CUDA 8.7 s. The 32x512 file produced different (also repetitive, less accurate) text from the same prompt: ` ______.\nA. Paris\nB. London ...`.
58
+
59
+ ## gigatoken on Windows
60
+
61
+ gigatoken 0.11.0 installs with pip and runs on native Windows. It was used as `gt.Tokenizer(hf_tokenizer)` and `encode(text)`.
62
+
63
+ On the WikiText-2 train split (10,966,424 characters, 2,518,423 tokens), with the Qwen2.5-0.5B tokenizer:
64
+
65
+ | Tokenizer | Seconds |
66
+ | --- | --- |
67
+ | Hugging Face `tokenizers` | 8.44 |
68
+ | gigatoken | 0.23 |
69
+
70
+ gigatoken was 36.8x faster, and its token ids were identical to Hugging Face's (`gigatoken_matches_hf: true`). Note: the software-versions field of the older `bench_backends.json` says gigatoken "not installed" because it was recorded before installation. This is a single measurement of one tokenizer on one text, not a general speed claim.
71
+
72
+ ## Not done
73
+
74
+ - **Qwen/Qwen2.5-1.5B** was skipped to avoid timeouts. Requirement 16.5 is conditional on time ("where time permits").
75
+ - No GGUF Q4 comparison was run (no GGUF file was provided).
76
+ - Conversion wall time was not recorded: the earlier attempt reused already-written `.quf` files.
77
+ - Marlin was not run on this device (optional backend).