vllm-quantpack 0.1.2__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {vllm_quantpack-0.1.2/vllm_quantpack.egg-info → vllm_quantpack-0.1.4}/PKG-INFO +55 -1
  2. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/README.md +54 -0
  3. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/pyproject.toml +1 -1
  4. vllm_quantpack-0.1.4/tests/test_exl3_moe.py +83 -0
  5. vllm_quantpack-0.1.4/tests/test_int4_kv.py +274 -0
  6. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/__init__.py +3 -1
  7. vllm_quantpack-0.1.4/vllm_quantpack/attention/int4_kv.py +283 -0
  8. vllm_quantpack-0.1.4/vllm_quantpack/attention/int4_prefill.py +97 -0
  9. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/moe.py +38 -2
  10. vllm_quantpack-0.1.4/vllm_quantpack/tools/__init__.py +0 -0
  11. vllm_quantpack-0.1.4/vllm_quantpack/tools/int4_kv_bench.py +161 -0
  12. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4/vllm_quantpack.egg-info}/PKG-INFO +55 -1
  13. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack.egg-info/SOURCES.txt +7 -1
  14. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/LICENSE +0 -0
  15. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/NOTICE +0 -0
  16. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/setup.cfg +0 -0
  17. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/tests/test_embed_codec.py +0 -0
  18. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/tests/test_pure.py +0 -0
  19. {vllm_quantpack-0.1.2/vllm_quantpack/exl3 → vllm_quantpack-0.1.4/vllm_quantpack/attention}/__init__.py +0 -0
  20. {vllm_quantpack-0.1.2/vllm_quantpack/families → vllm_quantpack-0.1.4/vllm_quantpack/exl3}/__init__.py +0 -0
  21. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/embed_codec.py +0 -0
  22. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/embedding.py +0 -0
  23. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/hooks.py +0 -0
  24. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/layers.py +0 -0
  25. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/linear.py +0 -0
  26. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/names.py +0 -0
  27. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/ops.py +0 -0
  28. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/spec_decode.py +0 -0
  29. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/tp_ranges.py +0 -0
  30. {vllm_quantpack-0.1.2/vllm_quantpack/tools → vllm_quantpack-0.1.4/vllm_quantpack/families}/__init__.py +0 -0
  31. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/flint/__init__.py +0 -0
  32. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/flint/config.py +0 -0
  33. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/flint/interm_div.py +0 -0
  34. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/flint/model.py +0 -0
  35. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/nemotron/__init__.py +0 -0
  36. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/nemotron/model.py +0 -0
  37. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/qwen.py +0 -0
  38. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/spark/__init__.py +0 -0
  39. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/spark/config.py +0 -0
  40. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/spark/model.py +0 -0
  41. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/tools/embed_quant.py +0 -0
  42. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack.egg-info/dependency_links.txt +0 -0
  43. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack.egg-info/entry_points.txt +0 -0
  44. {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: vllm-quantpack
3
- Version: 0.1.2
3
+ Version: 0.1.4
4
4
  Summary: vLLM plugin: serve EXL3 (exllamav3 trellis) checkpoints and the model-family fixes they need, with no patch to vLLM
5
5
  Author: Jorge Ramírez Espejo
6
6
  License-Expression: Apache-2.0
@@ -29,6 +29,59 @@ model-family fixes they need, in stock vLLM: no patch to vLLM, no per-model plug
29
29
  - Model-family fixes:
30
30
  - **Qwen 3.5 / 3.6 / 3.8**: the vision tower's duplicated attention tensors;
31
31
  - **Qwen3.8-Flash-Next**: the FP8 PLE lookup on GPUs below sm_89.
32
+ - vLLM's int4 KV cache (`--kv-cache-dtype int4_per_token_head`, TRITON_ATTN backend), made exact and fast enough to
33
+ serve 262k-token sessions: see [int4 KV cache](#int4-kv-cache).
34
+
35
+ ## int4 KV cache
36
+
37
+ `int4_per_token_head` halves the KV cache of fp8 (a 27B model fits 262,144 tokens in 16 GB), but in vLLM 0.31 its
38
+ attention is inaccurate at long context and slow. The plugin replaces it when the int4 mode is present:
39
+
40
+ - **Exact read kernel.** Stock tensor-core dots round to TF32 while the value zero-point correction uses the unrounded
41
+ probabilities: 11 % relative error at 32k context, 28 % at 229k. The plugin uses bf16 operands (the nibbles are
42
+ exact) and the same rounded values in the correction: 0.4 %.
43
+ - **Long prefills on FlashAttention.** For a chunk of 64+ queries over 8k+ cached tokens, the cache is unpacked to bf16
44
+ in segments of 16k tokens (~124 MiB of scratch, reused by every layer) and attended with FlashAttention 2, merging
45
+ the segments by log-sum-exp. Shorter chunks run on the Triton kernel with larger tiles.
46
+ - **Split-KV decode with speculative decoding.** Stock vLLM uses the split-KV path only for one query token per
47
+ request; MTP verification has two, so decode scanned the whole context in 4 programs. The plugin uses it for up to 4
48
+ query tokens, with 64-token tiles and a segment count sized to the GPU (next power of two of SMs / 4, from 16 to
49
+ 128; each extra 16 segments cost ~12.6 MiB at 24 query heads, head 256).
50
+
51
+ Measured with Swift-1.5 27B (EXL3 2.25 bpw), MTP 1, chunked prefill 1024, one request, tokens per second:
52
+
53
+ | RTX PRO 6000 Server, context | prefill: exact kernel only / plugin / fp8 KV | decode: exact kernel only / plugin / fp8 KV |
54
+ |---|---|---|
55
+ | 128k | 1431 / 2220 / 1887 | 48.6 / 95 / 99 |
56
+ | 259k | 901 / 1737 / 1397 | 30.2 / 75 / 82 |
57
+
58
+ Stock vLLM int4 for comparison: 651 prefill and 3.8 decode at 201k on the same GPU; 216 and 3.2 at 261k on an A40.
59
+
60
+ Options (environment variables of the vLLM process):
61
+
62
+ | Variable | Default | Effect |
63
+ |---|---|---|
64
+ | `VLLM_QUANTPACK_INT4_KV` | `1` | `0` leaves vLLM's int4 attention untouched |
65
+ | `VLLM_QUANTPACK_INT4_FA_PREFILL` | `1` | `0` keeps long prefills on the Triton kernel (no FlashAttention, no scratch) |
66
+ | `VLLM_QUANTPACK_INT4_DECODE` | `64,4,auto` | split-KV decode `tile,warps,segments`; `segments` is a power of two or `auto` (from the SM count) |
67
+ | `VLLM_QUANTPACK_INT4_TILES` | `64,64,8,2` | Triton prefill `BLOCK_M,TILE,warps,stages` (short chunks, or with FlashAttention prefill off) |
68
+ | `VLLM_QUANTPACK_CACHE` | `~/.cache/vllm-quantpack` | where the generated kernel module is written (Triton needs it on disk) |
69
+
70
+ The defaults come from an A40 (84 SMs) and an RTX PRO 6000 (188 SMs). On another GPU, time the options on it:
71
+
72
+ ```
73
+ python -m vllm_quantpack.tools.int4_kv_bench --contexts 32768,131072,262144
74
+ ```
75
+
76
+ It fills a random int4 cache and prints, per attention layer, vLLM's stock decode and each split-KV decode config
77
+ (`--decode 64,4,16:64,4,32:...`), and a prefill chunk on Triton and on FlashAttention, each with its relative error
78
+ against an fp64 reference. The model shape defaults to Qwen3.8-27B (`--q-heads`, `--kv-heads`, `--head-size`,
79
+ `--block-size`, `--mtp` change it). Then set `VLLM_QUANTPACK_INT4_DECODE` to the fastest config.
80
+
81
+ On Blackwell (RTX 50xx, RTX PRO 6000), vLLM 0.31's FlashAttention 2 build carries sm_80 code only, so the driver
82
+ compiles it for the GPU on its first use: the first long prefill takes ~30 s more. The driver keeps the result in
83
+ `~/.nv/ComputeCache`; a service without `HOME` (a systemd unit, for example) loses it and pays it at every start, so
84
+ give it `HOME` or `CUDA_CACHE_PATH`. Ampere and Ada run the sm_80 code directly.
32
85
 
33
86
  ## Requirements
34
87
 
@@ -70,6 +123,7 @@ environment, uninstall it, or load only this plugin with `VLLM_PLUGINS=quantpack
70
123
  |---|---|
71
124
  | EXL3 linear, int8 embedding, lm_head | validated: same scores as the previous stack (dense Qwen 27B) |
72
125
  | EXL3 int4 embedding | validated (dense Qwen 27B): KLD +0.16 % vs bf16 table, tool calls and agent tasks unchanged |
126
+ | int4 KV cache attention (`int4_per_token_head`) | validated (Swift-1.5 27B 2.25 bpw on an A40 limited to 16 GB, 262,144 tokens, MTP 1): tool calls 30/30, agent tasks 10/10, 9/10, 10/10; speed above, A40 and RTX PRO 6000 |
73
127
  | EXL3 MoE | validated on one GPU (Groot 19B-A3B): same as the previous stack within run-to-run noise |
74
128
  | Tensor / expert parallelism | validated with MoE TP2 + EP (Groot 19B-A3B); Flash-Next TP not tested yet |
75
129
  | Qwen vision, Flash-Next PLE | validated (Kestrel, Groot, Flash-Next 2.25) |
@@ -15,6 +15,59 @@ model-family fixes they need, in stock vLLM: no patch to vLLM, no per-model plug
15
15
  - Model-family fixes:
16
16
  - **Qwen 3.5 / 3.6 / 3.8**: the vision tower's duplicated attention tensors;
17
17
  - **Qwen3.8-Flash-Next**: the FP8 PLE lookup on GPUs below sm_89.
18
+ - vLLM's int4 KV cache (`--kv-cache-dtype int4_per_token_head`, TRITON_ATTN backend), made exact and fast enough to
19
+ serve 262k-token sessions: see [int4 KV cache](#int4-kv-cache).
20
+
21
+ ## int4 KV cache
22
+
23
+ `int4_per_token_head` halves the KV cache of fp8 (a 27B model fits 262,144 tokens in 16 GB), but in vLLM 0.31 its
24
+ attention is inaccurate at long context and slow. The plugin replaces it when the int4 mode is present:
25
+
26
+ - **Exact read kernel.** Stock tensor-core dots round to TF32 while the value zero-point correction uses the unrounded
27
+ probabilities: 11 % relative error at 32k context, 28 % at 229k. The plugin uses bf16 operands (the nibbles are
28
+ exact) and the same rounded values in the correction: 0.4 %.
29
+ - **Long prefills on FlashAttention.** For a chunk of 64+ queries over 8k+ cached tokens, the cache is unpacked to bf16
30
+ in segments of 16k tokens (~124 MiB of scratch, reused by every layer) and attended with FlashAttention 2, merging
31
+ the segments by log-sum-exp. Shorter chunks run on the Triton kernel with larger tiles.
32
+ - **Split-KV decode with speculative decoding.** Stock vLLM uses the split-KV path only for one query token per
33
+ request; MTP verification has two, so decode scanned the whole context in 4 programs. The plugin uses it for up to 4
34
+ query tokens, with 64-token tiles and a segment count sized to the GPU (next power of two of SMs / 4, from 16 to
35
+ 128; each extra 16 segments cost ~12.6 MiB at 24 query heads, head 256).
36
+
37
+ Measured with Swift-1.5 27B (EXL3 2.25 bpw), MTP 1, chunked prefill 1024, one request, tokens per second:
38
+
39
+ | RTX PRO 6000 Server, context | prefill: exact kernel only / plugin / fp8 KV | decode: exact kernel only / plugin / fp8 KV |
40
+ |---|---|---|
41
+ | 128k | 1431 / 2220 / 1887 | 48.6 / 95 / 99 |
42
+ | 259k | 901 / 1737 / 1397 | 30.2 / 75 / 82 |
43
+
44
+ Stock vLLM int4 for comparison: 651 prefill and 3.8 decode at 201k on the same GPU; 216 and 3.2 at 261k on an A40.
45
+
46
+ Options (environment variables of the vLLM process):
47
+
48
+ | Variable | Default | Effect |
49
+ |---|---|---|
50
+ | `VLLM_QUANTPACK_INT4_KV` | `1` | `0` leaves vLLM's int4 attention untouched |
51
+ | `VLLM_QUANTPACK_INT4_FA_PREFILL` | `1` | `0` keeps long prefills on the Triton kernel (no FlashAttention, no scratch) |
52
+ | `VLLM_QUANTPACK_INT4_DECODE` | `64,4,auto` | split-KV decode `tile,warps,segments`; `segments` is a power of two or `auto` (from the SM count) |
53
+ | `VLLM_QUANTPACK_INT4_TILES` | `64,64,8,2` | Triton prefill `BLOCK_M,TILE,warps,stages` (short chunks, or with FlashAttention prefill off) |
54
+ | `VLLM_QUANTPACK_CACHE` | `~/.cache/vllm-quantpack` | where the generated kernel module is written (Triton needs it on disk) |
55
+
56
+ The defaults come from an A40 (84 SMs) and an RTX PRO 6000 (188 SMs). On another GPU, time the options on it:
57
+
58
+ ```
59
+ python -m vllm_quantpack.tools.int4_kv_bench --contexts 32768,131072,262144
60
+ ```
61
+
62
+ It fills a random int4 cache and prints, per attention layer, vLLM's stock decode and each split-KV decode config
63
+ (`--decode 64,4,16:64,4,32:...`), and a prefill chunk on Triton and on FlashAttention, each with its relative error
64
+ against an fp64 reference. The model shape defaults to Qwen3.8-27B (`--q-heads`, `--kv-heads`, `--head-size`,
65
+ `--block-size`, `--mtp` change it). Then set `VLLM_QUANTPACK_INT4_DECODE` to the fastest config.
66
+
67
+ On Blackwell (RTX 50xx, RTX PRO 6000), vLLM 0.31's FlashAttention 2 build carries sm_80 code only, so the driver
68
+ compiles it for the GPU on its first use: the first long prefill takes ~30 s more. The driver keeps the result in
69
+ `~/.nv/ComputeCache`; a service without `HOME` (a systemd unit, for example) loses it and pays it at every start, so
70
+ give it `HOME` or `CUDA_CACHE_PATH`. Ampere and Ada run the sm_80 code directly.
18
71
 
19
72
  ## Requirements
20
73
 
@@ -56,6 +109,7 @@ environment, uninstall it, or load only this plugin with `VLLM_PLUGINS=quantpack
56
109
  |---|---|
57
110
  | EXL3 linear, int8 embedding, lm_head | validated: same scores as the previous stack (dense Qwen 27B) |
58
111
  | EXL3 int4 embedding | validated (dense Qwen 27B): KLD +0.16 % vs bf16 table, tool calls and agent tasks unchanged |
112
+ | int4 KV cache attention (`int4_per_token_head`) | validated (Swift-1.5 27B 2.25 bpw on an A40 limited to 16 GB, 262,144 tokens, MTP 1): tool calls 30/30, agent tasks 10/10, 9/10, 10/10; speed above, A40 and RTX PRO 6000 |
59
113
  | EXL3 MoE | validated on one GPU (Groot 19B-A3B): same as the previous stack within run-to-run noise |
60
114
  | Tensor / expert parallelism | validated with MoE TP2 + EP (Groot 19B-A3B); Flash-Next TP not tested yet |
61
115
  | Qwen vision, Flash-Next PLE | validated (Kestrel, Groot, Flash-Next 2.25) |
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "vllm-quantpack"
7
- version = "0.1.2"
7
+ version = "0.1.4"
8
8
  description = "vLLM plugin: serve EXL3 (exllamav3 trellis) checkpoints and the model-family fixes they need, with no patch to vLLM"
9
9
  requires-python = ">=3.10"
10
10
  license = "Apache-2.0"
@@ -0,0 +1,83 @@
1
+ """EXL3 MoE prefill path on real expert weights (GPU).
2
+
3
+ Set VLLM_QUANTPACK_TEST_MOE to one or more EXL3 MoE checkpoints (comma separated). One layer of experts is loaded
4
+ from each and the virtual-expert path (one fused launch per row tier, no CPU sync) is compared with the per-expert
5
+ path it replaces.
6
+ """
7
+ import json
8
+ import os
9
+ import re
10
+
11
+ import pytest
12
+
13
+ torch = pytest.importorskip("torch")
14
+ gpu = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
15
+ CKPTS = [p for p in os.environ.get("VLLM_QUANTPACK_TEST_MOE", "").split(",") if p]
16
+ _NAME = re.compile(r"^(.*\.experts)\.(\d+)\.(gate_proj|up_proj|down_proj)\.(trellis|suh|svh|mcg|mul1)$")
17
+
18
+
19
+ def _load_layer(ckpt):
20
+ """_Experts of the first MoE layer of an EXL3 checkpoint."""
21
+ from safetensors import safe_open
22
+ from vllm_quantpack.exl3 import moe, ops # noqa: F401 (ops registers quantpack::exl3_gemm)
23
+ wmap = json.load(open(os.path.join(ckpt, "model.safetensors.index.json")))["weight_map"]
24
+ names = [n for n in wmap if _NAME.match(n)]
25
+ prefix = min({_NAME.match(n).group(1) for n in names}, key=lambda p: int(re.search(r"layers\.(\d+)", p).group(1)))
26
+ store = {}
27
+ for n in names:
28
+ m = _NAME.match(n)
29
+ if m.group(1) != prefix:
30
+ continue
31
+ with safe_open(os.path.join(ckpt, wmap[n]), "pt") as f:
32
+ store[(int(m.group(2)), m.group(3), m.group(4))] = f.get_tensor(n)
33
+ E = 1 + max(e for e, _, _ in store)
34
+ gateless = (0, "gate_proj", "trellis") not in store
35
+ projs = {}
36
+ for proj in ("up_proj", "down_proj") if gateless else ("gate_proj", "up_proj", "down_proj"):
37
+ per = [{s: store[(e, proj, s)] for s in ("trellis", "suh", "svh", "mcg", "mul1") if (e, proj, s) in store}
38
+ for e in range(E)]
39
+ projs[proj] = moe._Proj(proj, per, torch.device("cuda", torch.cuda.current_device()))
40
+ if gateless:
41
+ projs["gate_proj"] = projs["up_proj"]
42
+ return moe._Experts(prefix, projs["gate_proj"], projs["up_proj"], projs["down_proj"], torch.device("cuda", torch.cuda.current_device()),
43
+ gateless=gateless)
44
+
45
+
46
+ def _routing(ex, T, k, seed):
47
+ """Skewed top-k routing (a few hot experts, as in real prefills) with distinct experts per token."""
48
+ g = torch.Generator(device="cuda").manual_seed(seed)
49
+ logits = torch.randn((T, ex.E), generator=g, device="cuda") + torch.linspace(3, 0, ex.E, device="cuda")
50
+ w, ids = logits.softmax(-1).topk(k, -1)
51
+ return ids.to(torch.int32), (w / w.sum(-1, keepdim=True)).to(torch.float32)
52
+
53
+
54
+ @pytest.fixture(scope="module", params=CKPTS or [None], ids=lambda p: os.path.basename(p) if p else "none")
55
+ def experts(request):
56
+ if not request.param or not torch.cuda.is_available():
57
+ pytest.skip("set VLLM_QUANTPACK_TEST_MOE to EXL3 MoE checkpoints")
58
+ pytest.importorskip("exllamav3")
59
+ return _load_layer(request.param)
60
+
61
+
62
+ @gpu
63
+ @pytest.mark.parametrize("T", [300, 2048, 8192])
64
+ def test_virtual_experts_match_per_expert_path(experts, T, monkeypatch):
65
+ ex = experts
66
+ x = torch.randn((T, ex.H), device="cuda", dtype=torch.bfloat16) * 0.5
67
+ ids, w = _routing(ex, T, 8, T)
68
+ ids[0, -1] = -1 # an EP padding slot goes to the sentinel bucket
69
+ monkeypatch.setenv("VLLM_QUANTPACK_MOE_VIRTUAL", "0")
70
+ ref = ex._sum(x, ids, w)
71
+ monkeypatch.setenv("VLLM_QUANTPACK_MOE_VIRTUAL", "1")
72
+ torch.cuda.synchronize()
73
+ torch.cuda.set_sync_debug_mode("error")
74
+ try:
75
+ out = ex._sum(x, ids, w)
76
+ finally:
77
+ torch.cuda.set_sync_debug_mode("default")
78
+ err = ((out - ref).norm() / ref.norm()).item()
79
+ # Both paths are deterministic (repeat runs differ by ~1e-8). With no expert above fused_rows (T=300 here) they run
80
+ # the same fused kernel and must agree exactly; hot experts went through a different kernel in the old path and
81
+ # differ by 1.1-1.7e-3 measured on both checkpoints (summation order of the bf16 products), so 5e-3 there.
82
+ tol = 1e-5 if T == 300 else 5e-3
83
+ assert err < tol, f"relative error {err:.2e}"
@@ -0,0 +1,274 @@
1
+ import logging
2
+ import sys
3
+ import types
4
+
5
+ import pytest
6
+ import torch
7
+
8
+ from vllm_quantpack.attention import int4_kv
9
+
10
+ LOG = logging.getLogger("test")
11
+
12
+
13
+ def test_missing_anchor_is_reported_not_patched():
14
+ with pytest.raises(LookupError):
15
+ int4_kv._edit("def f():\n return 1\n", (("return 2", "return 3"),), "kernel")
16
+
17
+
18
+ def test_duplicated_anchor_is_reported():
19
+ with pytest.raises(LookupError):
20
+ int4_kv._edit("a\na\n", (("a", "b"),), "kernel")
21
+
22
+
23
+ def test_disabled_by_environment(monkeypatch):
24
+ monkeypatch.setenv("VLLM_QUANTPACK_INT4_KV", "0")
25
+ assert int4_kv.patch_int4_kv(LOG) is False
26
+
27
+
28
+ def test_vllm_without_int4_mode_leaves_everything(monkeypatch):
29
+ monkeypatch.setattr(int4_kv, "MODULE", "vllm_quantpack_no_such_module")
30
+ assert int4_kv.patch_int4_kv(LOG) is False
31
+
32
+
33
+ def _fake_triton_attn(monkeypatch):
34
+ """A stand-in for vllm.v1.attention.backends.triton_attn whose builder records the segment count it saw."""
35
+ mod = types.ModuleType("vllm.v1.attention.backends.triton_attn")
36
+ mod.NUM_PAR_SOFTMAX_SEGMENTS = 16
37
+
38
+ class Builder:
39
+ def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
40
+ self.num_par_softmax_segments = mod.NUM_PAR_SOFTMAX_SEGMENTS
41
+
42
+ mod.TritonAttentionMetadataBuilder = Builder
43
+ names = ("vllm", "vllm.v1", "vllm.v1.attention", "vllm.v1.attention.backends")
44
+ for name in names:
45
+ monkeypatch.setitem(sys.modules, name, sys.modules.get(name) or types.ModuleType(name))
46
+ monkeypatch.setitem(sys.modules, mod.__name__, mod)
47
+ monkeypatch.setattr(sys.modules["vllm.v1.attention.backends"], "triton_attn", mod, raising=False)
48
+ return mod
49
+
50
+
51
+ def _config(dtype):
52
+ return types.SimpleNamespace(cache_config=types.SimpleNamespace(cache_dtype=dtype))
53
+
54
+
55
+ def test_decode_segments_only_for_int4_caches(monkeypatch):
56
+ mod = _fake_triton_attn(monkeypatch)
57
+ assert int4_kv._patch_decode_segments(64, LOG) == 64
58
+ assert mod.TritonAttentionMetadataBuilder(None, [], _config("int4_per_token_head"), None).num_par_softmax_segments == 64
59
+ assert mod.TritonAttentionMetadataBuilder(None, [], _config("auto"), None).num_par_softmax_segments == 16
60
+ assert mod.NUM_PAR_SOFTMAX_SEGMENTS == 16
61
+
62
+
63
+ def test_decode_segments_not_power_of_two_left_stock(monkeypatch):
64
+ mod = _fake_triton_attn(monkeypatch)
65
+ assert int4_kv._patch_decode_segments(48, LOG) == 0
66
+ assert mod.TritonAttentionMetadataBuilder(None, [], _config("int4_per_token_head"), None).num_par_softmax_segments == 16
67
+
68
+
69
+ def test_decode_override_from_environment(monkeypatch):
70
+ monkeypatch.setenv("VLLM_QUANTPACK_INT4_DECODE", "32,8,32")
71
+ assert int4_kv.decode_tiles(188) == (32, 8, 32)
72
+ monkeypatch.setenv("VLLM_QUANTPACK_INT4_DECODE", "64,4,auto")
73
+ assert int4_kv.decode_tiles(84) == (64, 4, 32)
74
+
75
+
76
+ @pytest.mark.parametrize("sms,segments", [(188, 64), (84, 32), (36, 16), (8, 16), (132, 64), (512, 128)],
77
+ ids=["rtx-pro-6000", "a40", "rtx-5060-ti", "tiny", "h100", "cap"])
78
+ def test_segments_follow_sm_count(monkeypatch, sms, segments):
79
+ monkeypatch.delenv("VLLM_QUANTPACK_INT4_DECODE", raising=False)
80
+ assert int4_kv.decode_tiles(sms) == (64, 4, segments)
81
+
82
+
83
+ # ---------------------------------------------------------------- GPU: exactness against fp64, stock vs patched
84
+ gpu = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
85
+ NQ, NKV, HS, BS = 24, 4, 256, 3072 # Qwen3.8-27B full-attention layers, hybrid-model attention block
86
+
87
+
88
+ def _cache(m, nblocks):
89
+ phs = HS // 2 + 4
90
+ kv = torch.zeros((nblocks, NKV, BS, 2 * phs), dtype=torch.uint8, device="cuda")
91
+ base = torch.tensor([], dtype=torch.float32, device="cuda").set_(kv.untyped_storage())
92
+ st = [s // 4 for s in kv.stride()]
93
+ ks = torch.as_strided(base, (nblocks, BS, NKV), (st[0], st[2], st[1]), HS // 2 // 4)
94
+ vs = torch.as_strided(base, (nblocks, BS, NKV), (st[0], st[2], st[1]), (phs + HS // 2) // 4)
95
+ kc, vc = kv.transpose(1, 2).split(phs, dim=-1)
96
+ return kc, vc, ks, vs
97
+
98
+
99
+ def _fill(m, cache, first_block, n, seed):
100
+ kc, vc, ks, vs = cache
101
+ g = torch.Generator(device="cuda").manual_seed(seed)
102
+ k = torch.randn((n, NKV, HS), generator=g, device="cuda", dtype=torch.bfloat16)
103
+ v = torch.randn((n, NKV, HS), generator=g, device="cuda", dtype=torch.bfloat16)
104
+ slots = first_block * BS + torch.arange(n, device="cuda")
105
+ m.reshape_and_cache_int4(k, v, kc, vc, slots, k_scale_cache=ks, v_scale_cache=vs)
106
+
107
+
108
+ def _reference(m, q, cache, first_block, n):
109
+ """fp64 attention of q (the last len(q) positions of a context of n tokens) from the cache."""
110
+ kc, vc, ks, vs = cache
111
+
112
+ def deq(c, sc):
113
+ b = c.reshape(-1, NKV, c.shape[-1])[first_block * BS:first_block * BS + n, :, : HS // 2].to(torch.int16)
114
+ u = torch.stack((b & 0xF, b >> 4), -1).reshape(n, NKV, HS).double()
115
+ bits = sc.reshape(-1, NKV)[first_block * BS:first_block * BS + n].contiguous().view(torch.int32)
116
+ return (u - (bits & 0xF).double()[..., None]) * (bits & -16).view(torch.float32).double()[..., None]
117
+
118
+ K, V = deq(kc, ks), deq(vc, vs)
119
+ qr = m.single_rht(q.float()).double()
120
+ pos = n - len(q) + torch.arange(len(q), device="cuda")
121
+ out = torch.empty((len(q), NQ, HS), dtype=torch.float64, device="cuda")
122
+ for h in range(NQ):
123
+ s = qr[:, h] @ K[:, h // (NQ // NKV)].T * (HS ** -0.5 / HS)
124
+ s = s.masked_fill(torch.arange(n, device="cuda")[None] > pos[:, None], float("-inf"))
125
+ out[:, h] = torch.softmax(s, -1) @ V[:, h // (NQ // NKV)]
126
+ return m.single_rht(out.float(), inverse=True).double() / HS
127
+
128
+
129
+ def _attend(m, qs, cache, seqs, segments=16):
130
+ """One attention call for a batch: seqs = [(first_block, n_blocks, context_len), ...], qs = per-seq queries."""
131
+ kc, vc, ks, vs = cache
132
+ q = torch.cat(qs)
133
+ out = torch.empty_like(q)
134
+ rows = 32
135
+ segm = (torch.empty((rows, NQ, segments, HS), dtype=torch.float32, device="cuda"),
136
+ torch.empty((rows, NQ, segments), dtype=torch.float32, device="cuda"),
137
+ torch.empty((rows, NQ, segments), dtype=torch.float32, device="cuda"))
138
+ width = max(nb for _, nb, _ in seqs)
139
+ table = torch.zeros((len(seqs), width), dtype=torch.int32, device="cuda")
140
+ for i, (fb, nb, _) in enumerate(seqs):
141
+ table[i, :nb] = torch.arange(fb, fb + nb, dtype=torch.int32, device="cuda")
142
+ cu = torch.tensor([0, *torch.tensor([len(x) for x in qs]).cumsum(0).tolist()], dtype=torch.int32, device="cuda")
143
+ m.unified_attention_int4(q, kc, vc, out, cu_seqlens_q=cu, max_seqlen_q=max(len(x) for x in qs),
144
+ seqused_k=torch.tensor([n for _, _, n in seqs], dtype=torch.int32, device="cuda"),
145
+ max_seqlen_k=max(n for _, _, n in seqs), softmax_scale=HS ** -0.5, window_size=(-1, -1),
146
+ block_table=table, softcap=0.0, sinks=None, alibi_slopes=None, use_alibi_sqrt=False,
147
+ qq_bias=None, output_scale=None, mm_prefix_range=None, k_scale_cache=ks,
148
+ v_scale_cache=vs, seq_threshold_3D=rows, num_par_softmax_segments=segments,
149
+ softmax_segm_output=segm[0], softmax_segm_max=segm[1], softmax_segm_expsum=segm[2])
150
+ return out
151
+
152
+
153
+ def _relerr(o, ref):
154
+ return ((o.double() - ref).norm() / ref.norm()).item()
155
+
156
+
157
+ @pytest.fixture
158
+ def int4_module(monkeypatch):
159
+ m = pytest.importorskip("vllm.v1.attention.ops.int4_per_token_head")
160
+ saved = m._launch_packed_attn, m.unified_attention_int4
161
+ yield m
162
+ m._launch_packed_attn, m.unified_attention_int4 = saved
163
+
164
+
165
+ @gpu
166
+ @pytest.mark.parametrize("qlen,n,fa", [(256, 16384, "1"), (256, 16384, "0"), (256, 40000, "1"), (2, 16384, "1"),
167
+ (1, 16384, "1")],
168
+ ids=["prefill-flashattn", "prefill-triton", "prefill-3-segments", "mtp-decode", "decode"])
169
+ def test_patched_int4_attention_matches_fp64(int4_module, monkeypatch, qlen, n, fa):
170
+ m = int4_module
171
+ nb = (n + BS - 1) // BS
172
+ cache = _cache(m, nb)
173
+ _fill(m, cache, 0, n, 0)
174
+ q = torch.randn((qlen, NQ, HS), device="cuda", dtype=torch.bfloat16)
175
+ ref = _reference(m, q, cache, 0, n)
176
+ before = _relerr(_attend(m, [q], cache, [(0, nb, n)]), ref)
177
+ monkeypatch.setenv("VLLM_QUANTPACK_INT4_FA_PREFILL", fa)
178
+ assert int4_kv.patch_int4_kv(LOG)
179
+ assert getattr(m._launch_packed_attn, "_quantpack", False)
180
+ # decode runs with the segment count the patched metadata builder allocates for int4 caches
181
+ after = _relerr(_attend(m, [q], cache, [(0, nb, n)], int4_kv.decode_tiles(torch.cuda.get_device_properties(0).multi_processor_count)[2]), ref)
182
+ assert after < 1e-2, f"patched relative error {after:.3e}"
183
+ assert after < before, f"stock {before:.3e} vs patched {after:.3e}"
184
+
185
+
186
+ @gpu
187
+ def test_mixed_batch_long_prefill_and_mtp_decode(int4_module):
188
+ """One step with a long prefill chunk (FlashAttention path) and an MTP decode (Triton sub-batch)."""
189
+ m = int4_module
190
+ n0, n1 = 40000, 20000
191
+ b0, b1 = (n0 + BS - 1) // BS, (n1 + BS - 1) // BS
192
+ cache = _cache(m, b0 + b1)
193
+ _fill(m, cache, 0, n0, 0)
194
+ _fill(m, cache, b0, n1, 1)
195
+ q0 = torch.randn((256, NQ, HS), device="cuda", dtype=torch.bfloat16)
196
+ q1 = torch.randn((2, NQ, HS), device="cuda", dtype=torch.bfloat16)
197
+ assert int4_kv.patch_int4_kv(LOG)
198
+ out = _attend(m, [q0, q1], cache, [(0, b0, n0), (b0, b1, n1)], int4_kv.decode_tiles(torch.cuda.get_device_properties(0).multi_processor_count)[2])
199
+ assert _relerr(out[:256], _reference(m, q0, cache, 0, n0)) < 1e-2
200
+ assert _relerr(out[256:], _reference(m, q1, cache, b0, n1)) < 1e-2
201
+
202
+
203
+ @gpu
204
+ def test_long_prefill_does_not_wait_for_the_gpu(int4_module):
205
+ """prefill_sequence loops over 16k segments per layer; a host-to-GPU copy per segment stalled the engine loop."""
206
+ from vllm.vllm_flash_attn import flash_attn_varlen_func
207
+ from vllm_quantpack.attention import int4_prefill
208
+ m = int4_module
209
+ n = 40000
210
+ nb = (n + BS - 1) // BS
211
+ cache = _cache(m, nb)
212
+ _fill(m, cache, 0, n, 0)
213
+ kc, vc, ks, vs = cache
214
+ q = torch.randn((256, NQ, HS), device="cuda", dtype=torch.bfloat16)
215
+ table = torch.arange(nb, dtype=torch.int32, device="cuda")
216
+ q_rot = m.single_rht(q.float()).to(q.dtype)
217
+ torch.cuda.synchronize()
218
+ torch.cuda.set_sync_debug_mode("error")
219
+ try:
220
+ o = int4_prefill.prefill_sequence(q_rot, kc, vc, ks, vs, table, n, HS ** -0.5 / HS, flash_attn_varlen_func)
221
+ finally:
222
+ torch.cuda.set_sync_debug_mode("default")
223
+ out = (m.single_rht(o.float(), inverse=True) / HS).double()
224
+ assert _relerr(out, _reference(m, q, cache, 0, n)) < 1e-2
225
+
226
+
227
+ @gpu
228
+ def test_long_prefill_reads_step_lengths_once_per_step(int4_module, monkeypatch):
229
+ """Every attention layer of a step gets the same cu_seqlens_q/seqused_k; only the first may copy them to the host."""
230
+ import types
231
+ import vllm.forward_context as fc
232
+ m = int4_module
233
+ n = 20000
234
+ nb = (n + BS - 1) // BS
235
+ cache = _cache(m, nb)
236
+ _fill(m, cache, 0, n, 0)
237
+ kc, vc, ks, vs = cache
238
+ q = torch.randn((256, NQ, HS), device="cuda", dtype=torch.bfloat16)
239
+ segm = [torch.empty(s, dtype=torch.float32, device="cuda") for s in ((32, NQ, 16, HS), (32, NQ, 16), (32, NQ, 16))]
240
+ args = dict(cu_seqlens_q=torch.tensor([0, 256], dtype=torch.int32, device="cuda"), max_seqlen_q=256,
241
+ seqused_k=torch.tensor([n], dtype=torch.int32, device="cuda"), max_seqlen_k=n, softmax_scale=HS ** -0.5,
242
+ window_size=(-1, -1), block_table=torch.arange(nb, dtype=torch.int32, device="cuda")[None],
243
+ softcap=0.0, sinks=None, alibi_slopes=None, use_alibi_sqrt=False, qq_bias=None, output_scale=None,
244
+ mm_prefix_range=None, k_scale_cache=ks, v_scale_cache=vs, seq_threshold_3D=32,
245
+ num_par_softmax_segments=16, softmax_segm_output=segm[0], softmax_segm_max=segm[1],
246
+ softmax_segm_expsum=segm[2])
247
+ step = types.SimpleNamespace()
248
+ monkeypatch.setattr(fc, "get_forward_context", lambda: step)
249
+ monkeypatch.setattr(fc, "is_forward_context_available", lambda: True)
250
+ assert int4_kv.patch_int4_kv(LOG)
251
+ ref = _reference(m, q, cache, 0, n)
252
+ first, second = torch.empty_like(q), torch.empty_like(q)
253
+ m.unified_attention_int4(q, kc, vc, first, **args) # first layer of the step: may sync
254
+ torch.cuda.synchronize()
255
+ torch.cuda.set_sync_debug_mode("error")
256
+ try:
257
+ m.unified_attention_int4(q, kc, vc, second, **args) # next layer, same step: must not
258
+ finally:
259
+ torch.cuda.set_sync_debug_mode("default")
260
+ assert _relerr(first, ref) < 1e-2 and _relerr(second, ref) < 1e-2
261
+ step2 = types.SimpleNamespace() # a new step re-reads the lengths
262
+ monkeypatch.setattr(fc, "get_forward_context", lambda: step2)
263
+ args["seqused_k"].fill_(n - 1000)
264
+ third = torch.empty_like(q)
265
+ m.unified_attention_int4(q[:256], kc, vc, third, **{**args, "max_seqlen_k": n - 1000})
266
+ assert _relerr(third, _reference(m, q, cache, 0, n - 1000)) < 1e-2
267
+
268
+
269
+ def test_bench_tool_help(capsys):
270
+ from vllm_quantpack.tools import int4_kv_bench
271
+ with pytest.raises(SystemExit) as e:
272
+ int4_kv_bench.main(["--help"])
273
+ assert e.value.code == 0
274
+ assert "VLLM_QUANTPACK_INT4_DECODE" in capsys.readouterr().out
@@ -11,7 +11,7 @@ Layout:
11
11
  It replaces OrcaSAQ2 (whose code it includes, Apache-2.0), qwen38_exl3_vision, flint_vllm and spark_vllm, which must
12
12
  not be installed alongside.
13
13
  """
14
- __version__ = "0.1.2"
14
+ __version__ = "0.1.4"
15
15
 
16
16
  # Entry-point names of the plugins this package replaces.
17
17
  _CONFLICTS = ("orcasaq2", "qwen38_exl3_vision", "tarninfer_exl3", "flint_vllm", "spark_vllm")
@@ -48,9 +48,11 @@ def register():
48
48
  return # already registered in this process
49
49
  patch_embedding_layers(logger)
50
50
 
51
+ from .attention.int4_kv import patch_int4_kv
51
52
  from .exl3.spec_decode import patch_should_share
52
53
  from .families import flint, nemotron, qwen, spark
53
54
  patch_should_share(logger)
55
+ patch_int4_kv(logger)
54
56
  qwen.register(logger)
55
57
  if flint.register(logger):
56
58
  logger.info("vllm-quantpack: Flint model registered")