vllm-quantpack 0.1.2__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vllm_quantpack-0.1.2/vllm_quantpack.egg-info → vllm_quantpack-0.1.4}/PKG-INFO +55 -1
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/README.md +54 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/pyproject.toml +1 -1
- vllm_quantpack-0.1.4/tests/test_exl3_moe.py +83 -0
- vllm_quantpack-0.1.4/tests/test_int4_kv.py +274 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/__init__.py +3 -1
- vllm_quantpack-0.1.4/vllm_quantpack/attention/int4_kv.py +283 -0
- vllm_quantpack-0.1.4/vllm_quantpack/attention/int4_prefill.py +97 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/moe.py +38 -2
- vllm_quantpack-0.1.4/vllm_quantpack/tools/__init__.py +0 -0
- vllm_quantpack-0.1.4/vllm_quantpack/tools/int4_kv_bench.py +161 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4/vllm_quantpack.egg-info}/PKG-INFO +55 -1
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack.egg-info/SOURCES.txt +7 -1
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/LICENSE +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/NOTICE +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/setup.cfg +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/tests/test_embed_codec.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/tests/test_pure.py +0 -0
- {vllm_quantpack-0.1.2/vllm_quantpack/exl3 → vllm_quantpack-0.1.4/vllm_quantpack/attention}/__init__.py +0 -0
- {vllm_quantpack-0.1.2/vllm_quantpack/families → vllm_quantpack-0.1.4/vllm_quantpack/exl3}/__init__.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/embed_codec.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/embedding.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/hooks.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/layers.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/linear.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/names.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/ops.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/spec_decode.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/exl3/tp_ranges.py +0 -0
- {vllm_quantpack-0.1.2/vllm_quantpack/tools → vllm_quantpack-0.1.4/vllm_quantpack/families}/__init__.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/flint/__init__.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/flint/config.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/flint/interm_div.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/flint/model.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/nemotron/__init__.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/nemotron/model.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/qwen.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/spark/__init__.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/spark/config.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/families/spark/model.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack/tools/embed_quant.py +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack.egg-info/dependency_links.txt +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack.egg-info/entry_points.txt +0 -0
- {vllm_quantpack-0.1.2 → vllm_quantpack-0.1.4}/vllm_quantpack.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vllm-quantpack
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.4
|
|
4
4
|
Summary: vLLM plugin: serve EXL3 (exllamav3 trellis) checkpoints and the model-family fixes they need, with no patch to vLLM
|
|
5
5
|
Author: Jorge Ramírez Espejo
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -29,6 +29,59 @@ model-family fixes they need, in stock vLLM: no patch to vLLM, no per-model plug
|
|
|
29
29
|
- Model-family fixes:
|
|
30
30
|
- **Qwen 3.5 / 3.6 / 3.8**: the vision tower's duplicated attention tensors;
|
|
31
31
|
- **Qwen3.8-Flash-Next**: the FP8 PLE lookup on GPUs below sm_89.
|
|
32
|
+
- vLLM's int4 KV cache (`--kv-cache-dtype int4_per_token_head`, TRITON_ATTN backend), made exact and fast enough to
|
|
33
|
+
serve 262k-token sessions: see [int4 KV cache](#int4-kv-cache).
|
|
34
|
+
|
|
35
|
+
## int4 KV cache
|
|
36
|
+
|
|
37
|
+
`int4_per_token_head` halves the KV cache of fp8 (a 27B model fits 262,144 tokens in 16 GB), but in vLLM 0.31 its
|
|
38
|
+
attention is inaccurate at long context and slow. The plugin replaces it when the int4 mode is present:
|
|
39
|
+
|
|
40
|
+
- **Exact read kernel.** Stock tensor-core dots round to TF32 while the value zero-point correction uses the unrounded
|
|
41
|
+
probabilities: 11 % relative error at 32k context, 28 % at 229k. The plugin uses bf16 operands (the nibbles are
|
|
42
|
+
exact) and the same rounded values in the correction: 0.4 %.
|
|
43
|
+
- **Long prefills on FlashAttention.** For a chunk of 64+ queries over 8k+ cached tokens, the cache is unpacked to bf16
|
|
44
|
+
in segments of 16k tokens (~124 MiB of scratch, reused by every layer) and attended with FlashAttention 2, merging
|
|
45
|
+
the segments by log-sum-exp. Shorter chunks run on the Triton kernel with larger tiles.
|
|
46
|
+
- **Split-KV decode with speculative decoding.** Stock vLLM uses the split-KV path only for one query token per
|
|
47
|
+
request; MTP verification has two, so decode scanned the whole context in 4 programs. The plugin uses it for up to 4
|
|
48
|
+
query tokens, with 64-token tiles and a segment count sized to the GPU (next power of two of SMs / 4, from 16 to
|
|
49
|
+
128; each extra 16 segments cost ~12.6 MiB at 24 query heads, head 256).
|
|
50
|
+
|
|
51
|
+
Measured with Swift-1.5 27B (EXL3 2.25 bpw), MTP 1, chunked prefill 1024, one request, tokens per second:
|
|
52
|
+
|
|
53
|
+
| RTX PRO 6000 Server, context | prefill: exact kernel only / plugin / fp8 KV | decode: exact kernel only / plugin / fp8 KV |
|
|
54
|
+
|---|---|---|
|
|
55
|
+
| 128k | 1431 / 2220 / 1887 | 48.6 / 95 / 99 |
|
|
56
|
+
| 259k | 901 / 1737 / 1397 | 30.2 / 75 / 82 |
|
|
57
|
+
|
|
58
|
+
Stock vLLM int4 for comparison: 651 prefill and 3.8 decode at 201k on the same GPU; 216 and 3.2 at 261k on an A40.
|
|
59
|
+
|
|
60
|
+
Options (environment variables of the vLLM process):
|
|
61
|
+
|
|
62
|
+
| Variable | Default | Effect |
|
|
63
|
+
|---|---|---|
|
|
64
|
+
| `VLLM_QUANTPACK_INT4_KV` | `1` | `0` leaves vLLM's int4 attention untouched |
|
|
65
|
+
| `VLLM_QUANTPACK_INT4_FA_PREFILL` | `1` | `0` keeps long prefills on the Triton kernel (no FlashAttention, no scratch) |
|
|
66
|
+
| `VLLM_QUANTPACK_INT4_DECODE` | `64,4,auto` | split-KV decode `tile,warps,segments`; `segments` is a power of two or `auto` (from the SM count) |
|
|
67
|
+
| `VLLM_QUANTPACK_INT4_TILES` | `64,64,8,2` | Triton prefill `BLOCK_M,TILE,warps,stages` (short chunks, or with FlashAttention prefill off) |
|
|
68
|
+
| `VLLM_QUANTPACK_CACHE` | `~/.cache/vllm-quantpack` | where the generated kernel module is written (Triton needs it on disk) |
|
|
69
|
+
|
|
70
|
+
The defaults come from an A40 (84 SMs) and an RTX PRO 6000 (188 SMs). On another GPU, time the options on it:
|
|
71
|
+
|
|
72
|
+
```
|
|
73
|
+
python -m vllm_quantpack.tools.int4_kv_bench --contexts 32768,131072,262144
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
It fills a random int4 cache and prints, per attention layer, vLLM's stock decode and each split-KV decode config
|
|
77
|
+
(`--decode 64,4,16:64,4,32:...`), and a prefill chunk on Triton and on FlashAttention, each with its relative error
|
|
78
|
+
against an fp64 reference. The model shape defaults to Qwen3.8-27B (`--q-heads`, `--kv-heads`, `--head-size`,
|
|
79
|
+
`--block-size`, `--mtp` change it). Then set `VLLM_QUANTPACK_INT4_DECODE` to the fastest config.
|
|
80
|
+
|
|
81
|
+
On Blackwell (RTX 50xx, RTX PRO 6000), vLLM 0.31's FlashAttention 2 build carries sm_80 code only, so the driver
|
|
82
|
+
compiles it for the GPU on its first use: the first long prefill takes ~30 s more. The driver keeps the result in
|
|
83
|
+
`~/.nv/ComputeCache`; a service without `HOME` (a systemd unit, for example) loses it and pays it at every start, so
|
|
84
|
+
give it `HOME` or `CUDA_CACHE_PATH`. Ampere and Ada run the sm_80 code directly.
|
|
32
85
|
|
|
33
86
|
## Requirements
|
|
34
87
|
|
|
@@ -70,6 +123,7 @@ environment, uninstall it, or load only this plugin with `VLLM_PLUGINS=quantpack
|
|
|
70
123
|
|---|---|
|
|
71
124
|
| EXL3 linear, int8 embedding, lm_head | validated: same scores as the previous stack (dense Qwen 27B) |
|
|
72
125
|
| EXL3 int4 embedding | validated (dense Qwen 27B): KLD +0.16 % vs bf16 table, tool calls and agent tasks unchanged |
|
|
126
|
+
| int4 KV cache attention (`int4_per_token_head`) | validated (Swift-1.5 27B 2.25 bpw on an A40 limited to 16 GB, 262,144 tokens, MTP 1): tool calls 30/30, agent tasks 10/10, 9/10, 10/10; speed above, A40 and RTX PRO 6000 |
|
|
73
127
|
| EXL3 MoE | validated on one GPU (Groot 19B-A3B): same as the previous stack within run-to-run noise |
|
|
74
128
|
| Tensor / expert parallelism | validated with MoE TP2 + EP (Groot 19B-A3B); Flash-Next TP not tested yet |
|
|
75
129
|
| Qwen vision, Flash-Next PLE | validated (Kestrel, Groot, Flash-Next 2.25) |
|
|
@@ -15,6 +15,59 @@ model-family fixes they need, in stock vLLM: no patch to vLLM, no per-model plug
|
|
|
15
15
|
- Model-family fixes:
|
|
16
16
|
- **Qwen 3.5 / 3.6 / 3.8**: the vision tower's duplicated attention tensors;
|
|
17
17
|
- **Qwen3.8-Flash-Next**: the FP8 PLE lookup on GPUs below sm_89.
|
|
18
|
+
- vLLM's int4 KV cache (`--kv-cache-dtype int4_per_token_head`, TRITON_ATTN backend), made exact and fast enough to
|
|
19
|
+
serve 262k-token sessions: see [int4 KV cache](#int4-kv-cache).
|
|
20
|
+
|
|
21
|
+
## int4 KV cache
|
|
22
|
+
|
|
23
|
+
`int4_per_token_head` halves the KV cache of fp8 (a 27B model fits 262,144 tokens in 16 GB), but in vLLM 0.31 its
|
|
24
|
+
attention is inaccurate at long context and slow. The plugin replaces it when the int4 mode is present:
|
|
25
|
+
|
|
26
|
+
- **Exact read kernel.** Stock tensor-core dots round to TF32 while the value zero-point correction uses the unrounded
|
|
27
|
+
probabilities: 11 % relative error at 32k context, 28 % at 229k. The plugin uses bf16 operands (the nibbles are
|
|
28
|
+
exact) and the same rounded values in the correction: 0.4 %.
|
|
29
|
+
- **Long prefills on FlashAttention.** For a chunk of 64+ queries over 8k+ cached tokens, the cache is unpacked to bf16
|
|
30
|
+
in segments of 16k tokens (~124 MiB of scratch, reused by every layer) and attended with FlashAttention 2, merging
|
|
31
|
+
the segments by log-sum-exp. Shorter chunks run on the Triton kernel with larger tiles.
|
|
32
|
+
- **Split-KV decode with speculative decoding.** Stock vLLM uses the split-KV path only for one query token per
|
|
33
|
+
request; MTP verification has two, so decode scanned the whole context in 4 programs. The plugin uses it for up to 4
|
|
34
|
+
query tokens, with 64-token tiles and a segment count sized to the GPU (next power of two of SMs / 4, from 16 to
|
|
35
|
+
128; each extra 16 segments cost ~12.6 MiB at 24 query heads, head 256).
|
|
36
|
+
|
|
37
|
+
Measured with Swift-1.5 27B (EXL3 2.25 bpw), MTP 1, chunked prefill 1024, one request, tokens per second:
|
|
38
|
+
|
|
39
|
+
| RTX PRO 6000 Server, context | prefill: exact kernel only / plugin / fp8 KV | decode: exact kernel only / plugin / fp8 KV |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| 128k | 1431 / 2220 / 1887 | 48.6 / 95 / 99 |
|
|
42
|
+
| 259k | 901 / 1737 / 1397 | 30.2 / 75 / 82 |
|
|
43
|
+
|
|
44
|
+
Stock vLLM int4 for comparison: 651 prefill and 3.8 decode at 201k on the same GPU; 216 and 3.2 at 261k on an A40.
|
|
45
|
+
|
|
46
|
+
Options (environment variables of the vLLM process):
|
|
47
|
+
|
|
48
|
+
| Variable | Default | Effect |
|
|
49
|
+
|---|---|---|
|
|
50
|
+
| `VLLM_QUANTPACK_INT4_KV` | `1` | `0` leaves vLLM's int4 attention untouched |
|
|
51
|
+
| `VLLM_QUANTPACK_INT4_FA_PREFILL` | `1` | `0` keeps long prefills on the Triton kernel (no FlashAttention, no scratch) |
|
|
52
|
+
| `VLLM_QUANTPACK_INT4_DECODE` | `64,4,auto` | split-KV decode `tile,warps,segments`; `segments` is a power of two or `auto` (from the SM count) |
|
|
53
|
+
| `VLLM_QUANTPACK_INT4_TILES` | `64,64,8,2` | Triton prefill `BLOCK_M,TILE,warps,stages` (short chunks, or with FlashAttention prefill off) |
|
|
54
|
+
| `VLLM_QUANTPACK_CACHE` | `~/.cache/vllm-quantpack` | where the generated kernel module is written (Triton needs it on disk) |
|
|
55
|
+
|
|
56
|
+
The defaults come from an A40 (84 SMs) and an RTX PRO 6000 (188 SMs). On another GPU, time the options on it:
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
python -m vllm_quantpack.tools.int4_kv_bench --contexts 32768,131072,262144
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
It fills a random int4 cache and prints, per attention layer, vLLM's stock decode and each split-KV decode config
|
|
63
|
+
(`--decode 64,4,16:64,4,32:...`), and a prefill chunk on Triton and on FlashAttention, each with its relative error
|
|
64
|
+
against an fp64 reference. The model shape defaults to Qwen3.8-27B (`--q-heads`, `--kv-heads`, `--head-size`,
|
|
65
|
+
`--block-size`, `--mtp` change it). Then set `VLLM_QUANTPACK_INT4_DECODE` to the fastest config.
|
|
66
|
+
|
|
67
|
+
On Blackwell (RTX 50xx, RTX PRO 6000), vLLM 0.31's FlashAttention 2 build carries sm_80 code only, so the driver
|
|
68
|
+
compiles it for the GPU on its first use: the first long prefill takes ~30 s more. The driver keeps the result in
|
|
69
|
+
`~/.nv/ComputeCache`; a service without `HOME` (a systemd unit, for example) loses it and pays it at every start, so
|
|
70
|
+
give it `HOME` or `CUDA_CACHE_PATH`. Ampere and Ada run the sm_80 code directly.
|
|
18
71
|
|
|
19
72
|
## Requirements
|
|
20
73
|
|
|
@@ -56,6 +109,7 @@ environment, uninstall it, or load only this plugin with `VLLM_PLUGINS=quantpack
|
|
|
56
109
|
|---|---|
|
|
57
110
|
| EXL3 linear, int8 embedding, lm_head | validated: same scores as the previous stack (dense Qwen 27B) |
|
|
58
111
|
| EXL3 int4 embedding | validated (dense Qwen 27B): KLD +0.16 % vs bf16 table, tool calls and agent tasks unchanged |
|
|
112
|
+
| int4 KV cache attention (`int4_per_token_head`) | validated (Swift-1.5 27B 2.25 bpw on an A40 limited to 16 GB, 262,144 tokens, MTP 1): tool calls 30/30, agent tasks 10/10, 9/10, 10/10; speed above, A40 and RTX PRO 6000 |
|
|
59
113
|
| EXL3 MoE | validated on one GPU (Groot 19B-A3B): same as the previous stack within run-to-run noise |
|
|
60
114
|
| Tensor / expert parallelism | validated with MoE TP2 + EP (Groot 19B-A3B); Flash-Next TP not tested yet |
|
|
61
115
|
| Qwen vision, Flash-Next PLE | validated (Kestrel, Groot, Flash-Next 2.25) |
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "vllm-quantpack"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.4"
|
|
8
8
|
description = "vLLM plugin: serve EXL3 (exllamav3 trellis) checkpoints and the model-family fixes they need, with no patch to vLLM"
|
|
9
9
|
requires-python = ">=3.10"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""EXL3 MoE prefill path on real expert weights (GPU).
|
|
2
|
+
|
|
3
|
+
Set VLLM_QUANTPACK_TEST_MOE to one or more EXL3 MoE checkpoints (comma separated). One layer of experts is loaded
|
|
4
|
+
from each and the virtual-expert path (one fused launch per row tier, no CPU sync) is compared with the per-expert
|
|
5
|
+
path it replaces.
|
|
6
|
+
"""
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
|
|
11
|
+
import pytest
|
|
12
|
+
|
|
13
|
+
torch = pytest.importorskip("torch")
|
|
14
|
+
gpu = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
|
|
15
|
+
CKPTS = [p for p in os.environ.get("VLLM_QUANTPACK_TEST_MOE", "").split(",") if p]
|
|
16
|
+
_NAME = re.compile(r"^(.*\.experts)\.(\d+)\.(gate_proj|up_proj|down_proj)\.(trellis|suh|svh|mcg|mul1)$")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _load_layer(ckpt):
|
|
20
|
+
"""_Experts of the first MoE layer of an EXL3 checkpoint."""
|
|
21
|
+
from safetensors import safe_open
|
|
22
|
+
from vllm_quantpack.exl3 import moe, ops # noqa: F401 (ops registers quantpack::exl3_gemm)
|
|
23
|
+
wmap = json.load(open(os.path.join(ckpt, "model.safetensors.index.json")))["weight_map"]
|
|
24
|
+
names = [n for n in wmap if _NAME.match(n)]
|
|
25
|
+
prefix = min({_NAME.match(n).group(1) for n in names}, key=lambda p: int(re.search(r"layers\.(\d+)", p).group(1)))
|
|
26
|
+
store = {}
|
|
27
|
+
for n in names:
|
|
28
|
+
m = _NAME.match(n)
|
|
29
|
+
if m.group(1) != prefix:
|
|
30
|
+
continue
|
|
31
|
+
with safe_open(os.path.join(ckpt, wmap[n]), "pt") as f:
|
|
32
|
+
store[(int(m.group(2)), m.group(3), m.group(4))] = f.get_tensor(n)
|
|
33
|
+
E = 1 + max(e for e, _, _ in store)
|
|
34
|
+
gateless = (0, "gate_proj", "trellis") not in store
|
|
35
|
+
projs = {}
|
|
36
|
+
for proj in ("up_proj", "down_proj") if gateless else ("gate_proj", "up_proj", "down_proj"):
|
|
37
|
+
per = [{s: store[(e, proj, s)] for s in ("trellis", "suh", "svh", "mcg", "mul1") if (e, proj, s) in store}
|
|
38
|
+
for e in range(E)]
|
|
39
|
+
projs[proj] = moe._Proj(proj, per, torch.device("cuda", torch.cuda.current_device()))
|
|
40
|
+
if gateless:
|
|
41
|
+
projs["gate_proj"] = projs["up_proj"]
|
|
42
|
+
return moe._Experts(prefix, projs["gate_proj"], projs["up_proj"], projs["down_proj"], torch.device("cuda", torch.cuda.current_device()),
|
|
43
|
+
gateless=gateless)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _routing(ex, T, k, seed):
|
|
47
|
+
"""Skewed top-k routing (a few hot experts, as in real prefills) with distinct experts per token."""
|
|
48
|
+
g = torch.Generator(device="cuda").manual_seed(seed)
|
|
49
|
+
logits = torch.randn((T, ex.E), generator=g, device="cuda") + torch.linspace(3, 0, ex.E, device="cuda")
|
|
50
|
+
w, ids = logits.softmax(-1).topk(k, -1)
|
|
51
|
+
return ids.to(torch.int32), (w / w.sum(-1, keepdim=True)).to(torch.float32)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@pytest.fixture(scope="module", params=CKPTS or [None], ids=lambda p: os.path.basename(p) if p else "none")
|
|
55
|
+
def experts(request):
|
|
56
|
+
if not request.param or not torch.cuda.is_available():
|
|
57
|
+
pytest.skip("set VLLM_QUANTPACK_TEST_MOE to EXL3 MoE checkpoints")
|
|
58
|
+
pytest.importorskip("exllamav3")
|
|
59
|
+
return _load_layer(request.param)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@gpu
|
|
63
|
+
@pytest.mark.parametrize("T", [300, 2048, 8192])
|
|
64
|
+
def test_virtual_experts_match_per_expert_path(experts, T, monkeypatch):
|
|
65
|
+
ex = experts
|
|
66
|
+
x = torch.randn((T, ex.H), device="cuda", dtype=torch.bfloat16) * 0.5
|
|
67
|
+
ids, w = _routing(ex, T, 8, T)
|
|
68
|
+
ids[0, -1] = -1 # an EP padding slot goes to the sentinel bucket
|
|
69
|
+
monkeypatch.setenv("VLLM_QUANTPACK_MOE_VIRTUAL", "0")
|
|
70
|
+
ref = ex._sum(x, ids, w)
|
|
71
|
+
monkeypatch.setenv("VLLM_QUANTPACK_MOE_VIRTUAL", "1")
|
|
72
|
+
torch.cuda.synchronize()
|
|
73
|
+
torch.cuda.set_sync_debug_mode("error")
|
|
74
|
+
try:
|
|
75
|
+
out = ex._sum(x, ids, w)
|
|
76
|
+
finally:
|
|
77
|
+
torch.cuda.set_sync_debug_mode("default")
|
|
78
|
+
err = ((out - ref).norm() / ref.norm()).item()
|
|
79
|
+
# Both paths are deterministic (repeat runs differ by ~1e-8). With no expert above fused_rows (T=300 here) they run
|
|
80
|
+
# the same fused kernel and must agree exactly; hot experts went through a different kernel in the old path and
|
|
81
|
+
# differ by 1.1-1.7e-3 measured on both checkpoints (summation order of the bf16 products), so 5e-3 there.
|
|
82
|
+
tol = 1e-5 if T == 300 else 5e-3
|
|
83
|
+
assert err < tol, f"relative error {err:.2e}"
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
import types
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
import torch
|
|
7
|
+
|
|
8
|
+
from vllm_quantpack.attention import int4_kv
|
|
9
|
+
|
|
10
|
+
LOG = logging.getLogger("test")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def test_missing_anchor_is_reported_not_patched():
|
|
14
|
+
with pytest.raises(LookupError):
|
|
15
|
+
int4_kv._edit("def f():\n return 1\n", (("return 2", "return 3"),), "kernel")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def test_duplicated_anchor_is_reported():
|
|
19
|
+
with pytest.raises(LookupError):
|
|
20
|
+
int4_kv._edit("a\na\n", (("a", "b"),), "kernel")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_disabled_by_environment(monkeypatch):
|
|
24
|
+
monkeypatch.setenv("VLLM_QUANTPACK_INT4_KV", "0")
|
|
25
|
+
assert int4_kv.patch_int4_kv(LOG) is False
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_vllm_without_int4_mode_leaves_everything(monkeypatch):
|
|
29
|
+
monkeypatch.setattr(int4_kv, "MODULE", "vllm_quantpack_no_such_module")
|
|
30
|
+
assert int4_kv.patch_int4_kv(LOG) is False
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _fake_triton_attn(monkeypatch):
|
|
34
|
+
"""A stand-in for vllm.v1.attention.backends.triton_attn whose builder records the segment count it saw."""
|
|
35
|
+
mod = types.ModuleType("vllm.v1.attention.backends.triton_attn")
|
|
36
|
+
mod.NUM_PAR_SOFTMAX_SEGMENTS = 16
|
|
37
|
+
|
|
38
|
+
class Builder:
|
|
39
|
+
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
|
|
40
|
+
self.num_par_softmax_segments = mod.NUM_PAR_SOFTMAX_SEGMENTS
|
|
41
|
+
|
|
42
|
+
mod.TritonAttentionMetadataBuilder = Builder
|
|
43
|
+
names = ("vllm", "vllm.v1", "vllm.v1.attention", "vllm.v1.attention.backends")
|
|
44
|
+
for name in names:
|
|
45
|
+
monkeypatch.setitem(sys.modules, name, sys.modules.get(name) or types.ModuleType(name))
|
|
46
|
+
monkeypatch.setitem(sys.modules, mod.__name__, mod)
|
|
47
|
+
monkeypatch.setattr(sys.modules["vllm.v1.attention.backends"], "triton_attn", mod, raising=False)
|
|
48
|
+
return mod
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _config(dtype):
|
|
52
|
+
return types.SimpleNamespace(cache_config=types.SimpleNamespace(cache_dtype=dtype))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_decode_segments_only_for_int4_caches(monkeypatch):
|
|
56
|
+
mod = _fake_triton_attn(monkeypatch)
|
|
57
|
+
assert int4_kv._patch_decode_segments(64, LOG) == 64
|
|
58
|
+
assert mod.TritonAttentionMetadataBuilder(None, [], _config("int4_per_token_head"), None).num_par_softmax_segments == 64
|
|
59
|
+
assert mod.TritonAttentionMetadataBuilder(None, [], _config("auto"), None).num_par_softmax_segments == 16
|
|
60
|
+
assert mod.NUM_PAR_SOFTMAX_SEGMENTS == 16
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_decode_segments_not_power_of_two_left_stock(monkeypatch):
|
|
64
|
+
mod = _fake_triton_attn(monkeypatch)
|
|
65
|
+
assert int4_kv._patch_decode_segments(48, LOG) == 0
|
|
66
|
+
assert mod.TritonAttentionMetadataBuilder(None, [], _config("int4_per_token_head"), None).num_par_softmax_segments == 16
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def test_decode_override_from_environment(monkeypatch):
|
|
70
|
+
monkeypatch.setenv("VLLM_QUANTPACK_INT4_DECODE", "32,8,32")
|
|
71
|
+
assert int4_kv.decode_tiles(188) == (32, 8, 32)
|
|
72
|
+
monkeypatch.setenv("VLLM_QUANTPACK_INT4_DECODE", "64,4,auto")
|
|
73
|
+
assert int4_kv.decode_tiles(84) == (64, 4, 32)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@pytest.mark.parametrize("sms,segments", [(188, 64), (84, 32), (36, 16), (8, 16), (132, 64), (512, 128)],
|
|
77
|
+
ids=["rtx-pro-6000", "a40", "rtx-5060-ti", "tiny", "h100", "cap"])
|
|
78
|
+
def test_segments_follow_sm_count(monkeypatch, sms, segments):
|
|
79
|
+
monkeypatch.delenv("VLLM_QUANTPACK_INT4_DECODE", raising=False)
|
|
80
|
+
assert int4_kv.decode_tiles(sms) == (64, 4, segments)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# ---------------------------------------------------------------- GPU: exactness against fp64, stock vs patched
|
|
84
|
+
gpu = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
|
85
|
+
NQ, NKV, HS, BS = 24, 4, 256, 3072 # Qwen3.8-27B full-attention layers, hybrid-model attention block
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _cache(m, nblocks):
|
|
89
|
+
phs = HS // 2 + 4
|
|
90
|
+
kv = torch.zeros((nblocks, NKV, BS, 2 * phs), dtype=torch.uint8, device="cuda")
|
|
91
|
+
base = torch.tensor([], dtype=torch.float32, device="cuda").set_(kv.untyped_storage())
|
|
92
|
+
st = [s // 4 for s in kv.stride()]
|
|
93
|
+
ks = torch.as_strided(base, (nblocks, BS, NKV), (st[0], st[2], st[1]), HS // 2 // 4)
|
|
94
|
+
vs = torch.as_strided(base, (nblocks, BS, NKV), (st[0], st[2], st[1]), (phs + HS // 2) // 4)
|
|
95
|
+
kc, vc = kv.transpose(1, 2).split(phs, dim=-1)
|
|
96
|
+
return kc, vc, ks, vs
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _fill(m, cache, first_block, n, seed):
|
|
100
|
+
kc, vc, ks, vs = cache
|
|
101
|
+
g = torch.Generator(device="cuda").manual_seed(seed)
|
|
102
|
+
k = torch.randn((n, NKV, HS), generator=g, device="cuda", dtype=torch.bfloat16)
|
|
103
|
+
v = torch.randn((n, NKV, HS), generator=g, device="cuda", dtype=torch.bfloat16)
|
|
104
|
+
slots = first_block * BS + torch.arange(n, device="cuda")
|
|
105
|
+
m.reshape_and_cache_int4(k, v, kc, vc, slots, k_scale_cache=ks, v_scale_cache=vs)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _reference(m, q, cache, first_block, n):
|
|
109
|
+
"""fp64 attention of q (the last len(q) positions of a context of n tokens) from the cache."""
|
|
110
|
+
kc, vc, ks, vs = cache
|
|
111
|
+
|
|
112
|
+
def deq(c, sc):
|
|
113
|
+
b = c.reshape(-1, NKV, c.shape[-1])[first_block * BS:first_block * BS + n, :, : HS // 2].to(torch.int16)
|
|
114
|
+
u = torch.stack((b & 0xF, b >> 4), -1).reshape(n, NKV, HS).double()
|
|
115
|
+
bits = sc.reshape(-1, NKV)[first_block * BS:first_block * BS + n].contiguous().view(torch.int32)
|
|
116
|
+
return (u - (bits & 0xF).double()[..., None]) * (bits & -16).view(torch.float32).double()[..., None]
|
|
117
|
+
|
|
118
|
+
K, V = deq(kc, ks), deq(vc, vs)
|
|
119
|
+
qr = m.single_rht(q.float()).double()
|
|
120
|
+
pos = n - len(q) + torch.arange(len(q), device="cuda")
|
|
121
|
+
out = torch.empty((len(q), NQ, HS), dtype=torch.float64, device="cuda")
|
|
122
|
+
for h in range(NQ):
|
|
123
|
+
s = qr[:, h] @ K[:, h // (NQ // NKV)].T * (HS ** -0.5 / HS)
|
|
124
|
+
s = s.masked_fill(torch.arange(n, device="cuda")[None] > pos[:, None], float("-inf"))
|
|
125
|
+
out[:, h] = torch.softmax(s, -1) @ V[:, h // (NQ // NKV)]
|
|
126
|
+
return m.single_rht(out.float(), inverse=True).double() / HS
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _attend(m, qs, cache, seqs, segments=16):
|
|
130
|
+
"""One attention call for a batch: seqs = [(first_block, n_blocks, context_len), ...], qs = per-seq queries."""
|
|
131
|
+
kc, vc, ks, vs = cache
|
|
132
|
+
q = torch.cat(qs)
|
|
133
|
+
out = torch.empty_like(q)
|
|
134
|
+
rows = 32
|
|
135
|
+
segm = (torch.empty((rows, NQ, segments, HS), dtype=torch.float32, device="cuda"),
|
|
136
|
+
torch.empty((rows, NQ, segments), dtype=torch.float32, device="cuda"),
|
|
137
|
+
torch.empty((rows, NQ, segments), dtype=torch.float32, device="cuda"))
|
|
138
|
+
width = max(nb for _, nb, _ in seqs)
|
|
139
|
+
table = torch.zeros((len(seqs), width), dtype=torch.int32, device="cuda")
|
|
140
|
+
for i, (fb, nb, _) in enumerate(seqs):
|
|
141
|
+
table[i, :nb] = torch.arange(fb, fb + nb, dtype=torch.int32, device="cuda")
|
|
142
|
+
cu = torch.tensor([0, *torch.tensor([len(x) for x in qs]).cumsum(0).tolist()], dtype=torch.int32, device="cuda")
|
|
143
|
+
m.unified_attention_int4(q, kc, vc, out, cu_seqlens_q=cu, max_seqlen_q=max(len(x) for x in qs),
|
|
144
|
+
seqused_k=torch.tensor([n for _, _, n in seqs], dtype=torch.int32, device="cuda"),
|
|
145
|
+
max_seqlen_k=max(n for _, _, n in seqs), softmax_scale=HS ** -0.5, window_size=(-1, -1),
|
|
146
|
+
block_table=table, softcap=0.0, sinks=None, alibi_slopes=None, use_alibi_sqrt=False,
|
|
147
|
+
qq_bias=None, output_scale=None, mm_prefix_range=None, k_scale_cache=ks,
|
|
148
|
+
v_scale_cache=vs, seq_threshold_3D=rows, num_par_softmax_segments=segments,
|
|
149
|
+
softmax_segm_output=segm[0], softmax_segm_max=segm[1], softmax_segm_expsum=segm[2])
|
|
150
|
+
return out
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _relerr(o, ref):
|
|
154
|
+
return ((o.double() - ref).norm() / ref.norm()).item()
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@pytest.fixture
|
|
158
|
+
def int4_module(monkeypatch):
|
|
159
|
+
m = pytest.importorskip("vllm.v1.attention.ops.int4_per_token_head")
|
|
160
|
+
saved = m._launch_packed_attn, m.unified_attention_int4
|
|
161
|
+
yield m
|
|
162
|
+
m._launch_packed_attn, m.unified_attention_int4 = saved
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@gpu
|
|
166
|
+
@pytest.mark.parametrize("qlen,n,fa", [(256, 16384, "1"), (256, 16384, "0"), (256, 40000, "1"), (2, 16384, "1"),
|
|
167
|
+
(1, 16384, "1")],
|
|
168
|
+
ids=["prefill-flashattn", "prefill-triton", "prefill-3-segments", "mtp-decode", "decode"])
|
|
169
|
+
def test_patched_int4_attention_matches_fp64(int4_module, monkeypatch, qlen, n, fa):
|
|
170
|
+
m = int4_module
|
|
171
|
+
nb = (n + BS - 1) // BS
|
|
172
|
+
cache = _cache(m, nb)
|
|
173
|
+
_fill(m, cache, 0, n, 0)
|
|
174
|
+
q = torch.randn((qlen, NQ, HS), device="cuda", dtype=torch.bfloat16)
|
|
175
|
+
ref = _reference(m, q, cache, 0, n)
|
|
176
|
+
before = _relerr(_attend(m, [q], cache, [(0, nb, n)]), ref)
|
|
177
|
+
monkeypatch.setenv("VLLM_QUANTPACK_INT4_FA_PREFILL", fa)
|
|
178
|
+
assert int4_kv.patch_int4_kv(LOG)
|
|
179
|
+
assert getattr(m._launch_packed_attn, "_quantpack", False)
|
|
180
|
+
# decode runs with the segment count the patched metadata builder allocates for int4 caches
|
|
181
|
+
after = _relerr(_attend(m, [q], cache, [(0, nb, n)], int4_kv.decode_tiles(torch.cuda.get_device_properties(0).multi_processor_count)[2]), ref)
|
|
182
|
+
assert after < 1e-2, f"patched relative error {after:.3e}"
|
|
183
|
+
assert after < before, f"stock {before:.3e} vs patched {after:.3e}"
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@gpu
|
|
187
|
+
def test_mixed_batch_long_prefill_and_mtp_decode(int4_module):
|
|
188
|
+
"""One step with a long prefill chunk (FlashAttention path) and an MTP decode (Triton sub-batch)."""
|
|
189
|
+
m = int4_module
|
|
190
|
+
n0, n1 = 40000, 20000
|
|
191
|
+
b0, b1 = (n0 + BS - 1) // BS, (n1 + BS - 1) // BS
|
|
192
|
+
cache = _cache(m, b0 + b1)
|
|
193
|
+
_fill(m, cache, 0, n0, 0)
|
|
194
|
+
_fill(m, cache, b0, n1, 1)
|
|
195
|
+
q0 = torch.randn((256, NQ, HS), device="cuda", dtype=torch.bfloat16)
|
|
196
|
+
q1 = torch.randn((2, NQ, HS), device="cuda", dtype=torch.bfloat16)
|
|
197
|
+
assert int4_kv.patch_int4_kv(LOG)
|
|
198
|
+
out = _attend(m, [q0, q1], cache, [(0, b0, n0), (b0, b1, n1)], int4_kv.decode_tiles(torch.cuda.get_device_properties(0).multi_processor_count)[2])
|
|
199
|
+
assert _relerr(out[:256], _reference(m, q0, cache, 0, n0)) < 1e-2
|
|
200
|
+
assert _relerr(out[256:], _reference(m, q1, cache, b0, n1)) < 1e-2
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
@gpu
|
|
204
|
+
def test_long_prefill_does_not_wait_for_the_gpu(int4_module):
|
|
205
|
+
"""prefill_sequence loops over 16k segments per layer; a host-to-GPU copy per segment stalled the engine loop."""
|
|
206
|
+
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
|
207
|
+
from vllm_quantpack.attention import int4_prefill
|
|
208
|
+
m = int4_module
|
|
209
|
+
n = 40000
|
|
210
|
+
nb = (n + BS - 1) // BS
|
|
211
|
+
cache = _cache(m, nb)
|
|
212
|
+
_fill(m, cache, 0, n, 0)
|
|
213
|
+
kc, vc, ks, vs = cache
|
|
214
|
+
q = torch.randn((256, NQ, HS), device="cuda", dtype=torch.bfloat16)
|
|
215
|
+
table = torch.arange(nb, dtype=torch.int32, device="cuda")
|
|
216
|
+
q_rot = m.single_rht(q.float()).to(q.dtype)
|
|
217
|
+
torch.cuda.synchronize()
|
|
218
|
+
torch.cuda.set_sync_debug_mode("error")
|
|
219
|
+
try:
|
|
220
|
+
o = int4_prefill.prefill_sequence(q_rot, kc, vc, ks, vs, table, n, HS ** -0.5 / HS, flash_attn_varlen_func)
|
|
221
|
+
finally:
|
|
222
|
+
torch.cuda.set_sync_debug_mode("default")
|
|
223
|
+
out = (m.single_rht(o.float(), inverse=True) / HS).double()
|
|
224
|
+
assert _relerr(out, _reference(m, q, cache, 0, n)) < 1e-2
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
@gpu
|
|
228
|
+
def test_long_prefill_reads_step_lengths_once_per_step(int4_module, monkeypatch):
|
|
229
|
+
"""Every attention layer of a step gets the same cu_seqlens_q/seqused_k; only the first may copy them to the host."""
|
|
230
|
+
import types
|
|
231
|
+
import vllm.forward_context as fc
|
|
232
|
+
m = int4_module
|
|
233
|
+
n = 20000
|
|
234
|
+
nb = (n + BS - 1) // BS
|
|
235
|
+
cache = _cache(m, nb)
|
|
236
|
+
_fill(m, cache, 0, n, 0)
|
|
237
|
+
kc, vc, ks, vs = cache
|
|
238
|
+
q = torch.randn((256, NQ, HS), device="cuda", dtype=torch.bfloat16)
|
|
239
|
+
segm = [torch.empty(s, dtype=torch.float32, device="cuda") for s in ((32, NQ, 16, HS), (32, NQ, 16), (32, NQ, 16))]
|
|
240
|
+
args = dict(cu_seqlens_q=torch.tensor([0, 256], dtype=torch.int32, device="cuda"), max_seqlen_q=256,
|
|
241
|
+
seqused_k=torch.tensor([n], dtype=torch.int32, device="cuda"), max_seqlen_k=n, softmax_scale=HS ** -0.5,
|
|
242
|
+
window_size=(-1, -1), block_table=torch.arange(nb, dtype=torch.int32, device="cuda")[None],
|
|
243
|
+
softcap=0.0, sinks=None, alibi_slopes=None, use_alibi_sqrt=False, qq_bias=None, output_scale=None,
|
|
244
|
+
mm_prefix_range=None, k_scale_cache=ks, v_scale_cache=vs, seq_threshold_3D=32,
|
|
245
|
+
num_par_softmax_segments=16, softmax_segm_output=segm[0], softmax_segm_max=segm[1],
|
|
246
|
+
softmax_segm_expsum=segm[2])
|
|
247
|
+
step = types.SimpleNamespace()
|
|
248
|
+
monkeypatch.setattr(fc, "get_forward_context", lambda: step)
|
|
249
|
+
monkeypatch.setattr(fc, "is_forward_context_available", lambda: True)
|
|
250
|
+
assert int4_kv.patch_int4_kv(LOG)
|
|
251
|
+
ref = _reference(m, q, cache, 0, n)
|
|
252
|
+
first, second = torch.empty_like(q), torch.empty_like(q)
|
|
253
|
+
m.unified_attention_int4(q, kc, vc, first, **args) # first layer of the step: may sync
|
|
254
|
+
torch.cuda.synchronize()
|
|
255
|
+
torch.cuda.set_sync_debug_mode("error")
|
|
256
|
+
try:
|
|
257
|
+
m.unified_attention_int4(q, kc, vc, second, **args) # next layer, same step: must not
|
|
258
|
+
finally:
|
|
259
|
+
torch.cuda.set_sync_debug_mode("default")
|
|
260
|
+
assert _relerr(first, ref) < 1e-2 and _relerr(second, ref) < 1e-2
|
|
261
|
+
step2 = types.SimpleNamespace() # a new step re-reads the lengths
|
|
262
|
+
monkeypatch.setattr(fc, "get_forward_context", lambda: step2)
|
|
263
|
+
args["seqused_k"].fill_(n - 1000)
|
|
264
|
+
third = torch.empty_like(q)
|
|
265
|
+
m.unified_attention_int4(q[:256], kc, vc, third, **{**args, "max_seqlen_k": n - 1000})
|
|
266
|
+
assert _relerr(third, _reference(m, q, cache, 0, n - 1000)) < 1e-2
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def test_bench_tool_help(capsys):
|
|
270
|
+
from vllm_quantpack.tools import int4_kv_bench
|
|
271
|
+
with pytest.raises(SystemExit) as e:
|
|
272
|
+
int4_kv_bench.main(["--help"])
|
|
273
|
+
assert e.value.code == 0
|
|
274
|
+
assert "VLLM_QUANTPACK_INT4_DECODE" in capsys.readouterr().out
|
|
@@ -11,7 +11,7 @@ Layout:
|
|
|
11
11
|
It replaces OrcaSAQ2 (whose code it includes, Apache-2.0), qwen38_exl3_vision, flint_vllm and spark_vllm, which must
|
|
12
12
|
not be installed alongside.
|
|
13
13
|
"""
|
|
14
|
-
__version__ = "0.1.
|
|
14
|
+
__version__ = "0.1.4"
|
|
15
15
|
|
|
16
16
|
# Entry-point names of the plugins this package replaces.
|
|
17
17
|
_CONFLICTS = ("orcasaq2", "qwen38_exl3_vision", "tarninfer_exl3", "flint_vllm", "spark_vllm")
|
|
@@ -48,9 +48,11 @@ def register():
|
|
|
48
48
|
return # already registered in this process
|
|
49
49
|
patch_embedding_layers(logger)
|
|
50
50
|
|
|
51
|
+
from .attention.int4_kv import patch_int4_kv
|
|
51
52
|
from .exl3.spec_decode import patch_should_share
|
|
52
53
|
from .families import flint, nemotron, qwen, spark
|
|
53
54
|
patch_should_share(logger)
|
|
55
|
+
patch_int4_kv(logger)
|
|
54
56
|
qwen.register(logger)
|
|
55
57
|
if flint.register(logger):
|
|
56
58
|
logger.info("vllm-quantpack: Flint model registered")
|