mjolnir-kernel 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/PKG-INFO +2 -1
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/README.md +1 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/__init__.py +1 -1
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/harness.py +279 -11
- mjolnir_kernel-0.3.3/mjolnir/arch/thor/qsa.py +272 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/backend.py +5 -4
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/lmhead.py +113 -5
- mjolnir_kernel-0.3.3/mjolnir/qsa_gather.py +118 -0
- mjolnir_kernel-0.3.3/mjolnir/qsa_impl.py +421 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/.gitignore +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/LICENSE +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/NOTICE +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/__init__.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/README.md +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/__init__.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/combine.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/harness.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/mma.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/plan.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/runner.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/__init__.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/call.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/const.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/decode.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/plan.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/runner.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/banner.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/builder.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/calibrate.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/draftvocab.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/kvdtype.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/kvfp6.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/kvscales.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/platforms.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/py.typed +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/qpack.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/quant.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/runner.py +0 -0
- {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/pyproject.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: mjolnir-kernel
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Mjolnir: a paged decode-attention kernel for NVIDIA Jetson AGX Thor (sm_110) on tcgen05 / TMEM / TMA (CuTe DSL), with fp8 and 6-bit (e2m3) KV caches — packaged as the vLLM MJOLNIR attention backend.
|
|
5
5
|
Project-URL: Homepage, https://github.com/GoBrrr-ai/mjolnir
|
|
6
6
|
Project-URL: Documentation, https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel
|
|
@@ -144,6 +144,7 @@ runs on the fallback backend.
|
|
|
144
144
|
| `MJOLNIR_ATTN_FUSED_DRAFT` | `1` | `0`: tell vLLM the backend cannot advance draft metadata in place → one graph replay per draft step instead of the fused draft loop |
|
|
145
145
|
| `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV). `MJOLNIR_LM_HEAD_TILES` overrides the GEMV launch config (`BN,BK,warps,stages`; default 64,256,8,5 — the 2026-11-04 serving A/B winner) |
|
|
146
146
|
| `MJOLNIR_MTP_FP8` | – | `1`: fp8-row-quantize the drafter's big unquantized linears (≥16M params) at load, same e4m3-per-row scheme + self-test (≤0.02 rel err) as the head; the bf16 originals are freed. Halves the MTP draft loop's streamed weight bytes |
|
|
147
|
+
| `MJOLNIR_MTP_LAYER` | – | `fp8`: the older, narrower variant — W8A8 via cuBLASLt rowwise `scaled_mm` on the MTP layer + eh_proj `fc` only. **Measured negative on sm_110** (cuBLASLt fp8 ≈60% of peak on layer shapes; 16–22% slower decode, acceptance safe) — see config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K_M8`; revisit only with a faster skinny-shape fp8 GEMM |
|
|
147
148
|
| `MJOLNIR_DRAFT_VOCAB` | – | path to a `draft_vocab.json` (`mjolnir calibrate --draft-vocab`): the drafter's logits only over those tokens (VocabTrim); `MJOLNIR_DRAFT_VOCAB_SIZE` keeps the first N (a shorter file is padded with the lowest unused ids, with a warning). The kept head stays in the drafter's native quant (NVFP4 4-bit in-kernel, FP8 row-slice) — never up-scaled to bf16; a bf16 head is the one exception (fp8 only when `MJOLNIR_LM_HEAD=fp8` is set). `MJOLNIR_DRAFT_VOCAB_NATIVE=0` forces the dequant→fp8 path |
|
|
148
149
|
| `MJOLNIR_ATTN_VERBOSE` | – | `1`: print the kernel's compile-time notes (ring stages, TMEM columns) — the standalone harness always does |
|
|
149
150
|
| `MJOLNIR_ATTN_BANNER` | `1` | `0`: skip the Mjolnir hammer banner in the vLLM boot log (`mjolnir.banner.emit`, once per start, coloured with the server's formatter) |
|
|
@@ -116,6 +116,7 @@ runs on the fallback backend.
|
|
|
116
116
|
| `MJOLNIR_ATTN_FUSED_DRAFT` | `1` | `0`: tell vLLM the backend cannot advance draft metadata in place → one graph replay per draft step instead of the fused draft loop |
|
|
117
117
|
| `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV). `MJOLNIR_LM_HEAD_TILES` overrides the GEMV launch config (`BN,BK,warps,stages`; default 64,256,8,5 — the 2026-11-04 serving A/B winner) |
|
|
118
118
|
| `MJOLNIR_MTP_FP8` | – | `1`: fp8-row-quantize the drafter's big unquantized linears (≥16M params) at load, same e4m3-per-row scheme + self-test (≤0.02 rel err) as the head; the bf16 originals are freed. Halves the MTP draft loop's streamed weight bytes |
|
|
119
|
+
| `MJOLNIR_MTP_LAYER` | – | `fp8`: the older, narrower variant — W8A8 via cuBLASLt rowwise `scaled_mm` on the MTP layer + eh_proj `fc` only. **Measured negative on sm_110** (cuBLASLt fp8 ≈60% of peak on layer shapes; 16–22% slower decode, acceptance safe) — see config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K_M8`; revisit only with a faster skinny-shape fp8 GEMM |
|
|
119
120
|
| `MJOLNIR_DRAFT_VOCAB` | – | path to a `draft_vocab.json` (`mjolnir calibrate --draft-vocab`): the drafter's logits only over those tokens (VocabTrim); `MJOLNIR_DRAFT_VOCAB_SIZE` keeps the first N (a shorter file is padded with the lowest unused ids, with a warning). The kept head stays in the drafter's native quant (NVFP4 4-bit in-kernel, FP8 row-slice) — never up-scaled to bf16; a bf16 head is the one exception (fp8 only when `MJOLNIR_LM_HEAD=fp8` is set). `MJOLNIR_DRAFT_VOCAB_NATIVE=0` forces the dequant→fp8 path |
|
|
120
121
|
| `MJOLNIR_ATTN_VERBOSE` | – | `1`: print the kernel's compile-time notes (ring stages, TMEM columns) — the standalone harness always does |
|
|
121
122
|
| `MJOLNIR_ATTN_BANNER` | `1` | `0`: skip the Mjolnir hammer banner in the vLLM boot log (`mjolnir.banner.emit`, once per start, coloured with the server's formatter) |
|
|
@@ -24,7 +24,7 @@ from __future__ import annotations
|
|
|
24
24
|
import os
|
|
25
25
|
|
|
26
26
|
__all__ = ["BACKEND_NAME", "BACKEND_PATH", "__version__", "register"]
|
|
27
|
-
__version__ = "0.3.
|
|
27
|
+
__version__ = "0.3.3"
|
|
28
28
|
|
|
29
29
|
BACKEND_NAME = "MJOLNIR"
|
|
30
30
|
BACKEND_PATH = "mjolnir.backend.MjolnirAttentionBackend"
|
|
@@ -63,10 +63,20 @@ def emulate_decode(
|
|
|
63
63
|
chunk_tiles,
|
|
64
64
|
fp8,
|
|
65
65
|
causal=True,
|
|
66
|
+
selection_blocks=None, # (b, budget_blocks) int32, -1 padded: QSA gather reference
|
|
67
|
+
compress_ratio=4,
|
|
66
68
|
):
|
|
67
69
|
"""Bit-faithful model of the kernel's arithmetic (contiguous chunk split, running
|
|
68
70
|
max, -inf guard, P rounding to e4m3(p * 448) or bf16, descale folding).
|
|
69
|
-
Also returns a plain fp32 attention for the fp8-vs-exact gap.
|
|
71
|
+
Also returns a plain fp32 attention for the fp8-vs-exact gap.
|
|
72
|
+
|
|
73
|
+
With ``selection_blocks`` the S columns are the SELECTED tokens (the QSA
|
|
74
|
+
gather path): block i covers rows [i*ratio, (i+1)*ratio) of the sequence,
|
|
75
|
+
gathered per (page, row) from the pool. Visibility is then owned by the
|
|
76
|
+
selection (every query row sees every selected column — the shared-
|
|
77
|
+
selection semantics of MTP verify), so ``causal`` is ignored; an
|
|
78
|
+
all-blocks selection at q_len == 1 is bit-identical to the dense path
|
|
79
|
+
(the identity oracle)."""
|
|
70
80
|
b, h_k, n_q, d = q_packed.shape
|
|
71
81
|
PS = k_pool.shape[1]
|
|
72
82
|
h_r = n_q // q_len
|
|
@@ -76,21 +86,42 @@ def emulate_decode(
|
|
|
76
86
|
o_exact = torch.zeros(b, h_k, n_q, d)
|
|
77
87
|
for bi in range(b):
|
|
78
88
|
L = int(seq_lens[bi])
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
89
|
+
if selection_blocks is not None:
|
|
90
|
+
ids = selection_blocks[bi]
|
|
91
|
+
ids = ids[ids >= 0].long()
|
|
92
|
+
rows = torch.cat(
|
|
93
|
+
[
|
|
94
|
+
torch.arange(min(int(i) * compress_ratio, L), min((int(i) + 1) * compress_ratio, L))
|
|
95
|
+
for i in ids.tolist()
|
|
96
|
+
]
|
|
97
|
+
or [torch.empty(0, dtype=torch.long)]
|
|
98
|
+
)
|
|
99
|
+
n_cols = len(rows)
|
|
100
|
+
# token row r of the sequence lives at pool page block_table[bi, r // PS]
|
|
101
|
+
pages_of_row = block_table[bi][rows // PS].long()
|
|
102
|
+
rows_in_page = rows % PS
|
|
103
|
+
Kb = k_pool[pages_of_row, rows_in_page].permute(1, 0, 2) # (h_k, n_cols, d)
|
|
104
|
+
Vb = v_pool[pages_of_row, rows_in_page].permute(1, 0, 2)
|
|
105
|
+
S = torch.einsum("gnd,gld->gnl", q_packed[bi], Kb) # raw scores
|
|
106
|
+
masked = torch.zeros(1, dtype=torch.bool) # no positional mask: the selection owns visibility
|
|
107
|
+
lim = None
|
|
108
|
+
tiles = math.ceil(n_cols / tile_s_rows)
|
|
109
|
+
else:
|
|
110
|
+
n_blk = math.ceil(L / PS)
|
|
111
|
+
pages = block_table[bi, :n_blk].long()
|
|
112
|
+
Kb = k_pool[pages].reshape(-1, h_k, d)[:L].permute(1, 0, 2) # (h_k, L, d)
|
|
113
|
+
Vb = v_pool[pages].reshape(-1, h_k, d)[:L].permute(1, 0, 2)
|
|
114
|
+
S = torch.einsum("gnd,gld->gnl", q_packed[bi], Kb) # raw scores
|
|
115
|
+
qi = torch.arange(n_q) // h_r
|
|
116
|
+
lim = L - q_len + qi if causal else torch.full((n_q,), L - 1) # (n_q,) last key each row sees
|
|
117
|
+
masked = torch.arange(L)[None, :] > lim[:, None] # (n_q, L)
|
|
118
|
+
tiles = math.ceil(L / tile_s_rows)
|
|
87
119
|
S = S.masked_fill(masked[None], -math.inf)
|
|
88
120
|
qk = q_descale[bi] * k_descale[bi] # (h_k,)
|
|
89
121
|
c = (scale_qs * log2_e * qk)[:, None] # (h_k, 1)
|
|
90
122
|
cta_scale = (scale_qs * qk)[:, None]
|
|
91
123
|
vd = v_descale[bi][:, None, None]
|
|
92
124
|
|
|
93
|
-
tiles = math.ceil(L / tile_s_rows)
|
|
94
125
|
o_parts, m_parts, l_parts = [], [], []
|
|
95
126
|
n_chunks = math.ceil(tiles / chunk_tiles)
|
|
96
127
|
for j in range(n_chunks): # contiguous chunks, as in the plan
|
|
@@ -98,7 +129,7 @@ def emulate_decode(
|
|
|
98
129
|
l = torch.zeros(h_k, n_q)
|
|
99
130
|
o = torch.zeros(h_k, n_q, d)
|
|
100
131
|
for t in range(j * chunk_tiles, min((j + 1) * chunk_tiles, tiles)):
|
|
101
|
-
sl = slice(t * tile_s_rows, min((t + 1) * tile_s_rows,
|
|
132
|
+
sl = slice(t * tile_s_rows, min((t + 1) * tile_s_rows, S.shape[-1]))
|
|
102
133
|
St = S[:, :, sl]
|
|
103
134
|
m_new = torch.maximum(m, St.amax(dim=-1))
|
|
104
135
|
m_use = torch.where(m_new == -math.inf, torch.zeros_like(m_new), m_new)
|
|
@@ -124,6 +155,230 @@ def emulate_decode(
|
|
|
124
155
|
return o_emu, lse_emu, o_exact
|
|
125
156
|
|
|
126
157
|
|
|
158
|
+
def run_qsa_gather(
|
|
159
|
+
batches: int = 2,
|
|
160
|
+
seqlen: int = 65536,
|
|
161
|
+
q_len: int = 1,
|
|
162
|
+
heads_q: int = 24,
|
|
163
|
+
heads_k: int = 4,
|
|
164
|
+
headdim: int = 256,
|
|
165
|
+
page_size: int = 128,
|
|
166
|
+
iterations: int = 50,
|
|
167
|
+
warmup_iterations: int = 5,
|
|
168
|
+
seed: int = 1111,
|
|
169
|
+
tolerance: float = 0.05,
|
|
170
|
+
**_,
|
|
171
|
+
):
|
|
172
|
+
"""The QSA scratch-gather decision experiment (MJOLNIR_ATTN_QSA path).
|
|
173
|
+
|
|
174
|
+
Dense decode over the whole context vs COPY the selected rows into a
|
|
175
|
+
page-shaped scratch (torch row gather) + the SAME dense kernel on the
|
|
176
|
+
scratch — the P0 gather with zero traced-body changes. Correctness:
|
|
177
|
+
(a) a full selection reproduces dense exactly, (b) a partial selection
|
|
178
|
+
matches the emulate_decode selection oracle. Timing: cuda-event medians
|
|
179
|
+
of dense / copy+gather / copy alone at the requested context — the
|
|
180
|
+
numbers the ``verify qsagain`` stage collects.
|
|
181
|
+
"""
|
|
182
|
+
from mjolnir.arch.thor import qsa
|
|
183
|
+
from mjolnir.arch.thor.runner import T5DecodeRunner
|
|
184
|
+
|
|
185
|
+
if not torch.cuda.is_available():
|
|
186
|
+
raise RuntimeError("GPU is required to run this example!")
|
|
187
|
+
cap = torch.cuda.get_device_capability()
|
|
188
|
+
if cap not in ((10, 0), (11, 0)):
|
|
189
|
+
raise RuntimeError(f"Thor sm_110 harness; device is sm_{cap[0]}{cap[1]}")
|
|
190
|
+
torch.manual_seed(seed)
|
|
191
|
+
dev = torch.device("cuda")
|
|
192
|
+
PS = page_size
|
|
193
|
+
spec = qsa.IndexerSpec()
|
|
194
|
+
spec.check_main_page_size(PS)
|
|
195
|
+
assert q_len >= 1
|
|
196
|
+
|
|
197
|
+
n_blocks = math.ceil(seqlen / PS)
|
|
198
|
+
num_pages = n_blocks * batches + 4
|
|
199
|
+
gen = torch.Generator().manual_seed(seed)
|
|
200
|
+
k_pool = (torch.randn(num_pages, PS, heads_k, headdim, generator=gen) * 0.5).to(torch.float8_e4m3fn).to(dev)
|
|
201
|
+
v_pool = (torch.randn(num_pages, PS, heads_k, headdim, generator=gen) * 0.5).to(torch.float8_e4m3fn).to(dev)
|
|
202
|
+
block_table = torch.zeros(batches, n_blocks, dtype=torch.int32)
|
|
203
|
+
for s in range(batches):
|
|
204
|
+
block_table[s] = torch.arange(s * n_blocks, (s + 1) * n_blocks, dtype=torch.int32)
|
|
205
|
+
block_table = block_table.to(dev)
|
|
206
|
+
seq_lens = torch.full((batches,), seqlen, dtype=torch.int32, device=dev)
|
|
207
|
+
|
|
208
|
+
# deterministic budget-limited selection over RATIO blocks (4 tokens each,
|
|
209
|
+
# NOT pages): evenly spaced + the local tail when over budget
|
|
210
|
+
n_blocks_total = math.ceil(seqlen / spec.ratio)
|
|
211
|
+
if n_blocks_total > spec.budget_blocks:
|
|
212
|
+
ids = torch.linspace(0, n_blocks_total - 17, spec.budget_blocks - 16).round().long()
|
|
213
|
+
ids = torch.cat([ids, torch.arange(n_blocks_total - 16, n_blocks_total)])
|
|
214
|
+
else:
|
|
215
|
+
ids = torch.arange(n_blocks_total)
|
|
216
|
+
selection = torch.full((batches, spec.budget_blocks), -1, dtype=torch.int32, device=dev)
|
|
217
|
+
selection[:, : len(ids)] = ids.to(dev)
|
|
218
|
+
|
|
219
|
+
eff = qsa.effective_seq_lens(selection, seq_lens.cpu(), spec)
|
|
220
|
+
src, dst, sbt, pages_max = qsa.scratch_plan(selection.cpu(), seq_lens.cpu(), block_table.cpu(), spec, PS)
|
|
221
|
+
scratch_pages = pages_max * batches + 2
|
|
222
|
+
scratch_k = torch.empty(scratch_pages, PS, heads_k, headdim, dtype=torch.float8_e4m3fn, device=dev)
|
|
223
|
+
scratch_v = torch.empty_like(scratch_k)
|
|
224
|
+
scratch_bt = torch.zeros(batches, sbt.shape[1], dtype=torch.int32, device=dev)
|
|
225
|
+
scratch_bt[:, : sbt.shape[1]] = sbt.to(dev)
|
|
226
|
+
src_dev, dst_dev = src.to(dev), dst.to(dev)
|
|
227
|
+
eff_cpu = eff.cpu()
|
|
228
|
+
|
|
229
|
+
# the copy path: fused Triton when available, torch indexing otherwise.
|
|
230
|
+
# BOTH paths feed correctness (b) — the timed path is the checked path.
|
|
231
|
+
from mjolnir import qsa_gather
|
|
232
|
+
|
|
233
|
+
if qsa_gather.available():
|
|
234
|
+
|
|
235
|
+
def copy():
|
|
236
|
+
# one Triton launch per K/V; strides in bytes of the logical layout
|
|
237
|
+
kb, vb = k_pool.view(torch.uint8), v_pool.view(torch.uint8)
|
|
238
|
+
qsa_gather.gather_rows(
|
|
239
|
+
kb, scratch_k.view(torch.uint8), src_dev, dst_dev, eff, PS, kb.stride(0), kb.stride(1), kb.stride(2)
|
|
240
|
+
)
|
|
241
|
+
qsa_gather.gather_rows(
|
|
242
|
+
vb, scratch_v.view(torch.uint8), src_dev, dst_dev, eff, PS, vb.stride(0), vb.stride(1), vb.stride(2)
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
print("qsa copy path: triton (fused)")
|
|
246
|
+
else:
|
|
247
|
+
|
|
248
|
+
def copy():
|
|
249
|
+
qsa.scratch_copy(k_pool, scratch_k, src_dev, dst_dev, eff_cpu)
|
|
250
|
+
qsa.scratch_copy(v_pool, scratch_v, src_dev, dst_dev, eff_cpu)
|
|
251
|
+
|
|
252
|
+
print("qsa copy path: torch (no Triton)")
|
|
253
|
+
|
|
254
|
+
tokens = batches * q_len
|
|
255
|
+
query = (torch.randn(tokens, heads_q, headdim, generator=gen) * 0.5).to(torch.bfloat16).to(dev)
|
|
256
|
+
out = torch.empty(tokens, heads_q, headdim, dtype=torch.bfloat16, device=dev)
|
|
257
|
+
one = torch.ones(1, device=dev)
|
|
258
|
+
softmax_scale = 1.0 / math.sqrt(headdim)
|
|
259
|
+
|
|
260
|
+
runner = T5DecodeRunner(
|
|
261
|
+
num_heads=heads_q,
|
|
262
|
+
num_kv_heads=heads_k,
|
|
263
|
+
head_size=headdim,
|
|
264
|
+
page_size=PS,
|
|
265
|
+
out_dtype=torch.bfloat16,
|
|
266
|
+
max_num_seqs=batches,
|
|
267
|
+
max_q_len=max(2, q_len),
|
|
268
|
+
q_mode="bf16",
|
|
269
|
+
kv_format="fp8",
|
|
270
|
+
device=dev,
|
|
271
|
+
)
|
|
272
|
+
# NOTE: runner.plan() writes into the runner's PERSISTENT shared device
|
|
273
|
+
# buffers — a plan is only valid until the next plan() call. Every
|
|
274
|
+
# run() below is immediately preceded by its own plan().
|
|
275
|
+
|
|
276
|
+
# ---- correctness (a): full selection == dense, same kernel ----
|
|
277
|
+
# only meaningful at L <= budget (a "full" selection cannot exceed it)
|
|
278
|
+
L1 = min(seqlen, spec.budget_tokens)
|
|
279
|
+
pages1 = math.ceil(L1 / PS)
|
|
280
|
+
sel_full = qsa.identity_selection(torch.full((batches,), L1, dtype=torch.int32), spec).to(dev)
|
|
281
|
+
src_f, dst_f, sbt_f, pages_f = qsa.scratch_plan(
|
|
282
|
+
sel_full.cpu(), torch.full((batches,), L1, dtype=torch.int32), block_table[:, :pages1].cpu(), spec, PS
|
|
283
|
+
)
|
|
284
|
+
full_k = torch.empty(pages_f * batches + 2, PS, heads_k, headdim, dtype=torch.float8_e4m3fn, device=dev)
|
|
285
|
+
full_v = torch.empty_like(full_k)
|
|
286
|
+
qsa.scratch_copy(k_pool, full_k, src_f.to(dev), dst_f.to(dev), torch.full((batches,), L1, dtype=torch.int32))
|
|
287
|
+
qsa.scratch_copy(v_pool, full_v, src_f.to(dev), dst_f.to(dev), torch.full((batches,), L1, dtype=torch.int32))
|
|
288
|
+
full_bt = torch.zeros(batches, sbt_f.shape[1], dtype=torch.int32, device=dev)
|
|
289
|
+
full_bt[:, : sbt_f.shape[1]] = sbt_f.to(dev)
|
|
290
|
+
out_full = torch.empty_like(out)
|
|
291
|
+
seqs1 = torch.full((batches,), L1, dtype=torch.int32, device=dev)
|
|
292
|
+
plan = runner.plan([L1] * batches, batches, q_len, full_bt, seqs1, causal=False)
|
|
293
|
+
runner.ensure_compiled(q_len, plan.with_reduction, causal=False)
|
|
294
|
+
runner.run(plan, query, out_full, full_k, full_v, one, one, softmax_scale)
|
|
295
|
+
# both sides causal=False: the gather's shared-selection visibility — the
|
|
296
|
+
# check is layout transparency, not causal semantics (q_len > 1 ladders differ)
|
|
297
|
+
plan = runner.plan([L1] * batches, batches, q_len, block_table[:, :pages1], seqs1, causal=False)
|
|
298
|
+
runner.ensure_compiled(q_len, plan.with_reduction, causal=False)
|
|
299
|
+
runner.run(plan, query, out, k_pool, v_pool, one, one, softmax_scale)
|
|
300
|
+
torch.cuda.synchronize()
|
|
301
|
+
err_full = (out.float() - out_full.float()).abs().max().item()
|
|
302
|
+
print(f"qsa gather: full-selection vs dense (L={L1}) max|err| = {err_full:.3g} (expect ~0)")
|
|
303
|
+
assert err_full == 0.0, "full-selection scratch decode diverged from dense"
|
|
304
|
+
|
|
305
|
+
# ---- correctness (b): partial selection vs the selection oracle ----
|
|
306
|
+
copy()
|
|
307
|
+
plan = runner.plan(eff.tolist(), batches, q_len, scratch_bt, eff.to(dev), causal=False)
|
|
308
|
+
runner.ensure_compiled(q_len, plan.with_reduction, causal=False)
|
|
309
|
+
runner.run(plan, query, out, scratch_k, scratch_v, one, one, softmax_scale)
|
|
310
|
+
torch.cuda.synchronize()
|
|
311
|
+
o_got = (
|
|
312
|
+
out.view(batches, q_len, heads_k, heads_q // heads_k, headdim)
|
|
313
|
+
.permute(0, 2, 1, 3, 4)
|
|
314
|
+
.reshape(batches, heads_k, q_len * (heads_q // heads_k), headdim)
|
|
315
|
+
.float()
|
|
316
|
+
.cpu()
|
|
317
|
+
)
|
|
318
|
+
o_emu, _, _ = emulate_decode(
|
|
319
|
+
query.view(batches, q_len, heads_k, -1, headdim)
|
|
320
|
+
.permute(0, 2, 1, 3, 4)
|
|
321
|
+
.reshape(batches, heads_k, q_len * (heads_q // heads_k), headdim)
|
|
322
|
+
.float()
|
|
323
|
+
.cpu(),
|
|
324
|
+
k_pool.float().cpu(),
|
|
325
|
+
v_pool.float().cpu(),
|
|
326
|
+
block_table.cpu(),
|
|
327
|
+
seq_lens.cpu(),
|
|
328
|
+
torch.ones(batches, heads_k),
|
|
329
|
+
torch.ones(batches, heads_k),
|
|
330
|
+
torch.ones(batches, heads_k),
|
|
331
|
+
softmax_scale,
|
|
332
|
+
1.0,
|
|
333
|
+
q_len,
|
|
334
|
+
2,
|
|
335
|
+
fp8=False,
|
|
336
|
+
selection_blocks=selection.cpu(),
|
|
337
|
+
)
|
|
338
|
+
err_sel = (o_got - o_emu).abs().max().item()
|
|
339
|
+
tol = tolerance * max(1.0, o_emu.abs().max().item())
|
|
340
|
+
print(f"qsa gather: partial-selection vs oracle max|err| = {err_sel:.3g} (tol {tol:.3g})")
|
|
341
|
+
assert err_sel <= tol, "scratch decode diverged from the selection oracle"
|
|
342
|
+
|
|
343
|
+
# ---- timing: dense / copy+gather / copy alone ----
|
|
344
|
+
def timed(fn, iters=iterations, warmup=warmup_iterations):
|
|
345
|
+
for _ in range(warmup):
|
|
346
|
+
fn()
|
|
347
|
+
torch.cuda.synchronize()
|
|
348
|
+
times = []
|
|
349
|
+
for _ in range(iters):
|
|
350
|
+
s, e = torch.cuda.Event(True), torch.cuda.Event(True)
|
|
351
|
+
s.record()
|
|
352
|
+
fn()
|
|
353
|
+
e.record()
|
|
354
|
+
e.synchronize()
|
|
355
|
+
times.append(s.elapsed_time(e) * 1000.0)
|
|
356
|
+
times.sort()
|
|
357
|
+
return times[len(times) // 2]
|
|
358
|
+
|
|
359
|
+
def gather_step():
|
|
360
|
+
# re-plan every step: the honest per-step cost (serving rebuilds
|
|
361
|
+
# metadata each step), and plan() owns the persistent buffers
|
|
362
|
+
copy()
|
|
363
|
+
plan = runner.plan(eff.tolist(), batches, q_len, scratch_bt, eff.to(dev), causal=False)
|
|
364
|
+
runner.run(plan, query, out, scratch_k, scratch_v, one, one, softmax_scale)
|
|
365
|
+
|
|
366
|
+
def dense_step():
|
|
367
|
+
plan = runner.plan(seq_lens.tolist(), batches, q_len, block_table, seq_lens)
|
|
368
|
+
runner.run(plan, query, out, k_pool, v_pool, one, one, softmax_scale)
|
|
369
|
+
|
|
370
|
+
t_dense = timed(dense_step)
|
|
371
|
+
t_gather = timed(gather_step)
|
|
372
|
+
t_copy = timed(copy)
|
|
373
|
+
kv_bytes = int(seq_lens.sum()) * heads_k * headdim * 2
|
|
374
|
+
sel_bytes = int(eff.sum()) * heads_k * headdim * 2
|
|
375
|
+
for name, us in (("dense", t_dense), ("gather", t_gather), ("copy", t_copy)):
|
|
376
|
+
print(f"median {name} {us:.2f} us")
|
|
377
|
+
print(f"qsa ratio dense/gather = {t_dense / t_gather:.3f} (copy share {t_copy / t_gather:.2f})")
|
|
378
|
+
print(f"qsa bytes: dense {kv_bytes} B vs selected {sel_bytes} B (+ indexer cache, not modeled here)")
|
|
379
|
+
print("PASS")
|
|
380
|
+
|
|
381
|
+
|
|
127
382
|
def run(
|
|
128
383
|
batches: int = 1,
|
|
129
384
|
seqlen: int = 1024,
|
|
@@ -705,8 +960,21 @@ def main():
|
|
|
705
960
|
action="store_true",
|
|
706
961
|
help="E1: the last CTA per (seq, q tile, kv head) merges the split partials in-kernel",
|
|
707
962
|
)
|
|
963
|
+
parser.add_argument(
|
|
964
|
+
"--qsa_gather",
|
|
965
|
+
action="store_true",
|
|
966
|
+
help="the QSA scratch-gather decision experiment (MJOLNIR_ATTN_QSA "
|
|
967
|
+
"path): dense decode vs copy-selected-rows + the same kernel on the "
|
|
968
|
+
"scratch — correctness vs the selection oracle + cuda-event medians; "
|
|
969
|
+
"uses --batches/--seqlen/--q_len/--heads_q/--heads_k/--headdim/"
|
|
970
|
+
"--page_size/--iterations/--seed/--tolerance, forces fp8 KV",
|
|
971
|
+
)
|
|
708
972
|
|
|
709
973
|
kwargs = vars(parser.parse_args())
|
|
974
|
+
if kwargs.pop("qsa_gather", False):
|
|
975
|
+
run_qsa_gather(**kwargs)
|
|
976
|
+
print("PASS")
|
|
977
|
+
return
|
|
710
978
|
if kwargs["kv_format"] == "auto": # sub-8-bit formats need fp8 Q
|
|
711
979
|
kwargs["kv_format"] = HARNESS_KV_FORMAT_FP8Q if kwargs["q_dtype"] is cutlass.Float8E4M3FN else "fp8"
|
|
712
980
|
run(**kwargs)
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
"""Qwen4 QSA (sparse-attention indexer) host-side plumbing for the Thor kernel.
|
|
2
|
+
|
|
3
|
+
Probe-gated: everything here is inert until ``MJOLNIR_ATTN_QSA`` is set —
|
|
4
|
+
the vLLM-side wiring arrives with the backend patch, no default flips.
|
|
5
|
+
|
|
6
|
+
Basis: ``task/research/qwen4-27b-upstream-prep/`` (the recovered Intel
|
|
7
|
+
``csrc/qwen38`` kernels + the qwen4_exp family config). Two additions over
|
|
8
|
+
today's kernel, both shape-compatible:
|
|
9
|
+
|
|
10
|
+
1. **Indexer decode** — the QSA indexer scores tokens for the sparse
|
|
11
|
+
selection with a small MQA over a *second* paged pool. It reuses
|
|
12
|
+
today's kernel unchanged at ``(b, h_k=1, n_q=indexer_heads*q_len,
|
|
13
|
+
d=indexer_dim)``: the pool's pooled rows are the single KV "head" and
|
|
14
|
+
the indexer q heads pack into ``n_q`` rows (``heads_q % heads_k == 0``,
|
|
15
|
+
``head_size % 128 == 0`` both hold).
|
|
16
|
+
2. **Gather decode** — the main attention reads a *selected* token set
|
|
17
|
+
(budget 2048 tokens = 512 blocks x compress ratio 4) instead of the
|
|
18
|
+
whole block table. Host side, the selection is coalesced into runs of
|
|
19
|
+
consecutive rows per page (:func:`selection_runs`); the device loader
|
|
20
|
+
that consumes runs is the only traced-body change and is flag-gated.
|
|
21
|
+
|
|
22
|
+
Under the budget (L <= budget tokens) the selection is every token, so QSA
|
|
23
|
+
is bit-identical to dense attention — the correctness oracle for both
|
|
24
|
+
paths (llama.cpp measured max logit delta 0.0 over all 2051 rows).
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import math
|
|
30
|
+
from dataclasses import dataclass
|
|
31
|
+
|
|
32
|
+
import torch
|
|
33
|
+
|
|
34
|
+
from mjolnir.runner import env_flag
|
|
35
|
+
|
|
36
|
+
# Family constants (qwen4_exp config + the recovered kernels; the indexer
|
|
37
|
+
# geometry is hardcoded family-wide: qsa_sycl.cpp kIndexHeads/kIndexDim).
|
|
38
|
+
INDEXER_HEADS = 4 # MQA query heads
|
|
39
|
+
INDEXER_DIM = 128 # indexer head dim (the main KV head dim is 256)
|
|
40
|
+
COMPRESS_RATIO = 4 # tokens pooled per compressed slot ("requires ratio=4")
|
|
41
|
+
BUDGET_TOKENS = 2048 # selection budget ("512 blocks or 2048 tokens")
|
|
42
|
+
BUDGET_BLOCKS = BUDGET_TOKENS // COMPRESS_RATIO
|
|
43
|
+
POOL_ROWS_PER_SLOT = 2 # raw key | pooled key (indexer_kpool_row = 2)
|
|
44
|
+
QSA_PAGE_SIZES = (64, 128) # compressed-pool slots per page (qsa_sycl.cpp:2084)
|
|
45
|
+
|
|
46
|
+
POOLED_ROW = 1 # the pooled key's row index inside a slot (raw is 0)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def qsa_enabled() -> bool:
|
|
50
|
+
"""``MJOLNIR_ATTN_QSA`` gate (default off): the whole QSA path is opt-in."""
|
|
51
|
+
return env_flag("MJOLNIR_ATTN_QSA").lower() not in ("", "0", "false", "off")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class IndexerSpec:
|
|
56
|
+
"""Per-model QSA indexer geometry and its derived shapes.
|
|
57
|
+
|
|
58
|
+
``page_size`` is the *compressed* pool's slots-per-page; the main KV
|
|
59
|
+
pool keeps its own page size (any of SUPPORTED_PAGE_SIZES, as long as
|
|
60
|
+
``main_page_size % ratio == 0`` so a block never crosses a page).
|
|
61
|
+
"""
|
|
62
|
+
|
|
63
|
+
heads: int = INDEXER_HEADS
|
|
64
|
+
dim: int = INDEXER_DIM
|
|
65
|
+
ratio: int = COMPRESS_RATIO
|
|
66
|
+
budget_tokens: int = BUDGET_TOKENS
|
|
67
|
+
page_size: int = 128
|
|
68
|
+
rows_per_slot: int = POOL_ROWS_PER_SLOT
|
|
69
|
+
|
|
70
|
+
def __post_init__(self):
|
|
71
|
+
assert self.ratio == COMPRESS_RATIO, "group compression requires ratio=4 (qsa_sycl.cpp:905)"
|
|
72
|
+
assert self.budget_tokens % self.ratio == 0, "budget must be a whole number of blocks"
|
|
73
|
+
assert self.page_size in QSA_PAGE_SIZES, f"compressed page size {self.page_size} unsupported"
|
|
74
|
+
assert self.dim % 128 == 0, "indexer head dim must be a multiple of 128 (kernel headdim rule)"
|
|
75
|
+
assert self.rows_per_slot == POOL_ROWS_PER_SLOT, "raw | pooled rows per slot"
|
|
76
|
+
|
|
77
|
+
@property
|
|
78
|
+
def budget_blocks(self) -> int:
|
|
79
|
+
return self.budget_tokens // self.ratio
|
|
80
|
+
|
|
81
|
+
def slots(self, tokens: int) -> int:
|
|
82
|
+
"""Compressed slots (KV rows of the indexer decode) for ``tokens``."""
|
|
83
|
+
return math.ceil(tokens / self.ratio)
|
|
84
|
+
|
|
85
|
+
def pool_pages(self, tokens: int) -> int:
|
|
86
|
+
return math.ceil(self.slots(tokens) / self.page_size)
|
|
87
|
+
|
|
88
|
+
def score_rows(self, q_len: int) -> int:
|
|
89
|
+
"""``n_q`` of the indexer decode: q tokens x indexer heads, packed."""
|
|
90
|
+
return q_len * self.heads
|
|
91
|
+
|
|
92
|
+
def check_main_page_size(self, main_page_size: int) -> None:
|
|
93
|
+
assert main_page_size % self.ratio == 0, (
|
|
94
|
+
f"main page size {main_page_size} must be a multiple of the compress "
|
|
95
|
+
f"ratio {self.ratio} or a selected block would cross a page"
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def effective_seq_lens(selection_blocks: torch.Tensor, seq_lens: torch.Tensor, spec: IndexerSpec) -> torch.Tensor:
|
|
100
|
+
"""The gather call's ``seq_lens``: tokens actually covered by the
|
|
101
|
+
selection (blocks clipped to L). The kernel's S masking uses it, so a
|
|
102
|
+
block id beyond L contributes nothing and a partial last block clips."""
|
|
103
|
+
ids = selection_blocks.to(torch.int64)
|
|
104
|
+
valid = ids >= 0
|
|
105
|
+
lo = ids.clamp_min(0) * spec.ratio
|
|
106
|
+
hi = (ids.clamp_min(0) + 1) * spec.ratio
|
|
107
|
+
L = seq_lens.to(ids.device, torch.int64).view(-1, 1)
|
|
108
|
+
contrib = (hi.clamp_max(L) - lo.clamp_max(L)).clamp_min(0) * valid
|
|
109
|
+
return contrib.sum(dim=1).to(torch.int32)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def scratch_plan(
|
|
113
|
+
selection_blocks: torch.Tensor,
|
|
114
|
+
seq_lens: torch.Tensor,
|
|
115
|
+
block_table: torch.Tensor,
|
|
116
|
+
spec: IndexerSpec,
|
|
117
|
+
main_page_size: int,
|
|
118
|
+
scratch_page_size: int | None = None,
|
|
119
|
+
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
|
|
120
|
+
"""Everything the scratch-gather copy and the follow-up dense decode need.
|
|
121
|
+
|
|
122
|
+
The scratch packs each sequence's SELECTED rows contiguously into
|
|
123
|
+
page-shaped scratch pages, so the EXISTING dense kernel + plan run on
|
|
124
|
+
the scratch unchanged (zero traced-body changes for P0; the in-loader
|
|
125
|
+
per-block TMA gather is the P2 refinement if ncu ever shows the copy).
|
|
126
|
+
|
|
127
|
+
``main_page_size`` is the POOL's page size (a hybrid stack may force it
|
|
128
|
+
large — GDN mamba pages set the floor); ``scratch_page_size`` is the
|
|
129
|
+
kernel's (default: same as the pool). Returns
|
|
130
|
+
``(src_rows, dst_rows, scratch_block_table, pages_per_seq_max)``:
|
|
131
|
+
src_rows/dst_rows are (b, max_rows) int64 gather/scatter indices into
|
|
132
|
+
the pool's and scratch's flat ``(pages * PS, ...)`` row space (rows
|
|
133
|
+
beyond a sequence's count are -1: skip), and scratch_block_table is the
|
|
134
|
+
(b, max_pages) int32 page table of the scratch pool.
|
|
135
|
+
"""
|
|
136
|
+
spec.check_main_page_size(main_page_size)
|
|
137
|
+
sps = scratch_page_size or main_page_size
|
|
138
|
+
b = selection_blocks.shape[0]
|
|
139
|
+
srcs, dsts, pages_max = [], [], 0
|
|
140
|
+
for i in range(b):
|
|
141
|
+
L = int(seq_lens[i])
|
|
142
|
+
ids = selection_blocks[i]
|
|
143
|
+
ids = ids[ids >= 0].long()
|
|
144
|
+
# pool flat row of each selected token row: token row r of the
|
|
145
|
+
# sequence lives at bt[r // PS] * PS + r % PS; a block's rows sit in
|
|
146
|
+
# ONE page (main_page_size % ratio == 0, checked above)
|
|
147
|
+
bt = block_table[i].long()
|
|
148
|
+
src = []
|
|
149
|
+
for bid in ids.tolist():
|
|
150
|
+
tok0 = min(int(bid) * spec.ratio, L)
|
|
151
|
+
n_rows = min((int(bid) + 1) * spec.ratio, L) - tok0
|
|
152
|
+
if n_rows > 0:
|
|
153
|
+
page = bt[tok0 // main_page_size]
|
|
154
|
+
src.append(page * main_page_size + tok0 % main_page_size + torch.arange(n_rows, dtype=torch.int64))
|
|
155
|
+
src = torch.cat(src) if src else torch.empty(0, dtype=torch.int64)
|
|
156
|
+
dst = torch.arange(len(src), dtype=torch.int64)
|
|
157
|
+
pages_max = max(pages_max, (len(src) + sps - 1) // sps)
|
|
158
|
+
srcs.append(src)
|
|
159
|
+
dsts.append(dst)
|
|
160
|
+
max_rows = max(1, max(len(s) for s in srcs))
|
|
161
|
+
src_rows = torch.full((b, max_rows), -1, dtype=torch.int64)
|
|
162
|
+
dst_rows = torch.full((b, max_rows), -1, dtype=torch.int64)
|
|
163
|
+
scratch_bt = torch.zeros(b, pages_max, dtype=torch.int32)
|
|
164
|
+
start = 0
|
|
165
|
+
for i in range(b):
|
|
166
|
+
n = len(srcs[i])
|
|
167
|
+
src_rows[i, :n] = srcs[i]
|
|
168
|
+
dst_rows[i, :n] = dsts[i] + start * sps
|
|
169
|
+
n_pages = (n + sps - 1) // sps
|
|
170
|
+
scratch_bt[i, :n_pages] = torch.arange(start, start + n_pages, dtype=torch.int32)
|
|
171
|
+
start += n_pages
|
|
172
|
+
return src_rows, dst_rows, scratch_bt, pages_max
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def scratch_copy(
|
|
176
|
+
pool: torch.Tensor,
|
|
177
|
+
scratch: torch.Tensor,
|
|
178
|
+
src_rows: torch.Tensor,
|
|
179
|
+
dst_rows: torch.Tensor,
|
|
180
|
+
counts: torch.Tensor,
|
|
181
|
+
) -> None:
|
|
182
|
+
"""Row-gather pool -> scratch on the flat ``(pages * PS, ...)`` row view.
|
|
183
|
+
``counts`` (host ints, from ``effective_seq_lens``) fix each sequence's
|
|
184
|
+
valid-row count so the copy is ONE index kernel with no device sync
|
|
185
|
+
(a data-dependent mask would sync every call). Works for fp8 element
|
|
186
|
+
views and packed uint8 byte views alike (rows are the unit)."""
|
|
187
|
+
pf = pool.view(-1, *pool.shape[2:])
|
|
188
|
+
sf = scratch.view(-1, *scratch.shape[2:])
|
|
189
|
+
for i, n in enumerate(counts.tolist()):
|
|
190
|
+
n = int(n)
|
|
191
|
+
if n:
|
|
192
|
+
sf[dst_rows[i, :n].long()] = pf[src_rows[i, :n].long()]
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def selection_runs(
|
|
196
|
+
selected_blocks: torch.Tensor,
|
|
197
|
+
seq_lens: torch.Tensor,
|
|
198
|
+
spec: IndexerSpec,
|
|
199
|
+
main_page_size: int,
|
|
200
|
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
201
|
+
"""Coalesce a per-sequence selection of *blocks* into per-page row runs.
|
|
202
|
+
|
|
203
|
+
``selected_blocks``: (b, <= budget_blocks) int32, -1 padded, UNORDERED
|
|
204
|
+
(a top-k emits arbitrary order). Block ``i`` covers tokens
|
|
205
|
+
``[i*ratio, (i+1)*ratio)``; since ``main_page_size % ratio == 0`` a
|
|
206
|
+
block sits inside one page, so consecutive selected blocks in a page
|
|
207
|
+
merge into one contiguous run — the unit the gather loader TMA-copies.
|
|
208
|
+
|
|
209
|
+
Returns ``(runs, counts)``: runs is (b, max_runs, 3) int32 columns
|
|
210
|
+
``(page, row, len)`` in ROWS of the main pool, len a multiple of the
|
|
211
|
+
ratio; counts is (b,) the number of valid runs per sequence. Padding
|
|
212
|
+
rows are zeroed.
|
|
213
|
+
"""
|
|
214
|
+
spec.check_main_page_size(main_page_size)
|
|
215
|
+
b = selected_blocks.shape[0]
|
|
216
|
+
assert selected_blocks.shape[1] <= spec.budget_blocks, "selection exceeds the budget"
|
|
217
|
+
max_runs = 1
|
|
218
|
+
runs_out = torch.zeros(b, spec.budget_blocks, 3, dtype=torch.int32)
|
|
219
|
+
counts = torch.zeros(b, dtype=torch.int32)
|
|
220
|
+
for i in range(b):
|
|
221
|
+
n_tok = int(seq_lens[i])
|
|
222
|
+
ids = selected_blocks[i]
|
|
223
|
+
ids = ids[ids >= 0].to(torch.int64)
|
|
224
|
+
ids, _ = ids.sort()
|
|
225
|
+
# a block may extend past L on the last partial block: clip its rows
|
|
226
|
+
rows = (ids * spec.ratio).clamp_max(max(n_tok - 1, 0))
|
|
227
|
+
pages = rows // main_page_size
|
|
228
|
+
# run boundary: page changes or block ids are not consecutive
|
|
229
|
+
new_run = torch.ones(len(ids) + 1, dtype=torch.bool)
|
|
230
|
+
if len(ids):
|
|
231
|
+
new_run[1:-1] = (pages[1:] != pages[:-1]) | (ids[1:] != ids[:-1] + 1)
|
|
232
|
+
boundaries = new_run[:-1].nonzero().flatten()
|
|
233
|
+
b_pages = pages[boundaries]
|
|
234
|
+
b_rows = rows[boundaries]
|
|
235
|
+
# a run ends at the page edge or at L, whichever comes first
|
|
236
|
+
upper = torch.minimum(
|
|
237
|
+
torch.full_like(b_rows, main_page_size) - (b_rows % main_page_size),
|
|
238
|
+
torch.full_like(b_rows, n_tok) - b_rows,
|
|
239
|
+
)
|
|
240
|
+
ends = torch.cat([boundaries[1:], torch.tensor([len(ids)])])
|
|
241
|
+
lens = ((ends - boundaries) * spec.ratio).clamp_max(upper)
|
|
242
|
+
n = len(boundaries)
|
|
243
|
+
runs_out[i, :n, 0] = b_pages.to(torch.int32)
|
|
244
|
+
runs_out[i, :n, 1] = (b_rows % main_page_size).to(torch.int32)
|
|
245
|
+
runs_out[i, :n, 2] = lens.to(torch.int32)
|
|
246
|
+
counts[i] = n
|
|
247
|
+
max_runs = max(max_runs, n)
|
|
248
|
+
return runs_out[:, :max_runs], counts
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def identity_selection(seq_lens: torch.Tensor, spec: IndexerSpec) -> torch.Tensor:
|
|
252
|
+
"""The under-budget selection: every block of every sequence (QSA is
|
|
253
|
+
then bit-identical to dense attention — the correctness oracle)."""
|
|
254
|
+
b = len(seq_lens)
|
|
255
|
+
out = torch.full((b, spec.budget_blocks), -1, dtype=torch.int32)
|
|
256
|
+
for i, L in enumerate(seq_lens.tolist()):
|
|
257
|
+
n_blocks = min(spec.slots(int(L)), spec.budget_blocks)
|
|
258
|
+
out[i, :n_blocks] = torch.arange(n_blocks, dtype=torch.int32)
|
|
259
|
+
return out
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def decoder_kwargs(spec: IndexerSpec) -> dict:
|
|
263
|
+
"""``T5DecodeRunner`` constructor args for the indexer decode: the same
|
|
264
|
+
kernel, re-parameterized (MQA 4x128 over the pooled rows)."""
|
|
265
|
+
return {
|
|
266
|
+
"num_heads": spec.heads,
|
|
267
|
+
"num_kv_heads": 1,
|
|
268
|
+
"head_size": spec.dim,
|
|
269
|
+
"page_size": spec.page_size,
|
|
270
|
+
"q_mode": "fp8",
|
|
271
|
+
"kv_format": "fp8",
|
|
272
|
+
}
|
|
@@ -18,8 +18,9 @@ Only two things change:
|
|
|
18
18
|
* the impl: forward() runs the T5 kernel when it gets T5 metadata,
|
|
19
19
|
otherwise it calls the base forward.
|
|
20
20
|
|
|
21
|
-
KV formats: fp8 (e4m3, vLLM's own cache write), and ``fp6_e2m3``
|
|
22
|
-
|
|
21
|
+
KV formats: fp8 (e4m3, vLLM's own cache write), and ``fp6_e2m3``
|
|
22
|
+
(plugin-registered on vLLM >= 0.33 by ``mjolnir.kvdtype`` from the entry
|
|
23
|
+
point; the 0.32-era image patch is retired): packed 6-bit codes
|
|
23
24
|
with static per-(layer, KV head, channel) scales from a calibration file
|
|
24
25
|
(``mjolnir.kvscales``). Mjolnir writes that cache itself (``mjolnir.kvfp6``),
|
|
25
26
|
T5 reads it directly (the channel scales folded into Q and the output), and
|
|
@@ -804,8 +805,8 @@ class MjolnirAttentionBackend(_Base): # type: ignore[misc, valid-type]
|
|
|
804
805
|
|
|
805
806
|
# -- KV cache dtypes --------------------------------------------------
|
|
806
807
|
# The fallback's own list, plus fp6_e2m3 where vLLM knows the dtype (the
|
|
807
|
-
#
|
|
808
|
-
# scratch (see _fallback_serves_fp8_scratch).
|
|
808
|
+
# mjolnir.kvdtype plugin registration on >= 0.33) and the fallback can
|
|
809
|
+
# read the fp8 scratch (see _fallback_serves_fp8_scratch).
|
|
809
810
|
supported_kv_cache_dtypes: ClassVar[list] = list(getattr(_Base, "supported_kv_cache_dtypes", [])) + (
|
|
810
811
|
list(FP6_CACHE_DTYPES) if _fallback_serves_fp8_scratch() else []
|
|
811
812
|
)
|