mjolnir-kernel 0.3.2__tar.gz → 0.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/PKG-INFO +2 -1
  2. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/README.md +1 -0
  3. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/__init__.py +1 -1
  4. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/harness.py +279 -11
  5. mjolnir_kernel-0.3.3/mjolnir/arch/thor/qsa.py +272 -0
  6. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/backend.py +5 -4
  7. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/lmhead.py +113 -5
  8. mjolnir_kernel-0.3.3/mjolnir/qsa_gather.py +118 -0
  9. mjolnir_kernel-0.3.3/mjolnir/qsa_impl.py +421 -0
  10. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/.gitignore +0 -0
  11. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/LICENSE +0 -0
  12. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/NOTICE +0 -0
  13. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/__init__.py +0 -0
  14. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/README.md +0 -0
  15. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/__init__.py +0 -0
  16. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/combine.py +0 -0
  17. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/harness.py +0 -0
  18. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/mma.py +0 -0
  19. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/plan.py +0 -0
  20. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/orin/runner.py +0 -0
  21. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/__init__.py +0 -0
  22. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/call.py +0 -0
  23. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/const.py +0 -0
  24. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/decode.py +0 -0
  25. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/plan.py +0 -0
  26. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/arch/thor/runner.py +0 -0
  27. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/banner.py +0 -0
  28. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/builder.py +0 -0
  29. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/calibrate.py +0 -0
  30. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/draftvocab.py +0 -0
  31. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/kvdtype.py +0 -0
  32. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/kvfp6.py +0 -0
  33. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/kvscales.py +0 -0
  34. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/platforms.py +0 -0
  35. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/py.typed +0 -0
  36. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/qpack.py +0 -0
  37. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/quant.py +0 -0
  38. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/mjolnir/runner.py +0 -0
  39. {mjolnir_kernel-0.3.2 → mjolnir_kernel-0.3.3}/pyproject.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: mjolnir-kernel
3
- Version: 0.3.2
3
+ Version: 0.3.3
4
4
  Summary: Mjolnir: a paged decode-attention kernel for NVIDIA Jetson AGX Thor (sm_110) on tcgen05 / TMEM / TMA (CuTe DSL), with fp8 and 6-bit (e2m3) KV caches — packaged as the vLLM MJOLNIR attention backend.
5
5
  Project-URL: Homepage, https://github.com/GoBrrr-ai/mjolnir
6
6
  Project-URL: Documentation, https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel
@@ -144,6 +144,7 @@ runs on the fallback backend.
144
144
  | `MJOLNIR_ATTN_FUSED_DRAFT` | `1` | `0`: tell vLLM the backend cannot advance draft metadata in place → one graph replay per draft step instead of the fused draft loop |
145
145
  | `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV). `MJOLNIR_LM_HEAD_TILES` overrides the GEMV launch config (`BN,BK,warps,stages`; default 64,256,8,5 — the 2026-11-04 serving A/B winner) |
146
146
  | `MJOLNIR_MTP_FP8` | – | `1`: fp8-row-quantize the drafter's big unquantized linears (≥16M params) at load, same e4m3-per-row scheme + self-test (≤0.02 rel err) as the head; the bf16 originals are freed. Halves the MTP draft loop's streamed weight bytes |
147
+ | `MJOLNIR_MTP_LAYER` | – | `fp8`: the older, narrower variant — W8A8 via cuBLASLt rowwise `scaled_mm` on the MTP layer + eh_proj `fc` only. **Measured negative on sm_110** (cuBLASLt fp8 ≈60% of peak on layer shapes; 16–22% slower decode, acceptance safe) — see config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K_M8`; revisit only with a faster skinny-shape fp8 GEMM |
147
148
  | `MJOLNIR_DRAFT_VOCAB` | – | path to a `draft_vocab.json` (`mjolnir calibrate --draft-vocab`): the drafter's logits only over those tokens (VocabTrim); `MJOLNIR_DRAFT_VOCAB_SIZE` keeps the first N (a shorter file is padded with the lowest unused ids, with a warning). The kept head stays in the drafter's native quant (NVFP4 4-bit in-kernel, FP8 row-slice) — never up-scaled to bf16; a bf16 head is the one exception (fp8 only when `MJOLNIR_LM_HEAD=fp8` is set). `MJOLNIR_DRAFT_VOCAB_NATIVE=0` forces the dequant→fp8 path |
148
149
  | `MJOLNIR_ATTN_VERBOSE` | – | `1`: print the kernel's compile-time notes (ring stages, TMEM columns) — the standalone harness always does |
149
150
  | `MJOLNIR_ATTN_BANNER` | `1` | `0`: skip the Mjolnir hammer banner in the vLLM boot log (`mjolnir.banner.emit`, once per start, coloured with the server's formatter) |
@@ -116,6 +116,7 @@ runs on the fallback backend.
116
116
  | `MJOLNIR_ATTN_FUSED_DRAFT` | `1` | `0`: tell vLLM the backend cannot advance draft metadata in place → one graph replay per draft step instead of the fused draft loop |
117
117
  | `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV). `MJOLNIR_LM_HEAD_TILES` overrides the GEMV launch config (`BN,BK,warps,stages`; default 64,256,8,5 — the 2026-11-04 serving A/B winner) |
118
118
  | `MJOLNIR_MTP_FP8` | – | `1`: fp8-row-quantize the drafter's big unquantized linears (≥16M params) at load, same e4m3-per-row scheme + self-test (≤0.02 rel err) as the head; the bf16 originals are freed. Halves the MTP draft loop's streamed weight bytes |
119
+ | `MJOLNIR_MTP_LAYER` | – | `fp8`: the older, narrower variant — W8A8 via cuBLASLt rowwise `scaled_mm` on the MTP layer + eh_proj `fc` only. **Measured negative on sm_110** (cuBLASLt fp8 ≈60% of peak on layer shapes; 16–22% slower decode, acceptance safe) — see config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K_M8`; revisit only with a faster skinny-shape fp8 GEMM |
119
120
  | `MJOLNIR_DRAFT_VOCAB` | – | path to a `draft_vocab.json` (`mjolnir calibrate --draft-vocab`): the drafter's logits only over those tokens (VocabTrim); `MJOLNIR_DRAFT_VOCAB_SIZE` keeps the first N (a shorter file is padded with the lowest unused ids, with a warning). The kept head stays in the drafter's native quant (NVFP4 4-bit in-kernel, FP8 row-slice) — never up-scaled to bf16; a bf16 head is the one exception (fp8 only when `MJOLNIR_LM_HEAD=fp8` is set). `MJOLNIR_DRAFT_VOCAB_NATIVE=0` forces the dequant→fp8 path |
120
121
  | `MJOLNIR_ATTN_VERBOSE` | – | `1`: print the kernel's compile-time notes (ring stages, TMEM columns) — the standalone harness always does |
121
122
  | `MJOLNIR_ATTN_BANNER` | `1` | `0`: skip the Mjolnir hammer banner in the vLLM boot log (`mjolnir.banner.emit`, once per start, coloured with the server's formatter) |
@@ -24,7 +24,7 @@ from __future__ import annotations
24
24
  import os
25
25
 
26
26
  __all__ = ["BACKEND_NAME", "BACKEND_PATH", "__version__", "register"]
27
- __version__ = "0.3.2"
27
+ __version__ = "0.3.3"
28
28
 
29
29
  BACKEND_NAME = "MJOLNIR"
30
30
  BACKEND_PATH = "mjolnir.backend.MjolnirAttentionBackend"
@@ -63,10 +63,20 @@ def emulate_decode(
63
63
  chunk_tiles,
64
64
  fp8,
65
65
  causal=True,
66
+ selection_blocks=None, # (b, budget_blocks) int32, -1 padded: QSA gather reference
67
+ compress_ratio=4,
66
68
  ):
67
69
  """Bit-faithful model of the kernel's arithmetic (contiguous chunk split, running
68
70
  max, -inf guard, P rounding to e4m3(p * 448) or bf16, descale folding).
69
- Also returns a plain fp32 attention for the fp8-vs-exact gap."""
71
+ Also returns a plain fp32 attention for the fp8-vs-exact gap.
72
+
73
+ With ``selection_blocks`` the S columns are the SELECTED tokens (the QSA
74
+ gather path): block i covers rows [i*ratio, (i+1)*ratio) of the sequence,
75
+ gathered per (page, row) from the pool. Visibility is then owned by the
76
+ selection (every query row sees every selected column — the shared-
77
+ selection semantics of MTP verify), so ``causal`` is ignored; an
78
+ all-blocks selection at q_len == 1 is bit-identical to the dense path
79
+ (the identity oracle)."""
70
80
  b, h_k, n_q, d = q_packed.shape
71
81
  PS = k_pool.shape[1]
72
82
  h_r = n_q // q_len
@@ -76,21 +86,42 @@ def emulate_decode(
76
86
  o_exact = torch.zeros(b, h_k, n_q, d)
77
87
  for bi in range(b):
78
88
  L = int(seq_lens[bi])
79
- n_blk = math.ceil(L / PS)
80
- pages = block_table[bi, :n_blk].long()
81
- Kb = k_pool[pages].reshape(-1, h_k, d)[:L].permute(1, 0, 2) # (h_k, L, d)
82
- Vb = v_pool[pages].reshape(-1, h_k, d)[:L].permute(1, 0, 2)
83
- S = torch.einsum("gnd,gld->gnl", q_packed[bi], Kb) # raw scores
84
- qi = torch.arange(n_q) // h_r
85
- lim = L - q_len + qi if causal else torch.full((n_q,), L - 1) # (n_q,) last key each row sees
86
- masked = torch.arange(L)[None, :] > lim[:, None] # (n_q, L)
89
+ if selection_blocks is not None:
90
+ ids = selection_blocks[bi]
91
+ ids = ids[ids >= 0].long()
92
+ rows = torch.cat(
93
+ [
94
+ torch.arange(min(int(i) * compress_ratio, L), min((int(i) + 1) * compress_ratio, L))
95
+ for i in ids.tolist()
96
+ ]
97
+ or [torch.empty(0, dtype=torch.long)]
98
+ )
99
+ n_cols = len(rows)
100
+ # token row r of the sequence lives at pool page block_table[bi, r // PS]
101
+ pages_of_row = block_table[bi][rows // PS].long()
102
+ rows_in_page = rows % PS
103
+ Kb = k_pool[pages_of_row, rows_in_page].permute(1, 0, 2) # (h_k, n_cols, d)
104
+ Vb = v_pool[pages_of_row, rows_in_page].permute(1, 0, 2)
105
+ S = torch.einsum("gnd,gld->gnl", q_packed[bi], Kb) # raw scores
106
+ masked = torch.zeros(1, dtype=torch.bool) # no positional mask: the selection owns visibility
107
+ lim = None
108
+ tiles = math.ceil(n_cols / tile_s_rows)
109
+ else:
110
+ n_blk = math.ceil(L / PS)
111
+ pages = block_table[bi, :n_blk].long()
112
+ Kb = k_pool[pages].reshape(-1, h_k, d)[:L].permute(1, 0, 2) # (h_k, L, d)
113
+ Vb = v_pool[pages].reshape(-1, h_k, d)[:L].permute(1, 0, 2)
114
+ S = torch.einsum("gnd,gld->gnl", q_packed[bi], Kb) # raw scores
115
+ qi = torch.arange(n_q) // h_r
116
+ lim = L - q_len + qi if causal else torch.full((n_q,), L - 1) # (n_q,) last key each row sees
117
+ masked = torch.arange(L)[None, :] > lim[:, None] # (n_q, L)
118
+ tiles = math.ceil(L / tile_s_rows)
87
119
  S = S.masked_fill(masked[None], -math.inf)
88
120
  qk = q_descale[bi] * k_descale[bi] # (h_k,)
89
121
  c = (scale_qs * log2_e * qk)[:, None] # (h_k, 1)
90
122
  cta_scale = (scale_qs * qk)[:, None]
91
123
  vd = v_descale[bi][:, None, None]
92
124
 
93
- tiles = math.ceil(L / tile_s_rows)
94
125
  o_parts, m_parts, l_parts = [], [], []
95
126
  n_chunks = math.ceil(tiles / chunk_tiles)
96
127
  for j in range(n_chunks): # contiguous chunks, as in the plan
@@ -98,7 +129,7 @@ def emulate_decode(
98
129
  l = torch.zeros(h_k, n_q)
99
130
  o = torch.zeros(h_k, n_q, d)
100
131
  for t in range(j * chunk_tiles, min((j + 1) * chunk_tiles, tiles)):
101
- sl = slice(t * tile_s_rows, min((t + 1) * tile_s_rows, L))
132
+ sl = slice(t * tile_s_rows, min((t + 1) * tile_s_rows, S.shape[-1]))
102
133
  St = S[:, :, sl]
103
134
  m_new = torch.maximum(m, St.amax(dim=-1))
104
135
  m_use = torch.where(m_new == -math.inf, torch.zeros_like(m_new), m_new)
@@ -124,6 +155,230 @@ def emulate_decode(
124
155
  return o_emu, lse_emu, o_exact
125
156
 
126
157
 
158
+ def run_qsa_gather(
159
+ batches: int = 2,
160
+ seqlen: int = 65536,
161
+ q_len: int = 1,
162
+ heads_q: int = 24,
163
+ heads_k: int = 4,
164
+ headdim: int = 256,
165
+ page_size: int = 128,
166
+ iterations: int = 50,
167
+ warmup_iterations: int = 5,
168
+ seed: int = 1111,
169
+ tolerance: float = 0.05,
170
+ **_,
171
+ ):
172
+ """The QSA scratch-gather decision experiment (MJOLNIR_ATTN_QSA path).
173
+
174
+ Dense decode over the whole context vs COPY the selected rows into a
175
+ page-shaped scratch (torch row gather) + the SAME dense kernel on the
176
+ scratch — the P0 gather with zero traced-body changes. Correctness:
177
+ (a) a full selection reproduces dense exactly, (b) a partial selection
178
+ matches the emulate_decode selection oracle. Timing: cuda-event medians
179
+ of dense / copy+gather / copy alone at the requested context — the
180
+ numbers the ``verify qsagain`` stage collects.
181
+ """
182
+ from mjolnir.arch.thor import qsa
183
+ from mjolnir.arch.thor.runner import T5DecodeRunner
184
+
185
+ if not torch.cuda.is_available():
186
+ raise RuntimeError("GPU is required to run this example!")
187
+ cap = torch.cuda.get_device_capability()
188
+ if cap not in ((10, 0), (11, 0)):
189
+ raise RuntimeError(f"Thor sm_110 harness; device is sm_{cap[0]}{cap[1]}")
190
+ torch.manual_seed(seed)
191
+ dev = torch.device("cuda")
192
+ PS = page_size
193
+ spec = qsa.IndexerSpec()
194
+ spec.check_main_page_size(PS)
195
+ assert q_len >= 1
196
+
197
+ n_blocks = math.ceil(seqlen / PS)
198
+ num_pages = n_blocks * batches + 4
199
+ gen = torch.Generator().manual_seed(seed)
200
+ k_pool = (torch.randn(num_pages, PS, heads_k, headdim, generator=gen) * 0.5).to(torch.float8_e4m3fn).to(dev)
201
+ v_pool = (torch.randn(num_pages, PS, heads_k, headdim, generator=gen) * 0.5).to(torch.float8_e4m3fn).to(dev)
202
+ block_table = torch.zeros(batches, n_blocks, dtype=torch.int32)
203
+ for s in range(batches):
204
+ block_table[s] = torch.arange(s * n_blocks, (s + 1) * n_blocks, dtype=torch.int32)
205
+ block_table = block_table.to(dev)
206
+ seq_lens = torch.full((batches,), seqlen, dtype=torch.int32, device=dev)
207
+
208
+ # deterministic budget-limited selection over RATIO blocks (4 tokens each,
209
+ # NOT pages): evenly spaced + the local tail when over budget
210
+ n_blocks_total = math.ceil(seqlen / spec.ratio)
211
+ if n_blocks_total > spec.budget_blocks:
212
+ ids = torch.linspace(0, n_blocks_total - 17, spec.budget_blocks - 16).round().long()
213
+ ids = torch.cat([ids, torch.arange(n_blocks_total - 16, n_blocks_total)])
214
+ else:
215
+ ids = torch.arange(n_blocks_total)
216
+ selection = torch.full((batches, spec.budget_blocks), -1, dtype=torch.int32, device=dev)
217
+ selection[:, : len(ids)] = ids.to(dev)
218
+
219
+ eff = qsa.effective_seq_lens(selection, seq_lens.cpu(), spec)
220
+ src, dst, sbt, pages_max = qsa.scratch_plan(selection.cpu(), seq_lens.cpu(), block_table.cpu(), spec, PS)
221
+ scratch_pages = pages_max * batches + 2
222
+ scratch_k = torch.empty(scratch_pages, PS, heads_k, headdim, dtype=torch.float8_e4m3fn, device=dev)
223
+ scratch_v = torch.empty_like(scratch_k)
224
+ scratch_bt = torch.zeros(batches, sbt.shape[1], dtype=torch.int32, device=dev)
225
+ scratch_bt[:, : sbt.shape[1]] = sbt.to(dev)
226
+ src_dev, dst_dev = src.to(dev), dst.to(dev)
227
+ eff_cpu = eff.cpu()
228
+
229
+ # the copy path: fused Triton when available, torch indexing otherwise.
230
+ # BOTH paths feed correctness (b) — the timed path is the checked path.
231
+ from mjolnir import qsa_gather
232
+
233
+ if qsa_gather.available():
234
+
235
+ def copy():
236
+ # one Triton launch per K/V; strides in bytes of the logical layout
237
+ kb, vb = k_pool.view(torch.uint8), v_pool.view(torch.uint8)
238
+ qsa_gather.gather_rows(
239
+ kb, scratch_k.view(torch.uint8), src_dev, dst_dev, eff, PS, kb.stride(0), kb.stride(1), kb.stride(2)
240
+ )
241
+ qsa_gather.gather_rows(
242
+ vb, scratch_v.view(torch.uint8), src_dev, dst_dev, eff, PS, vb.stride(0), vb.stride(1), vb.stride(2)
243
+ )
244
+
245
+ print("qsa copy path: triton (fused)")
246
+ else:
247
+
248
+ def copy():
249
+ qsa.scratch_copy(k_pool, scratch_k, src_dev, dst_dev, eff_cpu)
250
+ qsa.scratch_copy(v_pool, scratch_v, src_dev, dst_dev, eff_cpu)
251
+
252
+ print("qsa copy path: torch (no Triton)")
253
+
254
+ tokens = batches * q_len
255
+ query = (torch.randn(tokens, heads_q, headdim, generator=gen) * 0.5).to(torch.bfloat16).to(dev)
256
+ out = torch.empty(tokens, heads_q, headdim, dtype=torch.bfloat16, device=dev)
257
+ one = torch.ones(1, device=dev)
258
+ softmax_scale = 1.0 / math.sqrt(headdim)
259
+
260
+ runner = T5DecodeRunner(
261
+ num_heads=heads_q,
262
+ num_kv_heads=heads_k,
263
+ head_size=headdim,
264
+ page_size=PS,
265
+ out_dtype=torch.bfloat16,
266
+ max_num_seqs=batches,
267
+ max_q_len=max(2, q_len),
268
+ q_mode="bf16",
269
+ kv_format="fp8",
270
+ device=dev,
271
+ )
272
+ # NOTE: runner.plan() writes into the runner's PERSISTENT shared device
273
+ # buffers — a plan is only valid until the next plan() call. Every
274
+ # run() below is immediately preceded by its own plan().
275
+
276
+ # ---- correctness (a): full selection == dense, same kernel ----
277
+ # only meaningful at L <= budget (a "full" selection cannot exceed it)
278
+ L1 = min(seqlen, spec.budget_tokens)
279
+ pages1 = math.ceil(L1 / PS)
280
+ sel_full = qsa.identity_selection(torch.full((batches,), L1, dtype=torch.int32), spec).to(dev)
281
+ src_f, dst_f, sbt_f, pages_f = qsa.scratch_plan(
282
+ sel_full.cpu(), torch.full((batches,), L1, dtype=torch.int32), block_table[:, :pages1].cpu(), spec, PS
283
+ )
284
+ full_k = torch.empty(pages_f * batches + 2, PS, heads_k, headdim, dtype=torch.float8_e4m3fn, device=dev)
285
+ full_v = torch.empty_like(full_k)
286
+ qsa.scratch_copy(k_pool, full_k, src_f.to(dev), dst_f.to(dev), torch.full((batches,), L1, dtype=torch.int32))
287
+ qsa.scratch_copy(v_pool, full_v, src_f.to(dev), dst_f.to(dev), torch.full((batches,), L1, dtype=torch.int32))
288
+ full_bt = torch.zeros(batches, sbt_f.shape[1], dtype=torch.int32, device=dev)
289
+ full_bt[:, : sbt_f.shape[1]] = sbt_f.to(dev)
290
+ out_full = torch.empty_like(out)
291
+ seqs1 = torch.full((batches,), L1, dtype=torch.int32, device=dev)
292
+ plan = runner.plan([L1] * batches, batches, q_len, full_bt, seqs1, causal=False)
293
+ runner.ensure_compiled(q_len, plan.with_reduction, causal=False)
294
+ runner.run(plan, query, out_full, full_k, full_v, one, one, softmax_scale)
295
+ # both sides causal=False: the gather's shared-selection visibility — the
296
+ # check is layout transparency, not causal semantics (q_len > 1 ladders differ)
297
+ plan = runner.plan([L1] * batches, batches, q_len, block_table[:, :pages1], seqs1, causal=False)
298
+ runner.ensure_compiled(q_len, plan.with_reduction, causal=False)
299
+ runner.run(plan, query, out, k_pool, v_pool, one, one, softmax_scale)
300
+ torch.cuda.synchronize()
301
+ err_full = (out.float() - out_full.float()).abs().max().item()
302
+ print(f"qsa gather: full-selection vs dense (L={L1}) max|err| = {err_full:.3g} (expect ~0)")
303
+ assert err_full == 0.0, "full-selection scratch decode diverged from dense"
304
+
305
+ # ---- correctness (b): partial selection vs the selection oracle ----
306
+ copy()
307
+ plan = runner.plan(eff.tolist(), batches, q_len, scratch_bt, eff.to(dev), causal=False)
308
+ runner.ensure_compiled(q_len, plan.with_reduction, causal=False)
309
+ runner.run(plan, query, out, scratch_k, scratch_v, one, one, softmax_scale)
310
+ torch.cuda.synchronize()
311
+ o_got = (
312
+ out.view(batches, q_len, heads_k, heads_q // heads_k, headdim)
313
+ .permute(0, 2, 1, 3, 4)
314
+ .reshape(batches, heads_k, q_len * (heads_q // heads_k), headdim)
315
+ .float()
316
+ .cpu()
317
+ )
318
+ o_emu, _, _ = emulate_decode(
319
+ query.view(batches, q_len, heads_k, -1, headdim)
320
+ .permute(0, 2, 1, 3, 4)
321
+ .reshape(batches, heads_k, q_len * (heads_q // heads_k), headdim)
322
+ .float()
323
+ .cpu(),
324
+ k_pool.float().cpu(),
325
+ v_pool.float().cpu(),
326
+ block_table.cpu(),
327
+ seq_lens.cpu(),
328
+ torch.ones(batches, heads_k),
329
+ torch.ones(batches, heads_k),
330
+ torch.ones(batches, heads_k),
331
+ softmax_scale,
332
+ 1.0,
333
+ q_len,
334
+ 2,
335
+ fp8=False,
336
+ selection_blocks=selection.cpu(),
337
+ )
338
+ err_sel = (o_got - o_emu).abs().max().item()
339
+ tol = tolerance * max(1.0, o_emu.abs().max().item())
340
+ print(f"qsa gather: partial-selection vs oracle max|err| = {err_sel:.3g} (tol {tol:.3g})")
341
+ assert err_sel <= tol, "scratch decode diverged from the selection oracle"
342
+
343
+ # ---- timing: dense / copy+gather / copy alone ----
344
+ def timed(fn, iters=iterations, warmup=warmup_iterations):
345
+ for _ in range(warmup):
346
+ fn()
347
+ torch.cuda.synchronize()
348
+ times = []
349
+ for _ in range(iters):
350
+ s, e = torch.cuda.Event(True), torch.cuda.Event(True)
351
+ s.record()
352
+ fn()
353
+ e.record()
354
+ e.synchronize()
355
+ times.append(s.elapsed_time(e) * 1000.0)
356
+ times.sort()
357
+ return times[len(times) // 2]
358
+
359
+ def gather_step():
360
+ # re-plan every step: the honest per-step cost (serving rebuilds
361
+ # metadata each step), and plan() owns the persistent buffers
362
+ copy()
363
+ plan = runner.plan(eff.tolist(), batches, q_len, scratch_bt, eff.to(dev), causal=False)
364
+ runner.run(plan, query, out, scratch_k, scratch_v, one, one, softmax_scale)
365
+
366
+ def dense_step():
367
+ plan = runner.plan(seq_lens.tolist(), batches, q_len, block_table, seq_lens)
368
+ runner.run(plan, query, out, k_pool, v_pool, one, one, softmax_scale)
369
+
370
+ t_dense = timed(dense_step)
371
+ t_gather = timed(gather_step)
372
+ t_copy = timed(copy)
373
+ kv_bytes = int(seq_lens.sum()) * heads_k * headdim * 2
374
+ sel_bytes = int(eff.sum()) * heads_k * headdim * 2
375
+ for name, us in (("dense", t_dense), ("gather", t_gather), ("copy", t_copy)):
376
+ print(f"median {name} {us:.2f} us")
377
+ print(f"qsa ratio dense/gather = {t_dense / t_gather:.3f} (copy share {t_copy / t_gather:.2f})")
378
+ print(f"qsa bytes: dense {kv_bytes} B vs selected {sel_bytes} B (+ indexer cache, not modeled here)")
379
+ print("PASS")
380
+
381
+
127
382
  def run(
128
383
  batches: int = 1,
129
384
  seqlen: int = 1024,
@@ -705,8 +960,21 @@ def main():
705
960
  action="store_true",
706
961
  help="E1: the last CTA per (seq, q tile, kv head) merges the split partials in-kernel",
707
962
  )
963
+ parser.add_argument(
964
+ "--qsa_gather",
965
+ action="store_true",
966
+ help="the QSA scratch-gather decision experiment (MJOLNIR_ATTN_QSA "
967
+ "path): dense decode vs copy-selected-rows + the same kernel on the "
968
+ "scratch — correctness vs the selection oracle + cuda-event medians; "
969
+ "uses --batches/--seqlen/--q_len/--heads_q/--heads_k/--headdim/"
970
+ "--page_size/--iterations/--seed/--tolerance, forces fp8 KV",
971
+ )
708
972
 
709
973
  kwargs = vars(parser.parse_args())
974
+ if kwargs.pop("qsa_gather", False):
975
+ run_qsa_gather(**kwargs)
976
+ print("PASS")
977
+ return
710
978
  if kwargs["kv_format"] == "auto": # sub-8-bit formats need fp8 Q
711
979
  kwargs["kv_format"] = HARNESS_KV_FORMAT_FP8Q if kwargs["q_dtype"] is cutlass.Float8E4M3FN else "fp8"
712
980
  run(**kwargs)
@@ -0,0 +1,272 @@
1
+ """Qwen4 QSA (sparse-attention indexer) host-side plumbing for the Thor kernel.
2
+
3
+ Probe-gated: everything here is inert until ``MJOLNIR_ATTN_QSA`` is set —
4
+ the vLLM-side wiring arrives with the backend patch, no default flips.
5
+
6
+ Basis: ``task/research/qwen4-27b-upstream-prep/`` (the recovered Intel
7
+ ``csrc/qwen38`` kernels + the qwen4_exp family config). Two additions over
8
+ today's kernel, both shape-compatible:
9
+
10
+ 1. **Indexer decode** — the QSA indexer scores tokens for the sparse
11
+ selection with a small MQA over a *second* paged pool. It reuses
12
+ today's kernel unchanged at ``(b, h_k=1, n_q=indexer_heads*q_len,
13
+ d=indexer_dim)``: the pool's pooled rows are the single KV "head" and
14
+ the indexer q heads pack into ``n_q`` rows (``heads_q % heads_k == 0``,
15
+ ``head_size % 128 == 0`` both hold).
16
+ 2. **Gather decode** — the main attention reads a *selected* token set
17
+ (budget 2048 tokens = 512 blocks x compress ratio 4) instead of the
18
+ whole block table. Host side, the selection is coalesced into runs of
19
+ consecutive rows per page (:func:`selection_runs`); the device loader
20
+ that consumes runs is the only traced-body change and is flag-gated.
21
+
22
+ Under the budget (L <= budget tokens) the selection is every token, so QSA
23
+ is bit-identical to dense attention — the correctness oracle for both
24
+ paths (llama.cpp measured max logit delta 0.0 over all 2051 rows).
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import math
30
+ from dataclasses import dataclass
31
+
32
+ import torch
33
+
34
+ from mjolnir.runner import env_flag
35
+
36
+ # Family constants (qwen4_exp config + the recovered kernels; the indexer
37
+ # geometry is hardcoded family-wide: qsa_sycl.cpp kIndexHeads/kIndexDim).
38
+ INDEXER_HEADS = 4 # MQA query heads
39
+ INDEXER_DIM = 128 # indexer head dim (the main KV head dim is 256)
40
+ COMPRESS_RATIO = 4 # tokens pooled per compressed slot ("requires ratio=4")
41
+ BUDGET_TOKENS = 2048 # selection budget ("512 blocks or 2048 tokens")
42
+ BUDGET_BLOCKS = BUDGET_TOKENS // COMPRESS_RATIO
43
+ POOL_ROWS_PER_SLOT = 2 # raw key | pooled key (indexer_kpool_row = 2)
44
+ QSA_PAGE_SIZES = (64, 128) # compressed-pool slots per page (qsa_sycl.cpp:2084)
45
+
46
+ POOLED_ROW = 1 # the pooled key's row index inside a slot (raw is 0)
47
+
48
+
49
+ def qsa_enabled() -> bool:
50
+ """``MJOLNIR_ATTN_QSA`` gate (default off): the whole QSA path is opt-in."""
51
+ return env_flag("MJOLNIR_ATTN_QSA").lower() not in ("", "0", "false", "off")
52
+
53
+
54
+ @dataclass(frozen=True)
55
+ class IndexerSpec:
56
+ """Per-model QSA indexer geometry and its derived shapes.
57
+
58
+ ``page_size`` is the *compressed* pool's slots-per-page; the main KV
59
+ pool keeps its own page size (any of SUPPORTED_PAGE_SIZES, as long as
60
+ ``main_page_size % ratio == 0`` so a block never crosses a page).
61
+ """
62
+
63
+ heads: int = INDEXER_HEADS
64
+ dim: int = INDEXER_DIM
65
+ ratio: int = COMPRESS_RATIO
66
+ budget_tokens: int = BUDGET_TOKENS
67
+ page_size: int = 128
68
+ rows_per_slot: int = POOL_ROWS_PER_SLOT
69
+
70
+ def __post_init__(self):
71
+ assert self.ratio == COMPRESS_RATIO, "group compression requires ratio=4 (qsa_sycl.cpp:905)"
72
+ assert self.budget_tokens % self.ratio == 0, "budget must be a whole number of blocks"
73
+ assert self.page_size in QSA_PAGE_SIZES, f"compressed page size {self.page_size} unsupported"
74
+ assert self.dim % 128 == 0, "indexer head dim must be a multiple of 128 (kernel headdim rule)"
75
+ assert self.rows_per_slot == POOL_ROWS_PER_SLOT, "raw | pooled rows per slot"
76
+
77
+ @property
78
+ def budget_blocks(self) -> int:
79
+ return self.budget_tokens // self.ratio
80
+
81
+ def slots(self, tokens: int) -> int:
82
+ """Compressed slots (KV rows of the indexer decode) for ``tokens``."""
83
+ return math.ceil(tokens / self.ratio)
84
+
85
+ def pool_pages(self, tokens: int) -> int:
86
+ return math.ceil(self.slots(tokens) / self.page_size)
87
+
88
+ def score_rows(self, q_len: int) -> int:
89
+ """``n_q`` of the indexer decode: q tokens x indexer heads, packed."""
90
+ return q_len * self.heads
91
+
92
+ def check_main_page_size(self, main_page_size: int) -> None:
93
+ assert main_page_size % self.ratio == 0, (
94
+ f"main page size {main_page_size} must be a multiple of the compress "
95
+ f"ratio {self.ratio} or a selected block would cross a page"
96
+ )
97
+
98
+
99
+ def effective_seq_lens(selection_blocks: torch.Tensor, seq_lens: torch.Tensor, spec: IndexerSpec) -> torch.Tensor:
100
+ """The gather call's ``seq_lens``: tokens actually covered by the
101
+ selection (blocks clipped to L). The kernel's S masking uses it, so a
102
+ block id beyond L contributes nothing and a partial last block clips."""
103
+ ids = selection_blocks.to(torch.int64)
104
+ valid = ids >= 0
105
+ lo = ids.clamp_min(0) * spec.ratio
106
+ hi = (ids.clamp_min(0) + 1) * spec.ratio
107
+ L = seq_lens.to(ids.device, torch.int64).view(-1, 1)
108
+ contrib = (hi.clamp_max(L) - lo.clamp_max(L)).clamp_min(0) * valid
109
+ return contrib.sum(dim=1).to(torch.int32)
110
+
111
+
112
+ def scratch_plan(
113
+ selection_blocks: torch.Tensor,
114
+ seq_lens: torch.Tensor,
115
+ block_table: torch.Tensor,
116
+ spec: IndexerSpec,
117
+ main_page_size: int,
118
+ scratch_page_size: int | None = None,
119
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
120
+ """Everything the scratch-gather copy and the follow-up dense decode need.
121
+
122
+ The scratch packs each sequence's SELECTED rows contiguously into
123
+ page-shaped scratch pages, so the EXISTING dense kernel + plan run on
124
+ the scratch unchanged (zero traced-body changes for P0; the in-loader
125
+ per-block TMA gather is the P2 refinement if ncu ever shows the copy).
126
+
127
+ ``main_page_size`` is the POOL's page size (a hybrid stack may force it
128
+ large — GDN mamba pages set the floor); ``scratch_page_size`` is the
129
+ kernel's (default: same as the pool). Returns
130
+ ``(src_rows, dst_rows, scratch_block_table, pages_per_seq_max)``:
131
+ src_rows/dst_rows are (b, max_rows) int64 gather/scatter indices into
132
+ the pool's and scratch's flat ``(pages * PS, ...)`` row space (rows
133
+ beyond a sequence's count are -1: skip), and scratch_block_table is the
134
+ (b, max_pages) int32 page table of the scratch pool.
135
+ """
136
+ spec.check_main_page_size(main_page_size)
137
+ sps = scratch_page_size or main_page_size
138
+ b = selection_blocks.shape[0]
139
+ srcs, dsts, pages_max = [], [], 0
140
+ for i in range(b):
141
+ L = int(seq_lens[i])
142
+ ids = selection_blocks[i]
143
+ ids = ids[ids >= 0].long()
144
+ # pool flat row of each selected token row: token row r of the
145
+ # sequence lives at bt[r // PS] * PS + r % PS; a block's rows sit in
146
+ # ONE page (main_page_size % ratio == 0, checked above)
147
+ bt = block_table[i].long()
148
+ src = []
149
+ for bid in ids.tolist():
150
+ tok0 = min(int(bid) * spec.ratio, L)
151
+ n_rows = min((int(bid) + 1) * spec.ratio, L) - tok0
152
+ if n_rows > 0:
153
+ page = bt[tok0 // main_page_size]
154
+ src.append(page * main_page_size + tok0 % main_page_size + torch.arange(n_rows, dtype=torch.int64))
155
+ src = torch.cat(src) if src else torch.empty(0, dtype=torch.int64)
156
+ dst = torch.arange(len(src), dtype=torch.int64)
157
+ pages_max = max(pages_max, (len(src) + sps - 1) // sps)
158
+ srcs.append(src)
159
+ dsts.append(dst)
160
+ max_rows = max(1, max(len(s) for s in srcs))
161
+ src_rows = torch.full((b, max_rows), -1, dtype=torch.int64)
162
+ dst_rows = torch.full((b, max_rows), -1, dtype=torch.int64)
163
+ scratch_bt = torch.zeros(b, pages_max, dtype=torch.int32)
164
+ start = 0
165
+ for i in range(b):
166
+ n = len(srcs[i])
167
+ src_rows[i, :n] = srcs[i]
168
+ dst_rows[i, :n] = dsts[i] + start * sps
169
+ n_pages = (n + sps - 1) // sps
170
+ scratch_bt[i, :n_pages] = torch.arange(start, start + n_pages, dtype=torch.int32)
171
+ start += n_pages
172
+ return src_rows, dst_rows, scratch_bt, pages_max
173
+
174
+
175
+ def scratch_copy(
176
+ pool: torch.Tensor,
177
+ scratch: torch.Tensor,
178
+ src_rows: torch.Tensor,
179
+ dst_rows: torch.Tensor,
180
+ counts: torch.Tensor,
181
+ ) -> None:
182
+ """Row-gather pool -> scratch on the flat ``(pages * PS, ...)`` row view.
183
+ ``counts`` (host ints, from ``effective_seq_lens``) fix each sequence's
184
+ valid-row count so the copy is ONE index kernel with no device sync
185
+ (a data-dependent mask would sync every call). Works for fp8 element
186
+ views and packed uint8 byte views alike (rows are the unit)."""
187
+ pf = pool.view(-1, *pool.shape[2:])
188
+ sf = scratch.view(-1, *scratch.shape[2:])
189
+ for i, n in enumerate(counts.tolist()):
190
+ n = int(n)
191
+ if n:
192
+ sf[dst_rows[i, :n].long()] = pf[src_rows[i, :n].long()]
193
+
194
+
195
+ def selection_runs(
196
+ selected_blocks: torch.Tensor,
197
+ seq_lens: torch.Tensor,
198
+ spec: IndexerSpec,
199
+ main_page_size: int,
200
+ ) -> tuple[torch.Tensor, torch.Tensor]:
201
+ """Coalesce a per-sequence selection of *blocks* into per-page row runs.
202
+
203
+ ``selected_blocks``: (b, <= budget_blocks) int32, -1 padded, UNORDERED
204
+ (a top-k emits arbitrary order). Block ``i`` covers tokens
205
+ ``[i*ratio, (i+1)*ratio)``; since ``main_page_size % ratio == 0`` a
206
+ block sits inside one page, so consecutive selected blocks in a page
207
+ merge into one contiguous run — the unit the gather loader TMA-copies.
208
+
209
+ Returns ``(runs, counts)``: runs is (b, max_runs, 3) int32 columns
210
+ ``(page, row, len)`` in ROWS of the main pool, len a multiple of the
211
+ ratio; counts is (b,) the number of valid runs per sequence. Padding
212
+ rows are zeroed.
213
+ """
214
+ spec.check_main_page_size(main_page_size)
215
+ b = selected_blocks.shape[0]
216
+ assert selected_blocks.shape[1] <= spec.budget_blocks, "selection exceeds the budget"
217
+ max_runs = 1
218
+ runs_out = torch.zeros(b, spec.budget_blocks, 3, dtype=torch.int32)
219
+ counts = torch.zeros(b, dtype=torch.int32)
220
+ for i in range(b):
221
+ n_tok = int(seq_lens[i])
222
+ ids = selected_blocks[i]
223
+ ids = ids[ids >= 0].to(torch.int64)
224
+ ids, _ = ids.sort()
225
+ # a block may extend past L on the last partial block: clip its rows
226
+ rows = (ids * spec.ratio).clamp_max(max(n_tok - 1, 0))
227
+ pages = rows // main_page_size
228
+ # run boundary: page changes or block ids are not consecutive
229
+ new_run = torch.ones(len(ids) + 1, dtype=torch.bool)
230
+ if len(ids):
231
+ new_run[1:-1] = (pages[1:] != pages[:-1]) | (ids[1:] != ids[:-1] + 1)
232
+ boundaries = new_run[:-1].nonzero().flatten()
233
+ b_pages = pages[boundaries]
234
+ b_rows = rows[boundaries]
235
+ # a run ends at the page edge or at L, whichever comes first
236
+ upper = torch.minimum(
237
+ torch.full_like(b_rows, main_page_size) - (b_rows % main_page_size),
238
+ torch.full_like(b_rows, n_tok) - b_rows,
239
+ )
240
+ ends = torch.cat([boundaries[1:], torch.tensor([len(ids)])])
241
+ lens = ((ends - boundaries) * spec.ratio).clamp_max(upper)
242
+ n = len(boundaries)
243
+ runs_out[i, :n, 0] = b_pages.to(torch.int32)
244
+ runs_out[i, :n, 1] = (b_rows % main_page_size).to(torch.int32)
245
+ runs_out[i, :n, 2] = lens.to(torch.int32)
246
+ counts[i] = n
247
+ max_runs = max(max_runs, n)
248
+ return runs_out[:, :max_runs], counts
249
+
250
+
251
+ def identity_selection(seq_lens: torch.Tensor, spec: IndexerSpec) -> torch.Tensor:
252
+ """The under-budget selection: every block of every sequence (QSA is
253
+ then bit-identical to dense attention — the correctness oracle)."""
254
+ b = len(seq_lens)
255
+ out = torch.full((b, spec.budget_blocks), -1, dtype=torch.int32)
256
+ for i, L in enumerate(seq_lens.tolist()):
257
+ n_blocks = min(spec.slots(int(L)), spec.budget_blocks)
258
+ out[i, :n_blocks] = torch.arange(n_blocks, dtype=torch.int32)
259
+ return out
260
+
261
+
262
+ def decoder_kwargs(spec: IndexerSpec) -> dict:
263
+ """``T5DecodeRunner`` constructor args for the indexer decode: the same
264
+ kernel, re-parameterized (MQA 4x128 over the pooled rows)."""
265
+ return {
266
+ "num_heads": spec.heads,
267
+ "num_kv_heads": 1,
268
+ "head_size": spec.dim,
269
+ "page_size": spec.page_size,
270
+ "q_mode": "fp8",
271
+ "kv_format": "fp8",
272
+ }
@@ -18,8 +18,9 @@ Only two things change:
18
18
  * the impl: forward() runs the T5 kernel when it gets T5 metadata,
19
19
  otherwise it calls the base forward.
20
20
 
21
- KV formats: fp8 (e4m3, vLLM's own cache write), and ``fp6_e2m3`` on the
22
- image (``thor-fp6-e2m3-kv-cache.patch`` adds the dtype): packed 6-bit codes
21
+ KV formats: fp8 (e4m3, vLLM's own cache write), and ``fp6_e2m3``
22
+ (plugin-registered on vLLM >= 0.33 by ``mjolnir.kvdtype`` from the entry
23
+ point; the 0.32-era image patch is retired): packed 6-bit codes
23
24
  with static per-(layer, KV head, channel) scales from a calibration file
24
25
  (``mjolnir.kvscales``). Mjolnir writes that cache itself (``mjolnir.kvfp6``),
25
26
  T5 reads it directly (the channel scales folded into Q and the output), and
@@ -804,8 +805,8 @@ class MjolnirAttentionBackend(_Base): # type: ignore[misc, valid-type]
804
805
 
805
806
  # -- KV cache dtypes --------------------------------------------------
806
807
  # The fallback's own list, plus fp6_e2m3 where vLLM knows the dtype (the
807
- # image's thor-fp6-e2m3-kv-cache patch) and the fallback can read the fp8
808
- # scratch (see _fallback_serves_fp8_scratch).
808
+ # mjolnir.kvdtype plugin registration on >= 0.33) and the fallback can
809
+ # read the fp8 scratch (see _fallback_serves_fp8_scratch).
809
810
  supported_kv_cache_dtypes: ClassVar[list] = list(getattr(_Base, "supported_kv_cache_dtypes", [])) + (
810
811
  list(FP6_CACHE_DTYPES) if _fallback_serves_fp8_scratch() else []
811
812
  )