mjolnir-kernel 0.1.1__tar.gz → 0.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/.gitignore +8 -0
  2. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/NOTICE +1 -1
  3. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/PKG-INFO +51 -20
  4. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/README.md +45 -14
  5. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/__init__.py +18 -2
  6. mjolnir_kernel-0.3.2/mjolnir/arch/__init__.py +7 -0
  7. mjolnir_kernel-0.3.2/mjolnir/arch/orin/README.md +102 -0
  8. mjolnir_kernel-0.3.2/mjolnir/arch/orin/__init__.py +82 -0
  9. mjolnir_kernel-0.3.2/mjolnir/arch/orin/combine.py +111 -0
  10. mjolnir_kernel-0.3.2/mjolnir/arch/orin/harness.py +483 -0
  11. mjolnir_kernel-0.3.2/mjolnir/arch/orin/mma.py +2295 -0
  12. mjolnir_kernel-0.3.2/mjolnir/arch/orin/plan.py +67 -0
  13. mjolnir_kernel-0.3.2/mjolnir/arch/orin/runner.py +686 -0
  14. mjolnir_kernel-0.3.2/mjolnir/arch/thor/__init__.py +45 -0
  15. mjolnir_kernel-0.1.1/mjolnir/t5_call.py → mjolnir_kernel-0.3.2/mjolnir/arch/thor/call.py +2 -2
  16. mjolnir_kernel-0.1.1/mjolnir/t5_decode.py → mjolnir_kernel-0.3.2/mjolnir/arch/thor/decode.py +4 -4
  17. mjolnir_kernel-0.1.1/mjolnir/t5_harness.py → mjolnir_kernel-0.3.2/mjolnir/arch/thor/harness.py +17 -10
  18. {mjolnir_kernel-0.1.1/mjolnir → mjolnir_kernel-0.3.2/mjolnir/arch/thor}/runner.py +29 -292
  19. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/backend.py +290 -364
  20. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/banner.py +1 -1
  21. mjolnir_kernel-0.3.2/mjolnir/builder.py +529 -0
  22. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/calibrate.py +2 -2
  23. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/draftvocab.py +4 -0
  24. mjolnir_kernel-0.3.2/mjolnir/kvdtype.py +53 -0
  25. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/kvfp6.py +42 -3
  26. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/lmhead.py +69 -3
  27. mjolnir_kernel-0.3.2/mjolnir/platforms.py +120 -0
  28. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/quant.py +1 -1
  29. mjolnir_kernel-0.3.2/mjolnir/runner.py +329 -0
  30. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/pyproject.toml +5 -5
  31. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/LICENSE +0 -0
  32. /mjolnir_kernel-0.1.1/mjolnir/t5_const.py → /mjolnir_kernel-0.3.2/mjolnir/arch/thor/const.py +0 -0
  33. /mjolnir_kernel-0.1.1/mjolnir/t5_plan.py → /mjolnir_kernel-0.3.2/mjolnir/arch/thor/plan.py +0 -0
  34. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/kvscales.py +0 -0
  35. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/py.typed +0 -0
  36. {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/qpack.py +0 -0
@@ -1,9 +1,13 @@
1
1
  __pycache__/
2
2
  *.py[cod]
3
3
  .venv/
4
+ .playwright-mcp/
4
5
  .pytest_cache/
5
6
  .ruff_cache/
6
7
  .env
8
+ # the configs' env sidecars are plain mjolnir knobs (calibration paths, feature
9
+ # flags) and ride with the configs; real secrets stay out
10
+ !configs/**/.env
7
11
  # coverage artifacts (the CI test stage writes these; they regenerate)
8
12
  .coverage
9
13
  .coverage.*
@@ -24,6 +28,7 @@ benchmarks/raw/*
24
28
  !benchmarks/raw/accuracy/
25
29
  benchmarks/raw/accuracy/*/*/
26
30
  benchmarks/logs/
31
+ benchmarks/staging/
27
32
  task/
28
33
 
29
34
  # autoresearch: the live ledger is untracked (discards are `git reset --hard`);
@@ -32,3 +37,6 @@ research/ledger.jsonl
32
37
  research/ledger.tsv
33
38
  # …and the exported runs ARE committed (the generic runs/ rule above would hide them)
34
39
  !research/runs/
40
+ /cutlass/
41
+ /fa2/
42
+ /llamacpp/
@@ -8,7 +8,7 @@ works listed below; each keeps its own license and copyright notice.
8
8
  -------------------------------------------------------------------------------
9
9
  1. NVIDIA CUTLASS — CuTe DSL examples BSD-3-Clause
10
10
  -------------------------------------------------------------------------------
11
- src/mjolnir/mjolnir/t5_decode.py (the Mjolnir decode-attention kernel) is
11
+ src/mjolnir/mjolnir/arch/thor/decode.py (the Mjolnir decode-attention kernel) is
12
12
  derived from
13
13
 
14
14
  examples/python/CuTeDSL/cute/blackwell/kernel/attention/mixed_input_fmha/
@@ -1,12 +1,12 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: mjolnir-kernel
3
- Version: 0.1.1
3
+ Version: 0.3.2
4
4
  Summary: Mjolnir: a paged decode-attention kernel for NVIDIA Jetson AGX Thor (sm_110) on tcgen05 / TMEM / TMA (CuTe DSL), with fp8 and 6-bit (e2m3) KV caches — packaged as the vLLM MJOLNIR attention backend.
5
- Project-URL: Homepage, https://github.com/thor-arena/mjolnir
6
- Project-URL: Documentation, https://github.com/thor-arena/mjolnir/tree/main/docs/kernel
7
- Project-URL: Repository, https://github.com/thor-arena/mjolnir.git
8
- Project-URL: Issues, https://github.com/thor-arena/mjolnir/issues
9
- Project-URL: Changelog, https://github.com/thor-arena/mjolnir/releases
5
+ Project-URL: Homepage, https://github.com/GoBrrr-ai/mjolnir
6
+ Project-URL: Documentation, https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel
7
+ Project-URL: Repository, https://github.com/GoBrrr-ai/mjolnir.git
8
+ Project-URL: Issues, https://github.com/GoBrrr-ai/mjolnir/issues
9
+ Project-URL: Changelog, https://github.com/GoBrrr-ai/mjolnir/releases
10
10
  Author-email: Mieszko Syty <mieszko@ms1design.pl>
11
11
  License-Expression: Apache-2.0 AND BSD-3-Clause
12
12
  License-File: LICENSE
@@ -41,7 +41,7 @@ This package is the kernel only. Serving, calibration and benchmarking are the
41
41
  [`mjolnir-cli`](https://pypi.org/project/mjolnir-cli/) package; benchmarks
42
42
  (Mjolnir vs FlashInfer vs FlashAttention 2 across context lengths), the design,
43
43
  and one chapter per kernel feature live in the project repository:
44
- **[thor-arena/mjolnir](https://github.com/thor-arena/mjolnir)** — numbers there
44
+ **[GoBrrr-ai/mjolnir](https://github.com/GoBrrr-ai/mjolnir)** — numbers there
45
45
  are rendered from the raw results, so they are never stale.
46
46
 
47
47
  ## What it does
@@ -102,7 +102,7 @@ The package registers a `vllm.general_plugins` entry point (`mjolnir:register`)
102
102
  that maps `AttentionBackendEnum.CUSTOM` — and `MJOLNIR`, when the vLLM build has
103
103
  that member — to `mjolnir.backend.MjolnirAttentionBackend` in every vLLM process.
104
104
  The two-line enum patch for vLLM is
105
- [`thor-mjolnir-attention-backend.patch`](https://github.com/thor-arena/mjolnir/blob/main/docker/vllm-thor/patches/thor-mjolnir-attention-backend.patch).
105
+ [`thor-mjolnir-attention-backend.patch`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docker/vllm-thor/patches/thor-mjolnir-attention-backend.patch).
106
106
 
107
107
  Look for this line in the server log:
108
108
 
@@ -131,7 +131,8 @@ runs on the fallback backend.
131
131
  | `MJOLNIR_ATTN_Q_DTYPE` | `fp8` | `fp8` (dynamic per-(seq, KV head) quantization) or `bf16` |
132
132
  | `MJOLNIR_ATTN_MAX_QLEN` | `1 + num_speculative_tokens` | the largest q_len Mjolnir takes |
133
133
  | `MJOLNIR_ATTN_CTAS` | `2` | CTAs per SM (2 is fastest on Thor) |
134
- | `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel) |
134
+ | `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel). On Orin it caps the split capacity the kernel's occupancy picks (8 for the mma kernel on AGX Orin) |
135
+
135
136
  | `MJOLNIR_ATTN_KV_STAGES` | `0` | KV ring depth override (0 = the tuned default) |
136
137
  | `MJOLNIR_ATTN_PDL` | `0` | `1`: programmatic dependent launch |
137
138
  | `MJOLNIR_ATTN_L2_PREFETCH` | `0` | TMA-prefetch K/V this many tiles ahead into L2 |
@@ -141,7 +142,8 @@ runs on the fallback backend.
141
142
  | `MJOLNIR_ATTN_DRAFT_T5` | `1` | `0`: the drafter's layers (MTP / EAGLE) run on the fallback — isolates the draft-step cost (`mjolnir verify profile`). fp8 KV only: refused on fp6 KV, whose fallback cannot be CUDA-graphed |
142
143
  | `MJOLNIR_ATTN_NONCAUSAL` | `1` | `0`: a parallel drafter's non-causal block steps (DSpark / DFlash) go to the fallback instead of the decode kernel — fp8 KV only for the same reason as above |
143
144
  | `MJOLNIR_ATTN_FUSED_DRAFT` | `1` | `0`: tell vLLM the backend cannot advance draft metadata in place → one graph replay per draft step instead of the fused draft loop |
144
- | `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV) |
145
+ | `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV). `MJOLNIR_LM_HEAD_TILES` overrides the GEMV launch config (`BN,BK,warps,stages`; default 64,256,8,5 — the 2026-11-04 serving A/B winner) |
146
+ | `MJOLNIR_MTP_FP8` | – | `1`: fp8-row-quantize the drafter's big unquantized linears (≥16M params) at load, same e4m3-per-row scheme + self-test (≤0.02 rel err) as the head; the bf16 originals are freed. Halves the MTP draft loop's streamed weight bytes |
145
147
  | `MJOLNIR_DRAFT_VOCAB` | – | path to a `draft_vocab.json` (`mjolnir calibrate --draft-vocab`): the drafter's logits only over those tokens (VocabTrim); `MJOLNIR_DRAFT_VOCAB_SIZE` keeps the first N (a shorter file is padded with the lowest unused ids, with a warning). The kept head stays in the drafter's native quant (NVFP4 4-bit in-kernel, FP8 row-slice) — never up-scaled to bf16; a bf16 head is the one exception (fp8 only when `MJOLNIR_LM_HEAD=fp8` is set). `MJOLNIR_DRAFT_VOCAB_NATIVE=0` forces the dequant→fp8 path |
146
148
  | `MJOLNIR_ATTN_VERBOSE` | – | `1`: print the kernel's compile-time notes (ring stages, TMEM columns) — the standalone harness always does |
147
149
  | `MJOLNIR_ATTN_BANNER` | `1` | `0`: skip the Mjolnir hammer banner in the vLLM boot log (`mjolnir.banner.emit`, once per start, coloured with the server's formatter) |
@@ -153,7 +155,7 @@ runs on the fallback backend.
153
155
 
154
156
  The pre-rename `THOR_T5_*` spellings are still honored. Each default is backed
155
157
  by a measurement — see the
156
- [kernel chapters](https://github.com/thor-arena/mjolnir/tree/main/docs/kernel).
158
+ [kernel chapters](https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel).
157
159
 
158
160
  ## fp6 (e2m3) KV in vLLM
159
161
 
@@ -166,25 +168,52 @@ expands the pages a prefill / mixed step reads to e4m3 for the FA4 fallback
166
168
  (channel scales applied, one descale per head). The scales come from a calibration run on fp8 KV:
167
169
 
168
170
  ```bash
169
- python -m mjolnir.calibrate --config <serve.yaml> --out kv_scales.json # in the image
170
- MJOLNIR_ATTN_KV_SCALES=kv_scales.json vllm serve ... --kv-cache-dtype fp6_e2m3 --attention-backend MJOLNIR
171
+ python -m mjolnir.calibrate --config <serve.yaml> --out kv_scales/e2m3.json # in the image
172
+ MJOLNIR_ATTN_KV_SCALES=kv_scales/e2m3.json vllm serve ... --kv-cache-dtype fp6_e2m3 --attention-backend MJOLNIR
171
173
  ```
172
174
 
173
175
  (`mjolnir calibrate` / `mjolnir serve` do both for you —
174
- [docs/faq/fp6-kv.md](https://github.com/thor-arena/mjolnir/blob/main/docs/faq/fp6-kv.md).)
176
+ [docs/faq/fp6-kv.md](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/faq/fp6-kv.md).)
175
177
 
176
178
  ## Beyond fp8: the harness
177
179
 
178
- `mjolnir/t5_decode.py` is the kernel **and** its standalone harness
179
- (`python -m mjolnir.t5_decode --help`) with a bit-faithful emulator as the
180
+ `mjolnir/arch/thor/decode.py` is the kernel; `mjolnir/arch/thor/harness.py`
181
+ (`python -m mjolnir.arch.thor.harness --help`) is its standalone harness, with a bit-faithful emulator as the
180
182
  correctness reference. The harness can also read KV formats vLLM cannot write
181
183
  yet — fp6 (`e2m3`/`e3m2`) and fp4 (`e2m1`) read directly by `tcgen05
182
184
  kind::f8f6f4`, and `nvfp4` — and defaults to e2m3 on head-major pages for fp8 Q.
183
185
  How they compare in speed and accuracy:
184
- [`kvfmt`](https://github.com/thor-arena/mjolnir/blob/main/docs/kernel/kvfmt.md) and
185
- [`quant`](https://github.com/thor-arena/mjolnir/blob/main/docs/kernel/quant.md).
186
+ [`kvfmt`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/kvfmt.md) and
187
+ [`quant`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/quant.md).
186
188
  Wiring a vLLM cache-write path for fp6 is an open contribution.
187
189
 
190
+ ## Platforms and architectures
191
+
192
+ The kernel is written for one architecture today — Thor (`sm_110`) — but the
193
+ package is organized by platform: `mjolnir/arch/` is the implemented-kernel
194
+ registry (`mjolnir.arch.thor` re-exports the exact runner surface the backend
195
+ consumes), and `mjolnir/platforms.py` is the torch-free leaf that maps a
196
+ compute capability to it. Thor and Jetson Orin (`sm_87`) are implemented;
197
+ DGX Spark (`sm_121`) is declared research — named in the matrix, no kernel
198
+ yet. Orin ships one kernel: the `mma.m16n8k16` decode kernel (fp8 KV at
199
+ hd 128/256, fp6-e2m3 KV hd 256, M ≤ 8, bf16 Q) — see `mjolnir/arch/orin/README.md`.
200
+
201
+ `kernel_for((11, 0))` returns `mjolnir.arch.thor`; any other compute
202
+ capability raises `PlatformNotSupported` naming the requested `sm_XX` and the
203
+ shipped platforms. The vLLM backend never sees that exception: its metadata
204
+ builder consults the pure `unsupported_reason()` and takes the normal
205
+ ineligible-config path — that layer's decode steps run on the fallback
206
+ backend, the serve continues. On a host without a usable CUDA device,
207
+ `MJOLNIR_ARCH` (`sm_110` or `11.0`) pins the architecture instead of the
208
+ torch read (the standalone harness).
209
+
210
+ There is no remembered platform state anywhere in the stack: the CLI side
211
+ resolves its platform per invocation (flag > `$MJOLNIR_PLATFORM` > host
212
+ detection — four source classes with provenance, the `/tmp/nv_jetson_model`
213
+ model copy corroboration-only), so the kernel is always resolved from the
214
+ device the run actually happens on. See
215
+ [docs/faq/platforms.md](../../docs/faq/platforms.md).
216
+
188
217
  ## Tests
189
218
 
190
219
  From a checkout, on a Thor:
@@ -210,11 +239,13 @@ is printed as information.
210
239
 
211
240
  | file | what |
212
241
  |---|---|
213
- | `mjolnir/t5_decode.py` | the CuTe-DSL kernel, the split-KV merge, the harness and the emulator |
242
+ | `mjolnir/arch/thor/` | the Thor kernel family: `decode.py` (the CuTe-DSL kernel + merge), `call.py`, `const.py`, `plan.py`, `runner.py`, `harness.py` (driver + emulator) |
214
243
  | `mjolnir/runner.py` | plan buffers, compile cache, launch, KV views (no vLLM import) |
215
244
  | `mjolnir/backend.py` | the vLLM backend, metadata builder, impl |
245
+ | `mjolnir/platforms.py` | arch → kernel resolution (`kernel_for`, `PlatformNotSupported`), `MJOLNIR_ARCH` detection — torch-free |
246
+ | `mjolnir/arch/thor.py` | the Thor (`sm_110`) kernel surface: re-exports of the runner |
216
247
  | `mjolnir/__init__.py` | `register()` — the vLLM plugin entry point |
217
248
 
218
- Apache-2.0, except `mjolnir/t5_decode.py`, which is derived from an NVIDIA
249
+ Apache-2.0, except `mjolnir/arch/thor/decode.py`, which is derived from an NVIDIA
219
250
  CUTLASS CuTe DSL example and stays under BSD-3-Clause — see `LICENSE` and
220
251
  `NOTICE` in this package.
@@ -13,7 +13,7 @@ This package is the kernel only. Serving, calibration and benchmarking are the
13
13
  [`mjolnir-cli`](https://pypi.org/project/mjolnir-cli/) package; benchmarks
14
14
  (Mjolnir vs FlashInfer vs FlashAttention 2 across context lengths), the design,
15
15
  and one chapter per kernel feature live in the project repository:
16
- **[thor-arena/mjolnir](https://github.com/thor-arena/mjolnir)** — numbers there
16
+ **[GoBrrr-ai/mjolnir](https://github.com/GoBrrr-ai/mjolnir)** — numbers there
17
17
  are rendered from the raw results, so they are never stale.
18
18
 
19
19
  ## What it does
@@ -74,7 +74,7 @@ The package registers a `vllm.general_plugins` entry point (`mjolnir:register`)
74
74
  that maps `AttentionBackendEnum.CUSTOM` — and `MJOLNIR`, when the vLLM build has
75
75
  that member — to `mjolnir.backend.MjolnirAttentionBackend` in every vLLM process.
76
76
  The two-line enum patch for vLLM is
77
- [`thor-mjolnir-attention-backend.patch`](https://github.com/thor-arena/mjolnir/blob/main/docker/vllm-thor/patches/thor-mjolnir-attention-backend.patch).
77
+ [`thor-mjolnir-attention-backend.patch`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docker/vllm-thor/patches/thor-mjolnir-attention-backend.patch).
78
78
 
79
79
  Look for this line in the server log:
80
80
 
@@ -103,7 +103,8 @@ runs on the fallback backend.
103
103
  | `MJOLNIR_ATTN_Q_DTYPE` | `fp8` | `fp8` (dynamic per-(seq, KV head) quantization) or `bf16` |
104
104
  | `MJOLNIR_ATTN_MAX_QLEN` | `1 + num_speculative_tokens` | the largest q_len Mjolnir takes |
105
105
  | `MJOLNIR_ATTN_CTAS` | `2` | CTAs per SM (2 is fastest on Thor) |
106
- | `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel) |
106
+ | `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel). On Orin it caps the split capacity the kernel's occupancy picks (8 for the mma kernel on AGX Orin) |
107
+
107
108
  | `MJOLNIR_ATTN_KV_STAGES` | `0` | KV ring depth override (0 = the tuned default) |
108
109
  | `MJOLNIR_ATTN_PDL` | `0` | `1`: programmatic dependent launch |
109
110
  | `MJOLNIR_ATTN_L2_PREFETCH` | `0` | TMA-prefetch K/V this many tiles ahead into L2 |
@@ -113,7 +114,8 @@ runs on the fallback backend.
113
114
  | `MJOLNIR_ATTN_DRAFT_T5` | `1` | `0`: the drafter's layers (MTP / EAGLE) run on the fallback — isolates the draft-step cost (`mjolnir verify profile`). fp8 KV only: refused on fp6 KV, whose fallback cannot be CUDA-graphed |
114
115
  | `MJOLNIR_ATTN_NONCAUSAL` | `1` | `0`: a parallel drafter's non-causal block steps (DSpark / DFlash) go to the fallback instead of the decode kernel — fp8 KV only for the same reason as above |
115
116
  | `MJOLNIR_ATTN_FUSED_DRAFT` | `1` | `0`: tell vLLM the backend cannot advance draft metadata in place → one graph replay per draft step instead of the fused draft loop |
116
- | `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV) |
117
+ | `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV). `MJOLNIR_LM_HEAD_TILES` overrides the GEMV launch config (`BN,BK,warps,stages`; default 64,256,8,5 — the 2026-11-04 serving A/B winner) |
118
+ | `MJOLNIR_MTP_FP8` | – | `1`: fp8-row-quantize the drafter's big unquantized linears (≥16M params) at load, same e4m3-per-row scheme + self-test (≤0.02 rel err) as the head; the bf16 originals are freed. Halves the MTP draft loop's streamed weight bytes |
117
119
  | `MJOLNIR_DRAFT_VOCAB` | – | path to a `draft_vocab.json` (`mjolnir calibrate --draft-vocab`): the drafter's logits only over those tokens (VocabTrim); `MJOLNIR_DRAFT_VOCAB_SIZE` keeps the first N (a shorter file is padded with the lowest unused ids, with a warning). The kept head stays in the drafter's native quant (NVFP4 4-bit in-kernel, FP8 row-slice) — never up-scaled to bf16; a bf16 head is the one exception (fp8 only when `MJOLNIR_LM_HEAD=fp8` is set). `MJOLNIR_DRAFT_VOCAB_NATIVE=0` forces the dequant→fp8 path |
118
120
  | `MJOLNIR_ATTN_VERBOSE` | – | `1`: print the kernel's compile-time notes (ring stages, TMEM columns) — the standalone harness always does |
119
121
  | `MJOLNIR_ATTN_BANNER` | `1` | `0`: skip the Mjolnir hammer banner in the vLLM boot log (`mjolnir.banner.emit`, once per start, coloured with the server's formatter) |
@@ -125,7 +127,7 @@ runs on the fallback backend.
125
127
 
126
128
  The pre-rename `THOR_T5_*` spellings are still honored. Each default is backed
127
129
  by a measurement — see the
128
- [kernel chapters](https://github.com/thor-arena/mjolnir/tree/main/docs/kernel).
130
+ [kernel chapters](https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel).
129
131
 
130
132
  ## fp6 (e2m3) KV in vLLM
131
133
 
@@ -138,25 +140,52 @@ expands the pages a prefill / mixed step reads to e4m3 for the FA4 fallback
138
140
  (channel scales applied, one descale per head). The scales come from a calibration run on fp8 KV:
139
141
 
140
142
  ```bash
141
- python -m mjolnir.calibrate --config <serve.yaml> --out kv_scales.json # in the image
142
- MJOLNIR_ATTN_KV_SCALES=kv_scales.json vllm serve ... --kv-cache-dtype fp6_e2m3 --attention-backend MJOLNIR
143
+ python -m mjolnir.calibrate --config <serve.yaml> --out kv_scales/e2m3.json # in the image
144
+ MJOLNIR_ATTN_KV_SCALES=kv_scales/e2m3.json vllm serve ... --kv-cache-dtype fp6_e2m3 --attention-backend MJOLNIR
143
145
  ```
144
146
 
145
147
  (`mjolnir calibrate` / `mjolnir serve` do both for you —
146
- [docs/faq/fp6-kv.md](https://github.com/thor-arena/mjolnir/blob/main/docs/faq/fp6-kv.md).)
148
+ [docs/faq/fp6-kv.md](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/faq/fp6-kv.md).)
147
149
 
148
150
  ## Beyond fp8: the harness
149
151
 
150
- `mjolnir/t5_decode.py` is the kernel **and** its standalone harness
151
- (`python -m mjolnir.t5_decode --help`) with a bit-faithful emulator as the
152
+ `mjolnir/arch/thor/decode.py` is the kernel; `mjolnir/arch/thor/harness.py`
153
+ (`python -m mjolnir.arch.thor.harness --help`) is its standalone harness, with a bit-faithful emulator as the
152
154
  correctness reference. The harness can also read KV formats vLLM cannot write
153
155
  yet — fp6 (`e2m3`/`e3m2`) and fp4 (`e2m1`) read directly by `tcgen05
154
156
  kind::f8f6f4`, and `nvfp4` — and defaults to e2m3 on head-major pages for fp8 Q.
155
157
  How they compare in speed and accuracy:
156
- [`kvfmt`](https://github.com/thor-arena/mjolnir/blob/main/docs/kernel/kvfmt.md) and
157
- [`quant`](https://github.com/thor-arena/mjolnir/blob/main/docs/kernel/quant.md).
158
+ [`kvfmt`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/kvfmt.md) and
159
+ [`quant`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/quant.md).
158
160
  Wiring a vLLM cache-write path for fp6 is an open contribution.
159
161
 
162
+ ## Platforms and architectures
163
+
164
+ The kernel is written for one architecture today — Thor (`sm_110`) — but the
165
+ package is organized by platform: `mjolnir/arch/` is the implemented-kernel
166
+ registry (`mjolnir.arch.thor` re-exports the exact runner surface the backend
167
+ consumes), and `mjolnir/platforms.py` is the torch-free leaf that maps a
168
+ compute capability to it. Thor and Jetson Orin (`sm_87`) are implemented;
169
+ DGX Spark (`sm_121`) is declared research — named in the matrix, no kernel
170
+ yet. Orin ships one kernel: the `mma.m16n8k16` decode kernel (fp8 KV at
171
+ hd 128/256, fp6-e2m3 KV hd 256, M ≤ 8, bf16 Q) — see `mjolnir/arch/orin/README.md`.
172
+
173
+ `kernel_for((11, 0))` returns `mjolnir.arch.thor`; any other compute
174
+ capability raises `PlatformNotSupported` naming the requested `sm_XX` and the
175
+ shipped platforms. The vLLM backend never sees that exception: its metadata
176
+ builder consults the pure `unsupported_reason()` and takes the normal
177
+ ineligible-config path — that layer's decode steps run on the fallback
178
+ backend, the serve continues. On a host without a usable CUDA device,
179
+ `MJOLNIR_ARCH` (`sm_110` or `11.0`) pins the architecture instead of the
180
+ torch read (the standalone harness).
181
+
182
+ There is no remembered platform state anywhere in the stack: the CLI side
183
+ resolves its platform per invocation (flag > `$MJOLNIR_PLATFORM` > host
184
+ detection — four source classes with provenance, the `/tmp/nv_jetson_model`
185
+ model copy corroboration-only), so the kernel is always resolved from the
186
+ device the run actually happens on. See
187
+ [docs/faq/platforms.md](../../docs/faq/platforms.md).
188
+
160
189
  ## Tests
161
190
 
162
191
  From a checkout, on a Thor:
@@ -182,11 +211,13 @@ is printed as information.
182
211
 
183
212
  | file | what |
184
213
  |---|---|
185
- | `mjolnir/t5_decode.py` | the CuTe-DSL kernel, the split-KV merge, the harness and the emulator |
214
+ | `mjolnir/arch/thor/` | the Thor kernel family: `decode.py` (the CuTe-DSL kernel + merge), `call.py`, `const.py`, `plan.py`, `runner.py`, `harness.py` (driver + emulator) |
186
215
  | `mjolnir/runner.py` | plan buffers, compile cache, launch, KV views (no vLLM import) |
187
216
  | `mjolnir/backend.py` | the vLLM backend, metadata builder, impl |
217
+ | `mjolnir/platforms.py` | arch → kernel resolution (`kernel_for`, `PlatformNotSupported`), `MJOLNIR_ARCH` detection — torch-free |
218
+ | `mjolnir/arch/thor.py` | the Thor (`sm_110`) kernel surface: re-exports of the runner |
188
219
  | `mjolnir/__init__.py` | `register()` — the vLLM plugin entry point |
189
220
 
190
- Apache-2.0, except `mjolnir/t5_decode.py`, which is derived from an NVIDIA
221
+ Apache-2.0, except `mjolnir/arch/thor/decode.py`, which is derived from an NVIDIA
191
222
  CUTLASS CuTe DSL example and stays under BSD-3-Clause — see `LICENSE` and
192
223
  `NOTICE` in this package.
@@ -1,6 +1,6 @@
1
1
  """Mjolnir — fp8 paged decode attention for vLLM on Jetson AGX Thor (sm_110).
2
2
 
3
- The kernel (codename **T5**, ``mjolnir.t5_decode``) is a CuTe-DSL tcgen05 +
3
+ The kernel (codename **T5**, ``mjolnir.arch.thor.decode``) is a CuTe-DSL tcgen05 +
4
4
  TMEM + TMA decode kernel; ``mjolnir.backend`` wraps it as a vLLM attention
5
5
  backend and ``mjolnir.runner`` holds the plan buffers, compile cache and KV
6
6
  views.
@@ -24,7 +24,7 @@ from __future__ import annotations
24
24
  import os
25
25
 
26
26
  __all__ = ["BACKEND_NAME", "BACKEND_PATH", "__version__", "register"]
27
- __version__ = "0.1.1"
27
+ __version__ = "0.3.2"
28
28
 
29
29
  BACKEND_NAME = "MJOLNIR"
30
30
  BACKEND_PATH = "mjolnir.backend.MjolnirAttentionBackend"
@@ -60,6 +60,22 @@ def has_native_member() -> bool:
60
60
  def register() -> None:
61
61
  if _disabled():
62
62
  return
63
+ # vLLM > 0.32.0: the fp6_e2m3 KV dtype registers through the pluggable
64
+ # dtype API (vLLM #52670) instead of the image patch — a no-op on
65
+ # 0.32.0 and on a base where the patch already added it.
66
+ try:
67
+ from .kvdtype import register as _register_kv_dtype
68
+
69
+ _register_kv_dtype()
70
+ except Exception as e: # noqa: BLE001 - never break vLLM startup
71
+ import logging
72
+
73
+ logging.getLogger("vllm.mjolnir").warning("Mjolnir fp6 KV dtype not registered: %r", e)
74
+ if os.environ.get("MJOLNIR_ATTN_DEBUG", "").strip():
75
+ # The IMA hunt: the engine core is a child process that inherits this
76
+ # env before its CUDA runtime initializes, so faulting kernels are
77
+ # named at their launch instead of surfacing on a later sync.
78
+ os.environ.setdefault("CUDA_LAUNCH_BLOCKING", "1")
63
79
  enum, register_backend = _registry()
64
80
  if BACKEND_NAME in enum.__members__:
65
81
  register_backend(enum[BACKEND_NAME], BACKEND_PATH)
@@ -0,0 +1,7 @@
1
+ """Implemented-kernel registry: one subpackage per supported architecture
2
+ (``mjolnir.arch.thor``).
3
+
4
+ ``mjolnir.platforms.kernel_for`` maps a compute capability to a dotted module
5
+ path here. Importing a subpackage pulls the CUTLASS DSL (heavy) — only do it
6
+ when a device is real.
7
+ """
@@ -0,0 +1,102 @@
1
+ # Orin kernel family — the mma.sync decode kernel (sm_87)
2
+
3
+ This subpackage is the Mjolnir kernel for **Jetson AGX Orin 64GB (`sm_87`,
4
+ GA10B Ampere)** — targeted at that one architecture (Thor keeps the
5
+ `tcgen05` `t5_*` family; this family never runs on sm_100/sm_110). The
6
+ family's only kernel is the tensor-core `mma.m16n8k16` decode kernel.
7
+
8
+ ## What the kernel is
9
+
10
+ One CTA per `(kv-head, split, batch)`, 128 threads = 4 warps; the M tile
11
+ = 64 rows split across the warps (one 16-row mma atom each); the KV rows
12
+ stage through `cp.async` (32 rows per stage), dequantize fp8→bf16 in
13
+ smem, and ride the reference's `ldmatrix → mma.m16n8k16` path. `M ≤ 8`
14
+ (the MTP=7 verify shape), split-KV with chunk-normalized fp32 partials +
15
+ the fused LSE combine, causal and non-causal (the MTP drafts see one
16
+ fewer future key each). **KV formats: fp8 e4m3 and fp6 e2m3** — the fp6
17
+ rows read packed (d·3/4 = 192 bytes for d=256, 4 codes per 3 bytes LE)
18
+ through the 64-entry `kvfp6.e2m3_values` LUT; the per-channel scales
19
+ fold into Q (`q_mul`) and post-multiply the output (`o_mul`) in the
20
+ runner, so the kernel's math is untouched (M1+M2).
21
+
22
+ ## The target to beat
23
+
24
+ The native sm_87 path in vLLM 0.32.0 is **FlashInfer's tensor-core FA2
25
+ decode** — confirmed at both tags: `vllm/v1/attention/backends/flashinfer.py`
26
+ constructs `BatchDecodeWithPagedKVCacheWrapper(..., use_tensor_cores=True)`
27
+ unconditionally, which routes to the FA2 prefill kernel
28
+ (`get_batch_prefill_module("fa2", ...)`); XQA/trtllm-gen are SM90/100/120-only.
29
+ Its Ampere shape for our model (packed qo = q_len·g = 6..24 ≤ 16 →
30
+ `CTA_TILE_Q = 16`, 1 Q-warp + 4 KV-warps = 128 threads):
31
+
32
+ * **NUM_MMA_KV = 2 → 128 KV rows per iteration** (fp8: 32 KB K+V smem per
33
+ step; the launcher's 2-CTA/SM budget against the *100 KB-class* Ampere
34
+ parts — Orin's 163 KB is headroom it never uses);
35
+ * fp8 KV is converted to fp16 **in registers before every HMMA**
36
+ (`fast_dequant_f8f16x4`, ≈2–4 ALU ops per 4 values — no fp8 cvt or mma
37
+ exists below sm_89) — a per-value tax the mma pays once per loaded
38
+ vector in smem instead;
39
+ * Q stays **bf16** on SM87 + fp8 KV ("architectures with only fa2 cannot
40
+ consume FP8 queries");
41
+ * split-KV is capped at 2 CTAs/SM (32 CTAs on Orin's 16 SMs), the LSE
42
+ merge is a **second kernel launch**, and PDL does not exist on
43
+ Ampere (`device_support_pdl` requires major ≥ 9) — a hard serial
44
+ launch floor per decode step that Thor does not pay.
45
+
46
+ Hardware reference (AGX Orin 64GB): 2048 cores / **16 SMs** / 64 3rd-gen
47
+ Tensor Cores / 5.3 TFLOPS fp32 / ~42.5 TFLOPS fp16 (derived) /
48
+ **204.8 GB/s** LPDDR5 / 4 MB L2 / 163 KB smem per block / 1.3 GHz.
49
+
50
+ ## Measured status
51
+
52
+ * **The mma kernel**: `mjolnir verify sweep` same-run vs FlashInfer
53
+ FA2-TC across the M/L band (research/runs/orin1/notes/
54
+ mma-kernel-design-2.md).
55
+ * **The mma kernel (2026-10-26)**: serving-validated end-to-end — the
56
+ A/B/C matrix byte-identical correct, the spec sweep 14/14 (Mjolnir
57
+ 19.9 t/s/req vs FlashInfer 18.9 at c=1, +5.0% aggregate at c=4,
58
+ identical per-position acceptance), fp6 (M1+M2) probe-validated on
59
+ both pool layouts and the image canary (`kernel_test_orin_gates.py`).
60
+
61
+ ## The fp6 e2m3 KV path (M1+M2, landed)
62
+
63
+ * **M1 (in-kernel)**: the mma reads the packed rows directly — the
64
+ 192-byte row moves as 12 × 16-byte cp.async chunks; the codes
65
+ unpack at bit 6i and dequantize through the LUT into the bf16 mma
66
+ operands. The per-head descales ride the accumulator
67
+ (`scale_eff` / the epilogue) exactly like fp8.
68
+ * **M2 (per-channel)**: the per-(layer, KV head, channel) scales from
69
+ `mjolnir calibrate` fold into Q pre-launch (`q_mul`) and
70
+ post-multiply the output (`o_mul`) — the kernel untouched
71
+ (`mjolnir/kvfp6.py`, the calibration `kv_scales/e2m3-awq.json`).
72
+
73
+ ## hd128 (shipped, 2026-10-31)
74
+
75
+ The mma serves **head_size 128 and 256 for fp8** from one D-parameterized
76
+ body: the staging, the fragment counts and the PV ranges all derive from
77
+ `self.HEAD_DIM`, the constructor takes the head size, and the runner
78
+ gates `{128, 256}`, keys the binary on it (the TVM-FFI dims bake it) and
79
+ passes it to the split-KV combine — whose elements-per-thread is
80
+ `d / NUM_THREADS`. That combine was the whole hd128 bug: it baked
81
+ D=256, so at hd128 its second element per thread read and wrote past
82
+ the row — masked whenever `max_splits=1` (the write-through path), a
83
+ 9.2 rel-err otherwise. fp6 stays **hd256-only**: the e2m3 D2 recipe's
84
+ byte-lane phase is static per thread and assumes 64 groups per row
85
+ (a per-iteration phase would serve hd128's 32).
86
+
87
+ Verified on the harness (rel err 0.0016–0.0019 vs the fp64 reference):
88
+ L 1K–262K, q_len 3/4/5, HND + NHD, split caps 2/4/8; the hd256 fp8 and
89
+ fp6 regressions clean. At M=5 / L=64K the hd128 shape graphs 600 µs vs
90
+ hd256's 967 µs (1.61× — an hd128 model streams half the KV bytes), and
91
+ the kernel drops to 128 regs / 27 KB smem (4 CTAs/SM by registers) from
92
+ hd256's 236 regs / 83 KB (2 CTAs/SM).
93
+
94
+ ## Not portable from Thor (deliberate)
95
+
96
+ * `tcgen05 kind::f8f6f4` (the Thor fp6 read) — Ampere has no such MMA;
97
+ fp6 read here is unpack + `mma.sync`.
98
+ * PDL / programmatic dependent launch — sm_90+.
99
+ * fp8 lm_head (`MJOLNIR_LM_HEAD=fp8`) — no fp8 tensor cores on Ampere;
100
+ the Triton fp8 GEMM would be compute-bound. Orin keeps a bf16 lm_head.
101
+ * The Thor-tuned ring depths / split plans — retuned for 16 SMs,
102
+ 100 KiB smem and LPDDR5x latency; the sm_110 numbers do not transfer.
@@ -0,0 +1,82 @@
1
+ """The Orin (``sm_87``) kernel family: the mma.sync tensor-core decode kernel.
2
+
3
+ One subpackage per implemented architecture, sibling to
4
+ :mod:`mjolnir.arch.thor` (see that module's docstring for the surface
5
+ contract). The family mirrors the Thor layout:
6
+
7
+ * :mod:`mjolnir.arch.orin.mma` — the tensor-core (mma.sync m16n8k16)
8
+ decode kernel: fp8 (head_dim 128/256) and fp6-e2m3 (head_dim 256) KV,
9
+ bf16 Q, q_len <= 8
10
+ (decode and the MTP verify). The family's only kernel;
11
+ * :mod:`mjolnir.arch.orin.plan` — the split-KV planner (FlashInfer
12
+ Alg. 1, board-agnostic: the SM count is a device query, so AGX Orin
13
+ 64/32 GB, Orin NX and Orin Nano all plan correctly);
14
+ * :mod:`mjolnir.arch.orin.combine` — the fused split-KV LSE merge;
15
+ * :mod:`mjolnir.arch.orin.runner` — the plan buffers, compile cache and
16
+ launch; :class:`Runner` below is the surface the vLLM backend consumes.
17
+
18
+ Do not import this package on a device whose compute capability is not
19
+ (8, 7): the DSL compile and the module's budget asserts assume GA10B.
20
+ """
21
+
22
+ import logging
23
+
24
+ from mjolnir.runner import env_flag
25
+
26
+ from .runner import OrinDecodeRunner
27
+
28
+ __all__ = ["OrinDecodeRunner", "Runner"]
29
+
30
+ _log = logging.getLogger("vllm.mjolnir")
31
+
32
+ MMA_MAX_Q_LEN = 8 # mma.MMADecodeKernel.MAX_Q_LEN
33
+ MMA_KV_FORMATS = ("fp8", "e2m3") # the mma kernel's KV formats
34
+
35
+
36
+ class Runner(OrinDecodeRunner):
37
+ """The backend-facing runner surface (``mjolnir.platforms.kernel_for``
38
+ hands this to the vLLM backend). Accepts the shared backend kwargs and
39
+ absorbs the Thor-only levers as inert attributes: the Orin grid has no
40
+ CTAs/SM knob, and PDL / L2-prefetch / evict-first hints do not exist
41
+ below sm_90 — the attributes exist so the backend's builder logging and
42
+ metadata paths read them unchanged.
43
+
44
+ Two backend kwargs are translated to this family's meaning:
45
+
46
+ * ``max_splits`` (``MJOLNIR_ATTN_MAX_SPLITS``, "max KV chunks per
47
+ sequence") caps the auto split capacity — it used to pin it, so the
48
+ backend's default 16 silently replaced the mma's 2-CTA-occupancy
49
+ optimum of 8 (one wave on 16 SMs) with the over-split 16;
50
+ * ``q_mode``: the mma kernel takes a bf16 query, so the runner always
51
+ runs bf16 Q."""
52
+
53
+ def __init__(
54
+ self,
55
+ *,
56
+ ctas_per_sm: int = 2,
57
+ pdl: bool = False,
58
+ l2_prefetch: int = 0,
59
+ kv_evict_first: bool = False,
60
+ max_splits: int = 0,
61
+ **kw,
62
+ ):
63
+ head_size = kw["head_size"]
64
+ kv_format = kw.get("kv_format", "fp8")
65
+ max_q_len = kw.get("max_q_len", 1)
66
+ supported = (256,) if kv_format == "e2m3" else (128, 256)
67
+ if head_size not in supported:
68
+ raise ValueError(
69
+ f"the Orin mma kernel serves head_size {'/'.join(map(str, supported))} for {kv_format} KV (got {head_size})"
70
+ )
71
+ if kv_format not in MMA_KV_FORMATS:
72
+ raise ValueError(f"the Orin mma kernel serves {'/'.join(MMA_KV_FORMATS)} KV only (got {kv_format!r})")
73
+ if max_q_len > MMA_MAX_Q_LEN:
74
+ raise ValueError(f"the Orin mma kernel serves q_len <= {MMA_MAX_Q_LEN} (got {max_q_len})")
75
+ if env_flag("MJOLNIR_ATTN_ORIN_IMPL", "") not in ("", "mma"):
76
+ raise ValueError("MJOLNIR_ATTN_ORIN_IMPL: only 'mma' is supported on Orin")
77
+ kw["q_mode"] = "bf16"
78
+ super().__init__(split_cap=max_splits, **kw)
79
+ self.ctas_per_sm = ctas_per_sm
80
+ self.pdl = pdl
81
+ self.l2_prefetch = l2_prefetch
82
+ self.kv_evict_first = kv_evict_first
@@ -0,0 +1,111 @@
1
+ """The fused split-KV combine for the Orin MMA decode kernel.
2
+
3
+ The runner's partials are (ns, rows, h_q, d) fp32 O and (ns, b, h_q, ql)
4
+ fp32 LSE; the merge is O = Σ_s exp(lse_s − L)·O_s / Σ_s exp(lse_s − L)
5
+ with L = max_s lse_s (the chunk-normalized contract the kernels write).
6
+ The eager torch form issued ~20 small kernels (~150 us of GPU per call
7
+ — more than the decode kernel itself); this is one kernel, one pass
8
+ over the partials, the FlashInfer MergeStates cost class.
9
+
10
+ One block per (row, q-head): the block loads its ns LSE values, then
11
+ each thread accumulates its d/NUM_THREADS elements across the splits
12
+ (2 at hd256, 1 at hd128 — the head size rides the compile as a
13
+ constexpr). ns/rows/h_q/b/ql are runtime Int32s so one compile serves
14
+ every capture shape per head size. The tensor layouts
15
+ are the runner's capacity buffers (fixed at construction, so the baked
16
+ extents are stable). The empty-chunk splits write lse = -inf -> weight
17
+ 0; an all-empty row normalizes to 0 (the torch combine's clamp_min) —
18
+ L is floored at -FLT_MAX so that row never computes -inf - -inf.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import cutlass
24
+ import cutlass.cute as cute
25
+ import cuda.bindings.driver as cuda
26
+ from cutlass import Float32, Int32, const_expr
27
+
28
+ NUM_THREADS = 128
29
+ _TINY = 1.1754944e-35 # torch.finfo(torch.float32).tiny, rounded
30
+ _NEG_FLOOR = -3.4028234e38 # -torch.finfo(torch.float32).max: the finite L floor
31
+
32
+
33
+ @cute.kernel
34
+ def combine_kernel(
35
+ mO_part: cute.Tensor, # (ns_cap*rows_cap*h_q*D) fp32 flat
36
+ mLse: cute.Tensor, # (ns_cap*b_cap*h_q*ql_cap) fp32 flat
37
+ mOut: cute.Tensor, # (rows_cap*h_q*D) bf16 flat
38
+ d: cutlass.Constexpr[int], # the head size (128 or 256): the elements/thread = d/NUM_THREADS
39
+ max_ns: cutlass.Constexpr[int], # the unroll/predication bound (= ns_cap)
40
+ ns: Int32,
41
+ rows: Int32,
42
+ h_q: Int32,
43
+ b: Int32,
44
+ ql: Int32,
45
+ ):
46
+ bid = cute.arch.block_idx()[0]
47
+ tidx = cute.arch.thread_idx()[0]
48
+ r = bid // h_q
49
+ h = bid - r * h_q
50
+ bb = r // ql
51
+ qq = r - bb * ql
52
+ ls_s = b * h_q * ql
53
+ ls_b = h_q * ql
54
+
55
+ # Both loops unroll over max_ns with the s < ns predication: the
56
+ # dynamic-range form serialized every split's load behind the loop
57
+ # branch (~34 us; the FI merge overlaps all partial rows in flight
58
+ # and lands at 11). Unrolled, the loads issue independently and the
59
+ # latency hides under the acc chain.
60
+ # 1. L = max_s lse[s, h, r], floored at a FINITE -FLT_MAX: a row whose
61
+ # every split is empty (all lse = -inf — a CUDA-graph padding request
62
+ # with seq_len 0, or a dummy/profile step) would otherwise get
63
+ # L = -inf and w = exp2(-inf - -inf) = NaN, a NaN output row instead
64
+ # of the documented 0 (and a NaN row poisons any per-tensor reduction
65
+ # downstream). With the floor, every -inf lse weighs exactly 0 and a
66
+ # real lse (>= -FLT_MAX) is unaffected.
67
+ L = Float32(_NEG_FLOOR)
68
+ for s in cutlass.range_constexpr(max_ns):
69
+ if s < ns:
70
+ L = cutlass.max(L, mLse[s * ls_s + bb * ls_b + h * ql + qq])
71
+
72
+ # 2. the weighted sum over the splits; the exp weights recomputed per
73
+ # element pass (ns <= 64 exp2s are nothing next to the loads).
74
+ ln2e = Float32(1.4426950408889634) # 1/ln 2: exp(x) = exp2(x*log2 e)
75
+ sw = Float32(0.0)
76
+ acc0 = Float32(0.0)
77
+ acc1 = Float32(0.0)
78
+ for s in cutlass.range_constexpr(max_ns):
79
+ if s < ns:
80
+ w = cute.math.exp2((mLse[s * ls_s + bb * ls_b + h * ql + qq] - L) * ln2e, fastmath=True)
81
+ sw = sw + w
82
+ obase = s * rows * h_q * d + r * h_q * d + h * d
83
+ acc0 = acc0 + w * mO_part[obase + tidx]
84
+ if const_expr(d == 256):
85
+ acc1 = acc1 + w * mO_part[obase + tidx + NUM_THREADS]
86
+ scale = 1.0 / cutlass.max(sw, Float32(_TINY))
87
+ obase = r * h_q * d + h * d
88
+ mOut[obase + tidx] = (acc0 * scale).to(mOut.element_type)
89
+ if const_expr(d == 256):
90
+ mOut[obase + tidx + NUM_THREADS] = (acc1 * scale).to(mOut.element_type)
91
+
92
+
93
+ @cute.jit
94
+ def combine_launch(
95
+ mO_part: cute.Tensor,
96
+ mLse: cute.Tensor,
97
+ mOut: cute.Tensor,
98
+ d: cutlass.Constexpr[int],
99
+ max_ns: cutlass.Constexpr[int],
100
+ ns: Int32,
101
+ rows: Int32,
102
+ h_q: Int32,
103
+ b: Int32,
104
+ ql: Int32,
105
+ stream: cuda.CUstream = None,
106
+ ):
107
+ combine_kernel(mO_part, mLse, mOut, d, max_ns, ns, rows, h_q, b, ql).launch(
108
+ grid=(rows * h_q, 1, 1),
109
+ block=(NUM_THREADS, 1, 1),
110
+ stream=stream,
111
+ )