mjolnir-kernel 0.2.0__tar.gz → 0.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/.gitignore +8 -0
  2. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/NOTICE +1 -1
  3. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/PKG-INFO +25 -20
  4. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/README.md +19 -14
  5. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/__init__.py +18 -2
  6. mjolnir_kernel-0.3.3/mjolnir/arch/orin/README.md +102 -0
  7. mjolnir_kernel-0.3.3/mjolnir/arch/orin/__init__.py +82 -0
  8. mjolnir_kernel-0.3.3/mjolnir/arch/orin/combine.py +111 -0
  9. mjolnir_kernel-0.3.3/mjolnir/arch/orin/harness.py +483 -0
  10. mjolnir_kernel-0.3.3/mjolnir/arch/orin/mma.py +2295 -0
  11. mjolnir_kernel-0.3.3/mjolnir/arch/orin/plan.py +67 -0
  12. mjolnir_kernel-0.3.3/mjolnir/arch/orin/runner.py +686 -0
  13. mjolnir_kernel-0.3.3/mjolnir/arch/thor/__init__.py +45 -0
  14. mjolnir_kernel-0.2.0/mjolnir/t5_call.py → mjolnir_kernel-0.3.3/mjolnir/arch/thor/call.py +2 -2
  15. mjolnir_kernel-0.2.0/mjolnir/t5_decode.py → mjolnir_kernel-0.3.3/mjolnir/arch/thor/decode.py +4 -4
  16. mjolnir_kernel-0.2.0/mjolnir/t5_harness.py → mjolnir_kernel-0.3.3/mjolnir/arch/thor/harness.py +296 -21
  17. mjolnir_kernel-0.3.3/mjolnir/arch/thor/qsa.py +272 -0
  18. {mjolnir_kernel-0.2.0/mjolnir → mjolnir_kernel-0.3.3/mjolnir/arch/thor}/runner.py +29 -292
  19. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/backend.py +279 -370
  20. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/banner.py +1 -1
  21. mjolnir_kernel-0.3.3/mjolnir/builder.py +529 -0
  22. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/draftvocab.py +4 -0
  23. mjolnir_kernel-0.3.3/mjolnir/kvdtype.py +53 -0
  24. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/kvfp6.py +42 -3
  25. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/lmhead.py +182 -8
  26. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/platforms.py +22 -5
  27. mjolnir_kernel-0.3.3/mjolnir/qsa_gather.py +118 -0
  28. mjolnir_kernel-0.3.3/mjolnir/qsa_impl.py +421 -0
  29. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/quant.py +1 -1
  30. mjolnir_kernel-0.3.3/mjolnir/runner.py +329 -0
  31. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/pyproject.toml +5 -5
  32. mjolnir_kernel-0.2.0/mjolnir/arch/thor.py +0 -42
  33. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/LICENSE +0 -0
  34. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/arch/__init__.py +0 -0
  35. /mjolnir_kernel-0.2.0/mjolnir/t5_const.py → /mjolnir_kernel-0.3.3/mjolnir/arch/thor/const.py +0 -0
  36. /mjolnir_kernel-0.2.0/mjolnir/t5_plan.py → /mjolnir_kernel-0.3.3/mjolnir/arch/thor/plan.py +0 -0
  37. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/calibrate.py +0 -0
  38. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/kvscales.py +0 -0
  39. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/py.typed +0 -0
  40. {mjolnir_kernel-0.2.0 → mjolnir_kernel-0.3.3}/mjolnir/qpack.py +0 -0
@@ -1,9 +1,13 @@
1
1
  __pycache__/
2
2
  *.py[cod]
3
3
  .venv/
4
+ .playwright-mcp/
4
5
  .pytest_cache/
5
6
  .ruff_cache/
6
7
  .env
8
+ # the configs' env sidecars are plain mjolnir knobs (calibration paths, feature
9
+ # flags) and ride with the configs; real secrets stay out
10
+ !configs/**/.env
7
11
  # coverage artifacts (the CI test stage writes these; they regenerate)
8
12
  .coverage
9
13
  .coverage.*
@@ -24,6 +28,7 @@ benchmarks/raw/*
24
28
  !benchmarks/raw/accuracy/
25
29
  benchmarks/raw/accuracy/*/*/
26
30
  benchmarks/logs/
31
+ benchmarks/staging/
27
32
  task/
28
33
 
29
34
  # autoresearch: the live ledger is untracked (discards are `git reset --hard`);
@@ -32,3 +37,6 @@ research/ledger.jsonl
32
37
  research/ledger.tsv
33
38
  # …and the exported runs ARE committed (the generic runs/ rule above would hide them)
34
39
  !research/runs/
40
+ /cutlass/
41
+ /fa2/
42
+ /llamacpp/
@@ -8,7 +8,7 @@ works listed below; each keeps its own license and copyright notice.
8
8
  -------------------------------------------------------------------------------
9
9
  1. NVIDIA CUTLASS — CuTe DSL examples BSD-3-Clause
10
10
  -------------------------------------------------------------------------------
11
- src/mjolnir/mjolnir/t5_decode.py (the Mjolnir decode-attention kernel) is
11
+ src/mjolnir/mjolnir/arch/thor/decode.py (the Mjolnir decode-attention kernel) is
12
12
  derived from
13
13
 
14
14
  examples/python/CuTeDSL/cute/blackwell/kernel/attention/mixed_input_fmha/
@@ -1,12 +1,12 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: mjolnir-kernel
3
- Version: 0.2.0
3
+ Version: 0.3.3
4
4
  Summary: Mjolnir: a paged decode-attention kernel for NVIDIA Jetson AGX Thor (sm_110) on tcgen05 / TMEM / TMA (CuTe DSL), with fp8 and 6-bit (e2m3) KV caches — packaged as the vLLM MJOLNIR attention backend.
5
- Project-URL: Homepage, https://github.com/thor-arena/mjolnir
6
- Project-URL: Documentation, https://github.com/thor-arena/mjolnir/tree/main/docs/kernel
7
- Project-URL: Repository, https://github.com/thor-arena/mjolnir.git
8
- Project-URL: Issues, https://github.com/thor-arena/mjolnir/issues
9
- Project-URL: Changelog, https://github.com/thor-arena/mjolnir/releases
5
+ Project-URL: Homepage, https://github.com/GoBrrr-ai/mjolnir
6
+ Project-URL: Documentation, https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel
7
+ Project-URL: Repository, https://github.com/GoBrrr-ai/mjolnir.git
8
+ Project-URL: Issues, https://github.com/GoBrrr-ai/mjolnir/issues
9
+ Project-URL: Changelog, https://github.com/GoBrrr-ai/mjolnir/releases
10
10
  Author-email: Mieszko Syty <mieszko@ms1design.pl>
11
11
  License-Expression: Apache-2.0 AND BSD-3-Clause
12
12
  License-File: LICENSE
@@ -41,7 +41,7 @@ This package is the kernel only. Serving, calibration and benchmarking are the
41
41
  [`mjolnir-cli`](https://pypi.org/project/mjolnir-cli/) package; benchmarks
42
42
  (Mjolnir vs FlashInfer vs FlashAttention 2 across context lengths), the design,
43
43
  and one chapter per kernel feature live in the project repository:
44
- **[thor-arena/mjolnir](https://github.com/thor-arena/mjolnir)** — numbers there
44
+ **[GoBrrr-ai/mjolnir](https://github.com/GoBrrr-ai/mjolnir)** — numbers there
45
45
  are rendered from the raw results, so they are never stale.
46
46
 
47
47
  ## What it does
@@ -102,7 +102,7 @@ The package registers a `vllm.general_plugins` entry point (`mjolnir:register`)
102
102
  that maps `AttentionBackendEnum.CUSTOM` — and `MJOLNIR`, when the vLLM build has
103
103
  that member — to `mjolnir.backend.MjolnirAttentionBackend` in every vLLM process.
104
104
  The two-line enum patch for vLLM is
105
- [`thor-mjolnir-attention-backend.patch`](https://github.com/thor-arena/mjolnir/blob/main/docker/vllm-thor/patches/thor-mjolnir-attention-backend.patch).
105
+ [`thor-mjolnir-attention-backend.patch`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docker/vllm-thor/patches/thor-mjolnir-attention-backend.patch).
106
106
 
107
107
  Look for this line in the server log:
108
108
 
@@ -131,7 +131,8 @@ runs on the fallback backend.
131
131
  | `MJOLNIR_ATTN_Q_DTYPE` | `fp8` | `fp8` (dynamic per-(seq, KV head) quantization) or `bf16` |
132
132
  | `MJOLNIR_ATTN_MAX_QLEN` | `1 + num_speculative_tokens` | the largest q_len Mjolnir takes |
133
133
  | `MJOLNIR_ATTN_CTAS` | `2` | CTAs per SM (2 is fastest on Thor) |
134
- | `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel) |
134
+ | `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel). On Orin it caps the split capacity the kernel's occupancy picks (8 for the mma kernel on AGX Orin) |
135
+
135
136
  | `MJOLNIR_ATTN_KV_STAGES` | `0` | KV ring depth override (0 = the tuned default) |
136
137
  | `MJOLNIR_ATTN_PDL` | `0` | `1`: programmatic dependent launch |
137
138
  | `MJOLNIR_ATTN_L2_PREFETCH` | `0` | TMA-prefetch K/V this many tiles ahead into L2 |
@@ -141,7 +142,9 @@ runs on the fallback backend.
141
142
  | `MJOLNIR_ATTN_DRAFT_T5` | `1` | `0`: the drafter's layers (MTP / EAGLE) run on the fallback — isolates the draft-step cost (`mjolnir verify profile`). fp8 KV only: refused on fp6 KV, whose fallback cannot be CUDA-graphed |
142
143
  | `MJOLNIR_ATTN_NONCAUSAL` | `1` | `0`: a parallel drafter's non-causal block steps (DSpark / DFlash) go to the fallback instead of the decode kernel — fp8 KV only for the same reason as above |
143
144
  | `MJOLNIR_ATTN_FUSED_DRAFT` | `1` | `0`: tell vLLM the backend cannot advance draft metadata in place → one graph replay per draft step instead of the fused draft loop |
144
- | `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV) |
145
+ | `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV). `MJOLNIR_LM_HEAD_TILES` overrides the GEMV launch config (`BN,BK,warps,stages`; default 64,256,8,5 — the 2026-11-04 serving A/B winner) |
146
+ | `MJOLNIR_MTP_FP8` | – | `1`: fp8-row-quantize the drafter's big unquantized linears (≥16M params) at load, same e4m3-per-row scheme + self-test (≤0.02 rel err) as the head; the bf16 originals are freed. Halves the MTP draft loop's streamed weight bytes |
147
+ | `MJOLNIR_MTP_LAYER` | – | `fp8`: the older, narrower variant — W8A8 via cuBLASLt rowwise `scaled_mm` on the MTP layer + eh_proj `fc` only. **Measured negative on sm_110** (cuBLASLt fp8 ≈60% of peak on layer shapes; 16–22% slower decode, acceptance safe) — see config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K_M8`; revisit only with a faster skinny-shape fp8 GEMM |
145
148
  | `MJOLNIR_DRAFT_VOCAB` | – | path to a `draft_vocab.json` (`mjolnir calibrate --draft-vocab`): the drafter's logits only over those tokens (VocabTrim); `MJOLNIR_DRAFT_VOCAB_SIZE` keeps the first N (a shorter file is padded with the lowest unused ids, with a warning). The kept head stays in the drafter's native quant (NVFP4 4-bit in-kernel, FP8 row-slice) — never up-scaled to bf16; a bf16 head is the one exception (fp8 only when `MJOLNIR_LM_HEAD=fp8` is set). `MJOLNIR_DRAFT_VOCAB_NATIVE=0` forces the dequant→fp8 path |
146
149
  | `MJOLNIR_ATTN_VERBOSE` | – | `1`: print the kernel's compile-time notes (ring stages, TMEM columns) — the standalone harness always does |
147
150
  | `MJOLNIR_ATTN_BANNER` | `1` | `0`: skip the Mjolnir hammer banner in the vLLM boot log (`mjolnir.banner.emit`, once per start, coloured with the server's formatter) |
@@ -153,7 +156,7 @@ runs on the fallback backend.
153
156
 
154
157
  The pre-rename `THOR_T5_*` spellings are still honored. Each default is backed
155
158
  by a measurement — see the
156
- [kernel chapters](https://github.com/thor-arena/mjolnir/tree/main/docs/kernel).
159
+ [kernel chapters](https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel).
157
160
 
158
161
  ## fp6 (e2m3) KV in vLLM
159
162
 
@@ -171,18 +174,18 @@ MJOLNIR_ATTN_KV_SCALES=kv_scales/e2m3.json vllm serve ... --kv-cache-dtype fp6_e
171
174
  ```
172
175
 
173
176
  (`mjolnir calibrate` / `mjolnir serve` do both for you —
174
- [docs/faq/fp6-kv.md](https://github.com/thor-arena/mjolnir/blob/main/docs/faq/fp6-kv.md).)
177
+ [docs/faq/fp6-kv.md](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/faq/fp6-kv.md).)
175
178
 
176
179
  ## Beyond fp8: the harness
177
180
 
178
- `mjolnir/t5_decode.py` is the kernel **and** its standalone harness
179
- (`python -m mjolnir.t5_decode --help`) with a bit-faithful emulator as the
181
+ `mjolnir/arch/thor/decode.py` is the kernel; `mjolnir/arch/thor/harness.py`
182
+ (`python -m mjolnir.arch.thor.harness --help`) is its standalone harness, with a bit-faithful emulator as the
180
183
  correctness reference. The harness can also read KV formats vLLM cannot write
181
184
  yet — fp6 (`e2m3`/`e3m2`) and fp4 (`e2m1`) read directly by `tcgen05
182
185
  kind::f8f6f4`, and `nvfp4` — and defaults to e2m3 on head-major pages for fp8 Q.
183
186
  How they compare in speed and accuracy:
184
- [`kvfmt`](https://github.com/thor-arena/mjolnir/blob/main/docs/kernel/kvfmt.md) and
185
- [`quant`](https://github.com/thor-arena/mjolnir/blob/main/docs/kernel/quant.md).
187
+ [`kvfmt`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/kvfmt.md) and
188
+ [`quant`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/quant.md).
186
189
  Wiring a vLLM cache-write path for fp6 is an open contribution.
187
190
 
188
191
  ## Platforms and architectures
@@ -191,8 +194,10 @@ The kernel is written for one architecture today — Thor (`sm_110`) — but the
191
194
  package is organized by platform: `mjolnir/arch/` is the implemented-kernel
192
195
  registry (`mjolnir.arch.thor` re-exports the exact runner surface the backend
193
196
  consumes), and `mjolnir/platforms.py` is the torch-free leaf that maps a
194
- compute capability to it. Thor is implemented; DGX Spark (`sm_121`) and Jetson
195
- Orin (`sm_87`) are declared research — named in the matrix, no kernel yet.
197
+ compute capability to it. Thor and Jetson Orin (`sm_87`) are implemented;
198
+ DGX Spark (`sm_121`) is declared research — named in the matrix, no kernel
199
+ yet. Orin ships one kernel: the `mma.m16n8k16` decode kernel (fp8 KV at
200
+ hd 128/256, fp6-e2m3 KV hd 256, M ≤ 8, bf16 Q) — see `mjolnir/arch/orin/README.md`.
196
201
 
197
202
  `kernel_for((11, 0))` returns `mjolnir.arch.thor`; any other compute
198
203
  capability raises `PlatformNotSupported` naming the requested `sm_XX` and the
@@ -235,13 +240,13 @@ is printed as information.
235
240
 
236
241
  | file | what |
237
242
  |---|---|
238
- | `mjolnir/t5_decode.py` | the CuTe-DSL kernel, the split-KV merge, the harness and the emulator |
243
+ | `mjolnir/arch/thor/` | the Thor kernel family: `decode.py` (the CuTe-DSL kernel + merge), `call.py`, `const.py`, `plan.py`, `runner.py`, `harness.py` (driver + emulator) |
239
244
  | `mjolnir/runner.py` | plan buffers, compile cache, launch, KV views (no vLLM import) |
240
245
  | `mjolnir/backend.py` | the vLLM backend, metadata builder, impl |
241
246
  | `mjolnir/platforms.py` | arch → kernel resolution (`kernel_for`, `PlatformNotSupported`), `MJOLNIR_ARCH` detection — torch-free |
242
247
  | `mjolnir/arch/thor.py` | the Thor (`sm_110`) kernel surface: re-exports of the runner |
243
248
  | `mjolnir/__init__.py` | `register()` — the vLLM plugin entry point |
244
249
 
245
- Apache-2.0, except `mjolnir/t5_decode.py`, which is derived from an NVIDIA
250
+ Apache-2.0, except `mjolnir/arch/thor/decode.py`, which is derived from an NVIDIA
246
251
  CUTLASS CuTe DSL example and stays under BSD-3-Clause — see `LICENSE` and
247
252
  `NOTICE` in this package.
@@ -13,7 +13,7 @@ This package is the kernel only. Serving, calibration and benchmarking are the
13
13
  [`mjolnir-cli`](https://pypi.org/project/mjolnir-cli/) package; benchmarks
14
14
  (Mjolnir vs FlashInfer vs FlashAttention 2 across context lengths), the design,
15
15
  and one chapter per kernel feature live in the project repository:
16
- **[thor-arena/mjolnir](https://github.com/thor-arena/mjolnir)** — numbers there
16
+ **[GoBrrr-ai/mjolnir](https://github.com/GoBrrr-ai/mjolnir)** — numbers there
17
17
  are rendered from the raw results, so they are never stale.
18
18
 
19
19
  ## What it does
@@ -74,7 +74,7 @@ The package registers a `vllm.general_plugins` entry point (`mjolnir:register`)
74
74
  that maps `AttentionBackendEnum.CUSTOM` — and `MJOLNIR`, when the vLLM build has
75
75
  that member — to `mjolnir.backend.MjolnirAttentionBackend` in every vLLM process.
76
76
  The two-line enum patch for vLLM is
77
- [`thor-mjolnir-attention-backend.patch`](https://github.com/thor-arena/mjolnir/blob/main/docker/vllm-thor/patches/thor-mjolnir-attention-backend.patch).
77
+ [`thor-mjolnir-attention-backend.patch`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docker/vllm-thor/patches/thor-mjolnir-attention-backend.patch).
78
78
 
79
79
  Look for this line in the server log:
80
80
 
@@ -103,7 +103,8 @@ runs on the fallback backend.
103
103
  | `MJOLNIR_ATTN_Q_DTYPE` | `fp8` | `fp8` (dynamic per-(seq, KV head) quantization) or `bf16` |
104
104
  | `MJOLNIR_ATTN_MAX_QLEN` | `1 + num_speculative_tokens` | the largest q_len Mjolnir takes |
105
105
  | `MJOLNIR_ATTN_CTAS` | `2` | CTAs per SM (2 is fastest on Thor) |
106
- | `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel) |
106
+ | `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel). On Orin it caps the split capacity the kernel's occupancy picks (8 for the mma kernel on AGX Orin) |
107
+
107
108
  | `MJOLNIR_ATTN_KV_STAGES` | `0` | KV ring depth override (0 = the tuned default) |
108
109
  | `MJOLNIR_ATTN_PDL` | `0` | `1`: programmatic dependent launch |
109
110
  | `MJOLNIR_ATTN_L2_PREFETCH` | `0` | TMA-prefetch K/V this many tiles ahead into L2 |
@@ -113,7 +114,9 @@ runs on the fallback backend.
113
114
  | `MJOLNIR_ATTN_DRAFT_T5` | `1` | `0`: the drafter's layers (MTP / EAGLE) run on the fallback — isolates the draft-step cost (`mjolnir verify profile`). fp8 KV only: refused on fp6 KV, whose fallback cannot be CUDA-graphed |
114
115
  | `MJOLNIR_ATTN_NONCAUSAL` | `1` | `0`: a parallel drafter's non-causal block steps (DSpark / DFlash) go to the fallback instead of the decode kernel — fp8 KV only for the same reason as above |
115
116
  | `MJOLNIR_ATTN_FUSED_DRAFT` | `1` | `0`: tell vLLM the backend cannot advance draft metadata in place → one graph replay per draft step instead of the fused draft loop |
116
- | `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV) |
117
+ | `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV). `MJOLNIR_LM_HEAD_TILES` overrides the GEMV launch config (`BN,BK,warps,stages`; default 64,256,8,5 — the 2026-11-04 serving A/B winner) |
118
+ | `MJOLNIR_MTP_FP8` | – | `1`: fp8-row-quantize the drafter's big unquantized linears (≥16M params) at load, same e4m3-per-row scheme + self-test (≤0.02 rel err) as the head; the bf16 originals are freed. Halves the MTP draft loop's streamed weight bytes |
119
+ | `MJOLNIR_MTP_LAYER` | – | `fp8`: the older, narrower variant — W8A8 via cuBLASLt rowwise `scaled_mm` on the MTP layer + eh_proj `fc` only. **Measured negative on sm_110** (cuBLASLt fp8 ≈60% of peak on layer shapes; 16–22% slower decode, acceptance safe) — see config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K_M8`; revisit only with a faster skinny-shape fp8 GEMM |
117
120
  | `MJOLNIR_DRAFT_VOCAB` | – | path to a `draft_vocab.json` (`mjolnir calibrate --draft-vocab`): the drafter's logits only over those tokens (VocabTrim); `MJOLNIR_DRAFT_VOCAB_SIZE` keeps the first N (a shorter file is padded with the lowest unused ids, with a warning). The kept head stays in the drafter's native quant (NVFP4 4-bit in-kernel, FP8 row-slice) — never up-scaled to bf16; a bf16 head is the one exception (fp8 only when `MJOLNIR_LM_HEAD=fp8` is set). `MJOLNIR_DRAFT_VOCAB_NATIVE=0` forces the dequant→fp8 path |
118
121
  | `MJOLNIR_ATTN_VERBOSE` | – | `1`: print the kernel's compile-time notes (ring stages, TMEM columns) — the standalone harness always does |
119
122
  | `MJOLNIR_ATTN_BANNER` | `1` | `0`: skip the Mjolnir hammer banner in the vLLM boot log (`mjolnir.banner.emit`, once per start, coloured with the server's formatter) |
@@ -125,7 +128,7 @@ runs on the fallback backend.
125
128
 
126
129
  The pre-rename `THOR_T5_*` spellings are still honored. Each default is backed
127
130
  by a measurement — see the
128
- [kernel chapters](https://github.com/thor-arena/mjolnir/tree/main/docs/kernel).
131
+ [kernel chapters](https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel).
129
132
 
130
133
  ## fp6 (e2m3) KV in vLLM
131
134
 
@@ -143,18 +146,18 @@ MJOLNIR_ATTN_KV_SCALES=kv_scales/e2m3.json vllm serve ... --kv-cache-dtype fp6_e
143
146
  ```
144
147
 
145
148
  (`mjolnir calibrate` / `mjolnir serve` do both for you —
146
- [docs/faq/fp6-kv.md](https://github.com/thor-arena/mjolnir/blob/main/docs/faq/fp6-kv.md).)
149
+ [docs/faq/fp6-kv.md](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/faq/fp6-kv.md).)
147
150
 
148
151
  ## Beyond fp8: the harness
149
152
 
150
- `mjolnir/t5_decode.py` is the kernel **and** its standalone harness
151
- (`python -m mjolnir.t5_decode --help`) with a bit-faithful emulator as the
153
+ `mjolnir/arch/thor/decode.py` is the kernel; `mjolnir/arch/thor/harness.py`
154
+ (`python -m mjolnir.arch.thor.harness --help`) is its standalone harness, with a bit-faithful emulator as the
152
155
  correctness reference. The harness can also read KV formats vLLM cannot write
153
156
  yet — fp6 (`e2m3`/`e3m2`) and fp4 (`e2m1`) read directly by `tcgen05
154
157
  kind::f8f6f4`, and `nvfp4` — and defaults to e2m3 on head-major pages for fp8 Q.
155
158
  How they compare in speed and accuracy:
156
- [`kvfmt`](https://github.com/thor-arena/mjolnir/blob/main/docs/kernel/kvfmt.md) and
157
- [`quant`](https://github.com/thor-arena/mjolnir/blob/main/docs/kernel/quant.md).
159
+ [`kvfmt`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/kvfmt.md) and
160
+ [`quant`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/quant.md).
158
161
  Wiring a vLLM cache-write path for fp6 is an open contribution.
159
162
 
160
163
  ## Platforms and architectures
@@ -163,8 +166,10 @@ The kernel is written for one architecture today — Thor (`sm_110`) — but the
163
166
  package is organized by platform: `mjolnir/arch/` is the implemented-kernel
164
167
  registry (`mjolnir.arch.thor` re-exports the exact runner surface the backend
165
168
  consumes), and `mjolnir/platforms.py` is the torch-free leaf that maps a
166
- compute capability to it. Thor is implemented; DGX Spark (`sm_121`) and Jetson
167
- Orin (`sm_87`) are declared research — named in the matrix, no kernel yet.
169
+ compute capability to it. Thor and Jetson Orin (`sm_87`) are implemented;
170
+ DGX Spark (`sm_121`) is declared research — named in the matrix, no kernel
171
+ yet. Orin ships one kernel: the `mma.m16n8k16` decode kernel (fp8 KV at
172
+ hd 128/256, fp6-e2m3 KV hd 256, M ≤ 8, bf16 Q) — see `mjolnir/arch/orin/README.md`.
168
173
 
169
174
  `kernel_for((11, 0))` returns `mjolnir.arch.thor`; any other compute
170
175
  capability raises `PlatformNotSupported` naming the requested `sm_XX` and the
@@ -207,13 +212,13 @@ is printed as information.
207
212
 
208
213
  | file | what |
209
214
  |---|---|
210
- | `mjolnir/t5_decode.py` | the CuTe-DSL kernel, the split-KV merge, the harness and the emulator |
215
+ | `mjolnir/arch/thor/` | the Thor kernel family: `decode.py` (the CuTe-DSL kernel + merge), `call.py`, `const.py`, `plan.py`, `runner.py`, `harness.py` (driver + emulator) |
211
216
  | `mjolnir/runner.py` | plan buffers, compile cache, launch, KV views (no vLLM import) |
212
217
  | `mjolnir/backend.py` | the vLLM backend, metadata builder, impl |
213
218
  | `mjolnir/platforms.py` | arch → kernel resolution (`kernel_for`, `PlatformNotSupported`), `MJOLNIR_ARCH` detection — torch-free |
214
219
  | `mjolnir/arch/thor.py` | the Thor (`sm_110`) kernel surface: re-exports of the runner |
215
220
  | `mjolnir/__init__.py` | `register()` — the vLLM plugin entry point |
216
221
 
217
- Apache-2.0, except `mjolnir/t5_decode.py`, which is derived from an NVIDIA
222
+ Apache-2.0, except `mjolnir/arch/thor/decode.py`, which is derived from an NVIDIA
218
223
  CUTLASS CuTe DSL example and stays under BSD-3-Clause — see `LICENSE` and
219
224
  `NOTICE` in this package.
@@ -1,6 +1,6 @@
1
1
  """Mjolnir — fp8 paged decode attention for vLLM on Jetson AGX Thor (sm_110).
2
2
 
3
- The kernel (codename **T5**, ``mjolnir.t5_decode``) is a CuTe-DSL tcgen05 +
3
+ The kernel (codename **T5**, ``mjolnir.arch.thor.decode``) is a CuTe-DSL tcgen05 +
4
4
  TMEM + TMA decode kernel; ``mjolnir.backend`` wraps it as a vLLM attention
5
5
  backend and ``mjolnir.runner`` holds the plan buffers, compile cache and KV
6
6
  views.
@@ -24,7 +24,7 @@ from __future__ import annotations
24
24
  import os
25
25
 
26
26
  __all__ = ["BACKEND_NAME", "BACKEND_PATH", "__version__", "register"]
27
- __version__ = "0.2.0"
27
+ __version__ = "0.3.3"
28
28
 
29
29
  BACKEND_NAME = "MJOLNIR"
30
30
  BACKEND_PATH = "mjolnir.backend.MjolnirAttentionBackend"
@@ -60,6 +60,22 @@ def has_native_member() -> bool:
60
60
  def register() -> None:
61
61
  if _disabled():
62
62
  return
63
+ # vLLM > 0.32.0: the fp6_e2m3 KV dtype registers through the pluggable
64
+ # dtype API (vLLM #52670) instead of the image patch — a no-op on
65
+ # 0.32.0 and on a base where the patch already added it.
66
+ try:
67
+ from .kvdtype import register as _register_kv_dtype
68
+
69
+ _register_kv_dtype()
70
+ except Exception as e: # noqa: BLE001 - never break vLLM startup
71
+ import logging
72
+
73
+ logging.getLogger("vllm.mjolnir").warning("Mjolnir fp6 KV dtype not registered: %r", e)
74
+ if os.environ.get("MJOLNIR_ATTN_DEBUG", "").strip():
75
+ # The IMA hunt: the engine core is a child process that inherits this
76
+ # env before its CUDA runtime initializes, so faulting kernels are
77
+ # named at their launch instead of surfacing on a later sync.
78
+ os.environ.setdefault("CUDA_LAUNCH_BLOCKING", "1")
63
79
  enum, register_backend = _registry()
64
80
  if BACKEND_NAME in enum.__members__:
65
81
  register_backend(enum[BACKEND_NAME], BACKEND_PATH)
@@ -0,0 +1,102 @@
1
+ # Orin kernel family — the mma.sync decode kernel (sm_87)
2
+
3
+ This subpackage is the Mjolnir kernel for **Jetson AGX Orin 64GB (`sm_87`,
4
+ GA10B Ampere)** — targeted at that one architecture (Thor keeps the
5
+ `tcgen05` `t5_*` family; this family never runs on sm_100/sm_110). The
6
+ family's only kernel is the tensor-core `mma.m16n8k16` decode kernel.
7
+
8
+ ## What the kernel is
9
+
10
+ One CTA per `(kv-head, split, batch)`, 128 threads = 4 warps; the M tile
11
+ = 64 rows split across the warps (one 16-row mma atom each); the KV rows
12
+ stage through `cp.async` (32 rows per stage), dequantize fp8→bf16 in
13
+ smem, and ride the reference's `ldmatrix → mma.m16n8k16` path. `M ≤ 8`
14
+ (the MTP=7 verify shape), split-KV with chunk-normalized fp32 partials +
15
+ the fused LSE combine, causal and non-causal (the MTP drafts see one
16
+ fewer future key each). **KV formats: fp8 e4m3 and fp6 e2m3** — the fp6
17
+ rows read packed (d·3/4 = 192 bytes for d=256, 4 codes per 3 bytes LE)
18
+ through the 64-entry `kvfp6.e2m3_values` LUT; the per-channel scales
19
+ fold into Q (`q_mul`) and post-multiply the output (`o_mul`) in the
20
+ runner, so the kernel's math is untouched (M1+M2).
21
+
22
+ ## The target to beat
23
+
24
+ The native sm_87 path in vLLM 0.32.0 is **FlashInfer's tensor-core FA2
25
+ decode** — confirmed at both tags: `vllm/v1/attention/backends/flashinfer.py`
26
+ constructs `BatchDecodeWithPagedKVCacheWrapper(..., use_tensor_cores=True)`
27
+ unconditionally, which routes to the FA2 prefill kernel
28
+ (`get_batch_prefill_module("fa2", ...)`); XQA/trtllm-gen are SM90/100/120-only.
29
+ Its Ampere shape for our model (packed qo = q_len·g = 6..24 ≤ 16 →
30
+ `CTA_TILE_Q = 16`, 1 Q-warp + 4 KV-warps = 128 threads):
31
+
32
+ * **NUM_MMA_KV = 2 → 128 KV rows per iteration** (fp8: 32 KB K+V smem per
33
+ step; the launcher's 2-CTA/SM budget against the *100 KB-class* Ampere
34
+ parts — Orin's 163 KB is headroom it never uses);
35
+ * fp8 KV is converted to fp16 **in registers before every HMMA**
36
+ (`fast_dequant_f8f16x4`, ≈2–4 ALU ops per 4 values — no fp8 cvt or mma
37
+ exists below sm_89) — a per-value tax the mma pays once per loaded
38
+ vector in smem instead;
39
+ * Q stays **bf16** on SM87 + fp8 KV ("architectures with only fa2 cannot
40
+ consume FP8 queries");
41
+ * split-KV is capped at 2 CTAs/SM (32 CTAs on Orin's 16 SMs), the LSE
42
+ merge is a **second kernel launch**, and PDL does not exist on
43
+ Ampere (`device_support_pdl` requires major ≥ 9) — a hard serial
44
+ launch floor per decode step that Thor does not pay.
45
+
46
+ Hardware reference (AGX Orin 64GB): 2048 cores / **16 SMs** / 64 3rd-gen
47
+ Tensor Cores / 5.3 TFLOPS fp32 / ~42.5 TFLOPS fp16 (derived) /
48
+ **204.8 GB/s** LPDDR5 / 4 MB L2 / 163 KB smem per block / 1.3 GHz.
49
+
50
+ ## Measured status
51
+
52
+ * **The mma kernel**: `mjolnir verify sweep` same-run vs FlashInfer
53
+ FA2-TC across the M/L band (research/runs/orin1/notes/
54
+ mma-kernel-design-2.md).
55
+ * **The mma kernel (2026-10-26)**: serving-validated end-to-end — the
56
+ A/B/C matrix byte-identical correct, the spec sweep 14/14 (Mjolnir
57
+ 19.9 t/s/req vs FlashInfer 18.9 at c=1, +5.0% aggregate at c=4,
58
+ identical per-position acceptance), fp6 (M1+M2) probe-validated on
59
+ both pool layouts and the image canary (`kernel_test_orin_gates.py`).
60
+
61
+ ## The fp6 e2m3 KV path (M1+M2, landed)
62
+
63
+ * **M1 (in-kernel)**: the mma reads the packed rows directly — the
64
+ 192-byte row moves as 12 × 16-byte cp.async chunks; the codes
65
+ unpack at bit 6i and dequantize through the LUT into the bf16 mma
66
+ operands. The per-head descales ride the accumulator
67
+ (`scale_eff` / the epilogue) exactly like fp8.
68
+ * **M2 (per-channel)**: the per-(layer, KV head, channel) scales from
69
+ `mjolnir calibrate` fold into Q pre-launch (`q_mul`) and
70
+ post-multiply the output (`o_mul`) — the kernel untouched
71
+ (`mjolnir/kvfp6.py`, the calibration `kv_scales/e2m3-awq.json`).
72
+
73
+ ## hd128 (shipped, 2026-10-31)
74
+
75
+ The mma serves **head_size 128 and 256 for fp8** from one D-parameterized
76
+ body: the staging, the fragment counts and the PV ranges all derive from
77
+ `self.HEAD_DIM`, the constructor takes the head size, and the runner
78
+ gates `{128, 256}`, keys the binary on it (the TVM-FFI dims bake it) and
79
+ passes it to the split-KV combine — whose elements-per-thread is
80
+ `d / NUM_THREADS`. That combine was the whole hd128 bug: it baked
81
+ D=256, so at hd128 its second element per thread read and wrote past
82
+ the row — masked whenever `max_splits=1` (the write-through path), a
83
+ 9.2 rel-err otherwise. fp6 stays **hd256-only**: the e2m3 D2 recipe's
84
+ byte-lane phase is static per thread and assumes 64 groups per row
85
+ (a per-iteration phase would serve hd128's 32).
86
+
87
+ Verified on the harness (rel err 0.0016–0.0019 vs the fp64 reference):
88
+ L 1K–262K, q_len 3/4/5, HND + NHD, split caps 2/4/8; the hd256 fp8 and
89
+ fp6 regressions clean. At M=5 / L=64K the hd128 shape graphs 600 µs vs
90
+ hd256's 967 µs (1.61× — an hd128 model streams half the KV bytes), and
91
+ the kernel drops to 128 regs / 27 KB smem (4 CTAs/SM by registers) from
92
+ hd256's 236 regs / 83 KB (2 CTAs/SM).
93
+
94
+ ## Not portable from Thor (deliberate)
95
+
96
+ * `tcgen05 kind::f8f6f4` (the Thor fp6 read) — Ampere has no such MMA;
97
+ fp6 read here is unpack + `mma.sync`.
98
+ * PDL / programmatic dependent launch — sm_90+.
99
+ * fp8 lm_head (`MJOLNIR_LM_HEAD=fp8`) — no fp8 tensor cores on Ampere;
100
+ the Triton fp8 GEMM would be compute-bound. Orin keeps a bf16 lm_head.
101
+ * The Thor-tuned ring depths / split plans — retuned for 16 SMs,
102
+ 100 KiB smem and LPDDR5x latency; the sm_110 numbers do not transfer.
@@ -0,0 +1,82 @@
1
+ """The Orin (``sm_87``) kernel family: the mma.sync tensor-core decode kernel.
2
+
3
+ One subpackage per implemented architecture, sibling to
4
+ :mod:`mjolnir.arch.thor` (see that module's docstring for the surface
5
+ contract). The family mirrors the Thor layout:
6
+
7
+ * :mod:`mjolnir.arch.orin.mma` — the tensor-core (mma.sync m16n8k16)
8
+ decode kernel: fp8 (head_dim 128/256) and fp6-e2m3 (head_dim 256) KV,
9
+ bf16 Q, q_len <= 8
10
+ (decode and the MTP verify). The family's only kernel;
11
+ * :mod:`mjolnir.arch.orin.plan` — the split-KV planner (FlashInfer
12
+ Alg. 1, board-agnostic: the SM count is a device query, so AGX Orin
13
+ 64/32 GB, Orin NX and Orin Nano all plan correctly);
14
+ * :mod:`mjolnir.arch.orin.combine` — the fused split-KV LSE merge;
15
+ * :mod:`mjolnir.arch.orin.runner` — the plan buffers, compile cache and
16
+ launch; :class:`Runner` below is the surface the vLLM backend consumes.
17
+
18
+ Do not import this package on a device whose compute capability is not
19
+ (8, 7): the DSL compile and the module's budget asserts assume GA10B.
20
+ """
21
+
22
+ import logging
23
+
24
+ from mjolnir.runner import env_flag
25
+
26
+ from .runner import OrinDecodeRunner
27
+
28
+ __all__ = ["OrinDecodeRunner", "Runner"]
29
+
30
+ _log = logging.getLogger("vllm.mjolnir")
31
+
32
+ MMA_MAX_Q_LEN = 8 # mma.MMADecodeKernel.MAX_Q_LEN
33
+ MMA_KV_FORMATS = ("fp8", "e2m3") # the mma kernel's KV formats
34
+
35
+
36
+ class Runner(OrinDecodeRunner):
37
+ """The backend-facing runner surface (``mjolnir.platforms.kernel_for``
38
+ hands this to the vLLM backend). Accepts the shared backend kwargs and
39
+ absorbs the Thor-only levers as inert attributes: the Orin grid has no
40
+ CTAs/SM knob, and PDL / L2-prefetch / evict-first hints do not exist
41
+ below sm_90 — the attributes exist so the backend's builder logging and
42
+ metadata paths read them unchanged.
43
+
44
+ Two backend kwargs are translated to this family's meaning:
45
+
46
+ * ``max_splits`` (``MJOLNIR_ATTN_MAX_SPLITS``, "max KV chunks per
47
+ sequence") caps the auto split capacity — it used to pin it, so the
48
+ backend's default 16 silently replaced the mma's 2-CTA-occupancy
49
+ optimum of 8 (one wave on 16 SMs) with the over-split 16;
50
+ * ``q_mode``: the mma kernel takes a bf16 query, so the runner always
51
+ runs bf16 Q."""
52
+
53
+ def __init__(
54
+ self,
55
+ *,
56
+ ctas_per_sm: int = 2,
57
+ pdl: bool = False,
58
+ l2_prefetch: int = 0,
59
+ kv_evict_first: bool = False,
60
+ max_splits: int = 0,
61
+ **kw,
62
+ ):
63
+ head_size = kw["head_size"]
64
+ kv_format = kw.get("kv_format", "fp8")
65
+ max_q_len = kw.get("max_q_len", 1)
66
+ supported = (256,) if kv_format == "e2m3" else (128, 256)
67
+ if head_size not in supported:
68
+ raise ValueError(
69
+ f"the Orin mma kernel serves head_size {'/'.join(map(str, supported))} for {kv_format} KV (got {head_size})"
70
+ )
71
+ if kv_format not in MMA_KV_FORMATS:
72
+ raise ValueError(f"the Orin mma kernel serves {'/'.join(MMA_KV_FORMATS)} KV only (got {kv_format!r})")
73
+ if max_q_len > MMA_MAX_Q_LEN:
74
+ raise ValueError(f"the Orin mma kernel serves q_len <= {MMA_MAX_Q_LEN} (got {max_q_len})")
75
+ if env_flag("MJOLNIR_ATTN_ORIN_IMPL", "") not in ("", "mma"):
76
+ raise ValueError("MJOLNIR_ATTN_ORIN_IMPL: only 'mma' is supported on Orin")
77
+ kw["q_mode"] = "bf16"
78
+ super().__init__(split_cap=max_splits, **kw)
79
+ self.ctas_per_sm = ctas_per_sm
80
+ self.pdl = pdl
81
+ self.l2_prefetch = l2_prefetch
82
+ self.kv_evict_first = kv_evict_first
@@ -0,0 +1,111 @@
1
+ """The fused split-KV combine for the Orin MMA decode kernel.
2
+
3
+ The runner's partials are (ns, rows, h_q, d) fp32 O and (ns, b, h_q, ql)
4
+ fp32 LSE; the merge is O = Σ_s exp(lse_s − L)·O_s / Σ_s exp(lse_s − L)
5
+ with L = max_s lse_s (the chunk-normalized contract the kernels write).
6
+ The eager torch form issued ~20 small kernels (~150 us of GPU per call
7
+ — more than the decode kernel itself); this is one kernel, one pass
8
+ over the partials, the FlashInfer MergeStates cost class.
9
+
10
+ One block per (row, q-head): the block loads its ns LSE values, then
11
+ each thread accumulates its d/NUM_THREADS elements across the splits
12
+ (2 at hd256, 1 at hd128 — the head size rides the compile as a
13
+ constexpr). ns/rows/h_q/b/ql are runtime Int32s so one compile serves
14
+ every capture shape per head size. The tensor layouts
15
+ are the runner's capacity buffers (fixed at construction, so the baked
16
+ extents are stable). The empty-chunk splits write lse = -inf -> weight
17
+ 0; an all-empty row normalizes to 0 (the torch combine's clamp_min) —
18
+ L is floored at -FLT_MAX so that row never computes -inf - -inf.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import cutlass
24
+ import cutlass.cute as cute
25
+ import cuda.bindings.driver as cuda
26
+ from cutlass import Float32, Int32, const_expr
27
+
28
+ NUM_THREADS = 128
29
+ _TINY = 1.1754944e-35 # torch.finfo(torch.float32).tiny, rounded
30
+ _NEG_FLOOR = -3.4028234e38 # -torch.finfo(torch.float32).max: the finite L floor
31
+
32
+
33
+ @cute.kernel
34
+ def combine_kernel(
35
+ mO_part: cute.Tensor, # (ns_cap*rows_cap*h_q*D) fp32 flat
36
+ mLse: cute.Tensor, # (ns_cap*b_cap*h_q*ql_cap) fp32 flat
37
+ mOut: cute.Tensor, # (rows_cap*h_q*D) bf16 flat
38
+ d: cutlass.Constexpr[int], # the head size (128 or 256): the elements/thread = d/NUM_THREADS
39
+ max_ns: cutlass.Constexpr[int], # the unroll/predication bound (= ns_cap)
40
+ ns: Int32,
41
+ rows: Int32,
42
+ h_q: Int32,
43
+ b: Int32,
44
+ ql: Int32,
45
+ ):
46
+ bid = cute.arch.block_idx()[0]
47
+ tidx = cute.arch.thread_idx()[0]
48
+ r = bid // h_q
49
+ h = bid - r * h_q
50
+ bb = r // ql
51
+ qq = r - bb * ql
52
+ ls_s = b * h_q * ql
53
+ ls_b = h_q * ql
54
+
55
+ # Both loops unroll over max_ns with the s < ns predication: the
56
+ # dynamic-range form serialized every split's load behind the loop
57
+ # branch (~34 us; the FI merge overlaps all partial rows in flight
58
+ # and lands at 11). Unrolled, the loads issue independently and the
59
+ # latency hides under the acc chain.
60
+ # 1. L = max_s lse[s, h, r], floored at a FINITE -FLT_MAX: a row whose
61
+ # every split is empty (all lse = -inf — a CUDA-graph padding request
62
+ # with seq_len 0, or a dummy/profile step) would otherwise get
63
+ # L = -inf and w = exp2(-inf - -inf) = NaN, a NaN output row instead
64
+ # of the documented 0 (and a NaN row poisons any per-tensor reduction
65
+ # downstream). With the floor, every -inf lse weighs exactly 0 and a
66
+ # real lse (>= -FLT_MAX) is unaffected.
67
+ L = Float32(_NEG_FLOOR)
68
+ for s in cutlass.range_constexpr(max_ns):
69
+ if s < ns:
70
+ L = cutlass.max(L, mLse[s * ls_s + bb * ls_b + h * ql + qq])
71
+
72
+ # 2. the weighted sum over the splits; the exp weights recomputed per
73
+ # element pass (ns <= 64 exp2s are nothing next to the loads).
74
+ ln2e = Float32(1.4426950408889634) # 1/ln 2: exp(x) = exp2(x*log2 e)
75
+ sw = Float32(0.0)
76
+ acc0 = Float32(0.0)
77
+ acc1 = Float32(0.0)
78
+ for s in cutlass.range_constexpr(max_ns):
79
+ if s < ns:
80
+ w = cute.math.exp2((mLse[s * ls_s + bb * ls_b + h * ql + qq] - L) * ln2e, fastmath=True)
81
+ sw = sw + w
82
+ obase = s * rows * h_q * d + r * h_q * d + h * d
83
+ acc0 = acc0 + w * mO_part[obase + tidx]
84
+ if const_expr(d == 256):
85
+ acc1 = acc1 + w * mO_part[obase + tidx + NUM_THREADS]
86
+ scale = 1.0 / cutlass.max(sw, Float32(_TINY))
87
+ obase = r * h_q * d + h * d
88
+ mOut[obase + tidx] = (acc0 * scale).to(mOut.element_type)
89
+ if const_expr(d == 256):
90
+ mOut[obase + tidx + NUM_THREADS] = (acc1 * scale).to(mOut.element_type)
91
+
92
+
93
+ @cute.jit
94
+ def combine_launch(
95
+ mO_part: cute.Tensor,
96
+ mLse: cute.Tensor,
97
+ mOut: cute.Tensor,
98
+ d: cutlass.Constexpr[int],
99
+ max_ns: cutlass.Constexpr[int],
100
+ ns: Int32,
101
+ rows: Int32,
102
+ h_q: Int32,
103
+ b: Int32,
104
+ ql: Int32,
105
+ stream: cuda.CUstream = None,
106
+ ):
107
+ combine_kernel(mO_part, mLse, mOut, d, max_ns, ns, rows, h_q, b, ql).launch(
108
+ grid=(rows * h_q, 1, 1),
109
+ block=(NUM_THREADS, 1, 1),
110
+ stream=stream,
111
+ )