mjolnir-kernel 0.1.1__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/.gitignore +8 -0
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/NOTICE +1 -1
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/PKG-INFO +51 -20
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/README.md +45 -14
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/__init__.py +18 -2
- mjolnir_kernel-0.3.2/mjolnir/arch/__init__.py +7 -0
- mjolnir_kernel-0.3.2/mjolnir/arch/orin/README.md +102 -0
- mjolnir_kernel-0.3.2/mjolnir/arch/orin/__init__.py +82 -0
- mjolnir_kernel-0.3.2/mjolnir/arch/orin/combine.py +111 -0
- mjolnir_kernel-0.3.2/mjolnir/arch/orin/harness.py +483 -0
- mjolnir_kernel-0.3.2/mjolnir/arch/orin/mma.py +2295 -0
- mjolnir_kernel-0.3.2/mjolnir/arch/orin/plan.py +67 -0
- mjolnir_kernel-0.3.2/mjolnir/arch/orin/runner.py +686 -0
- mjolnir_kernel-0.3.2/mjolnir/arch/thor/__init__.py +45 -0
- mjolnir_kernel-0.1.1/mjolnir/t5_call.py → mjolnir_kernel-0.3.2/mjolnir/arch/thor/call.py +2 -2
- mjolnir_kernel-0.1.1/mjolnir/t5_decode.py → mjolnir_kernel-0.3.2/mjolnir/arch/thor/decode.py +4 -4
- mjolnir_kernel-0.1.1/mjolnir/t5_harness.py → mjolnir_kernel-0.3.2/mjolnir/arch/thor/harness.py +17 -10
- {mjolnir_kernel-0.1.1/mjolnir → mjolnir_kernel-0.3.2/mjolnir/arch/thor}/runner.py +29 -292
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/backend.py +290 -364
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/banner.py +1 -1
- mjolnir_kernel-0.3.2/mjolnir/builder.py +529 -0
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/calibrate.py +2 -2
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/draftvocab.py +4 -0
- mjolnir_kernel-0.3.2/mjolnir/kvdtype.py +53 -0
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/kvfp6.py +42 -3
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/lmhead.py +69 -3
- mjolnir_kernel-0.3.2/mjolnir/platforms.py +120 -0
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/quant.py +1 -1
- mjolnir_kernel-0.3.2/mjolnir/runner.py +329 -0
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/pyproject.toml +5 -5
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/LICENSE +0 -0
- /mjolnir_kernel-0.1.1/mjolnir/t5_const.py → /mjolnir_kernel-0.3.2/mjolnir/arch/thor/const.py +0 -0
- /mjolnir_kernel-0.1.1/mjolnir/t5_plan.py → /mjolnir_kernel-0.3.2/mjolnir/arch/thor/plan.py +0 -0
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/kvscales.py +0 -0
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/py.typed +0 -0
- {mjolnir_kernel-0.1.1 → mjolnir_kernel-0.3.2}/mjolnir/qpack.py +0 -0
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
__pycache__/
|
|
2
2
|
*.py[cod]
|
|
3
3
|
.venv/
|
|
4
|
+
.playwright-mcp/
|
|
4
5
|
.pytest_cache/
|
|
5
6
|
.ruff_cache/
|
|
6
7
|
.env
|
|
8
|
+
# the configs' env sidecars are plain mjolnir knobs (calibration paths, feature
|
|
9
|
+
# flags) and ride with the configs; real secrets stay out
|
|
10
|
+
!configs/**/.env
|
|
7
11
|
# coverage artifacts (the CI test stage writes these; they regenerate)
|
|
8
12
|
.coverage
|
|
9
13
|
.coverage.*
|
|
@@ -24,6 +28,7 @@ benchmarks/raw/*
|
|
|
24
28
|
!benchmarks/raw/accuracy/
|
|
25
29
|
benchmarks/raw/accuracy/*/*/
|
|
26
30
|
benchmarks/logs/
|
|
31
|
+
benchmarks/staging/
|
|
27
32
|
task/
|
|
28
33
|
|
|
29
34
|
# autoresearch: the live ledger is untracked (discards are `git reset --hard`);
|
|
@@ -32,3 +37,6 @@ research/ledger.jsonl
|
|
|
32
37
|
research/ledger.tsv
|
|
33
38
|
# …and the exported runs ARE committed (the generic runs/ rule above would hide them)
|
|
34
39
|
!research/runs/
|
|
40
|
+
/cutlass/
|
|
41
|
+
/fa2/
|
|
42
|
+
/llamacpp/
|
|
@@ -8,7 +8,7 @@ works listed below; each keeps its own license and copyright notice.
|
|
|
8
8
|
-------------------------------------------------------------------------------
|
|
9
9
|
1. NVIDIA CUTLASS — CuTe DSL examples BSD-3-Clause
|
|
10
10
|
-------------------------------------------------------------------------------
|
|
11
|
-
src/mjolnir/mjolnir/
|
|
11
|
+
src/mjolnir/mjolnir/arch/thor/decode.py (the Mjolnir decode-attention kernel) is
|
|
12
12
|
derived from
|
|
13
13
|
|
|
14
14
|
examples/python/CuTeDSL/cute/blackwell/kernel/attention/mixed_input_fmha/
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: mjolnir-kernel
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Mjolnir: a paged decode-attention kernel for NVIDIA Jetson AGX Thor (sm_110) on tcgen05 / TMEM / TMA (CuTe DSL), with fp8 and 6-bit (e2m3) KV caches — packaged as the vLLM MJOLNIR attention backend.
|
|
5
|
-
Project-URL: Homepage, https://github.com/
|
|
6
|
-
Project-URL: Documentation, https://github.com/
|
|
7
|
-
Project-URL: Repository, https://github.com/
|
|
8
|
-
Project-URL: Issues, https://github.com/
|
|
9
|
-
Project-URL: Changelog, https://github.com/
|
|
5
|
+
Project-URL: Homepage, https://github.com/GoBrrr-ai/mjolnir
|
|
6
|
+
Project-URL: Documentation, https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel
|
|
7
|
+
Project-URL: Repository, https://github.com/GoBrrr-ai/mjolnir.git
|
|
8
|
+
Project-URL: Issues, https://github.com/GoBrrr-ai/mjolnir/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/GoBrrr-ai/mjolnir/releases
|
|
10
10
|
Author-email: Mieszko Syty <mieszko@ms1design.pl>
|
|
11
11
|
License-Expression: Apache-2.0 AND BSD-3-Clause
|
|
12
12
|
License-File: LICENSE
|
|
@@ -41,7 +41,7 @@ This package is the kernel only. Serving, calibration and benchmarking are the
|
|
|
41
41
|
[`mjolnir-cli`](https://pypi.org/project/mjolnir-cli/) package; benchmarks
|
|
42
42
|
(Mjolnir vs FlashInfer vs FlashAttention 2 across context lengths), the design,
|
|
43
43
|
and one chapter per kernel feature live in the project repository:
|
|
44
|
-
**[
|
|
44
|
+
**[GoBrrr-ai/mjolnir](https://github.com/GoBrrr-ai/mjolnir)** — numbers there
|
|
45
45
|
are rendered from the raw results, so they are never stale.
|
|
46
46
|
|
|
47
47
|
## What it does
|
|
@@ -102,7 +102,7 @@ The package registers a `vllm.general_plugins` entry point (`mjolnir:register`)
|
|
|
102
102
|
that maps `AttentionBackendEnum.CUSTOM` — and `MJOLNIR`, when the vLLM build has
|
|
103
103
|
that member — to `mjolnir.backend.MjolnirAttentionBackend` in every vLLM process.
|
|
104
104
|
The two-line enum patch for vLLM is
|
|
105
|
-
[`thor-mjolnir-attention-backend.patch`](https://github.com/
|
|
105
|
+
[`thor-mjolnir-attention-backend.patch`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docker/vllm-thor/patches/thor-mjolnir-attention-backend.patch).
|
|
106
106
|
|
|
107
107
|
Look for this line in the server log:
|
|
108
108
|
|
|
@@ -131,7 +131,8 @@ runs on the fallback backend.
|
|
|
131
131
|
| `MJOLNIR_ATTN_Q_DTYPE` | `fp8` | `fp8` (dynamic per-(seq, KV head) quantization) or `bf16` |
|
|
132
132
|
| `MJOLNIR_ATTN_MAX_QLEN` | `1 + num_speculative_tokens` | the largest q_len Mjolnir takes |
|
|
133
133
|
| `MJOLNIR_ATTN_CTAS` | `2` | CTAs per SM (2 is fastest on Thor) |
|
|
134
|
-
| `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel) |
|
|
134
|
+
| `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel). On Orin it caps the split capacity the kernel's occupancy picks (8 for the mma kernel on AGX Orin) |
|
|
135
|
+
|
|
135
136
|
| `MJOLNIR_ATTN_KV_STAGES` | `0` | KV ring depth override (0 = the tuned default) |
|
|
136
137
|
| `MJOLNIR_ATTN_PDL` | `0` | `1`: programmatic dependent launch |
|
|
137
138
|
| `MJOLNIR_ATTN_L2_PREFETCH` | `0` | TMA-prefetch K/V this many tiles ahead into L2 |
|
|
@@ -141,7 +142,8 @@ runs on the fallback backend.
|
|
|
141
142
|
| `MJOLNIR_ATTN_DRAFT_T5` | `1` | `0`: the drafter's layers (MTP / EAGLE) run on the fallback — isolates the draft-step cost (`mjolnir verify profile`). fp8 KV only: refused on fp6 KV, whose fallback cannot be CUDA-graphed |
|
|
142
143
|
| `MJOLNIR_ATTN_NONCAUSAL` | `1` | `0`: a parallel drafter's non-causal block steps (DSpark / DFlash) go to the fallback instead of the decode kernel — fp8 KV only for the same reason as above |
|
|
143
144
|
| `MJOLNIR_ATTN_FUSED_DRAFT` | `1` | `0`: tell vLLM the backend cannot advance draft metadata in place → one graph replay per draft step instead of the fused draft loop |
|
|
144
|
-
| `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `
|
|
145
|
+
| `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV). `MJOLNIR_LM_HEAD_TILES` overrides the GEMV launch config (`BN,BK,warps,stages`; default 64,256,8,5 — the 2026-11-04 serving A/B winner) |
|
|
146
|
+
| `MJOLNIR_MTP_FP8` | – | `1`: fp8-row-quantize the drafter's big unquantized linears (≥16M params) at load, same e4m3-per-row scheme + self-test (≤0.02 rel err) as the head; the bf16 originals are freed. Halves the MTP draft loop's streamed weight bytes |
|
|
145
147
|
| `MJOLNIR_DRAFT_VOCAB` | – | path to a `draft_vocab.json` (`mjolnir calibrate --draft-vocab`): the drafter's logits only over those tokens (VocabTrim); `MJOLNIR_DRAFT_VOCAB_SIZE` keeps the first N (a shorter file is padded with the lowest unused ids, with a warning). The kept head stays in the drafter's native quant (NVFP4 4-bit in-kernel, FP8 row-slice) — never up-scaled to bf16; a bf16 head is the one exception (fp8 only when `MJOLNIR_LM_HEAD=fp8` is set). `MJOLNIR_DRAFT_VOCAB_NATIVE=0` forces the dequant→fp8 path |
|
|
146
148
|
| `MJOLNIR_ATTN_VERBOSE` | – | `1`: print the kernel's compile-time notes (ring stages, TMEM columns) — the standalone harness always does |
|
|
147
149
|
| `MJOLNIR_ATTN_BANNER` | `1` | `0`: skip the Mjolnir hammer banner in the vLLM boot log (`mjolnir.banner.emit`, once per start, coloured with the server's formatter) |
|
|
@@ -153,7 +155,7 @@ runs on the fallback backend.
|
|
|
153
155
|
|
|
154
156
|
The pre-rename `THOR_T5_*` spellings are still honored. Each default is backed
|
|
155
157
|
by a measurement — see the
|
|
156
|
-
[kernel chapters](https://github.com/
|
|
158
|
+
[kernel chapters](https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel).
|
|
157
159
|
|
|
158
160
|
## fp6 (e2m3) KV in vLLM
|
|
159
161
|
|
|
@@ -166,25 +168,52 @@ expands the pages a prefill / mixed step reads to e4m3 for the FA4 fallback
|
|
|
166
168
|
(channel scales applied, one descale per head). The scales come from a calibration run on fp8 KV:
|
|
167
169
|
|
|
168
170
|
```bash
|
|
169
|
-
python -m mjolnir.calibrate --config <serve.yaml> --out kv_scales.json # in the image
|
|
170
|
-
MJOLNIR_ATTN_KV_SCALES=kv_scales.json vllm serve ... --kv-cache-dtype fp6_e2m3 --attention-backend MJOLNIR
|
|
171
|
+
python -m mjolnir.calibrate --config <serve.yaml> --out kv_scales/e2m3.json # in the image
|
|
172
|
+
MJOLNIR_ATTN_KV_SCALES=kv_scales/e2m3.json vllm serve ... --kv-cache-dtype fp6_e2m3 --attention-backend MJOLNIR
|
|
171
173
|
```
|
|
172
174
|
|
|
173
175
|
(`mjolnir calibrate` / `mjolnir serve` do both for you —
|
|
174
|
-
[docs/faq/fp6-kv.md](https://github.com/
|
|
176
|
+
[docs/faq/fp6-kv.md](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/faq/fp6-kv.md).)
|
|
175
177
|
|
|
176
178
|
## Beyond fp8: the harness
|
|
177
179
|
|
|
178
|
-
`mjolnir/
|
|
179
|
-
(`python -m mjolnir.
|
|
180
|
+
`mjolnir/arch/thor/decode.py` is the kernel; `mjolnir/arch/thor/harness.py`
|
|
181
|
+
(`python -m mjolnir.arch.thor.harness --help`) is its standalone harness, with a bit-faithful emulator as the
|
|
180
182
|
correctness reference. The harness can also read KV formats vLLM cannot write
|
|
181
183
|
yet — fp6 (`e2m3`/`e3m2`) and fp4 (`e2m1`) read directly by `tcgen05
|
|
182
184
|
kind::f8f6f4`, and `nvfp4` — and defaults to e2m3 on head-major pages for fp8 Q.
|
|
183
185
|
How they compare in speed and accuracy:
|
|
184
|
-
[`kvfmt`](https://github.com/
|
|
185
|
-
[`quant`](https://github.com/
|
|
186
|
+
[`kvfmt`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/kvfmt.md) and
|
|
187
|
+
[`quant`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/quant.md).
|
|
186
188
|
Wiring a vLLM cache-write path for fp6 is an open contribution.
|
|
187
189
|
|
|
190
|
+
## Platforms and architectures
|
|
191
|
+
|
|
192
|
+
The kernel is written for one architecture today — Thor (`sm_110`) — but the
|
|
193
|
+
package is organized by platform: `mjolnir/arch/` is the implemented-kernel
|
|
194
|
+
registry (`mjolnir.arch.thor` re-exports the exact runner surface the backend
|
|
195
|
+
consumes), and `mjolnir/platforms.py` is the torch-free leaf that maps a
|
|
196
|
+
compute capability to it. Thor and Jetson Orin (`sm_87`) are implemented;
|
|
197
|
+
DGX Spark (`sm_121`) is declared research — named in the matrix, no kernel
|
|
198
|
+
yet. Orin ships one kernel: the `mma.m16n8k16` decode kernel (fp8 KV at
|
|
199
|
+
hd 128/256, fp6-e2m3 KV hd 256, M ≤ 8, bf16 Q) — see `mjolnir/arch/orin/README.md`.
|
|
200
|
+
|
|
201
|
+
`kernel_for((11, 0))` returns `mjolnir.arch.thor`; any other compute
|
|
202
|
+
capability raises `PlatformNotSupported` naming the requested `sm_XX` and the
|
|
203
|
+
shipped platforms. The vLLM backend never sees that exception: its metadata
|
|
204
|
+
builder consults the pure `unsupported_reason()` and takes the normal
|
|
205
|
+
ineligible-config path — that layer's decode steps run on the fallback
|
|
206
|
+
backend, the serve continues. On a host without a usable CUDA device,
|
|
207
|
+
`MJOLNIR_ARCH` (`sm_110` or `11.0`) pins the architecture instead of the
|
|
208
|
+
torch read (the standalone harness).
|
|
209
|
+
|
|
210
|
+
There is no remembered platform state anywhere in the stack: the CLI side
|
|
211
|
+
resolves its platform per invocation (flag > `$MJOLNIR_PLATFORM` > host
|
|
212
|
+
detection — four source classes with provenance, the `/tmp/nv_jetson_model`
|
|
213
|
+
model copy corroboration-only), so the kernel is always resolved from the
|
|
214
|
+
device the run actually happens on. See
|
|
215
|
+
[docs/faq/platforms.md](../../docs/faq/platforms.md).
|
|
216
|
+
|
|
188
217
|
## Tests
|
|
189
218
|
|
|
190
219
|
From a checkout, on a Thor:
|
|
@@ -210,11 +239,13 @@ is printed as information.
|
|
|
210
239
|
|
|
211
240
|
| file | what |
|
|
212
241
|
|---|---|
|
|
213
|
-
| `mjolnir/
|
|
242
|
+
| `mjolnir/arch/thor/` | the Thor kernel family: `decode.py` (the CuTe-DSL kernel + merge), `call.py`, `const.py`, `plan.py`, `runner.py`, `harness.py` (driver + emulator) |
|
|
214
243
|
| `mjolnir/runner.py` | plan buffers, compile cache, launch, KV views (no vLLM import) |
|
|
215
244
|
| `mjolnir/backend.py` | the vLLM backend, metadata builder, impl |
|
|
245
|
+
| `mjolnir/platforms.py` | arch → kernel resolution (`kernel_for`, `PlatformNotSupported`), `MJOLNIR_ARCH` detection — torch-free |
|
|
246
|
+
| `mjolnir/arch/thor.py` | the Thor (`sm_110`) kernel surface: re-exports of the runner |
|
|
216
247
|
| `mjolnir/__init__.py` | `register()` — the vLLM plugin entry point |
|
|
217
248
|
|
|
218
|
-
Apache-2.0, except `mjolnir/
|
|
249
|
+
Apache-2.0, except `mjolnir/arch/thor/decode.py`, which is derived from an NVIDIA
|
|
219
250
|
CUTLASS CuTe DSL example and stays under BSD-3-Clause — see `LICENSE` and
|
|
220
251
|
`NOTICE` in this package.
|
|
@@ -13,7 +13,7 @@ This package is the kernel only. Serving, calibration and benchmarking are the
|
|
|
13
13
|
[`mjolnir-cli`](https://pypi.org/project/mjolnir-cli/) package; benchmarks
|
|
14
14
|
(Mjolnir vs FlashInfer vs FlashAttention 2 across context lengths), the design,
|
|
15
15
|
and one chapter per kernel feature live in the project repository:
|
|
16
|
-
**[
|
|
16
|
+
**[GoBrrr-ai/mjolnir](https://github.com/GoBrrr-ai/mjolnir)** — numbers there
|
|
17
17
|
are rendered from the raw results, so they are never stale.
|
|
18
18
|
|
|
19
19
|
## What it does
|
|
@@ -74,7 +74,7 @@ The package registers a `vllm.general_plugins` entry point (`mjolnir:register`)
|
|
|
74
74
|
that maps `AttentionBackendEnum.CUSTOM` — and `MJOLNIR`, when the vLLM build has
|
|
75
75
|
that member — to `mjolnir.backend.MjolnirAttentionBackend` in every vLLM process.
|
|
76
76
|
The two-line enum patch for vLLM is
|
|
77
|
-
[`thor-mjolnir-attention-backend.patch`](https://github.com/
|
|
77
|
+
[`thor-mjolnir-attention-backend.patch`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docker/vllm-thor/patches/thor-mjolnir-attention-backend.patch).
|
|
78
78
|
|
|
79
79
|
Look for this line in the server log:
|
|
80
80
|
|
|
@@ -103,7 +103,8 @@ runs on the fallback backend.
|
|
|
103
103
|
| `MJOLNIR_ATTN_Q_DTYPE` | `fp8` | `fp8` (dynamic per-(seq, KV head) quantization) or `bf16` |
|
|
104
104
|
| `MJOLNIR_ATTN_MAX_QLEN` | `1 + num_speculative_tokens` | the largest q_len Mjolnir takes |
|
|
105
105
|
| `MJOLNIR_ATTN_CTAS` | `2` | CTAs per SM (2 is fastest on Thor) |
|
|
106
|
-
| `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel) |
|
|
106
|
+
| `MJOLNIR_ATTN_MAX_SPLITS` | `16` | max KV chunks per sequence (1 = never split, no merge kernel). On Orin it caps the split capacity the kernel's occupancy picks (8 for the mma kernel on AGX Orin) |
|
|
107
|
+
|
|
107
108
|
| `MJOLNIR_ATTN_KV_STAGES` | `0` | KV ring depth override (0 = the tuned default) |
|
|
108
109
|
| `MJOLNIR_ATTN_PDL` | `0` | `1`: programmatic dependent launch |
|
|
109
110
|
| `MJOLNIR_ATTN_L2_PREFETCH` | `0` | TMA-prefetch K/V this many tiles ahead into L2 |
|
|
@@ -113,7 +114,8 @@ runs on the fallback backend.
|
|
|
113
114
|
| `MJOLNIR_ATTN_DRAFT_T5` | `1` | `0`: the drafter's layers (MTP / EAGLE) run on the fallback — isolates the draft-step cost (`mjolnir verify profile`). fp8 KV only: refused on fp6 KV, whose fallback cannot be CUDA-graphed |
|
|
114
115
|
| `MJOLNIR_ATTN_NONCAUSAL` | `1` | `0`: a parallel drafter's non-causal block steps (DSpark / DFlash) go to the fallback instead of the decode kernel — fp8 KV only for the same reason as above |
|
|
115
116
|
| `MJOLNIR_ATTN_FUSED_DRAFT` | `1` | `0`: tell vLLM the backend cannot advance draft metadata in place → one graph replay per draft step instead of the fused draft loop |
|
|
116
|
-
| `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `
|
|
117
|
+
| `MJOLNIR_LM_HEAD` | – | `fp8`: quantize the lm_head to e4m3 per row at load, Triton fp8-weight GEMM for the verify and every draft (`mjolnir.lmhead`; config `NVFP4_MJOLNIR_E2M3_LMH8_VT64K`); the bf16 copy is freed automatically (untied heads only, +2.5 GB KV). `MJOLNIR_LM_HEAD_TILES` overrides the GEMV launch config (`BN,BK,warps,stages`; default 64,256,8,5 — the 2026-11-04 serving A/B winner) |
|
|
118
|
+
| `MJOLNIR_MTP_FP8` | – | `1`: fp8-row-quantize the drafter's big unquantized linears (≥16M params) at load, same e4m3-per-row scheme + self-test (≤0.02 rel err) as the head; the bf16 originals are freed. Halves the MTP draft loop's streamed weight bytes |
|
|
117
119
|
| `MJOLNIR_DRAFT_VOCAB` | – | path to a `draft_vocab.json` (`mjolnir calibrate --draft-vocab`): the drafter's logits only over those tokens (VocabTrim); `MJOLNIR_DRAFT_VOCAB_SIZE` keeps the first N (a shorter file is padded with the lowest unused ids, with a warning). The kept head stays in the drafter's native quant (NVFP4 4-bit in-kernel, FP8 row-slice) — never up-scaled to bf16; a bf16 head is the one exception (fp8 only when `MJOLNIR_LM_HEAD=fp8` is set). `MJOLNIR_DRAFT_VOCAB_NATIVE=0` forces the dequant→fp8 path |
|
|
118
120
|
| `MJOLNIR_ATTN_VERBOSE` | – | `1`: print the kernel's compile-time notes (ring stages, TMEM columns) — the standalone harness always does |
|
|
119
121
|
| `MJOLNIR_ATTN_BANNER` | `1` | `0`: skip the Mjolnir hammer banner in the vLLM boot log (`mjolnir.banner.emit`, once per start, coloured with the server's formatter) |
|
|
@@ -125,7 +127,7 @@ runs on the fallback backend.
|
|
|
125
127
|
|
|
126
128
|
The pre-rename `THOR_T5_*` spellings are still honored. Each default is backed
|
|
127
129
|
by a measurement — see the
|
|
128
|
-
[kernel chapters](https://github.com/
|
|
130
|
+
[kernel chapters](https://github.com/GoBrrr-ai/mjolnir/tree/main/docs/kernel).
|
|
129
131
|
|
|
130
132
|
## fp6 (e2m3) KV in vLLM
|
|
131
133
|
|
|
@@ -138,25 +140,52 @@ expands the pages a prefill / mixed step reads to e4m3 for the FA4 fallback
|
|
|
138
140
|
(channel scales applied, one descale per head). The scales come from a calibration run on fp8 KV:
|
|
139
141
|
|
|
140
142
|
```bash
|
|
141
|
-
python -m mjolnir.calibrate --config <serve.yaml> --out kv_scales.json # in the image
|
|
142
|
-
MJOLNIR_ATTN_KV_SCALES=kv_scales.json vllm serve ... --kv-cache-dtype fp6_e2m3 --attention-backend MJOLNIR
|
|
143
|
+
python -m mjolnir.calibrate --config <serve.yaml> --out kv_scales/e2m3.json # in the image
|
|
144
|
+
MJOLNIR_ATTN_KV_SCALES=kv_scales/e2m3.json vllm serve ... --kv-cache-dtype fp6_e2m3 --attention-backend MJOLNIR
|
|
143
145
|
```
|
|
144
146
|
|
|
145
147
|
(`mjolnir calibrate` / `mjolnir serve` do both for you —
|
|
146
|
-
[docs/faq/fp6-kv.md](https://github.com/
|
|
148
|
+
[docs/faq/fp6-kv.md](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/faq/fp6-kv.md).)
|
|
147
149
|
|
|
148
150
|
## Beyond fp8: the harness
|
|
149
151
|
|
|
150
|
-
`mjolnir/
|
|
151
|
-
(`python -m mjolnir.
|
|
152
|
+
`mjolnir/arch/thor/decode.py` is the kernel; `mjolnir/arch/thor/harness.py`
|
|
153
|
+
(`python -m mjolnir.arch.thor.harness --help`) is its standalone harness, with a bit-faithful emulator as the
|
|
152
154
|
correctness reference. The harness can also read KV formats vLLM cannot write
|
|
153
155
|
yet — fp6 (`e2m3`/`e3m2`) and fp4 (`e2m1`) read directly by `tcgen05
|
|
154
156
|
kind::f8f6f4`, and `nvfp4` — and defaults to e2m3 on head-major pages for fp8 Q.
|
|
155
157
|
How they compare in speed and accuracy:
|
|
156
|
-
[`kvfmt`](https://github.com/
|
|
157
|
-
[`quant`](https://github.com/
|
|
158
|
+
[`kvfmt`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/kvfmt.md) and
|
|
159
|
+
[`quant`](https://github.com/GoBrrr-ai/mjolnir/blob/main/docs/kernel/quant.md).
|
|
158
160
|
Wiring a vLLM cache-write path for fp6 is an open contribution.
|
|
159
161
|
|
|
162
|
+
## Platforms and architectures
|
|
163
|
+
|
|
164
|
+
The kernel is written for one architecture today — Thor (`sm_110`) — but the
|
|
165
|
+
package is organized by platform: `mjolnir/arch/` is the implemented-kernel
|
|
166
|
+
registry (`mjolnir.arch.thor` re-exports the exact runner surface the backend
|
|
167
|
+
consumes), and `mjolnir/platforms.py` is the torch-free leaf that maps a
|
|
168
|
+
compute capability to it. Thor and Jetson Orin (`sm_87`) are implemented;
|
|
169
|
+
DGX Spark (`sm_121`) is declared research — named in the matrix, no kernel
|
|
170
|
+
yet. Orin ships one kernel: the `mma.m16n8k16` decode kernel (fp8 KV at
|
|
171
|
+
hd 128/256, fp6-e2m3 KV hd 256, M ≤ 8, bf16 Q) — see `mjolnir/arch/orin/README.md`.
|
|
172
|
+
|
|
173
|
+
`kernel_for((11, 0))` returns `mjolnir.arch.thor`; any other compute
|
|
174
|
+
capability raises `PlatformNotSupported` naming the requested `sm_XX` and the
|
|
175
|
+
shipped platforms. The vLLM backend never sees that exception: its metadata
|
|
176
|
+
builder consults the pure `unsupported_reason()` and takes the normal
|
|
177
|
+
ineligible-config path — that layer's decode steps run on the fallback
|
|
178
|
+
backend, the serve continues. On a host without a usable CUDA device,
|
|
179
|
+
`MJOLNIR_ARCH` (`sm_110` or `11.0`) pins the architecture instead of the
|
|
180
|
+
torch read (the standalone harness).
|
|
181
|
+
|
|
182
|
+
There is no remembered platform state anywhere in the stack: the CLI side
|
|
183
|
+
resolves its platform per invocation (flag > `$MJOLNIR_PLATFORM` > host
|
|
184
|
+
detection — four source classes with provenance, the `/tmp/nv_jetson_model`
|
|
185
|
+
model copy corroboration-only), so the kernel is always resolved from the
|
|
186
|
+
device the run actually happens on. See
|
|
187
|
+
[docs/faq/platforms.md](../../docs/faq/platforms.md).
|
|
188
|
+
|
|
160
189
|
## Tests
|
|
161
190
|
|
|
162
191
|
From a checkout, on a Thor:
|
|
@@ -182,11 +211,13 @@ is printed as information.
|
|
|
182
211
|
|
|
183
212
|
| file | what |
|
|
184
213
|
|---|---|
|
|
185
|
-
| `mjolnir/
|
|
214
|
+
| `mjolnir/arch/thor/` | the Thor kernel family: `decode.py` (the CuTe-DSL kernel + merge), `call.py`, `const.py`, `plan.py`, `runner.py`, `harness.py` (driver + emulator) |
|
|
186
215
|
| `mjolnir/runner.py` | plan buffers, compile cache, launch, KV views (no vLLM import) |
|
|
187
216
|
| `mjolnir/backend.py` | the vLLM backend, metadata builder, impl |
|
|
217
|
+
| `mjolnir/platforms.py` | arch → kernel resolution (`kernel_for`, `PlatformNotSupported`), `MJOLNIR_ARCH` detection — torch-free |
|
|
218
|
+
| `mjolnir/arch/thor.py` | the Thor (`sm_110`) kernel surface: re-exports of the runner |
|
|
188
219
|
| `mjolnir/__init__.py` | `register()` — the vLLM plugin entry point |
|
|
189
220
|
|
|
190
|
-
Apache-2.0, except `mjolnir/
|
|
221
|
+
Apache-2.0, except `mjolnir/arch/thor/decode.py`, which is derived from an NVIDIA
|
|
191
222
|
CUTLASS CuTe DSL example and stays under BSD-3-Clause — see `LICENSE` and
|
|
192
223
|
`NOTICE` in this package.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Mjolnir — fp8 paged decode attention for vLLM on Jetson AGX Thor (sm_110).
|
|
2
2
|
|
|
3
|
-
The kernel (codename **T5**, ``mjolnir.
|
|
3
|
+
The kernel (codename **T5**, ``mjolnir.arch.thor.decode``) is a CuTe-DSL tcgen05 +
|
|
4
4
|
TMEM + TMA decode kernel; ``mjolnir.backend`` wraps it as a vLLM attention
|
|
5
5
|
backend and ``mjolnir.runner`` holds the plan buffers, compile cache and KV
|
|
6
6
|
views.
|
|
@@ -24,7 +24,7 @@ from __future__ import annotations
|
|
|
24
24
|
import os
|
|
25
25
|
|
|
26
26
|
__all__ = ["BACKEND_NAME", "BACKEND_PATH", "__version__", "register"]
|
|
27
|
-
__version__ = "0.
|
|
27
|
+
__version__ = "0.3.2"
|
|
28
28
|
|
|
29
29
|
BACKEND_NAME = "MJOLNIR"
|
|
30
30
|
BACKEND_PATH = "mjolnir.backend.MjolnirAttentionBackend"
|
|
@@ -60,6 +60,22 @@ def has_native_member() -> bool:
|
|
|
60
60
|
def register() -> None:
|
|
61
61
|
if _disabled():
|
|
62
62
|
return
|
|
63
|
+
# vLLM > 0.32.0: the fp6_e2m3 KV dtype registers through the pluggable
|
|
64
|
+
# dtype API (vLLM #52670) instead of the image patch — a no-op on
|
|
65
|
+
# 0.32.0 and on a base where the patch already added it.
|
|
66
|
+
try:
|
|
67
|
+
from .kvdtype import register as _register_kv_dtype
|
|
68
|
+
|
|
69
|
+
_register_kv_dtype()
|
|
70
|
+
except Exception as e: # noqa: BLE001 - never break vLLM startup
|
|
71
|
+
import logging
|
|
72
|
+
|
|
73
|
+
logging.getLogger("vllm.mjolnir").warning("Mjolnir fp6 KV dtype not registered: %r", e)
|
|
74
|
+
if os.environ.get("MJOLNIR_ATTN_DEBUG", "").strip():
|
|
75
|
+
# The IMA hunt: the engine core is a child process that inherits this
|
|
76
|
+
# env before its CUDA runtime initializes, so faulting kernels are
|
|
77
|
+
# named at their launch instead of surfacing on a later sync.
|
|
78
|
+
os.environ.setdefault("CUDA_LAUNCH_BLOCKING", "1")
|
|
63
79
|
enum, register_backend = _registry()
|
|
64
80
|
if BACKEND_NAME in enum.__members__:
|
|
65
81
|
register_backend(enum[BACKEND_NAME], BACKEND_PATH)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Implemented-kernel registry: one subpackage per supported architecture
|
|
2
|
+
(``mjolnir.arch.thor``).
|
|
3
|
+
|
|
4
|
+
``mjolnir.platforms.kernel_for`` maps a compute capability to a dotted module
|
|
5
|
+
path here. Importing a subpackage pulls the CUTLASS DSL (heavy) — only do it
|
|
6
|
+
when a device is real.
|
|
7
|
+
"""
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# Orin kernel family — the mma.sync decode kernel (sm_87)
|
|
2
|
+
|
|
3
|
+
This subpackage is the Mjolnir kernel for **Jetson AGX Orin 64GB (`sm_87`,
|
|
4
|
+
GA10B Ampere)** — targeted at that one architecture (Thor keeps the
|
|
5
|
+
`tcgen05` `t5_*` family; this family never runs on sm_100/sm_110). The
|
|
6
|
+
family's only kernel is the tensor-core `mma.m16n8k16` decode kernel.
|
|
7
|
+
|
|
8
|
+
## What the kernel is
|
|
9
|
+
|
|
10
|
+
One CTA per `(kv-head, split, batch)`, 128 threads = 4 warps; the M tile
|
|
11
|
+
= 64 rows split across the warps (one 16-row mma atom each); the KV rows
|
|
12
|
+
stage through `cp.async` (32 rows per stage), dequantize fp8→bf16 in
|
|
13
|
+
smem, and ride the reference's `ldmatrix → mma.m16n8k16` path. `M ≤ 8`
|
|
14
|
+
(the MTP=7 verify shape), split-KV with chunk-normalized fp32 partials +
|
|
15
|
+
the fused LSE combine, causal and non-causal (the MTP drafts see one
|
|
16
|
+
fewer future key each). **KV formats: fp8 e4m3 and fp6 e2m3** — the fp6
|
|
17
|
+
rows read packed (d·3/4 = 192 bytes for d=256, 4 codes per 3 bytes LE)
|
|
18
|
+
through the 64-entry `kvfp6.e2m3_values` LUT; the per-channel scales
|
|
19
|
+
fold into Q (`q_mul`) and post-multiply the output (`o_mul`) in the
|
|
20
|
+
runner, so the kernel's math is untouched (M1+M2).
|
|
21
|
+
|
|
22
|
+
## The target to beat
|
|
23
|
+
|
|
24
|
+
The native sm_87 path in vLLM 0.32.0 is **FlashInfer's tensor-core FA2
|
|
25
|
+
decode** — confirmed at both tags: `vllm/v1/attention/backends/flashinfer.py`
|
|
26
|
+
constructs `BatchDecodeWithPagedKVCacheWrapper(..., use_tensor_cores=True)`
|
|
27
|
+
unconditionally, which routes to the FA2 prefill kernel
|
|
28
|
+
(`get_batch_prefill_module("fa2", ...)`); XQA/trtllm-gen are SM90/100/120-only.
|
|
29
|
+
Its Ampere shape for our model (packed qo = q_len·g = 6..24 ≤ 16 →
|
|
30
|
+
`CTA_TILE_Q = 16`, 1 Q-warp + 4 KV-warps = 128 threads):
|
|
31
|
+
|
|
32
|
+
* **NUM_MMA_KV = 2 → 128 KV rows per iteration** (fp8: 32 KB K+V smem per
|
|
33
|
+
step; the launcher's 2-CTA/SM budget against the *100 KB-class* Ampere
|
|
34
|
+
parts — Orin's 163 KB is headroom it never uses);
|
|
35
|
+
* fp8 KV is converted to fp16 **in registers before every HMMA**
|
|
36
|
+
(`fast_dequant_f8f16x4`, ≈2–4 ALU ops per 4 values — no fp8 cvt or mma
|
|
37
|
+
exists below sm_89) — a per-value tax the mma pays once per loaded
|
|
38
|
+
vector in smem instead;
|
|
39
|
+
* Q stays **bf16** on SM87 + fp8 KV ("architectures with only fa2 cannot
|
|
40
|
+
consume FP8 queries");
|
|
41
|
+
* split-KV is capped at 2 CTAs/SM (32 CTAs on Orin's 16 SMs), the LSE
|
|
42
|
+
merge is a **second kernel launch**, and PDL does not exist on
|
|
43
|
+
Ampere (`device_support_pdl` requires major ≥ 9) — a hard serial
|
|
44
|
+
launch floor per decode step that Thor does not pay.
|
|
45
|
+
|
|
46
|
+
Hardware reference (AGX Orin 64GB): 2048 cores / **16 SMs** / 64 3rd-gen
|
|
47
|
+
Tensor Cores / 5.3 TFLOPS fp32 / ~42.5 TFLOPS fp16 (derived) /
|
|
48
|
+
**204.8 GB/s** LPDDR5 / 4 MB L2 / 163 KB smem per block / 1.3 GHz.
|
|
49
|
+
|
|
50
|
+
## Measured status
|
|
51
|
+
|
|
52
|
+
* **The mma kernel**: `mjolnir verify sweep` same-run vs FlashInfer
|
|
53
|
+
FA2-TC across the M/L band (research/runs/orin1/notes/
|
|
54
|
+
mma-kernel-design-2.md).
|
|
55
|
+
* **The mma kernel (2026-10-26)**: serving-validated end-to-end — the
|
|
56
|
+
A/B/C matrix byte-identical correct, the spec sweep 14/14 (Mjolnir
|
|
57
|
+
19.9 t/s/req vs FlashInfer 18.9 at c=1, +5.0% aggregate at c=4,
|
|
58
|
+
identical per-position acceptance), fp6 (M1+M2) probe-validated on
|
|
59
|
+
both pool layouts and the image canary (`kernel_test_orin_gates.py`).
|
|
60
|
+
|
|
61
|
+
## The fp6 e2m3 KV path (M1+M2, landed)
|
|
62
|
+
|
|
63
|
+
* **M1 (in-kernel)**: the mma reads the packed rows directly — the
|
|
64
|
+
192-byte row moves as 12 × 16-byte cp.async chunks; the codes
|
|
65
|
+
unpack at bit 6i and dequantize through the LUT into the bf16 mma
|
|
66
|
+
operands. The per-head descales ride the accumulator
|
|
67
|
+
(`scale_eff` / the epilogue) exactly like fp8.
|
|
68
|
+
* **M2 (per-channel)**: the per-(layer, KV head, channel) scales from
|
|
69
|
+
`mjolnir calibrate` fold into Q pre-launch (`q_mul`) and
|
|
70
|
+
post-multiply the output (`o_mul`) — the kernel untouched
|
|
71
|
+
(`mjolnir/kvfp6.py`, the calibration `kv_scales/e2m3-awq.json`).
|
|
72
|
+
|
|
73
|
+
## hd128 (shipped, 2026-10-31)
|
|
74
|
+
|
|
75
|
+
The mma serves **head_size 128 and 256 for fp8** from one D-parameterized
|
|
76
|
+
body: the staging, the fragment counts and the PV ranges all derive from
|
|
77
|
+
`self.HEAD_DIM`, the constructor takes the head size, and the runner
|
|
78
|
+
gates `{128, 256}`, keys the binary on it (the TVM-FFI dims bake it) and
|
|
79
|
+
passes it to the split-KV combine — whose elements-per-thread is
|
|
80
|
+
`d / NUM_THREADS`. That combine was the whole hd128 bug: it baked
|
|
81
|
+
D=256, so at hd128 its second element per thread read and wrote past
|
|
82
|
+
the row — masked whenever `max_splits=1` (the write-through path), a
|
|
83
|
+
9.2 rel-err otherwise. fp6 stays **hd256-only**: the e2m3 D2 recipe's
|
|
84
|
+
byte-lane phase is static per thread and assumes 64 groups per row
|
|
85
|
+
(a per-iteration phase would serve hd128's 32).
|
|
86
|
+
|
|
87
|
+
Verified on the harness (rel err 0.0016–0.0019 vs the fp64 reference):
|
|
88
|
+
L 1K–262K, q_len 3/4/5, HND + NHD, split caps 2/4/8; the hd256 fp8 and
|
|
89
|
+
fp6 regressions clean. At M=5 / L=64K the hd128 shape graphs 600 µs vs
|
|
90
|
+
hd256's 967 µs (1.61× — an hd128 model streams half the KV bytes), and
|
|
91
|
+
the kernel drops to 128 regs / 27 KB smem (4 CTAs/SM by registers) from
|
|
92
|
+
hd256's 236 regs / 83 KB (2 CTAs/SM).
|
|
93
|
+
|
|
94
|
+
## Not portable from Thor (deliberate)
|
|
95
|
+
|
|
96
|
+
* `tcgen05 kind::f8f6f4` (the Thor fp6 read) — Ampere has no such MMA;
|
|
97
|
+
fp6 read here is unpack + `mma.sync`.
|
|
98
|
+
* PDL / programmatic dependent launch — sm_90+.
|
|
99
|
+
* fp8 lm_head (`MJOLNIR_LM_HEAD=fp8`) — no fp8 tensor cores on Ampere;
|
|
100
|
+
the Triton fp8 GEMM would be compute-bound. Orin keeps a bf16 lm_head.
|
|
101
|
+
* The Thor-tuned ring depths / split plans — retuned for 16 SMs,
|
|
102
|
+
100 KiB smem and LPDDR5x latency; the sm_110 numbers do not transfer.
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""The Orin (``sm_87``) kernel family: the mma.sync tensor-core decode kernel.
|
|
2
|
+
|
|
3
|
+
One subpackage per implemented architecture, sibling to
|
|
4
|
+
:mod:`mjolnir.arch.thor` (see that module's docstring for the surface
|
|
5
|
+
contract). The family mirrors the Thor layout:
|
|
6
|
+
|
|
7
|
+
* :mod:`mjolnir.arch.orin.mma` — the tensor-core (mma.sync m16n8k16)
|
|
8
|
+
decode kernel: fp8 (head_dim 128/256) and fp6-e2m3 (head_dim 256) KV,
|
|
9
|
+
bf16 Q, q_len <= 8
|
|
10
|
+
(decode and the MTP verify). The family's only kernel;
|
|
11
|
+
* :mod:`mjolnir.arch.orin.plan` — the split-KV planner (FlashInfer
|
|
12
|
+
Alg. 1, board-agnostic: the SM count is a device query, so AGX Orin
|
|
13
|
+
64/32 GB, Orin NX and Orin Nano all plan correctly);
|
|
14
|
+
* :mod:`mjolnir.arch.orin.combine` — the fused split-KV LSE merge;
|
|
15
|
+
* :mod:`mjolnir.arch.orin.runner` — the plan buffers, compile cache and
|
|
16
|
+
launch; :class:`Runner` below is the surface the vLLM backend consumes.
|
|
17
|
+
|
|
18
|
+
Do not import this package on a device whose compute capability is not
|
|
19
|
+
(8, 7): the DSL compile and the module's budget asserts assume GA10B.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
import logging
|
|
23
|
+
|
|
24
|
+
from mjolnir.runner import env_flag
|
|
25
|
+
|
|
26
|
+
from .runner import OrinDecodeRunner
|
|
27
|
+
|
|
28
|
+
__all__ = ["OrinDecodeRunner", "Runner"]
|
|
29
|
+
|
|
30
|
+
_log = logging.getLogger("vllm.mjolnir")
|
|
31
|
+
|
|
32
|
+
MMA_MAX_Q_LEN = 8 # mma.MMADecodeKernel.MAX_Q_LEN
|
|
33
|
+
MMA_KV_FORMATS = ("fp8", "e2m3") # the mma kernel's KV formats
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class Runner(OrinDecodeRunner):
|
|
37
|
+
"""The backend-facing runner surface (``mjolnir.platforms.kernel_for``
|
|
38
|
+
hands this to the vLLM backend). Accepts the shared backend kwargs and
|
|
39
|
+
absorbs the Thor-only levers as inert attributes: the Orin grid has no
|
|
40
|
+
CTAs/SM knob, and PDL / L2-prefetch / evict-first hints do not exist
|
|
41
|
+
below sm_90 — the attributes exist so the backend's builder logging and
|
|
42
|
+
metadata paths read them unchanged.
|
|
43
|
+
|
|
44
|
+
Two backend kwargs are translated to this family's meaning:
|
|
45
|
+
|
|
46
|
+
* ``max_splits`` (``MJOLNIR_ATTN_MAX_SPLITS``, "max KV chunks per
|
|
47
|
+
sequence") caps the auto split capacity — it used to pin it, so the
|
|
48
|
+
backend's default 16 silently replaced the mma's 2-CTA-occupancy
|
|
49
|
+
optimum of 8 (one wave on 16 SMs) with the over-split 16;
|
|
50
|
+
* ``q_mode``: the mma kernel takes a bf16 query, so the runner always
|
|
51
|
+
runs bf16 Q."""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
*,
|
|
56
|
+
ctas_per_sm: int = 2,
|
|
57
|
+
pdl: bool = False,
|
|
58
|
+
l2_prefetch: int = 0,
|
|
59
|
+
kv_evict_first: bool = False,
|
|
60
|
+
max_splits: int = 0,
|
|
61
|
+
**kw,
|
|
62
|
+
):
|
|
63
|
+
head_size = kw["head_size"]
|
|
64
|
+
kv_format = kw.get("kv_format", "fp8")
|
|
65
|
+
max_q_len = kw.get("max_q_len", 1)
|
|
66
|
+
supported = (256,) if kv_format == "e2m3" else (128, 256)
|
|
67
|
+
if head_size not in supported:
|
|
68
|
+
raise ValueError(
|
|
69
|
+
f"the Orin mma kernel serves head_size {'/'.join(map(str, supported))} for {kv_format} KV (got {head_size})"
|
|
70
|
+
)
|
|
71
|
+
if kv_format not in MMA_KV_FORMATS:
|
|
72
|
+
raise ValueError(f"the Orin mma kernel serves {'/'.join(MMA_KV_FORMATS)} KV only (got {kv_format!r})")
|
|
73
|
+
if max_q_len > MMA_MAX_Q_LEN:
|
|
74
|
+
raise ValueError(f"the Orin mma kernel serves q_len <= {MMA_MAX_Q_LEN} (got {max_q_len})")
|
|
75
|
+
if env_flag("MJOLNIR_ATTN_ORIN_IMPL", "") not in ("", "mma"):
|
|
76
|
+
raise ValueError("MJOLNIR_ATTN_ORIN_IMPL: only 'mma' is supported on Orin")
|
|
77
|
+
kw["q_mode"] = "bf16"
|
|
78
|
+
super().__init__(split_cap=max_splits, **kw)
|
|
79
|
+
self.ctas_per_sm = ctas_per_sm
|
|
80
|
+
self.pdl = pdl
|
|
81
|
+
self.l2_prefetch = l2_prefetch
|
|
82
|
+
self.kv_evict_first = kv_evict_first
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""The fused split-KV combine for the Orin MMA decode kernel.
|
|
2
|
+
|
|
3
|
+
The runner's partials are (ns, rows, h_q, d) fp32 O and (ns, b, h_q, ql)
|
|
4
|
+
fp32 LSE; the merge is O = Σ_s exp(lse_s − L)·O_s / Σ_s exp(lse_s − L)
|
|
5
|
+
with L = max_s lse_s (the chunk-normalized contract the kernels write).
|
|
6
|
+
The eager torch form issued ~20 small kernels (~150 us of GPU per call
|
|
7
|
+
— more than the decode kernel itself); this is one kernel, one pass
|
|
8
|
+
over the partials, the FlashInfer MergeStates cost class.
|
|
9
|
+
|
|
10
|
+
One block per (row, q-head): the block loads its ns LSE values, then
|
|
11
|
+
each thread accumulates its d/NUM_THREADS elements across the splits
|
|
12
|
+
(2 at hd256, 1 at hd128 — the head size rides the compile as a
|
|
13
|
+
constexpr). ns/rows/h_q/b/ql are runtime Int32s so one compile serves
|
|
14
|
+
every capture shape per head size. The tensor layouts
|
|
15
|
+
are the runner's capacity buffers (fixed at construction, so the baked
|
|
16
|
+
extents are stable). The empty-chunk splits write lse = -inf -> weight
|
|
17
|
+
0; an all-empty row normalizes to 0 (the torch combine's clamp_min) —
|
|
18
|
+
L is floored at -FLT_MAX so that row never computes -inf - -inf.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import cutlass
|
|
24
|
+
import cutlass.cute as cute
|
|
25
|
+
import cuda.bindings.driver as cuda
|
|
26
|
+
from cutlass import Float32, Int32, const_expr
|
|
27
|
+
|
|
28
|
+
NUM_THREADS = 128
|
|
29
|
+
_TINY = 1.1754944e-35 # torch.finfo(torch.float32).tiny, rounded
|
|
30
|
+
_NEG_FLOOR = -3.4028234e38 # -torch.finfo(torch.float32).max: the finite L floor
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@cute.kernel
|
|
34
|
+
def combine_kernel(
|
|
35
|
+
mO_part: cute.Tensor, # (ns_cap*rows_cap*h_q*D) fp32 flat
|
|
36
|
+
mLse: cute.Tensor, # (ns_cap*b_cap*h_q*ql_cap) fp32 flat
|
|
37
|
+
mOut: cute.Tensor, # (rows_cap*h_q*D) bf16 flat
|
|
38
|
+
d: cutlass.Constexpr[int], # the head size (128 or 256): the elements/thread = d/NUM_THREADS
|
|
39
|
+
max_ns: cutlass.Constexpr[int], # the unroll/predication bound (= ns_cap)
|
|
40
|
+
ns: Int32,
|
|
41
|
+
rows: Int32,
|
|
42
|
+
h_q: Int32,
|
|
43
|
+
b: Int32,
|
|
44
|
+
ql: Int32,
|
|
45
|
+
):
|
|
46
|
+
bid = cute.arch.block_idx()[0]
|
|
47
|
+
tidx = cute.arch.thread_idx()[0]
|
|
48
|
+
r = bid // h_q
|
|
49
|
+
h = bid - r * h_q
|
|
50
|
+
bb = r // ql
|
|
51
|
+
qq = r - bb * ql
|
|
52
|
+
ls_s = b * h_q * ql
|
|
53
|
+
ls_b = h_q * ql
|
|
54
|
+
|
|
55
|
+
# Both loops unroll over max_ns with the s < ns predication: the
|
|
56
|
+
# dynamic-range form serialized every split's load behind the loop
|
|
57
|
+
# branch (~34 us; the FI merge overlaps all partial rows in flight
|
|
58
|
+
# and lands at 11). Unrolled, the loads issue independently and the
|
|
59
|
+
# latency hides under the acc chain.
|
|
60
|
+
# 1. L = max_s lse[s, h, r], floored at a FINITE -FLT_MAX: a row whose
|
|
61
|
+
# every split is empty (all lse = -inf — a CUDA-graph padding request
|
|
62
|
+
# with seq_len 0, or a dummy/profile step) would otherwise get
|
|
63
|
+
# L = -inf and w = exp2(-inf - -inf) = NaN, a NaN output row instead
|
|
64
|
+
# of the documented 0 (and a NaN row poisons any per-tensor reduction
|
|
65
|
+
# downstream). With the floor, every -inf lse weighs exactly 0 and a
|
|
66
|
+
# real lse (>= -FLT_MAX) is unaffected.
|
|
67
|
+
L = Float32(_NEG_FLOOR)
|
|
68
|
+
for s in cutlass.range_constexpr(max_ns):
|
|
69
|
+
if s < ns:
|
|
70
|
+
L = cutlass.max(L, mLse[s * ls_s + bb * ls_b + h * ql + qq])
|
|
71
|
+
|
|
72
|
+
# 2. the weighted sum over the splits; the exp weights recomputed per
|
|
73
|
+
# element pass (ns <= 64 exp2s are nothing next to the loads).
|
|
74
|
+
ln2e = Float32(1.4426950408889634) # 1/ln 2: exp(x) = exp2(x*log2 e)
|
|
75
|
+
sw = Float32(0.0)
|
|
76
|
+
acc0 = Float32(0.0)
|
|
77
|
+
acc1 = Float32(0.0)
|
|
78
|
+
for s in cutlass.range_constexpr(max_ns):
|
|
79
|
+
if s < ns:
|
|
80
|
+
w = cute.math.exp2((mLse[s * ls_s + bb * ls_b + h * ql + qq] - L) * ln2e, fastmath=True)
|
|
81
|
+
sw = sw + w
|
|
82
|
+
obase = s * rows * h_q * d + r * h_q * d + h * d
|
|
83
|
+
acc0 = acc0 + w * mO_part[obase + tidx]
|
|
84
|
+
if const_expr(d == 256):
|
|
85
|
+
acc1 = acc1 + w * mO_part[obase + tidx + NUM_THREADS]
|
|
86
|
+
scale = 1.0 / cutlass.max(sw, Float32(_TINY))
|
|
87
|
+
obase = r * h_q * d + h * d
|
|
88
|
+
mOut[obase + tidx] = (acc0 * scale).to(mOut.element_type)
|
|
89
|
+
if const_expr(d == 256):
|
|
90
|
+
mOut[obase + tidx + NUM_THREADS] = (acc1 * scale).to(mOut.element_type)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@cute.jit
|
|
94
|
+
def combine_launch(
|
|
95
|
+
mO_part: cute.Tensor,
|
|
96
|
+
mLse: cute.Tensor,
|
|
97
|
+
mOut: cute.Tensor,
|
|
98
|
+
d: cutlass.Constexpr[int],
|
|
99
|
+
max_ns: cutlass.Constexpr[int],
|
|
100
|
+
ns: Int32,
|
|
101
|
+
rows: Int32,
|
|
102
|
+
h_q: Int32,
|
|
103
|
+
b: Int32,
|
|
104
|
+
ql: Int32,
|
|
105
|
+
stream: cuda.CUstream = None,
|
|
106
|
+
):
|
|
107
|
+
combine_kernel(mO_part, mLse, mOut, d, max_ns, ns, rows, h_q, b, ql).launch(
|
|
108
|
+
grid=(rows * h_q, 1, 1),
|
|
109
|
+
block=(NUM_THREADS, 1, 1),
|
|
110
|
+
stream=stream,
|
|
111
|
+
)
|