kvkernels 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kvkernels/__init__.py +14 -0
- kvkernels/layouts.py +281 -0
- kvkernels/ops/__init__.py +17 -0
- kvkernels/ops/_registry.py +52 -0
- kvkernels/ops/cascade.py +899 -0
- kvkernels/ops/cascade_plan.py +813 -0
- kvkernels/ops/kv_gather.py +365 -0
- kvkernels/ops/merge_states.py +411 -0
- kvkernels/ops/plane_set.py +348 -0
- kvkernels/ops/prefix_attention.py +555 -0
- kvkernels/py.typed +0 -0
- kvkernels/reference/__init__.py +25 -0
- kvkernels/reference/attention.py +156 -0
- kvkernels/reference/cascade_plan.py +161 -0
- kvkernels/reference/kv_gather.py +40 -0
- kvkernels/roofline.py +90 -0
- kvkernels/specs.py +197 -0
- kvkernels/tolerance.py +75 -0
- kvkernels-0.1.0.dist-info/METADATA +160 -0
- kvkernels-0.1.0.dist-info/RECORD +22 -0
- kvkernels-0.1.0.dist-info/WHEEL +4 -0
- kvkernels-0.1.0.dist-info/licenses/LICENSE +202 -0
kvkernels/__init__.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Triton kernels for paged KV-cache movement and shared-prefix decode attention.
|
|
2
|
+
|
|
3
|
+
The public API lives in three modules: kvkernels.ops (the Triton ops),
|
|
4
|
+
kvkernels.layouts (engine KV cache layouts and their views) and
|
|
5
|
+
kvkernels.reference (PyTorch references for the ops). Importing this package
|
|
6
|
+
alone needs neither Triton nor a GPU.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
__version__ = version("kvkernels")
|
|
13
|
+
except PackageNotFoundError: # run from a source tree that was never installed
|
|
14
|
+
__version__ = "0.0.0+unknown"
|
kvkernels/layouts.py
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
"""Paged KV-cache layouts of vLLM and SGLang, and the views kvkernels takes of them.
|
|
2
|
+
|
|
3
|
+
A layout is a memory order (the order in which layer, block, head, slot and
|
|
4
|
+
content appear in memory) and a block stride (how far apart consecutive blocks
|
|
5
|
+
of one layer are). For each layout this module builds a cache the way the engine
|
|
6
|
+
allocates it and returns three views of the same memory, never a copy:
|
|
7
|
+
|
|
8
|
+
engine-shaped cache ──▶ planes each KV block one contiguous run of bytes (gather, scatter)
|
|
9
|
+
├──▶ kv_views K and V as [blocks, slots, kv_heads, head_dim], every layout
|
|
10
|
+
└──▶ make_index the block table or slot list the engine's attention reads
|
|
11
|
+
|
|
12
|
+
vLLM keeps one buffer whose per-layer views have axes (block, head, slot,
|
|
13
|
+
content), where content is a head's K values followed by its V values; the
|
|
14
|
+
memory order decides the strides. SGLang keeps one K and one V buffer per layer,
|
|
15
|
+
indexed by slot (default) or by page then head (heads first), and reserves page 0
|
|
16
|
+
for writes from padded tokens.
|
|
17
|
+
|
|
18
|
+
Block 0 is never given to a request: vLLM keeps it as its null block and SGLang
|
|
19
|
+
reserves it. Unused block-table entries are filled with it.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import math
|
|
25
|
+
from collections.abc import Sequence
|
|
26
|
+
from dataclasses import dataclass
|
|
27
|
+
from typing import Literal
|
|
28
|
+
|
|
29
|
+
import torch
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"LAYOUTS",
|
|
33
|
+
"CacheSpec",
|
|
34
|
+
"PagedKVLayout",
|
|
35
|
+
"kv_views",
|
|
36
|
+
"make_cache",
|
|
37
|
+
"make_index",
|
|
38
|
+
"planes",
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
VLLM_REPO = "vllm-project/vllm"
|
|
42
|
+
VLLM_COMMIT = "5df63a7203cdb9328d824a27b682e77c7b9928be" # v0.32.0
|
|
43
|
+
SGLANG_REPO = "sgl-project/sglang"
|
|
44
|
+
SGLANG_COMMIT = "baa9befa381511cd4d56f83980951e5a45fffc6d" # v0.5.22
|
|
45
|
+
|
|
46
|
+
RESERVED_BLOCK = 0
|
|
47
|
+
|
|
48
|
+
# K and V views: an fp8 cache is viewed as float8_e4m3fn (engines store its
|
|
49
|
+
# bytes as uint8; view the cache as float8_e4m3fn first).
|
|
50
|
+
_VIEW_DTYPES = (torch.float16, torch.bfloat16, torch.float32, torch.float8_e4m3fn)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass(frozen=True)
|
|
54
|
+
class Source:
|
|
55
|
+
"""The upstream code a layout follows."""
|
|
56
|
+
|
|
57
|
+
repo: str
|
|
58
|
+
commit: str
|
|
59
|
+
path: str
|
|
60
|
+
symbol: str
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True)
|
|
64
|
+
class CacheSpec:
|
|
65
|
+
num_layers: int
|
|
66
|
+
num_blocks: int # KV blocks (vLLM) or pages (SGLang), including reserved block 0
|
|
67
|
+
block_size: int # tokens per KV block or page
|
|
68
|
+
num_kv_heads: int
|
|
69
|
+
head_dim: int
|
|
70
|
+
dtype: torch.dtype
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class PagedKVLayout:
|
|
75
|
+
name: str
|
|
76
|
+
engine: Literal["vllm", "sglang"]
|
|
77
|
+
kv_split: Literal["joint", "separate"] # K and V in one buffer, or in two
|
|
78
|
+
index_kind: Literal["block_table", "slot_list"]
|
|
79
|
+
memory_order: str # outermost first; vLLM: letters L B H N C; SGLang: "nhd" or "hnd"
|
|
80
|
+
sources: tuple[Source, ...]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _vllm(path: str, symbol: str) -> Source:
|
|
84
|
+
return Source(VLLM_REPO, VLLM_COMMIT, path, symbol)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _sglang(path: str, symbol: str) -> Source:
|
|
88
|
+
return Source(SGLANG_REPO, SGLANG_COMMIT, path, symbol)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
_VLLM_SOURCES = (
|
|
92
|
+
_vllm("vllm/v1/kv_cache_layout.py", "KVCacheLayout"),
|
|
93
|
+
_vllm("vllm/v1/kv_cache_interface.py", "create_kv_cache_views"),
|
|
94
|
+
_vllm("vllm/v1/kv_cache_interface.py", "compute_layout_strides"),
|
|
95
|
+
_vllm("vllm/v1/attention/backends/triton_attn.py", "TritonAttentionImpl.forward"),
|
|
96
|
+
_vllm("vllm/v1/worker/gpu/block_table.py", "BlockTables"),
|
|
97
|
+
_vllm("vllm/v1/core/block_pool.py", "null_block"),
|
|
98
|
+
)
|
|
99
|
+
_SGLANG_SOURCES = (
|
|
100
|
+
_sglang("python/sglang/srt/mem_cache/memory_pool.py", "MHATokenToKVPool._kv_buffer_shapes"),
|
|
101
|
+
_sglang("python/sglang/srt/layers/attention/triton_backend.py", "kv_indptr, kv_indices"),
|
|
102
|
+
_sglang("python/sglang/srt/mem_cache/kv_index_translator.py", "pack_read_stream"),
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _vllm_layout(order: str) -> PagedKVLayout:
|
|
107
|
+
return PagedKVLayout(
|
|
108
|
+
name=f"vllm_{order}",
|
|
109
|
+
engine="vllm",
|
|
110
|
+
kv_split="joint",
|
|
111
|
+
index_kind="block_table",
|
|
112
|
+
memory_order=order,
|
|
113
|
+
sources=_VLLM_SOURCES,
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
LAYOUTS: dict[str, PagedKVLayout] = {
|
|
118
|
+
layout.name: layout
|
|
119
|
+
for layout in (
|
|
120
|
+
# Orders whose KV blocks are one contiguous run per layer. In vLLM's
|
|
121
|
+
# letters: L layer, B block, H head, N slot, C content.
|
|
122
|
+
_vllm_layout("LBNHC"), # vLLM's default
|
|
123
|
+
_vllm_layout("LBHNC"),
|
|
124
|
+
_vllm_layout("BLNHC"), # all layers of a block together: block stride spans every layer
|
|
125
|
+
_vllm_layout("BLHNC"),
|
|
126
|
+
PagedKVLayout("sglang_nhd", "sglang", "separate", "slot_list", "nhd", _SGLANG_SOURCES),
|
|
127
|
+
PagedKVLayout("sglang_hnd", "sglang", "separate", "slot_list", "hnd", _SGLANG_SOURCES),
|
|
128
|
+
)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _check_spec(spec: CacheSpec) -> None:
|
|
133
|
+
for name in ("num_layers", "num_blocks", "block_size", "num_kv_heads", "head_dim"):
|
|
134
|
+
if getattr(spec, name) < 1:
|
|
135
|
+
raise ValueError(f"{name} must be at least 1, got {getattr(spec, name)}")
|
|
136
|
+
if spec.num_blocks < 2:
|
|
137
|
+
raise ValueError("num_blocks must leave at least one block after reserved block 0")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _vllm_strides(spec: CacheSpec, order: str) -> tuple[dict[str, int], dict[str, int]]:
|
|
141
|
+
"""Sizes and element strides of the five axes, walking the memory order from innermost."""
|
|
142
|
+
sizes = {
|
|
143
|
+
"L": spec.num_layers,
|
|
144
|
+
"B": spec.num_blocks,
|
|
145
|
+
"H": spec.num_kv_heads,
|
|
146
|
+
"N": spec.block_size,
|
|
147
|
+
"C": 2 * spec.head_dim,
|
|
148
|
+
}
|
|
149
|
+
strides: dict[str, int] = {}
|
|
150
|
+
step = 1
|
|
151
|
+
for axis in reversed(order):
|
|
152
|
+
strides[axis] = step
|
|
153
|
+
step *= sizes[axis]
|
|
154
|
+
return sizes, strides
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def make_cache(
|
|
158
|
+
layout: PagedKVLayout, spec: CacheSpec, device: str | torch.device = "cpu"
|
|
159
|
+
) -> list[tuple[torch.Tensor, ...]]:
|
|
160
|
+
"""One entry per layer, holding the tensors the engine gives its kernels for that layer.
|
|
161
|
+
|
|
162
|
+
vLLM: one view per layer, axes (block, head, slot, content), over one buffer
|
|
163
|
+
shared by all layers. SGLang: (K buffer, V buffer); default order [slots,
|
|
164
|
+
heads, head_dim] with slots = num_blocks · block_size, heads first [pages,
|
|
165
|
+
heads, page_size, head_dim]. The contents are uninitialized.
|
|
166
|
+
"""
|
|
167
|
+
_check_spec(spec)
|
|
168
|
+
if layout.engine == "vllm":
|
|
169
|
+
sizes, strides = _vllm_strides(spec, layout.memory_order)
|
|
170
|
+
buffer = torch.empty(math.prod(sizes.values()), dtype=spec.dtype, device=device)
|
|
171
|
+
logical = "LBHNC"
|
|
172
|
+
everything = buffer.as_strided([sizes[a] for a in logical], [strides[a] for a in logical])
|
|
173
|
+
return [(everything[layer],) for layer in range(spec.num_layers)]
|
|
174
|
+
if layout.memory_order == "nhd":
|
|
175
|
+
shape: tuple[int, ...] = (
|
|
176
|
+
spec.num_blocks * spec.block_size,
|
|
177
|
+
spec.num_kv_heads,
|
|
178
|
+
spec.head_dim,
|
|
179
|
+
)
|
|
180
|
+
else:
|
|
181
|
+
shape = (spec.num_blocks, spec.num_kv_heads, spec.block_size, spec.head_dim)
|
|
182
|
+
return [
|
|
183
|
+
(
|
|
184
|
+
torch.empty(shape, dtype=spec.dtype, device=device),
|
|
185
|
+
torch.empty(shape, dtype=spec.dtype, device=device),
|
|
186
|
+
)
|
|
187
|
+
for _ in range(spec.num_layers)
|
|
188
|
+
]
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _slot_before_head(layout: PagedKVLayout) -> bool:
|
|
192
|
+
order = layout.memory_order
|
|
193
|
+
return order.index("N") < order.index("H")
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def planes(
|
|
197
|
+
layout: PagedKVLayout, spec: CacheSpec, layer_tensors: Sequence[torch.Tensor]
|
|
198
|
+
) -> list[torch.Tensor]:
|
|
199
|
+
"""The planes of one layer: first dimension indexes KV blocks, each block one contiguous run.
|
|
200
|
+
|
|
201
|
+
vLLM: one plane, K and V together. SGLang: two planes, K then V.
|
|
202
|
+
"""
|
|
203
|
+
if layout.engine == "vllm":
|
|
204
|
+
(view,) = layer_tensors
|
|
205
|
+
# The view's axes are (block, head, slot, content); reorder them to memory order.
|
|
206
|
+
return [view.permute(0, 2, 1, 3) if _slot_before_head(layout) else view]
|
|
207
|
+
if layout.memory_order == "nhd":
|
|
208
|
+
return [
|
|
209
|
+
t.view(spec.num_blocks, spec.block_size, spec.num_kv_heads, spec.head_dim)
|
|
210
|
+
for t in layer_tensors
|
|
211
|
+
]
|
|
212
|
+
return list(layer_tensors)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def kv_views(
|
|
216
|
+
layout: PagedKVLayout, spec: CacheSpec, layer_tensors: Sequence[torch.Tensor]
|
|
217
|
+
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
218
|
+
"""K and V of one layer, each [blocks, slots, kv_heads, head_dim]; views of the cache."""
|
|
219
|
+
if spec.dtype not in _VIEW_DTYPES:
|
|
220
|
+
raise ValueError(f"K and V views support {_VIEW_DTYPES}, got {spec.dtype}")
|
|
221
|
+
d = spec.head_dim
|
|
222
|
+
if layout.engine == "vllm":
|
|
223
|
+
(plane,) = planes(layout, spec, layer_tensors)
|
|
224
|
+
k, v = plane[..., :d], plane[..., d:]
|
|
225
|
+
if not _slot_before_head(layout): # plane is (block, head, slot, content)
|
|
226
|
+
k, v = k.transpose(1, 2), v.transpose(1, 2)
|
|
227
|
+
return k, v
|
|
228
|
+
k_plane, v_plane = planes(layout, spec, layer_tensors)
|
|
229
|
+
if layout.memory_order == "hnd": # (page, head, slot, head_dim)
|
|
230
|
+
return k_plane.transpose(1, 2), v_plane.transpose(1, 2)
|
|
231
|
+
return k_plane, v_plane
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def make_index(
|
|
235
|
+
layout: PagedKVLayout,
|
|
236
|
+
spec: CacheSpec,
|
|
237
|
+
request_blocks: Sequence[Sequence[int]],
|
|
238
|
+
seq_lens: Sequence[int],
|
|
239
|
+
device: str | torch.device = "cpu",
|
|
240
|
+
) -> dict[str, torch.Tensor]:
|
|
241
|
+
"""The index the engine's attention kernel reads, for requests owning the given blocks.
|
|
242
|
+
|
|
243
|
+
vLLM ("block_table", "seq_lens"): one int32 row per request; entries past a
|
|
244
|
+
request's blocks hold no defined value in the engine and are filled here
|
|
245
|
+
with block 0, the null block. "seq_lens" is int32.
|
|
246
|
+
|
|
247
|
+
SGLang ("kv_indptr", "kv_indices"): one int64 slot per token, at every page
|
|
248
|
+
size; kv_indptr (int32) is where each request's slots begin.
|
|
249
|
+
"""
|
|
250
|
+
if len(request_blocks) != len(seq_lens):
|
|
251
|
+
raise ValueError(f"{len(request_blocks)} block lists for {len(seq_lens)} lengths")
|
|
252
|
+
for r, (blocks, length) in enumerate(zip(request_blocks, seq_lens, strict=True)):
|
|
253
|
+
needed = math.ceil(length / spec.block_size)
|
|
254
|
+
if length < 1 or len(blocks) < needed:
|
|
255
|
+
raise ValueError(
|
|
256
|
+
f"request {r}: {length} tokens need {needed} blocks, got {len(blocks)}"
|
|
257
|
+
)
|
|
258
|
+
for b in blocks:
|
|
259
|
+
if not RESERVED_BLOCK < b < spec.num_blocks:
|
|
260
|
+
raise ValueError(f"request {r}: block {b} is outside 1 … {spec.num_blocks - 1}")
|
|
261
|
+
if layout.index_kind == "block_table":
|
|
262
|
+
width = max(len(b) for b in request_blocks) if request_blocks else 0
|
|
263
|
+
table = torch.full((len(request_blocks), width), RESERVED_BLOCK, dtype=torch.int32)
|
|
264
|
+
for r, blocks in enumerate(request_blocks):
|
|
265
|
+
table[r, : len(blocks)] = torch.tensor(blocks, dtype=torch.int32)
|
|
266
|
+
return {
|
|
267
|
+
"block_table": table.to(device),
|
|
268
|
+
"seq_lens": torch.tensor(list(seq_lens), dtype=torch.int32, device=device),
|
|
269
|
+
}
|
|
270
|
+
slots: list[int] = []
|
|
271
|
+
starts = [0]
|
|
272
|
+
for blocks, length in zip(request_blocks, seq_lens, strict=True):
|
|
273
|
+
slots.extend(
|
|
274
|
+
blocks[t // spec.block_size] * spec.block_size + t % spec.block_size
|
|
275
|
+
for t in range(length)
|
|
276
|
+
)
|
|
277
|
+
starts.append(len(slots))
|
|
278
|
+
return {
|
|
279
|
+
"kv_indptr": torch.tensor(starts, dtype=torch.int32, device=device),
|
|
280
|
+
"kv_indices": torch.tensor(slots, dtype=torch.int64, device=device),
|
|
281
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Triton ops. Each op has a pure-PyTorch reference in kvkernels.reference."""
|
|
2
|
+
|
|
3
|
+
from kvkernels.ops.cascade import CascadePlan, PrefixGroup, cascade_decode_attention
|
|
4
|
+
from kvkernels.ops.kv_gather import gather_kv_blocks, scatter_kv_blocks
|
|
5
|
+
from kvkernels.ops.merge_states import merge_attention_states, merge_attention_states_n
|
|
6
|
+
from kvkernels.ops.plane_set import PlaneSet
|
|
7
|
+
|
|
8
|
+
__all__ = [
|
|
9
|
+
"CascadePlan",
|
|
10
|
+
"PlaneSet",
|
|
11
|
+
"PrefixGroup",
|
|
12
|
+
"cascade_decode_attention",
|
|
13
|
+
"gather_kv_blocks",
|
|
14
|
+
"merge_attention_states",
|
|
15
|
+
"merge_attention_states_n",
|
|
16
|
+
"scatter_kv_blocks",
|
|
17
|
+
]
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Register an op with PyTorch, and call it without PyTorch's dispatch when nothing needs it.
|
|
2
|
+
|
|
3
|
+
Each op is registered as a PyTorch custom op, so torch.compile and engines treat a
|
|
4
|
+
call as one opaque operation that writes into the tensors named in mutates_args.
|
|
5
|
+
Going through the registration costs CPU time on every call: on the host of a
|
|
6
|
+
Kaggle T4, a gather call spent 67 µs of its 164 µs there (bench/host_cost.py).
|
|
7
|
+
A plain eager call needs none of it, so the op runs its body directly:
|
|
8
|
+
|
|
9
|
+
torch.compile tracing ───────────┐
|
|
10
|
+
a dispatch mode (fake tensors) ──┴──▶ the registered custom op
|
|
11
|
+
otherwise ───────────────────────────▶ the body, called directly
|
|
12
|
+
|
|
13
|
+
The direct call skips the version-counter bump PyTorch's autograd uses to detect
|
|
14
|
+
in-place changes to saved tensors; the ops are for inference and define no
|
|
15
|
+
gradient.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from collections.abc import Callable, Sequence
|
|
21
|
+
from typing import Generic, ParamSpec
|
|
22
|
+
|
|
23
|
+
import torch
|
|
24
|
+
|
|
25
|
+
P = ParamSpec("P")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class RegisteredOp(Generic[P]):
|
|
29
|
+
def __init__(self, name: str, body: Callable[P, None], mutates_args: Sequence[str]) -> None:
|
|
30
|
+
self.body = body
|
|
31
|
+
self.custom_op = torch.library.custom_op(name, body, mutates_args=tuple(mutates_args))
|
|
32
|
+
|
|
33
|
+
def register_fake(self, fake: Callable[P, None]) -> Callable[P, None]:
|
|
34
|
+
self.custom_op.register_fake(fake)
|
|
35
|
+
return fake
|
|
36
|
+
|
|
37
|
+
def __call__(self, *args: P.args, **kwargs: P.kwargs) -> None:
|
|
38
|
+
if torch.compiler.is_compiling() or torch._C._len_torch_dispatch_stack() > 0:
|
|
39
|
+
self.custom_op(*args, **kwargs)
|
|
40
|
+
else:
|
|
41
|
+
self.body(*args, **kwargs)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def registered_op(
|
|
45
|
+
name: str, *, mutates_args: Sequence[str]
|
|
46
|
+
) -> Callable[[Callable[P, None]], RegisteredOp[P]]:
|
|
47
|
+
"""Decorator: register the function as custom op `name` and return a RegisteredOp."""
|
|
48
|
+
|
|
49
|
+
def wrap(body: Callable[P, None]) -> RegisteredOp[P]:
|
|
50
|
+
return RegisteredOp(name, body, mutates_args)
|
|
51
|
+
|
|
52
|
+
return wrap
|