kvkernels 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
kvkernels/__init__.py ADDED
@@ -0,0 +1,14 @@
1
+ """Triton kernels for paged KV-cache movement and shared-prefix decode attention.
2
+
3
+ The public API lives in three modules: kvkernels.ops (the Triton ops),
4
+ kvkernels.layouts (engine KV cache layouts and their views) and
5
+ kvkernels.reference (PyTorch references for the ops). Importing this package
6
+ alone needs neither Triton nor a GPU.
7
+ """
8
+
9
+ from importlib.metadata import PackageNotFoundError, version
10
+
11
+ try:
12
+ __version__ = version("kvkernels")
13
+ except PackageNotFoundError: # run from a source tree that was never installed
14
+ __version__ = "0.0.0+unknown"
kvkernels/layouts.py ADDED
@@ -0,0 +1,281 @@
1
+ """Paged KV-cache layouts of vLLM and SGLang, and the views kvkernels takes of them.
2
+
3
+ A layout is a memory order (the order in which layer, block, head, slot and
4
+ content appear in memory) and a block stride (how far apart consecutive blocks
5
+ of one layer are). For each layout this module builds a cache the way the engine
6
+ allocates it and returns three views of the same memory, never a copy:
7
+
8
+ engine-shaped cache ──▶ planes each KV block one contiguous run of bytes (gather, scatter)
9
+ ├──▶ kv_views K and V as [blocks, slots, kv_heads, head_dim], every layout
10
+ └──▶ make_index the block table or slot list the engine's attention reads
11
+
12
+ vLLM keeps one buffer whose per-layer views have axes (block, head, slot,
13
+ content), where content is a head's K values followed by its V values; the
14
+ memory order decides the strides. SGLang keeps one K and one V buffer per layer,
15
+ indexed by slot (default) or by page then head (heads first), and reserves page 0
16
+ for writes from padded tokens.
17
+
18
+ Block 0 is never given to a request: vLLM keeps it as its null block and SGLang
19
+ reserves it. Unused block-table entries are filled with it.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import math
25
+ from collections.abc import Sequence
26
+ from dataclasses import dataclass
27
+ from typing import Literal
28
+
29
+ import torch
30
+
31
+ __all__ = [
32
+ "LAYOUTS",
33
+ "CacheSpec",
34
+ "PagedKVLayout",
35
+ "kv_views",
36
+ "make_cache",
37
+ "make_index",
38
+ "planes",
39
+ ]
40
+
41
+ VLLM_REPO = "vllm-project/vllm"
42
+ VLLM_COMMIT = "5df63a7203cdb9328d824a27b682e77c7b9928be" # v0.32.0
43
+ SGLANG_REPO = "sgl-project/sglang"
44
+ SGLANG_COMMIT = "baa9befa381511cd4d56f83980951e5a45fffc6d" # v0.5.22
45
+
46
+ RESERVED_BLOCK = 0
47
+
48
+ # K and V views: an fp8 cache is viewed as float8_e4m3fn (engines store its
49
+ # bytes as uint8; view the cache as float8_e4m3fn first).
50
+ _VIEW_DTYPES = (torch.float16, torch.bfloat16, torch.float32, torch.float8_e4m3fn)
51
+
52
+
53
+ @dataclass(frozen=True)
54
+ class Source:
55
+ """The upstream code a layout follows."""
56
+
57
+ repo: str
58
+ commit: str
59
+ path: str
60
+ symbol: str
61
+
62
+
63
+ @dataclass(frozen=True)
64
+ class CacheSpec:
65
+ num_layers: int
66
+ num_blocks: int # KV blocks (vLLM) or pages (SGLang), including reserved block 0
67
+ block_size: int # tokens per KV block or page
68
+ num_kv_heads: int
69
+ head_dim: int
70
+ dtype: torch.dtype
71
+
72
+
73
+ @dataclass(frozen=True)
74
+ class PagedKVLayout:
75
+ name: str
76
+ engine: Literal["vllm", "sglang"]
77
+ kv_split: Literal["joint", "separate"] # K and V in one buffer, or in two
78
+ index_kind: Literal["block_table", "slot_list"]
79
+ memory_order: str # outermost first; vLLM: letters L B H N C; SGLang: "nhd" or "hnd"
80
+ sources: tuple[Source, ...]
81
+
82
+
83
+ def _vllm(path: str, symbol: str) -> Source:
84
+ return Source(VLLM_REPO, VLLM_COMMIT, path, symbol)
85
+
86
+
87
+ def _sglang(path: str, symbol: str) -> Source:
88
+ return Source(SGLANG_REPO, SGLANG_COMMIT, path, symbol)
89
+
90
+
91
+ _VLLM_SOURCES = (
92
+ _vllm("vllm/v1/kv_cache_layout.py", "KVCacheLayout"),
93
+ _vllm("vllm/v1/kv_cache_interface.py", "create_kv_cache_views"),
94
+ _vllm("vllm/v1/kv_cache_interface.py", "compute_layout_strides"),
95
+ _vllm("vllm/v1/attention/backends/triton_attn.py", "TritonAttentionImpl.forward"),
96
+ _vllm("vllm/v1/worker/gpu/block_table.py", "BlockTables"),
97
+ _vllm("vllm/v1/core/block_pool.py", "null_block"),
98
+ )
99
+ _SGLANG_SOURCES = (
100
+ _sglang("python/sglang/srt/mem_cache/memory_pool.py", "MHATokenToKVPool._kv_buffer_shapes"),
101
+ _sglang("python/sglang/srt/layers/attention/triton_backend.py", "kv_indptr, kv_indices"),
102
+ _sglang("python/sglang/srt/mem_cache/kv_index_translator.py", "pack_read_stream"),
103
+ )
104
+
105
+
106
+ def _vllm_layout(order: str) -> PagedKVLayout:
107
+ return PagedKVLayout(
108
+ name=f"vllm_{order}",
109
+ engine="vllm",
110
+ kv_split="joint",
111
+ index_kind="block_table",
112
+ memory_order=order,
113
+ sources=_VLLM_SOURCES,
114
+ )
115
+
116
+
117
+ LAYOUTS: dict[str, PagedKVLayout] = {
118
+ layout.name: layout
119
+ for layout in (
120
+ # Orders whose KV blocks are one contiguous run per layer. In vLLM's
121
+ # letters: L layer, B block, H head, N slot, C content.
122
+ _vllm_layout("LBNHC"), # vLLM's default
123
+ _vllm_layout("LBHNC"),
124
+ _vllm_layout("BLNHC"), # all layers of a block together: block stride spans every layer
125
+ _vllm_layout("BLHNC"),
126
+ PagedKVLayout("sglang_nhd", "sglang", "separate", "slot_list", "nhd", _SGLANG_SOURCES),
127
+ PagedKVLayout("sglang_hnd", "sglang", "separate", "slot_list", "hnd", _SGLANG_SOURCES),
128
+ )
129
+ }
130
+
131
+
132
+ def _check_spec(spec: CacheSpec) -> None:
133
+ for name in ("num_layers", "num_blocks", "block_size", "num_kv_heads", "head_dim"):
134
+ if getattr(spec, name) < 1:
135
+ raise ValueError(f"{name} must be at least 1, got {getattr(spec, name)}")
136
+ if spec.num_blocks < 2:
137
+ raise ValueError("num_blocks must leave at least one block after reserved block 0")
138
+
139
+
140
+ def _vllm_strides(spec: CacheSpec, order: str) -> tuple[dict[str, int], dict[str, int]]:
141
+ """Sizes and element strides of the five axes, walking the memory order from innermost."""
142
+ sizes = {
143
+ "L": spec.num_layers,
144
+ "B": spec.num_blocks,
145
+ "H": spec.num_kv_heads,
146
+ "N": spec.block_size,
147
+ "C": 2 * spec.head_dim,
148
+ }
149
+ strides: dict[str, int] = {}
150
+ step = 1
151
+ for axis in reversed(order):
152
+ strides[axis] = step
153
+ step *= sizes[axis]
154
+ return sizes, strides
155
+
156
+
157
+ def make_cache(
158
+ layout: PagedKVLayout, spec: CacheSpec, device: str | torch.device = "cpu"
159
+ ) -> list[tuple[torch.Tensor, ...]]:
160
+ """One entry per layer, holding the tensors the engine gives its kernels for that layer.
161
+
162
+ vLLM: one view per layer, axes (block, head, slot, content), over one buffer
163
+ shared by all layers. SGLang: (K buffer, V buffer); default order [slots,
164
+ heads, head_dim] with slots = num_blocks · block_size, heads first [pages,
165
+ heads, page_size, head_dim]. The contents are uninitialized.
166
+ """
167
+ _check_spec(spec)
168
+ if layout.engine == "vllm":
169
+ sizes, strides = _vllm_strides(spec, layout.memory_order)
170
+ buffer = torch.empty(math.prod(sizes.values()), dtype=spec.dtype, device=device)
171
+ logical = "LBHNC"
172
+ everything = buffer.as_strided([sizes[a] for a in logical], [strides[a] for a in logical])
173
+ return [(everything[layer],) for layer in range(spec.num_layers)]
174
+ if layout.memory_order == "nhd":
175
+ shape: tuple[int, ...] = (
176
+ spec.num_blocks * spec.block_size,
177
+ spec.num_kv_heads,
178
+ spec.head_dim,
179
+ )
180
+ else:
181
+ shape = (spec.num_blocks, spec.num_kv_heads, spec.block_size, spec.head_dim)
182
+ return [
183
+ (
184
+ torch.empty(shape, dtype=spec.dtype, device=device),
185
+ torch.empty(shape, dtype=spec.dtype, device=device),
186
+ )
187
+ for _ in range(spec.num_layers)
188
+ ]
189
+
190
+
191
+ def _slot_before_head(layout: PagedKVLayout) -> bool:
192
+ order = layout.memory_order
193
+ return order.index("N") < order.index("H")
194
+
195
+
196
+ def planes(
197
+ layout: PagedKVLayout, spec: CacheSpec, layer_tensors: Sequence[torch.Tensor]
198
+ ) -> list[torch.Tensor]:
199
+ """The planes of one layer: first dimension indexes KV blocks, each block one contiguous run.
200
+
201
+ vLLM: one plane, K and V together. SGLang: two planes, K then V.
202
+ """
203
+ if layout.engine == "vllm":
204
+ (view,) = layer_tensors
205
+ # The view's axes are (block, head, slot, content); reorder them to memory order.
206
+ return [view.permute(0, 2, 1, 3) if _slot_before_head(layout) else view]
207
+ if layout.memory_order == "nhd":
208
+ return [
209
+ t.view(spec.num_blocks, spec.block_size, spec.num_kv_heads, spec.head_dim)
210
+ for t in layer_tensors
211
+ ]
212
+ return list(layer_tensors)
213
+
214
+
215
+ def kv_views(
216
+ layout: PagedKVLayout, spec: CacheSpec, layer_tensors: Sequence[torch.Tensor]
217
+ ) -> tuple[torch.Tensor, torch.Tensor]:
218
+ """K and V of one layer, each [blocks, slots, kv_heads, head_dim]; views of the cache."""
219
+ if spec.dtype not in _VIEW_DTYPES:
220
+ raise ValueError(f"K and V views support {_VIEW_DTYPES}, got {spec.dtype}")
221
+ d = spec.head_dim
222
+ if layout.engine == "vllm":
223
+ (plane,) = planes(layout, spec, layer_tensors)
224
+ k, v = plane[..., :d], plane[..., d:]
225
+ if not _slot_before_head(layout): # plane is (block, head, slot, content)
226
+ k, v = k.transpose(1, 2), v.transpose(1, 2)
227
+ return k, v
228
+ k_plane, v_plane = planes(layout, spec, layer_tensors)
229
+ if layout.memory_order == "hnd": # (page, head, slot, head_dim)
230
+ return k_plane.transpose(1, 2), v_plane.transpose(1, 2)
231
+ return k_plane, v_plane
232
+
233
+
234
+ def make_index(
235
+ layout: PagedKVLayout,
236
+ spec: CacheSpec,
237
+ request_blocks: Sequence[Sequence[int]],
238
+ seq_lens: Sequence[int],
239
+ device: str | torch.device = "cpu",
240
+ ) -> dict[str, torch.Tensor]:
241
+ """The index the engine's attention kernel reads, for requests owning the given blocks.
242
+
243
+ vLLM ("block_table", "seq_lens"): one int32 row per request; entries past a
244
+ request's blocks hold no defined value in the engine and are filled here
245
+ with block 0, the null block. "seq_lens" is int32.
246
+
247
+ SGLang ("kv_indptr", "kv_indices"): one int64 slot per token, at every page
248
+ size; kv_indptr (int32) is where each request's slots begin.
249
+ """
250
+ if len(request_blocks) != len(seq_lens):
251
+ raise ValueError(f"{len(request_blocks)} block lists for {len(seq_lens)} lengths")
252
+ for r, (blocks, length) in enumerate(zip(request_blocks, seq_lens, strict=True)):
253
+ needed = math.ceil(length / spec.block_size)
254
+ if length < 1 or len(blocks) < needed:
255
+ raise ValueError(
256
+ f"request {r}: {length} tokens need {needed} blocks, got {len(blocks)}"
257
+ )
258
+ for b in blocks:
259
+ if not RESERVED_BLOCK < b < spec.num_blocks:
260
+ raise ValueError(f"request {r}: block {b} is outside 1 … {spec.num_blocks - 1}")
261
+ if layout.index_kind == "block_table":
262
+ width = max(len(b) for b in request_blocks) if request_blocks else 0
263
+ table = torch.full((len(request_blocks), width), RESERVED_BLOCK, dtype=torch.int32)
264
+ for r, blocks in enumerate(request_blocks):
265
+ table[r, : len(blocks)] = torch.tensor(blocks, dtype=torch.int32)
266
+ return {
267
+ "block_table": table.to(device),
268
+ "seq_lens": torch.tensor(list(seq_lens), dtype=torch.int32, device=device),
269
+ }
270
+ slots: list[int] = []
271
+ starts = [0]
272
+ for blocks, length in zip(request_blocks, seq_lens, strict=True):
273
+ slots.extend(
274
+ blocks[t // spec.block_size] * spec.block_size + t % spec.block_size
275
+ for t in range(length)
276
+ )
277
+ starts.append(len(slots))
278
+ return {
279
+ "kv_indptr": torch.tensor(starts, dtype=torch.int32, device=device),
280
+ "kv_indices": torch.tensor(slots, dtype=torch.int64, device=device),
281
+ }
@@ -0,0 +1,17 @@
1
+ """Triton ops. Each op has a pure-PyTorch reference in kvkernels.reference."""
2
+
3
+ from kvkernels.ops.cascade import CascadePlan, PrefixGroup, cascade_decode_attention
4
+ from kvkernels.ops.kv_gather import gather_kv_blocks, scatter_kv_blocks
5
+ from kvkernels.ops.merge_states import merge_attention_states, merge_attention_states_n
6
+ from kvkernels.ops.plane_set import PlaneSet
7
+
8
+ __all__ = [
9
+ "CascadePlan",
10
+ "PlaneSet",
11
+ "PrefixGroup",
12
+ "cascade_decode_attention",
13
+ "gather_kv_blocks",
14
+ "merge_attention_states",
15
+ "merge_attention_states_n",
16
+ "scatter_kv_blocks",
17
+ ]
@@ -0,0 +1,52 @@
1
+ """Register an op with PyTorch, and call it without PyTorch's dispatch when nothing needs it.
2
+
3
+ Each op is registered as a PyTorch custom op, so torch.compile and engines treat a
4
+ call as one opaque operation that writes into the tensors named in mutates_args.
5
+ Going through the registration costs CPU time on every call: on the host of a
6
+ Kaggle T4, a gather call spent 67 µs of its 164 µs there (bench/host_cost.py).
7
+ A plain eager call needs none of it, so the op runs its body directly:
8
+
9
+ torch.compile tracing ───────────┐
10
+ a dispatch mode (fake tensors) ──┴──▶ the registered custom op
11
+ otherwise ───────────────────────────▶ the body, called directly
12
+
13
+ The direct call skips the version-counter bump PyTorch's autograd uses to detect
14
+ in-place changes to saved tensors; the ops are for inference and define no
15
+ gradient.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ from collections.abc import Callable, Sequence
21
+ from typing import Generic, ParamSpec
22
+
23
+ import torch
24
+
25
+ P = ParamSpec("P")
26
+
27
+
28
+ class RegisteredOp(Generic[P]):
29
+ def __init__(self, name: str, body: Callable[P, None], mutates_args: Sequence[str]) -> None:
30
+ self.body = body
31
+ self.custom_op = torch.library.custom_op(name, body, mutates_args=tuple(mutates_args))
32
+
33
+ def register_fake(self, fake: Callable[P, None]) -> Callable[P, None]:
34
+ self.custom_op.register_fake(fake)
35
+ return fake
36
+
37
+ def __call__(self, *args: P.args, **kwargs: P.kwargs) -> None:
38
+ if torch.compiler.is_compiling() or torch._C._len_torch_dispatch_stack() > 0:
39
+ self.custom_op(*args, **kwargs)
40
+ else:
41
+ self.body(*args, **kwargs)
42
+
43
+
44
+ def registered_op(
45
+ name: str, *, mutates_args: Sequence[str]
46
+ ) -> Callable[[Callable[P, None]], RegisteredOp[P]]:
47
+ """Decorator: register the function as custom op `name` and return a RegisteredOp."""
48
+
49
+ def wrap(body: Callable[P, None]) -> RegisteredOp[P]:
50
+ return RegisteredOp(name, body, mutates_args)
51
+
52
+ return wrap