ax-engine-sglang 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ax_engine_sglang/__init__.py +5 -0
- ax_engine_sglang/ax_chain.py +640 -0
- ax_engine_sglang/ax_dflash_sampling.py +43 -0
- ax_engine_sglang/ax_eagle_sampling.py +73 -0
- ax_engine_sglang/ax_fast.py +116 -0
- ax_engine_sglang/ax_fp8_probe.py +93 -0
- ax_engine_sglang/ax_gdn_recurrent.py +59 -0
- ax_engine_sglang/ax_lattice.py +182 -0
- ax_engine_sglang/ax_nvfp4_verify.py +78 -0
- ax_engine_sglang/ax_nxt_debug.py +103 -0
- ax_engine_sglang/ax_qsa.py +327 -0
- ax_engine_sglang/ax_rng.py +70 -0
- ax_engine_sglang/ax_state_cap.py +102 -0
- ax_engine_sglang/ax_verify_attention.py +138 -0
- ax_engine_sglang/cli.py +57 -0
- ax_engine_sglang/compat.py +80 -0
- ax_engine_sglang/config.py +150 -0
- ax_engine_sglang/contracts.py +67 -0
- ax_engine_sglang/defaults.py +284 -0
- ax_engine_sglang/fc1_block_inverse.py +83 -0
- ax_engine_sglang/fc1_branch64.py +81 -0
- ax_engine_sglang/fc1_builder.py +54 -0
- ax_engine_sglang/fc1_conv.py +47 -0
- ax_engine_sglang/fc1_fused.py +196 -0
- ax_engine_sglang/fc1_plan.py +44 -0
- ax_engine_sglang/fc1_verify.py +26 -0
- ax_engine_sglang/ngram_gpu.py +70 -0
- ax_engine_sglang/plugin.py +130 -0
- ax_engine_sglang/policy.py +31 -0
- ax_engine_sglang/qsa_topk.py +32 -0
- ax_engine_sglang/runtime.py +50 -0
- ax_engine_sglang/tree_kernels.py +208 -0
- ax_engine_sglang/tree_ops.py +102 -0
- ax_engine_sglang/worker_gen.py +123 -0
- ax_engine_sglang-0.2.0.dist-info/METADATA +350 -0
- ax_engine_sglang-0.2.0.dist-info/RECORD +42 -0
- ax_engine_sglang-0.2.0.dist-info/WHEEL +5 -0
- ax_engine_sglang-0.2.0.dist-info/entry_points.txt +5 -0
- ax_engine_sglang-0.2.0.dist-info/licenses/LICENSE +202 -0
- ax_engine_sglang-0.2.0.dist-info/licenses/NOTICE +35 -0
- ax_engine_sglang-0.2.0.dist-info/licenses/licenses/FLASH_LINEAR_ATTENTION-LICENSE +21 -0
- ax_engine_sglang-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"""ax-engine-sglang: lossless tree speculative decoding (DFlash / DFlash2 drafters).
|
|
2
|
+
|
|
3
|
+
Launch with `ax-engine-sglang serve` or select `SGLANG_PLUGINS=ax-engine` in SGLang.
|
|
4
|
+
The import package is `ax_engine_sglang`; `ax_engine` is the separate Apple Silicon runtime."""
|
|
5
|
+
__version__ = "0.2.0"
|
|
@@ -0,0 +1,640 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Several requests at once: chain rounds next to the one-request tree (multi-user Stage 0).
|
|
3
|
+
|
|
4
|
+
Design: docs/decisions/2026-11-15-multiuser-width-policy.md. One server, drafter block B
|
|
5
|
+
(`FC1_DRAFT_BLOCK`, 16) and `speculative_num_draft_tokens` = tree nodes (`FC1_TREE_N`, 16). Each
|
|
6
|
+
decode round is one of two kinds, decided here before the target verify and kept in `current_runtime().kind` so the
|
|
7
|
+
fused tree hooks (fc1_fused, fc1_conv, fc1_plan) run only in tree rounds:
|
|
8
|
+
|
|
9
|
+
tree one running request and the tree enabled: the fused tree verifier (fc1_branch64)
|
|
10
|
+
chain otherwise: SGLang's own DFlash chain verify on the first W drafted tokens
|
|
11
|
+
|
|
12
|
+
Stage 0 has no width rule. The width is pinned with `AX_CHAIN_WIDTH` (8 or 16, default B), or a
|
|
13
|
+
forced schedule `AX_CHAIN_SCHEDULE="16x5,8x3"` (width x rounds, cycled) for the switching checks.
|
|
14
|
+
`AX_TREE=0` turns tree rounds off (every round is a chain round, also with one request).
|
|
15
|
+
|
|
16
|
+
A chain round of width W = B runs the stock code unchanged. For W < B, `narrow` cuts the drafted
|
|
17
|
+
block, its positions and its KV slots to W columns, plans a W-slot verify window (the stock worker
|
|
18
|
+
plans one B-slot window for draft and verify), and sets the worker's `block_size` to W for the
|
|
19
|
+
rest of the round, so every stock read after the draft step (accept reshaping, logit adjustments,
|
|
20
|
+
logprob stride, hidden-state view, the result's `speculative_num_draft_tokens`) uses W.
|
|
21
|
+
`begin_round` puts B back at the start of the next call. The accept/bonus buffers are sized by
|
|
22
|
+
`block_size` when first used, so each width keeps its own set (`_swap_accept_buffers`).
|
|
23
|
+
|
|
24
|
+
`record_accept` (after the stock accept) keeps, per request, the token before the next anchor, which
|
|
25
|
+
the tree builder conditions on when the request is alone again (`pre1_for`), and appends one line
|
|
26
|
+
per round to `AX_ROUND_DUMP` (if set) for the offline accept oracle: request id, kind, width, the
|
|
27
|
+
verified tokens and the accepted length.
|
|
28
|
+
|
|
29
|
+
Two attention backends (docs/decisions/2026-11-16-dual-attention-backend.md): with
|
|
30
|
+
`AX_CHAIN_ATTN=<backend>` (for example trtllm_mha) the private chain runners are built with that
|
|
31
|
+
backend, in that backend's role, while the main runner (tree rounds, prefill) keeps the server's
|
|
32
|
+
`--attention-backend` (triton, which has the tree mask). Every chain round must then run on a
|
|
33
|
+
private runner: the mixed mode needs CUDA graphs and a graph runner for every chain width, and a
|
|
34
|
+
chain round that would run on the main backend stops the server. The round dump records, per
|
|
35
|
+
round, the backend that ran it and whether the graph replayed; `[ax_chain]` stats count graph and
|
|
36
|
+
eager rounds per (width, batch size).
|
|
37
|
+
|
|
38
|
+
Tree rounds serve one greedy request without grammar or logprobs (the tree verify accepts the
|
|
39
|
+
argmax path only). Any other decode batch, including a single sampled request, runs a chain round,
|
|
40
|
+
which is SGLang's own DFlash verify: sampled (selector sampling), grammar-constrained and logprob
|
|
41
|
+
requests get stock behaviour there instead of an error. (Until 2026-11-16 such batches raised in
|
|
42
|
+
the scheduler and stopped the server; with the model card's default temperature 1.0 that was any
|
|
43
|
+
two concurrent requests that did not set temperature 0.)
|
|
44
|
+
"""
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
import json
|
|
48
|
+
import os
|
|
49
|
+
|
|
50
|
+
import torch
|
|
51
|
+
|
|
52
|
+
from ax_engine_sglang.config import get_config
|
|
53
|
+
from ax_engine_sglang.policy import RoundFeatures, chain_width, use_tree
|
|
54
|
+
from ax_engine_sglang.runtime import activate_worker, current_runtime, runtime_statistics
|
|
55
|
+
from ax_engine_sglang.compat import patch_attribute
|
|
56
|
+
|
|
57
|
+
_CONFIG = get_config()
|
|
58
|
+
_TREE = _CONFIG.tree
|
|
59
|
+
_PINNED = _CONFIG.chain_width
|
|
60
|
+
_SCHED = _CONFIG.chain_schedule
|
|
61
|
+
_DUMP = _CONFIG.round_dump
|
|
62
|
+
_GRAPH_WIDTHS = _CONFIG.graph_widths
|
|
63
|
+
_DRAFT_WIDTHS = _CONFIG.draft_widths
|
|
64
|
+
_CHECK = _CONFIG.chain_check
|
|
65
|
+
_CHAIN_ATTN = _CONFIG.chain_attention
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def is_tree():
|
|
69
|
+
state = current_runtime()
|
|
70
|
+
if state.kind is not None:
|
|
71
|
+
state.stats["hook_tree" if state.kind == "tree" else "hook_chain"] += 1
|
|
72
|
+
return state.kind == "tree"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _check_alloc(batch):
|
|
76
|
+
"""AX_CHAIN_CHECK=1, sampled when the round's kind is chosen (after prepare_for_decode and the
|
|
77
|
+
draft step): every running request has this round's 16-slot window allocated beyond its committed
|
|
78
|
+
length (prepare reserves 2 x 16 = 32; with overlap scheduling the committed length can advance by
|
|
79
|
+
one round's commit before this sample, so the gap seen here is 16..32), and its allocation never
|
|
80
|
+
shrinks while it runs. The smallest gap seen is reported."""
|
|
81
|
+
for req in batch.reqs:
|
|
82
|
+
alloc, committed = int(req.kv.kv_allocated_len), int(req.kv.kv_committed_len)
|
|
83
|
+
current_runtime().stats["alloc_checked"] += 1
|
|
84
|
+
gap = alloc - committed
|
|
85
|
+
current_runtime().stats["alloc_min_gap"] = min(current_runtime().stats.get("alloc_min_gap", gap), gap)
|
|
86
|
+
if gap < 16:
|
|
87
|
+
current_runtime().stats["alloc_short"] += 1
|
|
88
|
+
if alloc < current_runtime().alloc_seen.get(req.rid, 0):
|
|
89
|
+
current_runtime().stats["alloc_shrunk"] += 1
|
|
90
|
+
current_runtime().alloc_seen[req.rid] = alloc
|
|
91
|
+
if req.finished():
|
|
92
|
+
current_runtime().alloc_seen.pop(req.rid, None)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def width(block):
|
|
96
|
+
"""Verify width of this chain round (anchor included)."""
|
|
97
|
+
return chain_width(_CONFIG, block, current_runtime().round_index)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _configured_widths(block):
|
|
101
|
+
"""Chain widths this server can choose (Stage 0: pinned, schedule or the block)."""
|
|
102
|
+
if _SCHED:
|
|
103
|
+
return sorted(set(_SCHED))
|
|
104
|
+
if _PINNED:
|
|
105
|
+
return [int(_PINNED)]
|
|
106
|
+
return [block]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def tree_round(batch):
|
|
110
|
+
"""Translate SGLang sampling capabilities into the pure round policy."""
|
|
111
|
+
if not _CONFIG.tree or len(batch.reqs) != 1 or batch.has_grammar or batch.return_logprob:
|
|
112
|
+
return False
|
|
113
|
+
info = batch.sampling_info
|
|
114
|
+
return use_tree(_CONFIG, RoundFeatures(
|
|
115
|
+
requests=len(batch.reqs), grammar=batch.has_grammar, logprobs=batch.return_logprob,
|
|
116
|
+
custom_processor=bool(info is not None and info.has_custom_logit_processor),
|
|
117
|
+
greedy=info is None or info.is_all_greedy,
|
|
118
|
+
min_p=bool(info is not None and not info.is_all_greedy and info.need_min_p_sampling),
|
|
119
|
+
))
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def begin_round(w, batch=None):
|
|
123
|
+
"""Start of forward_batch_generation: restore the drafter block after a narrowed chain round,
|
|
124
|
+
the main attention backend and graph runner after a chain round on a private pair, and the
|
|
125
|
+
drafter's state after a round drafted at a narrower width. Then, for a decode round that will be
|
|
126
|
+
a chain round drafted at width W (AX_CHAIN_DRAFT_WIDTHS), switch the drafter to W."""
|
|
127
|
+
state = activate_worker(w)
|
|
128
|
+
_leave_draft_width(w)
|
|
129
|
+
block = getattr(w, "_ax_block", None)
|
|
130
|
+
if block is None:
|
|
131
|
+
block = w._ax_block = int(w.block_size)
|
|
132
|
+
w.block_size = block
|
|
133
|
+
saved = getattr(w, "_ax_saved_runner", None)
|
|
134
|
+
if saved is not None:
|
|
135
|
+
mr = w.target_worker.model_runner
|
|
136
|
+
mr.attn_backend, mr.decode_cuda_graph_runner = saved
|
|
137
|
+
w._ax_saved_runner = None
|
|
138
|
+
w._ax_round_width = None
|
|
139
|
+
w._ax_tree_round = None
|
|
140
|
+
state.kind = None
|
|
141
|
+
state.host_kv_len = None
|
|
142
|
+
state.round_info.update(backend=None, graph=None, width=None)
|
|
143
|
+
sets = getattr(w, "_ax_draft_sets", None)
|
|
144
|
+
if not sets or batch is None or batch.forward_mode.is_extend() or batch.is_extend_in_batch:
|
|
145
|
+
return
|
|
146
|
+
w._ax_tree_round = (id(batch), tree_round(batch))
|
|
147
|
+
if w._ax_tree_round[1]:
|
|
148
|
+
return
|
|
149
|
+
wid = width(block)
|
|
150
|
+
w._ax_round_width = wid
|
|
151
|
+
if wid in sets:
|
|
152
|
+
_enter_draft_width(w, wid)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def install():
|
|
156
|
+
"""With CUDA graphs for one request only (tree rounds), SGLang's in-graph draft sampler has
|
|
157
|
+
buffers for one request, but the stock draft step stages its sampling parameters for every
|
|
158
|
+
batch. A batch larger than the sampler runs the drafter eager and never reads the sampler, so the
|
|
159
|
+
staging is skipped there."""
|
|
160
|
+
import fc1_dflash_worker as W
|
|
161
|
+
cls = getattr(W, "_SelectorDraftSampler", None)
|
|
162
|
+
if cls is None or getattr(cls, "_ax_chain_guard", False):
|
|
163
|
+
return
|
|
164
|
+
original = cls.stage_sampling_params
|
|
165
|
+
|
|
166
|
+
def stage_sampling_params(self, *, bs, sampling_info):
|
|
167
|
+
if bs > self.greedy_mask.shape[0]:
|
|
168
|
+
return None
|
|
169
|
+
return original(self, bs=bs, sampling_info=sampling_info)
|
|
170
|
+
|
|
171
|
+
patch_attribute(cls, 'stage_sampling_params', stage_sampling_params)
|
|
172
|
+
patch_attribute(cls, '_ax_chain_guard', True)
|
|
173
|
+
if _CHAIN_ATTN and not _GRAPH_WIDTHS:
|
|
174
|
+
raise RuntimeError(f"ax-engine: AX_CHAIN_ATTN={_CHAIN_ATTN} needs AX_CHAIN_GRAPH_WIDTHS (a graph runner per "
|
|
175
|
+
"chain width); without them every chain round would run on the main backend")
|
|
176
|
+
if _GRAPH_WIDTHS:
|
|
177
|
+
_install_graphs(W)
|
|
178
|
+
else:
|
|
179
|
+
_install_main_only()
|
|
180
|
+
if not getattr(cls_w := W.DFlashTreeWorkerV2, "_ax_checked", False):
|
|
181
|
+
init0 = cls_w.init_cuda_graphs
|
|
182
|
+
|
|
183
|
+
def init_cuda_graphs(self):
|
|
184
|
+
activate_worker(self)
|
|
185
|
+
return init0(self)
|
|
186
|
+
patch_attribute(cls_w, 'init_cuda_graphs', init_cuda_graphs)
|
|
187
|
+
patch_attribute(cls_w, '_ax_checked', True)
|
|
188
|
+
if os.environ.get("AX_NXT_DEBUG") == "1":
|
|
189
|
+
import ax_nxt_debug # loaded next to ax_chain (the plugin puts this directory on sys.path)
|
|
190
|
+
ax_nxt_debug.install(W.DFlashTreeWorkerV2)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def is_tree_runner(runner):
|
|
194
|
+
"""True for graph runners of a server running the tree algorithm. The capture patches below are
|
|
195
|
+
global (SGLang's DecodeCudaGraphRunner), so a server that loads the plugin for something else (stock
|
|
196
|
+
EAGLE / MTP, e.g. with AX_QSA_RING) must keep SGLang's own batch sizes."""
|
|
197
|
+
algo = getattr(getattr(runner.model_runner, "server_args", None), "speculative_algorithm", None)
|
|
198
|
+
return str(algo or "").upper() == "FC1_DFLASH_TREE"
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _install_main_only():
|
|
202
|
+
"""Servers without private chain runners (AX_CHAIN_GRAPH_WIDTHS unset): SGLang's main target runner
|
|
203
|
+
is the only graph runner.
|
|
204
|
+
|
|
205
|
+
- Tree on: it captures batch 1 only, with the tree verify input (the fused tree hooks handle one
|
|
206
|
+
request; a capture at a larger batch with the tree input fails), and every chain round runs
|
|
207
|
+
eager (it would otherwise be admitted to the tree graph at the tree's width).
|
|
208
|
+
- Tree off: it captures the stock chain input at the block width, and a chain round narrowed
|
|
209
|
+
below that width runs eager (the graph holds block-width buffers).
|
|
210
|
+
"""
|
|
211
|
+
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import DecodeCudaGraphRunner as R
|
|
212
|
+
if getattr(R, "_ax_main_only", False):
|
|
213
|
+
return
|
|
214
|
+
original_capture = R.capture
|
|
215
|
+
|
|
216
|
+
def capture(self):
|
|
217
|
+
|
|
218
|
+
if not is_tree_runner(self):
|
|
219
|
+
return original_capture(self)
|
|
220
|
+
if not self.model_runner.is_draft_worker:
|
|
221
|
+
activate_worker(self.model_runner)
|
|
222
|
+
previous_kind = current_runtime().kind
|
|
223
|
+
main = not self.model_runner.is_draft_worker
|
|
224
|
+
if main and _TREE:
|
|
225
|
+
current_runtime().kind = "tree"
|
|
226
|
+
self.capture_bs = [b for b in self.capture_bs if b == 1]
|
|
227
|
+
self.compile_bs = [b for b in (self.compile_bs or []) if b == 1]
|
|
228
|
+
else:
|
|
229
|
+
current_runtime().kind = "chain" # stock capture input (plugin.tree_capture_info)
|
|
230
|
+
try:
|
|
231
|
+
return original_capture(self)
|
|
232
|
+
finally:
|
|
233
|
+
current_runtime().kind = previous_kind
|
|
234
|
+
if main and _TREE:
|
|
235
|
+
self.max_bs = 1
|
|
236
|
+
|
|
237
|
+
can_run = R.can_run_graph
|
|
238
|
+
|
|
239
|
+
def can_run_graph(self, forward_batch):
|
|
240
|
+
if is_tree_runner(self) and current_runtime().kind == "chain" and not self.model_runner.is_draft_worker:
|
|
241
|
+
captured = getattr(self, "speculative_num_draft_tokens", None)
|
|
242
|
+
if _TREE or captured is None or current_runtime().round_info.get("width") not in (None, captured):
|
|
243
|
+
return False
|
|
244
|
+
return can_run(self, forward_batch)
|
|
245
|
+
|
|
246
|
+
patch_attribute(R, 'capture', capture)
|
|
247
|
+
patch_attribute(R, 'can_run_graph', can_run_graph)
|
|
248
|
+
patch_attribute(R, '_ax_main_only', True)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _install_graphs(W):
|
|
252
|
+
"""CUDA graphs per round kind and width (design, Stage 0 item 3).
|
|
253
|
+
|
|
254
|
+
- The main target runner (SGLang's) captures only batch 1, with the tree verify input when the
|
|
255
|
+
tree is on: the tree graph.
|
|
256
|
+
- Each chain width W in AX_CHAIN_GRAPH_WIDTHS gets its own runner with its own attention backend,
|
|
257
|
+
built with speculative_eagle_topk = 1 and speculative_num_draft_tokens = W (as SGLang's UNO
|
|
258
|
+
worker builds its private runner), capturing the stock DFlash chain verify at every batch size.
|
|
259
|
+
A private backend keeps the main runner's captured buffers untouched (a second
|
|
260
|
+
init_cuda_graph_state on a shared backend would reallocate them under the captured tree graph).
|
|
261
|
+
- A chain round swaps model_runner.attn_backend and decode_cuda_graph_runner to its width's pair
|
|
262
|
+
for the whole round (verify and the recurrent-state commit, which reads the backend's forward
|
|
263
|
+
metadata); begin_round restores them at the next call.
|
|
264
|
+
"""
|
|
265
|
+
from sglang.srt.model_executor.runner.decode_cuda_graph_runner import DecodeCudaGraphRunner as R
|
|
266
|
+
if getattr(R, "_ax_chain_graphs", False):
|
|
267
|
+
return
|
|
268
|
+
original_capture = R.capture
|
|
269
|
+
|
|
270
|
+
def capture(self):
|
|
271
|
+
|
|
272
|
+
if not is_tree_runner(self):
|
|
273
|
+
return original_capture(self)
|
|
274
|
+
if not self.model_runner.is_draft_worker:
|
|
275
|
+
activate_worker(self.model_runner)
|
|
276
|
+
previous_kind = current_runtime().kind
|
|
277
|
+
target = not self.model_runner.is_draft_worker
|
|
278
|
+
width = getattr(self, "_ax_chain_width", None)
|
|
279
|
+
main = target and width is None
|
|
280
|
+
if main:
|
|
281
|
+
# one request only: the tree graph (tree on) or an unused batch-1 chain graph (tree off;
|
|
282
|
+
# every chain round runs on a private runner)
|
|
283
|
+
self.capture_bs = [b for b in self.capture_bs if b == 1]
|
|
284
|
+
self.compile_bs = [b for b in (self.compile_bs or []) if b == 1]
|
|
285
|
+
current_runtime().kind = "tree" if main and _TREE else "chain"
|
|
286
|
+
try:
|
|
287
|
+
return original_capture(self)
|
|
288
|
+
finally:
|
|
289
|
+
current_runtime().kind = previous_kind
|
|
290
|
+
if main:
|
|
291
|
+
self.max_bs = 1
|
|
292
|
+
|
|
293
|
+
patch_attribute(R, 'capture', capture)
|
|
294
|
+
patch_attribute(R, '_ax_chain_graphs', True)
|
|
295
|
+
|
|
296
|
+
cls = W.DFlashTreeWorkerV2
|
|
297
|
+
original_init = cls.init_cuda_graphs
|
|
298
|
+
|
|
299
|
+
def init_cuda_graphs(self):
|
|
300
|
+
out = original_init(self)
|
|
301
|
+
_build_chain_runners(self, R)
|
|
302
|
+
|
|
303
|
+
mr = self.target_worker.model_runner
|
|
304
|
+
current_runtime().main_backend = _backend_name(mr, main=True)
|
|
305
|
+
if mr.decode_cuda_graph_runner is not None:
|
|
306
|
+
_count_replays(mr.decode_cuda_graph_runner, None, _backend_name(mr, main=True))
|
|
307
|
+
if os.environ.get("AX_FP8_PROBE") == "1":
|
|
308
|
+
import ax_fp8_probe
|
|
309
|
+
ax_fp8_probe.run(mr)
|
|
310
|
+
return out
|
|
311
|
+
|
|
312
|
+
patch_attribute(cls, 'init_cuda_graphs', init_cuda_graphs)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _backend_name(mr, main=False):
|
|
316
|
+
"""Name of the backend a target round runs on: the main runner's decode backend, or AX_CHAIN_ATTN."""
|
|
317
|
+
if main or not _CHAIN_ATTN:
|
|
318
|
+
return getattr(mr, "decode_attention_backend_str", None)
|
|
319
|
+
return _CHAIN_ATTN
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _count_replays(runner, wid, name):
|
|
323
|
+
"""Record, per target verify on this graph runner, whether the captured graph replays (SGLang
|
|
324
|
+
asks can_run_graph once per forward) or the round runs eager."""
|
|
325
|
+
original = runner.can_run_graph
|
|
326
|
+
|
|
327
|
+
def can_run_graph(forward_batch):
|
|
328
|
+
ok = bool(original(forward_batch))
|
|
329
|
+
key = f"{'tree' if wid is None else wid}:{forward_batch.batch_size}:{'graph' if ok else 'eager'}"
|
|
330
|
+
current_runtime().stats["replay"][key] = current_runtime().stats["replay"].get(key, 0) + 1
|
|
331
|
+
current_runtime().round_info["graph"] = ok
|
|
332
|
+
return ok
|
|
333
|
+
|
|
334
|
+
runner.can_run_graph = can_run_graph
|
|
335
|
+
runner._ax_backend_name = name
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _build_backend(mr, wid):
|
|
339
|
+
"""Private attention backend of a chain runner. With AX_CHAIN_ATTN it is built by name in that
|
|
340
|
+
backend's role: trtllm_mha decides its KV access checks and native-FP4 setup from the runner's
|
|
341
|
+
prefill/decode role strings (trtllm_mha_backend.py:218-240) and the Gated-DeltaNet wrapper
|
|
342
|
+
asserts on them (attention_registry.py:471-475), so both are set for the construction (as in an
|
|
343
|
+
all-trtllm_mha server) and restored; SGLang records the pair on the backend object too."""
|
|
344
|
+
if not _CHAIN_ATTN:
|
|
345
|
+
return mr._get_attention_backend(init_new_workspace=True)
|
|
346
|
+
from sglang.srt.model_executor.model_runner_components.attention_backend_setup import _build_backend_from_str
|
|
347
|
+
roles = (mr.prefill_attention_backend_str, mr.decode_attention_backend_str)
|
|
348
|
+
try:
|
|
349
|
+
mr.prefill_attention_backend_str = mr.decode_attention_backend_str = _CHAIN_ATTN
|
|
350
|
+
backend = _build_backend_from_str(model_runner=mr, backend_str=_CHAIN_ATTN, init_new_workspace=True)
|
|
351
|
+
finally:
|
|
352
|
+
mr.prefill_attention_backend_str, mr.decode_attention_backend_str = roles
|
|
353
|
+
backend.prefill_attention_backend_str = backend.decode_attention_backend_str = _CHAIN_ATTN
|
|
354
|
+
return backend
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def _build_chain_runners(w, R):
|
|
358
|
+
from sglang.srt.runtime_context import get_spec
|
|
359
|
+
mr = w.target_worker.model_runner
|
|
360
|
+
w._ax_chain_runners = {}
|
|
361
|
+
if mr.decode_cuda_graph_runner is None:
|
|
362
|
+
if _CHAIN_ATTN:
|
|
363
|
+
raise RuntimeError("ax-engine: AX_CHAIN_ATTN needs CUDA graphs (no private chain runner is built without them)")
|
|
364
|
+
return
|
|
365
|
+
missing = [x for x in _configured_widths(int(w.block_size)) if x not in _GRAPH_WIDTHS]
|
|
366
|
+
if missing:
|
|
367
|
+
# a chain round of a width without a private runner would fail in _use_chain_runner
|
|
368
|
+
raise RuntimeError(f"ax-engine: chain widths {missing} have no graph runner "
|
|
369
|
+
f"(AX_CHAIN_GRAPH_WIDTHS={_GRAPH_WIDTHS}); add them or unset AX_CHAIN_GRAPH_WIDTHS")
|
|
370
|
+
for wid in _GRAPH_WIDTHS:
|
|
371
|
+
free0 = torch.cuda.mem_get_info()[0]
|
|
372
|
+
flag = mr.init_new_workspace
|
|
373
|
+
try:
|
|
374
|
+
with get_spec().override(speculative_eagle_topk=1, speculative_num_draft_tokens=wid):
|
|
375
|
+
backend = _build_backend(mr, wid)
|
|
376
|
+
runner = R.__new__(R)
|
|
377
|
+
runner._ax_chain_width = wid
|
|
378
|
+
R.__init__(runner, mr, attn_backend=backend, speculative_num_draft_tokens=wid)
|
|
379
|
+
finally:
|
|
380
|
+
mr.init_new_workspace = flag
|
|
381
|
+
_count_replays(runner, wid, _backend_name(mr))
|
|
382
|
+
w._ax_chain_runners[wid] = runner
|
|
383
|
+
used = (free0 - torch.cuda.mem_get_info()[0]) / 2 ** 30
|
|
384
|
+
print(f"[ax_chain] chain graph runner width {wid}: backend {_backend_name(mr)} "
|
|
385
|
+
f"({type(backend).__name__}), batch sizes {runner.capture_bs}, {used:.2f} GB", flush=True)
|
|
386
|
+
_build_draft_runners(w, R)
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
# every buffer `_ensure_draft_block_buffers` (re)allocates together: it sizes them all from the capacity
|
|
390
|
+
# of _draft_block_ids_buf, so a set that swapped only some of them would mix capacities (a width-16 host
|
|
391
|
+
# length buffer for 14 requests next to width-8 buffers for 15 crashed the stress run)
|
|
392
|
+
_DRAFT_BUFFERS = ("_draft_block_ids_buf", "_draft_block_positions_buf", "_draft_block_tokens_buf",
|
|
393
|
+
"_draft_verify_out_cache_loc_buf", "_draft_block_end_buf", "_draft_seq_lens_cpu_buf")
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _check_buffer_set(w, where):
|
|
397
|
+
caps = {n: int(getattr(w, n).shape[0]) for n in _DRAFT_BUFFERS if getattr(w, n, None) is not None}
|
|
398
|
+
if caps and (len(set(caps.values())) > 1 or len(caps) != len(_DRAFT_BUFFERS)):
|
|
399
|
+
raise RuntimeError(f"ax-engine: draft buffers out of step {where}: {caps}")
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _build_draft_runners(w, R):
|
|
403
|
+
"""Per drafting width W < B: a W-token sampler folded into a private drafter graph runner (own
|
|
404
|
+
draft attention backend), as SGLang builds the drafter graph for a block-W server."""
|
|
405
|
+
import fc1_dflash_worker as W
|
|
406
|
+
from sglang.srt.runtime_context import get_spec
|
|
407
|
+
from sglang.srt.speculative.draft_worker_common import make_draft_sampler_capture_hook
|
|
408
|
+
|
|
409
|
+
w._ax_draft_sets = {}
|
|
410
|
+
dmr = w.draft_model_runner
|
|
411
|
+
if not _DRAFT_WIDTHS or dmr.decode_cuda_graph_runner is None or w._draft_sampler is None:
|
|
412
|
+
return
|
|
413
|
+
block = int(w.block_size)
|
|
414
|
+
for wid in _DRAFT_WIDTHS:
|
|
415
|
+
if wid not in _GRAPH_WIDTHS or not 2 <= wid < block:
|
|
416
|
+
raise ValueError(f"ax-engine: drafting width {wid} needs a chain graph width below the block {block}")
|
|
417
|
+
free0 = torch.cuda.mem_get_info()[0]
|
|
418
|
+
hooks, flag = dmr.capture_tail_hooks, dmr.init_new_workspace
|
|
419
|
+
try:
|
|
420
|
+
w.block_size = wid
|
|
421
|
+
sampler = w._maybe_build_draft_sampler()
|
|
422
|
+
if sampler is None:
|
|
423
|
+
raise RuntimeError("ax-engine: no in-graph draft sampler for a narrower drafting width")
|
|
424
|
+
sampler.ax_chain_only = True # ax_lattice: chain rounds only, no tree lattice needed
|
|
425
|
+
dmr.capture_tail_hooks = [make_draft_sampler_capture_hook(sampler)]
|
|
426
|
+
current_runtime().draft_width_override = wid
|
|
427
|
+
with W.draft_tp_context(w.draft_parallel_layout), get_spec().override(speculative_num_draft_tokens=wid):
|
|
428
|
+
backend = dmr._get_attention_backend(init_new_workspace=True)
|
|
429
|
+
runner = R.__new__(R)
|
|
430
|
+
runner._ax_chain_width = wid
|
|
431
|
+
R.__init__(runner, dmr, attn_backend=backend, speculative_num_draft_tokens=wid)
|
|
432
|
+
finally:
|
|
433
|
+
current_runtime().draft_width_override = None
|
|
434
|
+
dmr.capture_tail_hooks, dmr.init_new_workspace = hooks, flag
|
|
435
|
+
w.block_size = block
|
|
436
|
+
spec_info = W.make_draft_block_spec_info(draft_token_num=wid, device=w.device)
|
|
437
|
+
w._ax_draft_sets[wid] = {"runner": runner, "sampler": sampler, "buffers": None, "spec_info": spec_info}
|
|
438
|
+
used = (free0 - torch.cuda.mem_get_info()[0]) / 2 ** 30
|
|
439
|
+
print(f"[ax_chain] drafter graph runner width {wid}: batch sizes {runner.capture_bs}, {used:.2f} GB", flush=True)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _enter_draft_width(w, wid):
|
|
443
|
+
"""Before the draft step of a chain round that drafts at width W: swap block size, drafter graph
|
|
444
|
+
runner and attention backend, in-graph sampler and the block-sized draft buffers."""
|
|
445
|
+
d = w._ax_draft_sets[wid]
|
|
446
|
+
dmr = w.draft_model_runner
|
|
447
|
+
w._ax_draft_saved = (dmr.attn_backend, dmr.decode_cuda_graph_runner, w._draft_sampler,
|
|
448
|
+
w._draft_block_spec_info, {n: getattr(w, n, None) for n in _DRAFT_BUFFERS})
|
|
449
|
+
dmr.attn_backend, dmr.decode_cuda_graph_runner = d["runner"].attn_backend, d["runner"]
|
|
450
|
+
w._draft_sampler = d["sampler"]
|
|
451
|
+
w._draft_block_spec_info = d["spec_info"]
|
|
452
|
+
bufs = d["buffers"] or {n: None for n in _DRAFT_BUFFERS} # None: _ensure_draft_block_buffers allocates at W
|
|
453
|
+
for n, v in bufs.items():
|
|
454
|
+
setattr(w, n, v)
|
|
455
|
+
_check_buffer_set(w, f"entering width {wid}")
|
|
456
|
+
w.block_size = wid
|
|
457
|
+
w._ax_draft_width = wid
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _leave_draft_width(w):
|
|
461
|
+
wid = getattr(w, "_ax_draft_width", None)
|
|
462
|
+
if wid is None:
|
|
463
|
+
return
|
|
464
|
+
d = w._ax_draft_sets[wid]
|
|
465
|
+
d["buffers"] = {n: getattr(w, n, None) for n in _DRAFT_BUFFERS}
|
|
466
|
+
dmr = w.draft_model_runner
|
|
467
|
+
dmr.attn_backend, dmr.decode_cuda_graph_runner, w._draft_sampler, w._draft_block_spec_info, bufs = w._ax_draft_saved
|
|
468
|
+
for n, v in bufs.items():
|
|
469
|
+
setattr(w, n, v)
|
|
470
|
+
_check_buffer_set(w, f"leaving width {wid}")
|
|
471
|
+
w._ax_draft_width = None
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _use_chain_runner(w, wid):
|
|
475
|
+
runners = getattr(w, "_ax_chain_runners", None)
|
|
476
|
+
if not runners:
|
|
477
|
+
return
|
|
478
|
+
runner = runners.get(wid)
|
|
479
|
+
if runner is None:
|
|
480
|
+
raise ValueError(f"ax-engine: no chain graph runner for width {wid} (AX_CHAIN_GRAPH_WIDTHS)")
|
|
481
|
+
mr = w.target_worker.model_runner
|
|
482
|
+
w._ax_saved_runner = (mr.attn_backend, mr.decode_cuda_graph_runner)
|
|
483
|
+
mr.attn_backend, mr.decode_cuda_graph_runner = runner.attn_backend, runner
|
|
484
|
+
current_runtime().round_info["backend"] = runner._ax_backend_name
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _topk_backends(w):
|
|
488
|
+
"""Target attention backends that read `topk` at run time. The plugin declares
|
|
489
|
+
speculative_eagle_topk = 2 (a tree server), and SGLang's hybrid linear-attention backend then
|
|
490
|
+
takes its tree verify path (parent links from spec_info) for every target verify; a chain round
|
|
491
|
+
must take the chain path (topk 1) to run the same kernels as stock DFlash."""
|
|
492
|
+
found = getattr(w, "_ax_topk_backends", None)
|
|
493
|
+
if found is None:
|
|
494
|
+
found, seen, todo = [], set(), [w.target_worker.model_runner.attn_backend]
|
|
495
|
+
while todo:
|
|
496
|
+
b = todo.pop()
|
|
497
|
+
if b is None or id(b) in seen:
|
|
498
|
+
continue
|
|
499
|
+
seen.add(id(b))
|
|
500
|
+
if isinstance(getattr(b, "topk", None), int):
|
|
501
|
+
found.append((b, b.topk))
|
|
502
|
+
for name in ("full_attn_backend", "linear_attn_backend", "attn_backend_list", "primary", "decode_backend", "prefill_backend"):
|
|
503
|
+
v = getattr(b, name, None)
|
|
504
|
+
todo.extend(v if isinstance(v, (list, tuple)) else [v])
|
|
505
|
+
w._ax_topk_backends = found
|
|
506
|
+
return found
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _set_topk(w, chain):
|
|
510
|
+
for b, tree_topk in _topk_backends(w):
|
|
511
|
+
b.topk = 1 if chain else tree_topk
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def choose(w, batch, *, draft_features_ready):
|
|
515
|
+
"""Kind of this decode round; resets the single-request tree plan state for chain rounds."""
|
|
516
|
+
|
|
517
|
+
import fc1_plan
|
|
518
|
+
decided = getattr(w, "_ax_tree_round", None)
|
|
519
|
+
wants_tree = decided[1] if decided is not None and decided[0] == id(batch) else tree_round(batch)
|
|
520
|
+
# The builder reads this round's logits or lattice from the folded sampler.
|
|
521
|
+
is_tree = wants_tree and draft_features_ready
|
|
522
|
+
if is_tree and getattr(w, "_ax_draft_width", None) is not None:
|
|
523
|
+
raise RuntimeError("ax-engine: tree round after drafting at a chain width") # begin_round disagreed
|
|
524
|
+
if is_tree:
|
|
525
|
+
current_runtime().kind = "tree"
|
|
526
|
+
current_runtime().stats["tree"] += 1
|
|
527
|
+
else:
|
|
528
|
+
current_runtime().kind = "chain"
|
|
529
|
+
current_runtime().stats["chain"] += 1
|
|
530
|
+
current_runtime().host_kv_len = None
|
|
531
|
+
_set_topk(w, current_runtime().kind == "chain")
|
|
532
|
+
if _CHECK:
|
|
533
|
+
_check_alloc(batch)
|
|
534
|
+
current_runtime().round_index += 1
|
|
535
|
+
if current_runtime().round_index % 1000 == 0:
|
|
536
|
+
print(f"[ax_chain] round {current_runtime().round_index} {current_runtime().stats}", flush=True)
|
|
537
|
+
return current_runtime().kind
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _swap_accept_buffers(w, wid):
|
|
541
|
+
"""Keep one set of stock accept/bonus buffers per width (they are sized by block_size)."""
|
|
542
|
+
names = ("_accept_bonus_buffer_cap", "_accept_len_buf", "_commit_lens_bufs", "_bonus_id_bufs",
|
|
543
|
+
"_out_tokens_bufs", "_new_seq_lens_bufs", "_accept_bonus_buffer_slot")
|
|
544
|
+
cur = getattr(w, "_ax_accept_width", None)
|
|
545
|
+
if cur == wid:
|
|
546
|
+
return
|
|
547
|
+
store = w.__dict__.setdefault("_ax_accept_sets", {})
|
|
548
|
+
if cur is not None:
|
|
549
|
+
store[cur] = {n: getattr(w, n, None) for n in names}
|
|
550
|
+
saved = store.get(wid)
|
|
551
|
+
if saved is not None:
|
|
552
|
+
for n, v in saved.items():
|
|
553
|
+
setattr(w, n, v)
|
|
554
|
+
else:
|
|
555
|
+
w._accept_bonus_buffer_cap = 0 # _ensure_accept_bonus_buffers allocates at this width
|
|
556
|
+
w._accept_bonus_buffer_slot = 0
|
|
557
|
+
w._ax_accept_width = wid
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def narrow(w, batch, draft_tokens, positions, cache_loc_2d, prefix_lens, kv_loc_plan, block):
|
|
561
|
+
"""Cut the round to its chain width. Returns the stock locals the verify uses next."""
|
|
562
|
+
bs = draft_tokens.shape[0]
|
|
563
|
+
wid = w._ax_round_width or width(block) # decided in begin_round when the round drafted at its width
|
|
564
|
+
current_runtime().stats["chain_width"][wid] = current_runtime().stats["chain_width"].get(wid, 0) + 1
|
|
565
|
+
current_runtime().round_info["width"] = wid
|
|
566
|
+
_swap_accept_buffers(w, wid)
|
|
567
|
+
_use_chain_runner(w, wid)
|
|
568
|
+
if _CHAIN_ATTN and current_runtime().round_info["backend"] != _CHAIN_ATTN:
|
|
569
|
+
current_runtime().stats["chain_on_main"] = current_runtime().stats.get("chain_on_main", 0) + 1
|
|
570
|
+
raise RuntimeError(f"ax-engine: chain round of width {wid} would run on the main backend, not {_CHAIN_ATTN}")
|
|
571
|
+
if wid == block:
|
|
572
|
+
w.block_size = block
|
|
573
|
+
return draft_tokens, positions, cache_loc_2d.reshape(-1), cache_loc_2d, kv_loc_plan, block
|
|
574
|
+
tokens = draft_tokens[:, :wid].contiguous()
|
|
575
|
+
if w._selector_sample is not None:
|
|
576
|
+
candidate_ids, proposal_probs = w._selector_sample
|
|
577
|
+
# The first token is the anchor. Keep the original proposal law for each retained token.
|
|
578
|
+
w._selector_sample = (candidate_ids[:, :wid - 1], proposal_probs[:, :wid - 1])
|
|
579
|
+
pos = positions.view(bs, block)[:, :wid].contiguous().reshape(-1)
|
|
580
|
+
loc2d = cache_loc_2d[:, :wid].contiguous()
|
|
581
|
+
loc = loc2d.reshape(-1)
|
|
582
|
+
draft_input = batch.spec_info
|
|
583
|
+
plan = w.target_worker.model_runner.kv_index_translator.plan(
|
|
584
|
+
req_pool_indices=batch.req_pool_indices,
|
|
585
|
+
seq_lens=prefix_lens,
|
|
586
|
+
seq_lens_cpu=batch.seq_lens_cpu if batch.seq_lens_cpu is not None else draft_input.nxt_kv_lens_cpu,
|
|
587
|
+
write_virtual=loc,
|
|
588
|
+
read_extent=wid,
|
|
589
|
+
)
|
|
590
|
+
w.block_size = wid
|
|
591
|
+
return tokens, pos, loc, loc2d, plan, wid
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def record_accept(w, batch, candidates, accept_len):
|
|
595
|
+
"""After the stock accept of a chain round: token before the next anchor, per request; oracle dump."""
|
|
596
|
+
if current_runtime().kind != "chain":
|
|
597
|
+
return
|
|
598
|
+
before = candidates.gather(1, accept_len.to(torch.int64)[:, None])[:, 0]
|
|
599
|
+
for i, req in enumerate(batch.reqs):
|
|
600
|
+
req._ax_pre1 = before[i:i + 1]
|
|
601
|
+
if _DUMP:
|
|
602
|
+
_dump(batch, "chain", candidates, accept_len)
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def record_tree(batch, pre1, tree_tokens, accept_index, accept_lens):
|
|
606
|
+
"""After a tree round: same bookkeeping as record_accept."""
|
|
607
|
+
batch.reqs[0]._ax_pre1 = pre1
|
|
608
|
+
if current_runtime().round_info["backend"] is None:
|
|
609
|
+
current_runtime().round_info["backend"] = current_runtime().main_backend
|
|
610
|
+
if _DUMP:
|
|
611
|
+
path = tree_tokens.gather(1, accept_index.clamp(min=0).long())
|
|
612
|
+
_dump(batch, "tree", path, accept_lens.to(torch.int64) - 1, nodes=tree_tokens)
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def pre1_for(req):
|
|
616
|
+
"""Token before the anchor for the tree builder: kept from the request's last round of either
|
|
617
|
+
kind; on its first decode round the last prompt token."""
|
|
618
|
+
return getattr(req, "_ax_pre1", None)
|
|
619
|
+
|
|
620
|
+
|
|
621
|
+
def _dump(batch, kind, tokens, accept_len, nodes=None):
|
|
622
|
+
toks = tokens.tolist()
|
|
623
|
+
acc = accept_len.tolist()
|
|
624
|
+
with open(_DUMP, "a") as f:
|
|
625
|
+
for i, req in enumerate(batch.reqs):
|
|
626
|
+
row = {"round": current_runtime().round_index, "rid": req.rid, "kind": kind, "bs": len(batch.reqs),
|
|
627
|
+
"tokens": toks[i], "accept": int(acc[i]),
|
|
628
|
+
"backend": current_runtime().round_info["backend"], "graph": current_runtime().round_info["graph"]}
|
|
629
|
+
if nodes is not None:
|
|
630
|
+
row["nodes"] = nodes.tolist()[i]
|
|
631
|
+
f.write(json.dumps(row) + "\n")
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
import atexit # noqa: E402
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
@atexit.register
|
|
638
|
+
def _report():
|
|
639
|
+
for stats in runtime_statistics():
|
|
640
|
+
print(f"[ax_chain] rounds {stats}", flush=True)
|