ax-engine-sglang 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. ax_engine_sglang/__init__.py +5 -0
  2. ax_engine_sglang/ax_chain.py +640 -0
  3. ax_engine_sglang/ax_dflash_sampling.py +43 -0
  4. ax_engine_sglang/ax_eagle_sampling.py +73 -0
  5. ax_engine_sglang/ax_fast.py +116 -0
  6. ax_engine_sglang/ax_fp8_probe.py +93 -0
  7. ax_engine_sglang/ax_gdn_recurrent.py +59 -0
  8. ax_engine_sglang/ax_lattice.py +182 -0
  9. ax_engine_sglang/ax_nvfp4_verify.py +78 -0
  10. ax_engine_sglang/ax_nxt_debug.py +103 -0
  11. ax_engine_sglang/ax_qsa.py +327 -0
  12. ax_engine_sglang/ax_rng.py +70 -0
  13. ax_engine_sglang/ax_state_cap.py +102 -0
  14. ax_engine_sglang/ax_verify_attention.py +138 -0
  15. ax_engine_sglang/cli.py +57 -0
  16. ax_engine_sglang/compat.py +80 -0
  17. ax_engine_sglang/config.py +150 -0
  18. ax_engine_sglang/contracts.py +67 -0
  19. ax_engine_sglang/defaults.py +284 -0
  20. ax_engine_sglang/fc1_block_inverse.py +83 -0
  21. ax_engine_sglang/fc1_branch64.py +81 -0
  22. ax_engine_sglang/fc1_builder.py +54 -0
  23. ax_engine_sglang/fc1_conv.py +47 -0
  24. ax_engine_sglang/fc1_fused.py +196 -0
  25. ax_engine_sglang/fc1_plan.py +44 -0
  26. ax_engine_sglang/fc1_verify.py +26 -0
  27. ax_engine_sglang/ngram_gpu.py +70 -0
  28. ax_engine_sglang/plugin.py +130 -0
  29. ax_engine_sglang/policy.py +31 -0
  30. ax_engine_sglang/qsa_topk.py +32 -0
  31. ax_engine_sglang/runtime.py +50 -0
  32. ax_engine_sglang/tree_kernels.py +208 -0
  33. ax_engine_sglang/tree_ops.py +102 -0
  34. ax_engine_sglang/worker_gen.py +123 -0
  35. ax_engine_sglang-0.2.0.dist-info/METADATA +350 -0
  36. ax_engine_sglang-0.2.0.dist-info/RECORD +42 -0
  37. ax_engine_sglang-0.2.0.dist-info/WHEEL +5 -0
  38. ax_engine_sglang-0.2.0.dist-info/entry_points.txt +5 -0
  39. ax_engine_sglang-0.2.0.dist-info/licenses/LICENSE +202 -0
  40. ax_engine_sglang-0.2.0.dist-info/licenses/NOTICE +35 -0
  41. ax_engine_sglang-0.2.0.dist-info/licenses/licenses/FLASH_LINEAR_ATTENTION-LICENSE +21 -0
  42. ax_engine_sglang-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,5 @@
1
+ """ax-engine-sglang: lossless tree speculative decoding (DFlash / DFlash2 drafters).
2
+
3
+ Launch with `ax-engine-sglang serve` or select `SGLANG_PLUGINS=ax-engine` in SGLang.
4
+ The import package is `ax_engine_sglang`; `ax_engine` is the separate Apple Silicon runtime."""
5
+ __version__ = "0.2.0"
@@ -0,0 +1,640 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Several requests at once: chain rounds next to the one-request tree (multi-user Stage 0).
3
+
4
+ Design: docs/decisions/2026-11-15-multiuser-width-policy.md. One server, drafter block B
5
+ (`FC1_DRAFT_BLOCK`, 16) and `speculative_num_draft_tokens` = tree nodes (`FC1_TREE_N`, 16). Each
6
+ decode round is one of two kinds, decided here before the target verify and kept in `current_runtime().kind` so the
7
+ fused tree hooks (fc1_fused, fc1_conv, fc1_plan) run only in tree rounds:
8
+
9
+ tree one running request and the tree enabled: the fused tree verifier (fc1_branch64)
10
+ chain otherwise: SGLang's own DFlash chain verify on the first W drafted tokens
11
+
12
+ Stage 0 has no width rule. The width is pinned with `AX_CHAIN_WIDTH` (8 or 16, default B), or a
13
+ forced schedule `AX_CHAIN_SCHEDULE="16x5,8x3"` (width x rounds, cycled) for the switching checks.
14
+ `AX_TREE=0` turns tree rounds off (every round is a chain round, also with one request).
15
+
16
+ A chain round of width W = B runs the stock code unchanged. For W < B, `narrow` cuts the drafted
17
+ block, its positions and its KV slots to W columns, plans a W-slot verify window (the stock worker
18
+ plans one B-slot window for draft and verify), and sets the worker's `block_size` to W for the
19
+ rest of the round, so every stock read after the draft step (accept reshaping, logit adjustments,
20
+ logprob stride, hidden-state view, the result's `speculative_num_draft_tokens`) uses W.
21
+ `begin_round` puts B back at the start of the next call. The accept/bonus buffers are sized by
22
+ `block_size` when first used, so each width keeps its own set (`_swap_accept_buffers`).
23
+
24
+ `record_accept` (after the stock accept) keeps, per request, the token before the next anchor, which
25
+ the tree builder conditions on when the request is alone again (`pre1_for`), and appends one line
26
+ per round to `AX_ROUND_DUMP` (if set) for the offline accept oracle: request id, kind, width, the
27
+ verified tokens and the accepted length.
28
+
29
+ Two attention backends (docs/decisions/2026-11-16-dual-attention-backend.md): with
30
+ `AX_CHAIN_ATTN=<backend>` (for example trtllm_mha) the private chain runners are built with that
31
+ backend, in that backend's role, while the main runner (tree rounds, prefill) keeps the server's
32
+ `--attention-backend` (triton, which has the tree mask). Every chain round must then run on a
33
+ private runner: the mixed mode needs CUDA graphs and a graph runner for every chain width, and a
34
+ chain round that would run on the main backend stops the server. The round dump records, per
35
+ round, the backend that ran it and whether the graph replayed; `[ax_chain]` stats count graph and
36
+ eager rounds per (width, batch size).
37
+
38
+ Tree rounds serve one greedy request without grammar or logprobs (the tree verify accepts the
39
+ argmax path only). Any other decode batch, including a single sampled request, runs a chain round,
40
+ which is SGLang's own DFlash verify: sampled (selector sampling), grammar-constrained and logprob
41
+ requests get stock behaviour there instead of an error. (Until 2026-11-16 such batches raised in
42
+ the scheduler and stopped the server; with the model card's default temperature 1.0 that was any
43
+ two concurrent requests that did not set temperature 0.)
44
+ """
45
+ from __future__ import annotations
46
+
47
+ import json
48
+ import os
49
+
50
+ import torch
51
+
52
+ from ax_engine_sglang.config import get_config
53
+ from ax_engine_sglang.policy import RoundFeatures, chain_width, use_tree
54
+ from ax_engine_sglang.runtime import activate_worker, current_runtime, runtime_statistics
55
+ from ax_engine_sglang.compat import patch_attribute
56
+
57
+ _CONFIG = get_config()
58
+ _TREE = _CONFIG.tree
59
+ _PINNED = _CONFIG.chain_width
60
+ _SCHED = _CONFIG.chain_schedule
61
+ _DUMP = _CONFIG.round_dump
62
+ _GRAPH_WIDTHS = _CONFIG.graph_widths
63
+ _DRAFT_WIDTHS = _CONFIG.draft_widths
64
+ _CHECK = _CONFIG.chain_check
65
+ _CHAIN_ATTN = _CONFIG.chain_attention
66
+
67
+
68
+ def is_tree():
69
+ state = current_runtime()
70
+ if state.kind is not None:
71
+ state.stats["hook_tree" if state.kind == "tree" else "hook_chain"] += 1
72
+ return state.kind == "tree"
73
+
74
+
75
+ def _check_alloc(batch):
76
+ """AX_CHAIN_CHECK=1, sampled when the round's kind is chosen (after prepare_for_decode and the
77
+ draft step): every running request has this round's 16-slot window allocated beyond its committed
78
+ length (prepare reserves 2 x 16 = 32; with overlap scheduling the committed length can advance by
79
+ one round's commit before this sample, so the gap seen here is 16..32), and its allocation never
80
+ shrinks while it runs. The smallest gap seen is reported."""
81
+ for req in batch.reqs:
82
+ alloc, committed = int(req.kv.kv_allocated_len), int(req.kv.kv_committed_len)
83
+ current_runtime().stats["alloc_checked"] += 1
84
+ gap = alloc - committed
85
+ current_runtime().stats["alloc_min_gap"] = min(current_runtime().stats.get("alloc_min_gap", gap), gap)
86
+ if gap < 16:
87
+ current_runtime().stats["alloc_short"] += 1
88
+ if alloc < current_runtime().alloc_seen.get(req.rid, 0):
89
+ current_runtime().stats["alloc_shrunk"] += 1
90
+ current_runtime().alloc_seen[req.rid] = alloc
91
+ if req.finished():
92
+ current_runtime().alloc_seen.pop(req.rid, None)
93
+
94
+
95
+ def width(block):
96
+ """Verify width of this chain round (anchor included)."""
97
+ return chain_width(_CONFIG, block, current_runtime().round_index)
98
+
99
+
100
+ def _configured_widths(block):
101
+ """Chain widths this server can choose (Stage 0: pinned, schedule or the block)."""
102
+ if _SCHED:
103
+ return sorted(set(_SCHED))
104
+ if _PINNED:
105
+ return [int(_PINNED)]
106
+ return [block]
107
+
108
+
109
+ def tree_round(batch):
110
+ """Translate SGLang sampling capabilities into the pure round policy."""
111
+ if not _CONFIG.tree or len(batch.reqs) != 1 or batch.has_grammar or batch.return_logprob:
112
+ return False
113
+ info = batch.sampling_info
114
+ return use_tree(_CONFIG, RoundFeatures(
115
+ requests=len(batch.reqs), grammar=batch.has_grammar, logprobs=batch.return_logprob,
116
+ custom_processor=bool(info is not None and info.has_custom_logit_processor),
117
+ greedy=info is None or info.is_all_greedy,
118
+ min_p=bool(info is not None and not info.is_all_greedy and info.need_min_p_sampling),
119
+ ))
120
+
121
+
122
+ def begin_round(w, batch=None):
123
+ """Start of forward_batch_generation: restore the drafter block after a narrowed chain round,
124
+ the main attention backend and graph runner after a chain round on a private pair, and the
125
+ drafter's state after a round drafted at a narrower width. Then, for a decode round that will be
126
+ a chain round drafted at width W (AX_CHAIN_DRAFT_WIDTHS), switch the drafter to W."""
127
+ state = activate_worker(w)
128
+ _leave_draft_width(w)
129
+ block = getattr(w, "_ax_block", None)
130
+ if block is None:
131
+ block = w._ax_block = int(w.block_size)
132
+ w.block_size = block
133
+ saved = getattr(w, "_ax_saved_runner", None)
134
+ if saved is not None:
135
+ mr = w.target_worker.model_runner
136
+ mr.attn_backend, mr.decode_cuda_graph_runner = saved
137
+ w._ax_saved_runner = None
138
+ w._ax_round_width = None
139
+ w._ax_tree_round = None
140
+ state.kind = None
141
+ state.host_kv_len = None
142
+ state.round_info.update(backend=None, graph=None, width=None)
143
+ sets = getattr(w, "_ax_draft_sets", None)
144
+ if not sets or batch is None or batch.forward_mode.is_extend() or batch.is_extend_in_batch:
145
+ return
146
+ w._ax_tree_round = (id(batch), tree_round(batch))
147
+ if w._ax_tree_round[1]:
148
+ return
149
+ wid = width(block)
150
+ w._ax_round_width = wid
151
+ if wid in sets:
152
+ _enter_draft_width(w, wid)
153
+
154
+
155
+ def install():
156
+ """With CUDA graphs for one request only (tree rounds), SGLang's in-graph draft sampler has
157
+ buffers for one request, but the stock draft step stages its sampling parameters for every
158
+ batch. A batch larger than the sampler runs the drafter eager and never reads the sampler, so the
159
+ staging is skipped there."""
160
+ import fc1_dflash_worker as W
161
+ cls = getattr(W, "_SelectorDraftSampler", None)
162
+ if cls is None or getattr(cls, "_ax_chain_guard", False):
163
+ return
164
+ original = cls.stage_sampling_params
165
+
166
+ def stage_sampling_params(self, *, bs, sampling_info):
167
+ if bs > self.greedy_mask.shape[0]:
168
+ return None
169
+ return original(self, bs=bs, sampling_info=sampling_info)
170
+
171
+ patch_attribute(cls, 'stage_sampling_params', stage_sampling_params)
172
+ patch_attribute(cls, '_ax_chain_guard', True)
173
+ if _CHAIN_ATTN and not _GRAPH_WIDTHS:
174
+ raise RuntimeError(f"ax-engine: AX_CHAIN_ATTN={_CHAIN_ATTN} needs AX_CHAIN_GRAPH_WIDTHS (a graph runner per "
175
+ "chain width); without them every chain round would run on the main backend")
176
+ if _GRAPH_WIDTHS:
177
+ _install_graphs(W)
178
+ else:
179
+ _install_main_only()
180
+ if not getattr(cls_w := W.DFlashTreeWorkerV2, "_ax_checked", False):
181
+ init0 = cls_w.init_cuda_graphs
182
+
183
+ def init_cuda_graphs(self):
184
+ activate_worker(self)
185
+ return init0(self)
186
+ patch_attribute(cls_w, 'init_cuda_graphs', init_cuda_graphs)
187
+ patch_attribute(cls_w, '_ax_checked', True)
188
+ if os.environ.get("AX_NXT_DEBUG") == "1":
189
+ import ax_nxt_debug # loaded next to ax_chain (the plugin puts this directory on sys.path)
190
+ ax_nxt_debug.install(W.DFlashTreeWorkerV2)
191
+
192
+
193
+ def is_tree_runner(runner):
194
+ """True for graph runners of a server running the tree algorithm. The capture patches below are
195
+ global (SGLang's DecodeCudaGraphRunner), so a server that loads the plugin for something else (stock
196
+ EAGLE / MTP, e.g. with AX_QSA_RING) must keep SGLang's own batch sizes."""
197
+ algo = getattr(getattr(runner.model_runner, "server_args", None), "speculative_algorithm", None)
198
+ return str(algo or "").upper() == "FC1_DFLASH_TREE"
199
+
200
+
201
+ def _install_main_only():
202
+ """Servers without private chain runners (AX_CHAIN_GRAPH_WIDTHS unset): SGLang's main target runner
203
+ is the only graph runner.
204
+
205
+ - Tree on: it captures batch 1 only, with the tree verify input (the fused tree hooks handle one
206
+ request; a capture at a larger batch with the tree input fails), and every chain round runs
207
+ eager (it would otherwise be admitted to the tree graph at the tree's width).
208
+ - Tree off: it captures the stock chain input at the block width, and a chain round narrowed
209
+ below that width runs eager (the graph holds block-width buffers).
210
+ """
211
+ from sglang.srt.model_executor.runner.decode_cuda_graph_runner import DecodeCudaGraphRunner as R
212
+ if getattr(R, "_ax_main_only", False):
213
+ return
214
+ original_capture = R.capture
215
+
216
+ def capture(self):
217
+
218
+ if not is_tree_runner(self):
219
+ return original_capture(self)
220
+ if not self.model_runner.is_draft_worker:
221
+ activate_worker(self.model_runner)
222
+ previous_kind = current_runtime().kind
223
+ main = not self.model_runner.is_draft_worker
224
+ if main and _TREE:
225
+ current_runtime().kind = "tree"
226
+ self.capture_bs = [b for b in self.capture_bs if b == 1]
227
+ self.compile_bs = [b for b in (self.compile_bs or []) if b == 1]
228
+ else:
229
+ current_runtime().kind = "chain" # stock capture input (plugin.tree_capture_info)
230
+ try:
231
+ return original_capture(self)
232
+ finally:
233
+ current_runtime().kind = previous_kind
234
+ if main and _TREE:
235
+ self.max_bs = 1
236
+
237
+ can_run = R.can_run_graph
238
+
239
+ def can_run_graph(self, forward_batch):
240
+ if is_tree_runner(self) and current_runtime().kind == "chain" and not self.model_runner.is_draft_worker:
241
+ captured = getattr(self, "speculative_num_draft_tokens", None)
242
+ if _TREE or captured is None or current_runtime().round_info.get("width") not in (None, captured):
243
+ return False
244
+ return can_run(self, forward_batch)
245
+
246
+ patch_attribute(R, 'capture', capture)
247
+ patch_attribute(R, 'can_run_graph', can_run_graph)
248
+ patch_attribute(R, '_ax_main_only', True)
249
+
250
+
251
+ def _install_graphs(W):
252
+ """CUDA graphs per round kind and width (design, Stage 0 item 3).
253
+
254
+ - The main target runner (SGLang's) captures only batch 1, with the tree verify input when the
255
+ tree is on: the tree graph.
256
+ - Each chain width W in AX_CHAIN_GRAPH_WIDTHS gets its own runner with its own attention backend,
257
+ built with speculative_eagle_topk = 1 and speculative_num_draft_tokens = W (as SGLang's UNO
258
+ worker builds its private runner), capturing the stock DFlash chain verify at every batch size.
259
+ A private backend keeps the main runner's captured buffers untouched (a second
260
+ init_cuda_graph_state on a shared backend would reallocate them under the captured tree graph).
261
+ - A chain round swaps model_runner.attn_backend and decode_cuda_graph_runner to its width's pair
262
+ for the whole round (verify and the recurrent-state commit, which reads the backend's forward
263
+ metadata); begin_round restores them at the next call.
264
+ """
265
+ from sglang.srt.model_executor.runner.decode_cuda_graph_runner import DecodeCudaGraphRunner as R
266
+ if getattr(R, "_ax_chain_graphs", False):
267
+ return
268
+ original_capture = R.capture
269
+
270
+ def capture(self):
271
+
272
+ if not is_tree_runner(self):
273
+ return original_capture(self)
274
+ if not self.model_runner.is_draft_worker:
275
+ activate_worker(self.model_runner)
276
+ previous_kind = current_runtime().kind
277
+ target = not self.model_runner.is_draft_worker
278
+ width = getattr(self, "_ax_chain_width", None)
279
+ main = target and width is None
280
+ if main:
281
+ # one request only: the tree graph (tree on) or an unused batch-1 chain graph (tree off;
282
+ # every chain round runs on a private runner)
283
+ self.capture_bs = [b for b in self.capture_bs if b == 1]
284
+ self.compile_bs = [b for b in (self.compile_bs or []) if b == 1]
285
+ current_runtime().kind = "tree" if main and _TREE else "chain"
286
+ try:
287
+ return original_capture(self)
288
+ finally:
289
+ current_runtime().kind = previous_kind
290
+ if main:
291
+ self.max_bs = 1
292
+
293
+ patch_attribute(R, 'capture', capture)
294
+ patch_attribute(R, '_ax_chain_graphs', True)
295
+
296
+ cls = W.DFlashTreeWorkerV2
297
+ original_init = cls.init_cuda_graphs
298
+
299
+ def init_cuda_graphs(self):
300
+ out = original_init(self)
301
+ _build_chain_runners(self, R)
302
+
303
+ mr = self.target_worker.model_runner
304
+ current_runtime().main_backend = _backend_name(mr, main=True)
305
+ if mr.decode_cuda_graph_runner is not None:
306
+ _count_replays(mr.decode_cuda_graph_runner, None, _backend_name(mr, main=True))
307
+ if os.environ.get("AX_FP8_PROBE") == "1":
308
+ import ax_fp8_probe
309
+ ax_fp8_probe.run(mr)
310
+ return out
311
+
312
+ patch_attribute(cls, 'init_cuda_graphs', init_cuda_graphs)
313
+
314
+
315
+ def _backend_name(mr, main=False):
316
+ """Name of the backend a target round runs on: the main runner's decode backend, or AX_CHAIN_ATTN."""
317
+ if main or not _CHAIN_ATTN:
318
+ return getattr(mr, "decode_attention_backend_str", None)
319
+ return _CHAIN_ATTN
320
+
321
+
322
+ def _count_replays(runner, wid, name):
323
+ """Record, per target verify on this graph runner, whether the captured graph replays (SGLang
324
+ asks can_run_graph once per forward) or the round runs eager."""
325
+ original = runner.can_run_graph
326
+
327
+ def can_run_graph(forward_batch):
328
+ ok = bool(original(forward_batch))
329
+ key = f"{'tree' if wid is None else wid}:{forward_batch.batch_size}:{'graph' if ok else 'eager'}"
330
+ current_runtime().stats["replay"][key] = current_runtime().stats["replay"].get(key, 0) + 1
331
+ current_runtime().round_info["graph"] = ok
332
+ return ok
333
+
334
+ runner.can_run_graph = can_run_graph
335
+ runner._ax_backend_name = name
336
+
337
+
338
+ def _build_backend(mr, wid):
339
+ """Private attention backend of a chain runner. With AX_CHAIN_ATTN it is built by name in that
340
+ backend's role: trtllm_mha decides its KV access checks and native-FP4 setup from the runner's
341
+ prefill/decode role strings (trtllm_mha_backend.py:218-240) and the Gated-DeltaNet wrapper
342
+ asserts on them (attention_registry.py:471-475), so both are set for the construction (as in an
343
+ all-trtllm_mha server) and restored; SGLang records the pair on the backend object too."""
344
+ if not _CHAIN_ATTN:
345
+ return mr._get_attention_backend(init_new_workspace=True)
346
+ from sglang.srt.model_executor.model_runner_components.attention_backend_setup import _build_backend_from_str
347
+ roles = (mr.prefill_attention_backend_str, mr.decode_attention_backend_str)
348
+ try:
349
+ mr.prefill_attention_backend_str = mr.decode_attention_backend_str = _CHAIN_ATTN
350
+ backend = _build_backend_from_str(model_runner=mr, backend_str=_CHAIN_ATTN, init_new_workspace=True)
351
+ finally:
352
+ mr.prefill_attention_backend_str, mr.decode_attention_backend_str = roles
353
+ backend.prefill_attention_backend_str = backend.decode_attention_backend_str = _CHAIN_ATTN
354
+ return backend
355
+
356
+
357
+ def _build_chain_runners(w, R):
358
+ from sglang.srt.runtime_context import get_spec
359
+ mr = w.target_worker.model_runner
360
+ w._ax_chain_runners = {}
361
+ if mr.decode_cuda_graph_runner is None:
362
+ if _CHAIN_ATTN:
363
+ raise RuntimeError("ax-engine: AX_CHAIN_ATTN needs CUDA graphs (no private chain runner is built without them)")
364
+ return
365
+ missing = [x for x in _configured_widths(int(w.block_size)) if x not in _GRAPH_WIDTHS]
366
+ if missing:
367
+ # a chain round of a width without a private runner would fail in _use_chain_runner
368
+ raise RuntimeError(f"ax-engine: chain widths {missing} have no graph runner "
369
+ f"(AX_CHAIN_GRAPH_WIDTHS={_GRAPH_WIDTHS}); add them or unset AX_CHAIN_GRAPH_WIDTHS")
370
+ for wid in _GRAPH_WIDTHS:
371
+ free0 = torch.cuda.mem_get_info()[0]
372
+ flag = mr.init_new_workspace
373
+ try:
374
+ with get_spec().override(speculative_eagle_topk=1, speculative_num_draft_tokens=wid):
375
+ backend = _build_backend(mr, wid)
376
+ runner = R.__new__(R)
377
+ runner._ax_chain_width = wid
378
+ R.__init__(runner, mr, attn_backend=backend, speculative_num_draft_tokens=wid)
379
+ finally:
380
+ mr.init_new_workspace = flag
381
+ _count_replays(runner, wid, _backend_name(mr))
382
+ w._ax_chain_runners[wid] = runner
383
+ used = (free0 - torch.cuda.mem_get_info()[0]) / 2 ** 30
384
+ print(f"[ax_chain] chain graph runner width {wid}: backend {_backend_name(mr)} "
385
+ f"({type(backend).__name__}), batch sizes {runner.capture_bs}, {used:.2f} GB", flush=True)
386
+ _build_draft_runners(w, R)
387
+
388
+
389
+ # every buffer `_ensure_draft_block_buffers` (re)allocates together: it sizes them all from the capacity
390
+ # of _draft_block_ids_buf, so a set that swapped only some of them would mix capacities (a width-16 host
391
+ # length buffer for 14 requests next to width-8 buffers for 15 crashed the stress run)
392
+ _DRAFT_BUFFERS = ("_draft_block_ids_buf", "_draft_block_positions_buf", "_draft_block_tokens_buf",
393
+ "_draft_verify_out_cache_loc_buf", "_draft_block_end_buf", "_draft_seq_lens_cpu_buf")
394
+
395
+
396
+ def _check_buffer_set(w, where):
397
+ caps = {n: int(getattr(w, n).shape[0]) for n in _DRAFT_BUFFERS if getattr(w, n, None) is not None}
398
+ if caps and (len(set(caps.values())) > 1 or len(caps) != len(_DRAFT_BUFFERS)):
399
+ raise RuntimeError(f"ax-engine: draft buffers out of step {where}: {caps}")
400
+
401
+
402
+ def _build_draft_runners(w, R):
403
+ """Per drafting width W < B: a W-token sampler folded into a private drafter graph runner (own
404
+ draft attention backend), as SGLang builds the drafter graph for a block-W server."""
405
+ import fc1_dflash_worker as W
406
+ from sglang.srt.runtime_context import get_spec
407
+ from sglang.srt.speculative.draft_worker_common import make_draft_sampler_capture_hook
408
+
409
+ w._ax_draft_sets = {}
410
+ dmr = w.draft_model_runner
411
+ if not _DRAFT_WIDTHS or dmr.decode_cuda_graph_runner is None or w._draft_sampler is None:
412
+ return
413
+ block = int(w.block_size)
414
+ for wid in _DRAFT_WIDTHS:
415
+ if wid not in _GRAPH_WIDTHS or not 2 <= wid < block:
416
+ raise ValueError(f"ax-engine: drafting width {wid} needs a chain graph width below the block {block}")
417
+ free0 = torch.cuda.mem_get_info()[0]
418
+ hooks, flag = dmr.capture_tail_hooks, dmr.init_new_workspace
419
+ try:
420
+ w.block_size = wid
421
+ sampler = w._maybe_build_draft_sampler()
422
+ if sampler is None:
423
+ raise RuntimeError("ax-engine: no in-graph draft sampler for a narrower drafting width")
424
+ sampler.ax_chain_only = True # ax_lattice: chain rounds only, no tree lattice needed
425
+ dmr.capture_tail_hooks = [make_draft_sampler_capture_hook(sampler)]
426
+ current_runtime().draft_width_override = wid
427
+ with W.draft_tp_context(w.draft_parallel_layout), get_spec().override(speculative_num_draft_tokens=wid):
428
+ backend = dmr._get_attention_backend(init_new_workspace=True)
429
+ runner = R.__new__(R)
430
+ runner._ax_chain_width = wid
431
+ R.__init__(runner, dmr, attn_backend=backend, speculative_num_draft_tokens=wid)
432
+ finally:
433
+ current_runtime().draft_width_override = None
434
+ dmr.capture_tail_hooks, dmr.init_new_workspace = hooks, flag
435
+ w.block_size = block
436
+ spec_info = W.make_draft_block_spec_info(draft_token_num=wid, device=w.device)
437
+ w._ax_draft_sets[wid] = {"runner": runner, "sampler": sampler, "buffers": None, "spec_info": spec_info}
438
+ used = (free0 - torch.cuda.mem_get_info()[0]) / 2 ** 30
439
+ print(f"[ax_chain] drafter graph runner width {wid}: batch sizes {runner.capture_bs}, {used:.2f} GB", flush=True)
440
+
441
+
442
+ def _enter_draft_width(w, wid):
443
+ """Before the draft step of a chain round that drafts at width W: swap block size, drafter graph
444
+ runner and attention backend, in-graph sampler and the block-sized draft buffers."""
445
+ d = w._ax_draft_sets[wid]
446
+ dmr = w.draft_model_runner
447
+ w._ax_draft_saved = (dmr.attn_backend, dmr.decode_cuda_graph_runner, w._draft_sampler,
448
+ w._draft_block_spec_info, {n: getattr(w, n, None) for n in _DRAFT_BUFFERS})
449
+ dmr.attn_backend, dmr.decode_cuda_graph_runner = d["runner"].attn_backend, d["runner"]
450
+ w._draft_sampler = d["sampler"]
451
+ w._draft_block_spec_info = d["spec_info"]
452
+ bufs = d["buffers"] or {n: None for n in _DRAFT_BUFFERS} # None: _ensure_draft_block_buffers allocates at W
453
+ for n, v in bufs.items():
454
+ setattr(w, n, v)
455
+ _check_buffer_set(w, f"entering width {wid}")
456
+ w.block_size = wid
457
+ w._ax_draft_width = wid
458
+
459
+
460
+ def _leave_draft_width(w):
461
+ wid = getattr(w, "_ax_draft_width", None)
462
+ if wid is None:
463
+ return
464
+ d = w._ax_draft_sets[wid]
465
+ d["buffers"] = {n: getattr(w, n, None) for n in _DRAFT_BUFFERS}
466
+ dmr = w.draft_model_runner
467
+ dmr.attn_backend, dmr.decode_cuda_graph_runner, w._draft_sampler, w._draft_block_spec_info, bufs = w._ax_draft_saved
468
+ for n, v in bufs.items():
469
+ setattr(w, n, v)
470
+ _check_buffer_set(w, f"leaving width {wid}")
471
+ w._ax_draft_width = None
472
+
473
+
474
+ def _use_chain_runner(w, wid):
475
+ runners = getattr(w, "_ax_chain_runners", None)
476
+ if not runners:
477
+ return
478
+ runner = runners.get(wid)
479
+ if runner is None:
480
+ raise ValueError(f"ax-engine: no chain graph runner for width {wid} (AX_CHAIN_GRAPH_WIDTHS)")
481
+ mr = w.target_worker.model_runner
482
+ w._ax_saved_runner = (mr.attn_backend, mr.decode_cuda_graph_runner)
483
+ mr.attn_backend, mr.decode_cuda_graph_runner = runner.attn_backend, runner
484
+ current_runtime().round_info["backend"] = runner._ax_backend_name
485
+
486
+
487
+ def _topk_backends(w):
488
+ """Target attention backends that read `topk` at run time. The plugin declares
489
+ speculative_eagle_topk = 2 (a tree server), and SGLang's hybrid linear-attention backend then
490
+ takes its tree verify path (parent links from spec_info) for every target verify; a chain round
491
+ must take the chain path (topk 1) to run the same kernels as stock DFlash."""
492
+ found = getattr(w, "_ax_topk_backends", None)
493
+ if found is None:
494
+ found, seen, todo = [], set(), [w.target_worker.model_runner.attn_backend]
495
+ while todo:
496
+ b = todo.pop()
497
+ if b is None or id(b) in seen:
498
+ continue
499
+ seen.add(id(b))
500
+ if isinstance(getattr(b, "topk", None), int):
501
+ found.append((b, b.topk))
502
+ for name in ("full_attn_backend", "linear_attn_backend", "attn_backend_list", "primary", "decode_backend", "prefill_backend"):
503
+ v = getattr(b, name, None)
504
+ todo.extend(v if isinstance(v, (list, tuple)) else [v])
505
+ w._ax_topk_backends = found
506
+ return found
507
+
508
+
509
+ def _set_topk(w, chain):
510
+ for b, tree_topk in _topk_backends(w):
511
+ b.topk = 1 if chain else tree_topk
512
+
513
+
514
+ def choose(w, batch, *, draft_features_ready):
515
+ """Kind of this decode round; resets the single-request tree plan state for chain rounds."""
516
+
517
+ import fc1_plan
518
+ decided = getattr(w, "_ax_tree_round", None)
519
+ wants_tree = decided[1] if decided is not None and decided[0] == id(batch) else tree_round(batch)
520
+ # The builder reads this round's logits or lattice from the folded sampler.
521
+ is_tree = wants_tree and draft_features_ready
522
+ if is_tree and getattr(w, "_ax_draft_width", None) is not None:
523
+ raise RuntimeError("ax-engine: tree round after drafting at a chain width") # begin_round disagreed
524
+ if is_tree:
525
+ current_runtime().kind = "tree"
526
+ current_runtime().stats["tree"] += 1
527
+ else:
528
+ current_runtime().kind = "chain"
529
+ current_runtime().stats["chain"] += 1
530
+ current_runtime().host_kv_len = None
531
+ _set_topk(w, current_runtime().kind == "chain")
532
+ if _CHECK:
533
+ _check_alloc(batch)
534
+ current_runtime().round_index += 1
535
+ if current_runtime().round_index % 1000 == 0:
536
+ print(f"[ax_chain] round {current_runtime().round_index} {current_runtime().stats}", flush=True)
537
+ return current_runtime().kind
538
+
539
+
540
+ def _swap_accept_buffers(w, wid):
541
+ """Keep one set of stock accept/bonus buffers per width (they are sized by block_size)."""
542
+ names = ("_accept_bonus_buffer_cap", "_accept_len_buf", "_commit_lens_bufs", "_bonus_id_bufs",
543
+ "_out_tokens_bufs", "_new_seq_lens_bufs", "_accept_bonus_buffer_slot")
544
+ cur = getattr(w, "_ax_accept_width", None)
545
+ if cur == wid:
546
+ return
547
+ store = w.__dict__.setdefault("_ax_accept_sets", {})
548
+ if cur is not None:
549
+ store[cur] = {n: getattr(w, n, None) for n in names}
550
+ saved = store.get(wid)
551
+ if saved is not None:
552
+ for n, v in saved.items():
553
+ setattr(w, n, v)
554
+ else:
555
+ w._accept_bonus_buffer_cap = 0 # _ensure_accept_bonus_buffers allocates at this width
556
+ w._accept_bonus_buffer_slot = 0
557
+ w._ax_accept_width = wid
558
+
559
+
560
+ def narrow(w, batch, draft_tokens, positions, cache_loc_2d, prefix_lens, kv_loc_plan, block):
561
+ """Cut the round to its chain width. Returns the stock locals the verify uses next."""
562
+ bs = draft_tokens.shape[0]
563
+ wid = w._ax_round_width or width(block) # decided in begin_round when the round drafted at its width
564
+ current_runtime().stats["chain_width"][wid] = current_runtime().stats["chain_width"].get(wid, 0) + 1
565
+ current_runtime().round_info["width"] = wid
566
+ _swap_accept_buffers(w, wid)
567
+ _use_chain_runner(w, wid)
568
+ if _CHAIN_ATTN and current_runtime().round_info["backend"] != _CHAIN_ATTN:
569
+ current_runtime().stats["chain_on_main"] = current_runtime().stats.get("chain_on_main", 0) + 1
570
+ raise RuntimeError(f"ax-engine: chain round of width {wid} would run on the main backend, not {_CHAIN_ATTN}")
571
+ if wid == block:
572
+ w.block_size = block
573
+ return draft_tokens, positions, cache_loc_2d.reshape(-1), cache_loc_2d, kv_loc_plan, block
574
+ tokens = draft_tokens[:, :wid].contiguous()
575
+ if w._selector_sample is not None:
576
+ candidate_ids, proposal_probs = w._selector_sample
577
+ # The first token is the anchor. Keep the original proposal law for each retained token.
578
+ w._selector_sample = (candidate_ids[:, :wid - 1], proposal_probs[:, :wid - 1])
579
+ pos = positions.view(bs, block)[:, :wid].contiguous().reshape(-1)
580
+ loc2d = cache_loc_2d[:, :wid].contiguous()
581
+ loc = loc2d.reshape(-1)
582
+ draft_input = batch.spec_info
583
+ plan = w.target_worker.model_runner.kv_index_translator.plan(
584
+ req_pool_indices=batch.req_pool_indices,
585
+ seq_lens=prefix_lens,
586
+ seq_lens_cpu=batch.seq_lens_cpu if batch.seq_lens_cpu is not None else draft_input.nxt_kv_lens_cpu,
587
+ write_virtual=loc,
588
+ read_extent=wid,
589
+ )
590
+ w.block_size = wid
591
+ return tokens, pos, loc, loc2d, plan, wid
592
+
593
+
594
+ def record_accept(w, batch, candidates, accept_len):
595
+ """After the stock accept of a chain round: token before the next anchor, per request; oracle dump."""
596
+ if current_runtime().kind != "chain":
597
+ return
598
+ before = candidates.gather(1, accept_len.to(torch.int64)[:, None])[:, 0]
599
+ for i, req in enumerate(batch.reqs):
600
+ req._ax_pre1 = before[i:i + 1]
601
+ if _DUMP:
602
+ _dump(batch, "chain", candidates, accept_len)
603
+
604
+
605
+ def record_tree(batch, pre1, tree_tokens, accept_index, accept_lens):
606
+ """After a tree round: same bookkeeping as record_accept."""
607
+ batch.reqs[0]._ax_pre1 = pre1
608
+ if current_runtime().round_info["backend"] is None:
609
+ current_runtime().round_info["backend"] = current_runtime().main_backend
610
+ if _DUMP:
611
+ path = tree_tokens.gather(1, accept_index.clamp(min=0).long())
612
+ _dump(batch, "tree", path, accept_lens.to(torch.int64) - 1, nodes=tree_tokens)
613
+
614
+
615
+ def pre1_for(req):
616
+ """Token before the anchor for the tree builder: kept from the request's last round of either
617
+ kind; on its first decode round the last prompt token."""
618
+ return getattr(req, "_ax_pre1", None)
619
+
620
+
621
+ def _dump(batch, kind, tokens, accept_len, nodes=None):
622
+ toks = tokens.tolist()
623
+ acc = accept_len.tolist()
624
+ with open(_DUMP, "a") as f:
625
+ for i, req in enumerate(batch.reqs):
626
+ row = {"round": current_runtime().round_index, "rid": req.rid, "kind": kind, "bs": len(batch.reqs),
627
+ "tokens": toks[i], "accept": int(acc[i]),
628
+ "backend": current_runtime().round_info["backend"], "graph": current_runtime().round_info["graph"]}
629
+ if nodes is not None:
630
+ row["nodes"] = nodes.tolist()[i]
631
+ f.write(json.dumps(row) + "\n")
632
+
633
+
634
+ import atexit # noqa: E402
635
+
636
+
637
+ @atexit.register
638
+ def _report():
639
+ for stats in runtime_statistics():
640
+ print(f"[ax_chain] rounds {stats}", flush=True)