stackgrad 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stackgrad/serve/api.py ADDED
@@ -0,0 +1,434 @@
1
+ import ast
2
+ import asyncio
3
+ import json
4
+ import operator
5
+ import sys
6
+ import threading
7
+ import time
8
+ import uuid
9
+ from functools import lru_cache
10
+ from pathlib import Path
11
+ from typing import Annotated, Literal
12
+
13
+ import numpy as np
14
+ from fastapi import FastAPI, HTTPException
15
+ from fastapi.responses import StreamingResponse
16
+ from fastapi.staticfiles import StaticFiles
17
+ from pydantic import BaseModel, Field
18
+
19
+ from stackgrad import Tensor, TransformerLM, no_grad
20
+ from stackgrad.tokenizer import Tokenizer
21
+
22
+ ROOT = Path(__file__).resolve().parents[2]
23
+ RUNS = ROOT / "runs"
24
+ IDLE_S = 120
25
+ MAX_PROMPT_CHARS = 64_000 # past what a 4k-token context holds
26
+ EOT = "<|endoftext|>"
27
+ app = FastAPI(title="stackgrad")
28
+ # one model call at a time: the endpoints and the engine thread share the models and device memory
29
+ _model_lock = threading.Lock()
30
+
31
+
32
+ def _run_dir(name):
33
+ path = (RUNS / name).resolve()
34
+ if path.parent != RUNS.resolve() or not (path / "metrics.jsonl").exists():
35
+ raise HTTPException(404, f"no run named {name!r}")
36
+ return path
37
+
38
+
39
+ def _parse(line):
40
+ # a diverged run logs NaN, which python's json writes but strict JSON rejects: it becomes null
41
+ return json.loads(line, parse_constant=lambda _: None)
42
+
43
+
44
+ def _read_jsonl(path):
45
+ return [_parse(line) for line in path.open()] if path.exists() else []
46
+
47
+
48
+ @app.get("/api/runs")
49
+ def runs():
50
+ out = []
51
+ for d in sorted(RUNS.glob("*/metrics.jsonl")):
52
+ rows = _read_jsonl(d)
53
+ out.append({"name": d.parent.name, "config": json.loads((d.parent / "config.json").read_text()),
54
+ "steps": len(rows), "last": rows[-1] if rows else None,
55
+ "has_checkpoint": (d.parent / "ckpt.npz").exists()})
56
+ return out
57
+
58
+
59
+ @app.get("/api/runs/{name}/samples")
60
+ def samples(name: str):
61
+ return _read_jsonl(_run_dir(name) / "samples.jsonl")
62
+
63
+
64
+ @app.get("/api/runs/{name}/stream")
65
+ async def stream(name: str, offset: int = 0):
66
+ """Server-sent events: every metrics line from byte `offset` on, as the training script appends them."""
67
+ path = _run_dir(name) / "metrics.jsonl"
68
+
69
+ async def lines():
70
+ pos = offset
71
+ while True:
72
+ with path.open() as f:
73
+ f.seek(pos)
74
+ chunk = f.read()
75
+ # a partial line waits for its newline
76
+ end = chunk.rfind("\n") + 1
77
+ for line in chunk[:end].splitlines():
78
+ yield f"data: {json.dumps(_parse(line))}\n\n"
79
+ pos += len(chunk[:end].encode())
80
+ # end idle streams, or finished runs hold the browser's 6 connections per host (GRPO writes every ~47 s)
81
+ if time.time() - path.stat().st_mtime > IDLE_S:
82
+ yield "event: end\ndata: {}\n\n"
83
+ return
84
+ await asyncio.sleep(1.0)
85
+
86
+ return StreamingResponse(lines(), media_type="text/event-stream")
87
+
88
+
89
+ BINOPS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv,
90
+ ast.Pow: operator.pow, ast.MatMult: operator.matmul}
91
+ FUNCS = {"exp": Tensor.exp, "log": Tensor.log, "relu": Tensor.relu, "sigmoid": Tensor.sigmoid,
92
+ "tanh": Tensor.tanh, "sum": Tensor.sum, "mean": Tensor.mean}
93
+
94
+
95
+ def _eval(node, env):
96
+ # user text: walk an allowlist of AST nodes, never eval it
97
+ if isinstance(node, ast.BinOp) and type(node.op) in BINOPS:
98
+ left, right = _eval(node.left, env), _eval(node.right, env)
99
+ if isinstance(node.op, ast.Pow) and not isinstance(left, Tensor):
100
+ left = Tensor(left) # 2 ** x: a constant base needs the tensor-exponent path
101
+ return BINOPS[type(node.op)](left, right)
102
+ if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
103
+ return -_eval(node.operand, env)
104
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in FUNCS \
105
+ and len(node.args) == 1 and not node.keywords:
106
+ arg = _eval(node.args[0], env)
107
+ return FUNCS[node.func.id](arg if isinstance(arg, Tensor) else Tensor(arg))
108
+ if isinstance(node, ast.Name) and node.id in env:
109
+ return env[node.id]
110
+ if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)) and not isinstance(node.value, bool):
111
+ return float(node.value) # stays a float so x ** 2 is one pow node, not exp(2 log x)
112
+ raise HTTPException(400, f"unsupported: {ast.unparse(node)}")
113
+
114
+
115
+ class GraphRequest(BaseModel):
116
+ expr: str = Field(max_length=300)
117
+ variables: dict[str, float | Annotated[list[float], Field(max_length=64)]] = Field(max_length=8)
118
+
119
+
120
+ def _small(a):
121
+ if a is None or a.size > 16:
122
+ return None
123
+ a = np.asarray(a, dtype=float)
124
+ return np.where(np.isfinite(a), np.round(a, 6), None).tolist()
125
+
126
+
127
+ @app.post("/api/graph")
128
+ def graph(req: GraphRequest):
129
+ try:
130
+ with _model_lock:
131
+ env = {k: Tensor(v, requires_grad=True) for k, v in req.variables.items() if k.isidentifier()}
132
+ out = _eval(ast.parse(req.expr, mode="eval").body, env)
133
+ if not isinstance(out, Tensor):
134
+ raise TypeError("the expression must use at least one variable")
135
+ root = out if out.data.size == 1 else out.sum()
136
+ root.backward(keep_grads=True)
137
+ except (SyntaxError, ValueError, TypeError, ArithmeticError, AttributeError) as e: # AttributeError: x @ 2
138
+ raise HTTPException(400, str(e)) from e
139
+
140
+ names = {id(t): k for k, t in env.items()}
141
+ order = _walk(root)
142
+ ids = {id(t): i for i, t in enumerate(order)}
143
+ nodes, edges = [], []
144
+ for t in order:
145
+ label = names.get(id(t)) or (t._op if t._ctx else "const")
146
+ nodes.append({"id": ids[id(t)], "label": label, "op": t._op, "shape": list(t.shape),
147
+ "value": _small(t.data), "grad": _small(t.grad), "leaf": t._ctx is None,
148
+ "requires_grad": t.requires_grad})
149
+ for p in t._ctx.parents if t._ctx else []:
150
+ edges.append({"source": ids[id(p)], "target": ids[id(t)]})
151
+ return {"nodes": nodes, "edges": edges, "root": ids[id(root)]}
152
+
153
+
154
+ def _walk(root):
155
+ seen, order, stack = set(), [], [root]
156
+ while stack:
157
+ t = stack.pop()
158
+ if id(t) not in seen:
159
+ seen.add(id(t))
160
+ order.append(t)
161
+ stack.extend(t._ctx.parents if t._ctx else [])
162
+ return order
163
+
164
+
165
+ @lru_cache(maxsize=1)
166
+ def _tokenizer():
167
+ bpe = ROOT / "data" / "tinystories_bpe"
168
+ if (bpe / "vocab.json").exists():
169
+ return "tinystories-10k", Tokenizer.from_files(bpe / "vocab.json", bpe / "merges.txt", [EOT])
170
+ from huggingface_hub import hf_hub_download
171
+ return "gpt2", Tokenizer.from_files(hf_hub_download("openai-community/gpt2", "vocab.json"),
172
+ hf_hub_download("openai-community/gpt2", "merges.txt"), [EOT])
173
+
174
+
175
+ class TextRequest(BaseModel):
176
+ text: str = Field(max_length=2000)
177
+
178
+
179
+ @app.post("/api/bpe")
180
+ def bpe(req: TextRequest):
181
+ name, tok = _tokenizer()
182
+ ids = tok.encode(req.text)
183
+ return {"tokenizer": name, "ids": ids,
184
+ "tokens": [tok.vocab[i].decode("utf-8", errors="replace") for i in ids],
185
+ "trace": tok.trace(req.text)}
186
+
187
+
188
+ @app.post("/api/gradcheck")
189
+ def gradcheck():
190
+ sys.path.insert(0, str(ROOT))
191
+ import test_grads
192
+ np.random.seed(0)
193
+ results = []
194
+ with _model_lock:
195
+ for name, fn in test_grads.CHECKS:
196
+ try:
197
+ fn()
198
+ results.append({"name": name, "ok": True})
199
+ except Exception as e: # noqa: BLE001 - a crashing check is a failed check
200
+ results.append({"name": name, "ok": False, "error": repr(e)[:300]})
201
+ return results
202
+
203
+
204
+ @lru_cache(maxsize=2)
205
+ def _model(name, mtime):
206
+ cfg = json.loads((RUNS / name / "config.json").read_text())
207
+ model = TransformerLM(cfg["vocab_size"], cfg["context"], cfg["d_model"], cfg["layers"], cfg["heads"], cfg["d_ff"],
208
+ norm=cfg.get("norm", "pre"), rope=not cfg.get("no_rope", False), ffn=cfg.get("ffn", "swiglu"))
209
+ return model.load(RUNS / name / "ckpt.npz").eval()
210
+
211
+
212
+ class GenerateRequest(BaseModel):
213
+ run: str
214
+ prompt: str = Field(max_length=1000)
215
+ max_tokens: int = Field(100, ge=1, le=512)
216
+ temperature: float = Field(0.8, ge=0.0, le=2.0)
217
+ top_p: float = Field(0.95, gt=0.0, le=1.0)
218
+
219
+
220
+ @app.post("/api/generate")
221
+ def generate(req: GenerateRequest):
222
+ ckpt = _run_dir(req.run) / "ckpt.npz"
223
+ if not ckpt.exists():
224
+ raise HTTPException(404, f"run {req.run!r} has no checkpoint yet")
225
+ model = _model(req.run, ckpt.stat().st_mtime)
226
+ _, tok = _tokenizer()
227
+ prompt = tok.encode(req.prompt) or [tok.ids[EOT.encode()]]
228
+ max_new = min(req.max_tokens, model.context_length - len(prompt))
229
+ if max_new < 1:
230
+ raise HTTPException(400, "prompt fills the whole context")
231
+ with _model_lock:
232
+ ids = model.generate(np.array([prompt]), max_new, req.temperature, top_p=req.top_p, eos_id=tok.ids[EOT.encode()])
233
+ return {"text": tok.decode(ids[0, len(prompt):].tolist()), "prompt_tokens": len(prompt),
234
+ "new_tokens": int(ids.shape[1] - len(prompt))}
235
+
236
+
237
+ QWEN = ROOT / "data" / "Qwen3-0.6B"
238
+ _chat = None
239
+ _chat_lock = threading.Lock()
240
+
241
+
242
+ def _qwen():
243
+ global _chat
244
+ with _chat_lock:
245
+ if _chat is None:
246
+ if not QWEN.exists():
247
+ raise HTTPException(503, "Qwen3-0.6B is not downloaded into data/Qwen3-0.6B")
248
+ from stackgrad.qwen3 import load_qwen3, load_tokenizer
249
+ from stackgrad.serve.engine import Engine
250
+ from stackgrad.serve.kv_cache import BlockManager
251
+ from stackgrad.serve.runner import StackgradRunner
252
+ from stackgrad.serve.scheduler import Scheduler
253
+ model = load_qwen3(QWEN).to("metal", "bfloat16")
254
+ _chat = load_tokenizer(QWEN), Engine(Scheduler(StackgradRunner(model), BlockManager(512, 16)), _model_lock)
255
+ return _chat
256
+
257
+
258
+ class Message(BaseModel):
259
+ role: Literal["system", "user", "assistant", "tool"]
260
+ content: str = Field(max_length=16000) # ~4k tokens: three of SQuAD's longest paragraphs and a question
261
+
262
+
263
+ class ChatRequest(BaseModel):
264
+ model: str = "qwen3-0.6b"
265
+ messages: list[Message] = Field(min_length=1, max_length=50)
266
+ max_tokens: int = Field(default=512, ge=1, le=2048)
267
+ temperature: float = Field(default=0.7, ge=0, le=2)
268
+ top_p: float = Field(default=0.95, gt=0, le=1)
269
+ stream: bool = False
270
+ # tool calls come back as <tool_call> blocks inside the message text, as Qwen3 writes them
271
+ tools: list[dict] | None = Field(default=None, max_length=16)
272
+
273
+
274
+ def _replies(tok, tokens, eos):
275
+ ids, finish = [], "length"
276
+ for t in iter(tokens.get, None):
277
+ if isinstance(t, Exception):
278
+ raise t
279
+ if t == eos:
280
+ finish = "stop"
281
+ continue
282
+ ids.append(t)
283
+ text = tok.decode(ids)
284
+ if not text.endswith("\ufffd"): # a character split across tokens waits for its other bytes
285
+ yield text, None, len(ids)
286
+ yield tok.decode(ids), finish, len(ids)
287
+
288
+
289
+ @app.post("/v1/chat/completions")
290
+ def chat_completions(req: ChatRequest):
291
+ from stackgrad.qwen3 import chat_prompt
292
+
293
+ tok, engine = _qwen()
294
+ text = chat_prompt([m.model_dump() for m in req.messages], tools=req.tools)
295
+ if len(text) > MAX_PROMPT_CHARS: # before encoding: BPE on 16k characters of one word takes seconds
296
+ raise HTTPException(400, f"the prompt has {len(text):,} characters; the limit is {MAX_PROMPT_CHARS:,}")
297
+ prompt = tok.encode(text)
298
+ eos = tok.ids[b"<|im_end|>"]
299
+ try:
300
+ tokens = engine.submit(prompt, req.max_tokens, req.temperature, req.top_p, eos)
301
+ except ValueError as e:
302
+ raise HTTPException(400, str(e)) from e
303
+ head = {"id": f"chatcmpl-{uuid.uuid4().hex[:12]}", "created": int(time.time()), "model": req.model}
304
+
305
+ def usage(n):
306
+ return {"prompt_tokens": len(prompt), "completion_tokens": n, "total_tokens": len(prompt) + n}
307
+
308
+ if not req.stream:
309
+ text, finish, n = list(_replies(tok, tokens, eos))[-1]
310
+ return head | {"object": "chat.completion", "usage": usage(n),
311
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": finish}]}
312
+
313
+ def events():
314
+ # shortcut: a disconnected client's request still generates to max_tokens; cancel it in the engine if that costs real time
315
+ def chunk(delta, finish=None, **extra):
316
+ body = head | {"object": "chat.completion.chunk", "choices": [{"index": 0, "delta": delta, "finish_reason": finish}]} | extra
317
+ return f"data: {json.dumps(body)}\n\n"
318
+
319
+ yield chunk({"role": "assistant"})
320
+ sent = ""
321
+ try:
322
+ for text, finish, n in _replies(tok, tokens, eos):
323
+ if text != sent:
324
+ yield chunk({"content": text[len(sent):]})
325
+ sent = text
326
+ if finish:
327
+ yield chunk({}, finish, usage=usage(n))
328
+ except Exception as e: # noqa: BLE001 - the stream has started, so the error travels in it
329
+ yield f"data: {json.dumps({'error': {'message': str(e)}})}\n\n"
330
+ yield "data: [DONE]\n\n"
331
+
332
+ return StreamingResponse(events(), media_type="text/event-stream")
333
+
334
+
335
+ class AttentionRequest(BaseModel):
336
+ run: str = "tinystories"
337
+ text: str = Field(max_length=400)
338
+
339
+
340
+ @app.post("/api/attention")
341
+ def attention_patterns(req: AttentionRequest):
342
+ """Every head's attention weights for a text: softmax(q k^T / sqrt(d)) with the causal mask, per layer."""
343
+ ckpt = _run_dir(req.run) / "ckpt.npz"
344
+ if not ckpt.exists():
345
+ raise HTTPException(404, f"run {req.run!r} has no checkpoint yet")
346
+ model = _model(req.run, ckpt.stat().st_mtime)
347
+ _, tok = _tokenizer()
348
+ ids = tok.encode(req.text)[:40] or [tok.ids[EOT.encode()]]
349
+ blocks = [b.attn for b in model.blocks]
350
+ with _model_lock, no_grad():
351
+ for a in blocks:
352
+ a.record = True
353
+ try:
354
+ model(np.array([ids]))
355
+ recorded = [a.recorded for a in blocks]
356
+ finally: # left on, every later generate would copy each layer's queries and keys
357
+ for a in blocks:
358
+ a.record, a.recorded = False, None
359
+ layers = []
360
+ for q, k in recorded: # (1, H, T, D), (1, Hkv, T, D)
361
+ k = np.repeat(k, q.shape[1] // k.shape[1], axis=1)
362
+ s = q[0] @ k[0].swapaxes(-1, -2) / np.sqrt(q.shape[-1]) + np.triu(np.full((len(ids),) * 2, -np.inf), k=1)
363
+ p = np.exp(s - s.max(-1, keepdims=True))
364
+ layers.append(np.round(p / p.sum(-1, keepdims=True), 3).tolist())
365
+ return {"tokens": [tok.decode([i]) for i in ids], "layers": layers}
366
+
367
+
368
+ class AskRequest(BaseModel):
369
+ question: str = Field(min_length=1, max_length=1000)
370
+
371
+
372
+ def _complete(messages, max_tokens, tools=None):
373
+ from stackgrad.qwen3 import chat_prompt
374
+
375
+ tok, engine = _qwen()
376
+ prompt = tok.encode(chat_prompt(messages, tools=tools))
377
+ eos = tok.ids[b"<|im_end|>"]
378
+ start = time.perf_counter()
379
+ try:
380
+ tokens = engine.submit(prompt, max_tokens, 0.0, 1.0, eos)
381
+ except ValueError as e:
382
+ raise HTTPException(400, str(e)) from e
383
+ text, _, n = list(_replies(tok, tokens, eos))[-1]
384
+ return text, len(prompt), n, time.perf_counter() - start
385
+
386
+
387
+ @lru_cache(maxsize=1)
388
+ def _retriever():
389
+ # apps sit on top of stackgrad: import them here, not at module top
390
+ from apps.rag import EMBEDDER, INDEX, corpus
391
+ from stackgrad.qwen3 import load_qwen3, load_tokenizer
392
+
393
+ if not INDEX.exists():
394
+ raise HTTPException(503, "no index yet: run uv run python apps/rag.py once to build it")
395
+ return load_tokenizer(EMBEDDER), load_qwen3(EMBEDDER).to("metal"), corpus()[0], np.load(INDEX)
396
+
397
+
398
+ @app.post("/api/rag")
399
+ def rag(req: AskRequest):
400
+ """Embed the question, take the three nearest SQuAD paragraphs, and answer from them."""
401
+ from apps.rag import INSTRUCT, embed, rag_prompt
402
+
403
+ with _model_lock:
404
+ tok, model, paragraphs, index = _retriever()
405
+ start = time.perf_counter()
406
+ query = embed(model, tok, [INSTRUCT + req.question], "metal")[0]
407
+ embed_ms = (time.perf_counter() - start) * 1000
408
+ start = time.perf_counter()
409
+ scores = index @ query
410
+ top = np.argsort(-scores)[:3]
411
+ search_ms = (time.perf_counter() - start) * 1000
412
+ answer, _, _, seconds = _complete([{"role": "user", "content": rag_prompt(req.question, [paragraphs[i][1] for i in top])}], 32)
413
+ return {"passages": [{"title": paragraphs[i][0], "text": paragraphs[i][1], "score": round(float(scores[i]), 4)} for i in top],
414
+ "answer": answer, "embed_ms": round(embed_ms, 1), "search_ms": round(search_ms, 2), "answer_s": round(seconds, 2)}
415
+
416
+
417
+ @app.post("/api/agent")
418
+ def agent(req: AskRequest):
419
+ """apps/agent.py's loop on this server: the model calls the calculator until it answers without a call."""
420
+ from apps.agent import TOOLS, solve
421
+ from bench.gsm8k import final_number
422
+
423
+ def ask(messages):
424
+ return _complete(messages, 320, tools=TOOLS)
425
+ reply, trace = solve(ask, req.question)
426
+ return {"reply": reply, "answer": final_number(reply), "trace": trace}
427
+
428
+
429
+ @app.get("/api/bench")
430
+ def bench():
431
+ return {p.stem: json.loads(p.read_text()) for p in sorted((ROOT / "bench" / "results").glob("*.json"))}
432
+
433
+
434
+ app.mount("/results", StaticFiles(directory=ROOT / "bench" / "results", check_dir=False), name="results")
@@ -0,0 +1,53 @@
1
+ import itertools
2
+ import queue
3
+ import threading
4
+
5
+ from .scheduler import Request
6
+
7
+
8
+ class Engine:
9
+ """A background thread steps the scheduler under `lock`; each request's tokens stream back through a queue."""
10
+ def __init__(self, scheduler, lock):
11
+ self.scheduler, self.lock = scheduler, lock
12
+ self.inbox, self.outputs, self.ids = queue.Queue(), {}, itertools.count()
13
+ threading.Thread(target=self._loop, daemon=True, name="engine").start()
14
+
15
+ def submit(self, prompt, max_tokens, temperature=0.7, top_p=0.95, eos_id=None):
16
+ """Queue a request. Returns a queue that receives each new token, then None (or an exception if a step failed)."""
17
+ req = Request(next(self.ids), list(prompt), max_tokens, temperature, top_p, eos_id)
18
+ if not self.scheduler.fits(req):
19
+ raise ValueError(f"{len(prompt)} prompt tokens plus {max_tokens} new ones do not fit")
20
+ out = self.outputs[req.id] = queue.Queue()
21
+ self.inbox.put(req)
22
+ return out
23
+
24
+ def _loop(self):
25
+ s = self.scheduler
26
+ while True:
27
+ s.add(self.inbox.get())
28
+ while s.has_work():
29
+ while not self.inbox.empty():
30
+ s.add(self.inbox.get_nowait())
31
+ try:
32
+ with self.lock:
33
+ emitted = s.step()
34
+ except Exception as e: # noqa: BLE001 - every caller must hear about it, or it waits forever
35
+ # self.outputs, not s.running: it also holds a request popped from waiting whose prefill raised
36
+ for rid in list(self.outputs): # snapshot: submit() may add one from another thread
37
+ s.blocks.release(rid)
38
+ s.runner.release(rid)
39
+ self.outputs.pop(rid).put(e)
40
+ s.waiting.clear()
41
+ s.running.clear()
42
+ while not self.inbox.empty(): # submitted during the failed step: tell any caller still listening
43
+ out = self.outputs.pop(self.inbox.get_nowait().id, None)
44
+ if out is not None:
45
+ out.put(e)
46
+ continue
47
+ for req, token in emitted:
48
+ out = self.outputs.get(req.id)
49
+ if out is None: # submitted while a step failed: its caller already has the error
50
+ continue
51
+ out.put(token)
52
+ if req.done:
53
+ self.outputs.pop(req.id).put(None)
@@ -0,0 +1,35 @@
1
+ class BlockManager:
2
+ """Paged KV-cache bookkeeping: which fixed-size blocks of cache memory each sequence owns."""
3
+ def __init__(self, num_blocks, block_size=16):
4
+ self.block_size = block_size
5
+ self.num_blocks = num_blocks
6
+ self.free = list(range(num_blocks - 1, -1, -1)) # pop() hands out block 0 first
7
+ self.tables = {} # seq_id -> [block ids]
8
+ self.lengths = {} # seq_id -> tokens stored
9
+
10
+ def blocks_needed(self, seq_id, n_new):
11
+ """How many more blocks `seq_id` needs to store `n_new` more tokens."""
12
+ have = len(self.tables.get(seq_id, ()))
13
+ need = -(-(self.lengths.get(seq_id, 0) + n_new) // self.block_size) # ceil
14
+ return max(0, need - have)
15
+
16
+ def can_append(self, seq_id, n_new):
17
+ return self.blocks_needed(seq_id, n_new) <= len(self.free)
18
+
19
+ def append(self, seq_id, n_new):
20
+ """Record `n_new` more tokens for `seq_id`, taking blocks as needed. Returns its block table."""
21
+ n_blocks = self.blocks_needed(seq_id, n_new)
22
+ if n_blocks > len(self.free):
23
+ raise MemoryError(f"sequence {seq_id} needs {n_blocks} blocks, {len(self.free)} free")
24
+ table = self.tables.setdefault(seq_id, [])
25
+ table.extend(self.free.pop() for _ in range(n_blocks))
26
+ self.lengths[seq_id] = self.lengths.get(seq_id, 0) + n_new
27
+ return table
28
+
29
+ def release(self, seq_id):
30
+ """Return every block of `seq_id` to the pool."""
31
+ self.free.extend(reversed(self.tables.pop(seq_id, [])))
32
+ self.lengths.pop(seq_id, None)
33
+
34
+ def used(self):
35
+ return self.num_blocks - len(self.free)
@@ -0,0 +1,63 @@
1
+ import numpy as np
2
+
3
+ from .. import backend
4
+ from ..attention import KVCache
5
+ from ..tensor import Tensor, no_grad
6
+
7
+
8
+ class StackgradRunner:
9
+ """The scheduler's view of the model: prefill one sequence, decode the batch over left-padded KV caches."""
10
+ def __init__(self, model):
11
+ self.model = model
12
+ self.context_length = model.context_length
13
+ self.caches = {} # seq_id -> one KVCache per layer, for sequences outside the batch
14
+ self.batch, self.batched, self.pad = (), None, None # the batch's seq_ids, its padded caches, each row's padding
15
+
16
+ def prefill(self, seq_id, tokens):
17
+ self.caches[seq_id] = [KVCache() for _ in self.model.blocks]
18
+ with no_grad():
19
+ h = self.model.hidden(np.array([tokens]), self.caches[seq_id])
20
+ return self.model.project(h[:, -1])[0].numpy() # last position only: the other logits go unread
21
+
22
+ def decode(self, seq_ids, tokens):
23
+ # shortcut: a membership change copies every live cache twice (split, pad); paged attention over block tables avoids it
24
+ if tuple(seq_ids) != self.batch:
25
+ self._split()
26
+ self._join(seq_ids)
27
+ with no_grad():
28
+ logits = self.model(np.array(tokens)[:, None], self.batched, self.pad if self.pad.any() else None)
29
+ return logits[:, -1].numpy()
30
+
31
+ def release(self, seq_id):
32
+ if seq_id in self.batch:
33
+ self._split()
34
+ self.caches.pop(seq_id, None)
35
+
36
+ def _join(self, seq_ids):
37
+ rows = [self.caches.pop(s) for s in seq_ids]
38
+ lengths = np.array([len(r[0]) for r in rows])
39
+ self.pad = lengths.max() - lengths
40
+ device = rows[0][0].k.device
41
+ xp = backend.get(device)
42
+
43
+ def padded(layer, name):
44
+ parts = [xp.pad(getattr(r[layer], name).data, ((0, 0), (0, 0), (int(p), 0), (0, 0))) for r, p in zip(rows, self.pad)]
45
+ return Tensor(xp.concatenate(parts), device=device)
46
+
47
+ self.batched = []
48
+ for i in range(len(self.model.blocks)):
49
+ cache = KVCache()
50
+ cache.k, cache.v = padded(i, "k"), padded(i, "v")
51
+ self.batched.append(cache)
52
+ self.batch = tuple(seq_ids)
53
+
54
+ def _split(self):
55
+ for b, s in enumerate(self.batch):
56
+ p = int(self.pad[b])
57
+ self.caches[s] = []
58
+ for cache in self.batched:
59
+ own = KVCache()
60
+ own.k = Tensor(cache.k.data[b:b + 1, :, p:], device=cache.k.device)
61
+ own.v = Tensor(cache.v.data[b:b + 1, :, p:], device=cache.v.device)
62
+ self.caches[s].append(own)
63
+ self.batch, self.batched, self.pad = (), None, None
@@ -0,0 +1,88 @@
1
+ from collections import deque
2
+ from dataclasses import dataclass, field
3
+
4
+ from ..sampling import sample
5
+
6
+
7
+ @dataclass(eq=False) # requests compare by identity: two can hold the same tokens
8
+ class Request:
9
+ id: int
10
+ prompt: list
11
+ max_tokens: int
12
+ temperature: float = 1.0
13
+ top_p: float | None = None
14
+ eos_id: int | None = None
15
+ out: list = field(default_factory=list)
16
+ done: bool = False
17
+
18
+
19
+ class Scheduler:
20
+ """Continuous batching: requests join and leave the running batch between decode steps."""
21
+ def __init__(self, runner, blocks, max_batch=8):
22
+ self.runner, self.blocks, self.max_batch = runner, blocks, max_batch
23
+ self.waiting, self.running = deque(), []
24
+ self.preemptions = 0
25
+
26
+ def fits(self, request):
27
+ """Whether the request could ever run: its prompt plus all its new tokens within the context and the cache."""
28
+ total = len(request.prompt) + request.max_tokens
29
+ return total <= self.runner.context_length and -(-total // self.blocks.block_size) <= self.blocks.num_blocks
30
+
31
+ def add(self, request):
32
+ if not self.fits(request):
33
+ raise ValueError(f"request {request.id}: {len(request.prompt) + request.max_tokens} tokens can never fit")
34
+ self.waiting.append(request)
35
+
36
+ def has_work(self):
37
+ return bool(self.waiting or self.running)
38
+
39
+ def step(self):
40
+ """Admit, prefill, decode once. Returns the (request, token) pairs produced this step."""
41
+ emitted = []
42
+ while self.waiting and len(self.running) < self.max_batch:
43
+ req = self.waiting[0]
44
+ tokens = req.prompt + req.out # a preempted request also replays what it already generated
45
+ if not self.blocks.can_append(req.id, len(tokens)):
46
+ break
47
+ self.waiting.popleft()
48
+ self.blocks.append(req.id, len(tokens))
49
+ emitted.append(self._emit(req, self.runner.prefill(req.id, tokens)[None]))
50
+ self.running.append(req)
51
+ self._retire()
52
+
53
+ # requests admitted this step already got their token from prefill
54
+ fresh = {id(r) for r, _ in emitted}
55
+ decode = [r for r in self.running if id(r) not in fresh]
56
+ for req in list(decode):
57
+ # room for the fed-back token: preempt the newest request until it fits; it recomputes on readmission
58
+ while req in self.running and not self.blocks.can_append(req.id, 1):
59
+ victim = self.running[-1]
60
+ self._preempt(victim)
61
+ if victim in decode:
62
+ decode.remove(victim)
63
+ if req in self.running:
64
+ self.blocks.append(req.id, 1)
65
+ if decode:
66
+ logits = self.runner.decode([r.id for r in decode], [r.out[-1] for r in decode])
67
+ emitted += [self._emit(r, logits[i:i + 1]) for i, r in enumerate(decode)]
68
+ self._retire()
69
+ return emitted
70
+
71
+ def _retire(self):
72
+ for req in [r for r in self.running if r.done]:
73
+ self.running.remove(req)
74
+ self.blocks.release(req.id)
75
+ self.runner.release(req.id)
76
+
77
+ def _emit(self, req, logits):
78
+ token = int(sample(logits, req.temperature, top_p=req.top_p)[0])
79
+ req.out.append(token)
80
+ req.done = token == req.eos_id or len(req.out) >= req.max_tokens
81
+ return req, token
82
+
83
+ def _preempt(self, req):
84
+ self.running.remove(req)
85
+ self.blocks.release(req.id)
86
+ self.runner.release(req.id)
87
+ self.waiting.appendleft(req)
88
+ self.preemptions += 1