stackgrad 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stackgrad/__init__.py +75 -0
- stackgrad/attention.py +86 -0
- stackgrad/backend/__init__.py +11 -0
- stackgrad/backend/cpu.py +116 -0
- stackgrad/backend/cuda.py +96 -0
- stackgrad/backend/cuda_attention.py +138 -0
- stackgrad/backend/metal.py +106 -0
- stackgrad/backend/metal_attention.py +195 -0
- stackgrad/backend/metal_quant.py +44 -0
- stackgrad/checkpoint.py +38 -0
- stackgrad/conv.py +63 -0
- stackgrad/data.py +28 -0
- stackgrad/functional.py +41 -0
- stackgrad/gpt.py +120 -0
- stackgrad/grad_check.py +55 -0
- stackgrad/layers.py +215 -0
- stackgrad/lora.py +54 -0
- stackgrad/loss.py +28 -0
- stackgrad/mlops.py +352 -0
- stackgrad/module.py +120 -0
- stackgrad/optim.py +121 -0
- stackgrad/profile.py +45 -0
- stackgrad/quant.py +66 -0
- stackgrad/qwen3.py +104 -0
- stackgrad/sampling.py +39 -0
- stackgrad/serve/api.py +434 -0
- stackgrad/serve/engine.py +53 -0
- stackgrad/serve/kv_cache.py +35 -0
- stackgrad/serve/runner.py +63 -0
- stackgrad/serve/scheduler.py +88 -0
- stackgrad/speculative.py +68 -0
- stackgrad/state.py +27 -0
- stackgrad/tensor.py +377 -0
- stackgrad/tokenizer.py +202 -0
- stackgrad-0.2.0.dist-info/METADATA +172 -0
- stackgrad-0.2.0.dist-info/RECORD +38 -0
- stackgrad-0.2.0.dist-info/WHEEL +4 -0
- stackgrad-0.2.0.dist-info/licenses/LICENSE +21 -0
stackgrad/serve/api.py
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
import ast
|
|
2
|
+
import asyncio
|
|
3
|
+
import json
|
|
4
|
+
import operator
|
|
5
|
+
import sys
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
import uuid
|
|
9
|
+
from functools import lru_cache
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Annotated, Literal
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
from fastapi import FastAPI, HTTPException
|
|
15
|
+
from fastapi.responses import StreamingResponse
|
|
16
|
+
from fastapi.staticfiles import StaticFiles
|
|
17
|
+
from pydantic import BaseModel, Field
|
|
18
|
+
|
|
19
|
+
from stackgrad import Tensor, TransformerLM, no_grad
|
|
20
|
+
from stackgrad.tokenizer import Tokenizer
|
|
21
|
+
|
|
22
|
+
ROOT = Path(__file__).resolve().parents[2]
|
|
23
|
+
RUNS = ROOT / "runs"
|
|
24
|
+
IDLE_S = 120
|
|
25
|
+
MAX_PROMPT_CHARS = 64_000 # past what a 4k-token context holds
|
|
26
|
+
EOT = "<|endoftext|>"
|
|
27
|
+
app = FastAPI(title="stackgrad")
|
|
28
|
+
# one model call at a time: the endpoints and the engine thread share the models and device memory
|
|
29
|
+
_model_lock = threading.Lock()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _run_dir(name):
|
|
33
|
+
path = (RUNS / name).resolve()
|
|
34
|
+
if path.parent != RUNS.resolve() or not (path / "metrics.jsonl").exists():
|
|
35
|
+
raise HTTPException(404, f"no run named {name!r}")
|
|
36
|
+
return path
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _parse(line):
|
|
40
|
+
# a diverged run logs NaN, which python's json writes but strict JSON rejects: it becomes null
|
|
41
|
+
return json.loads(line, parse_constant=lambda _: None)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _read_jsonl(path):
|
|
45
|
+
return [_parse(line) for line in path.open()] if path.exists() else []
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@app.get("/api/runs")
|
|
49
|
+
def runs():
|
|
50
|
+
out = []
|
|
51
|
+
for d in sorted(RUNS.glob("*/metrics.jsonl")):
|
|
52
|
+
rows = _read_jsonl(d)
|
|
53
|
+
out.append({"name": d.parent.name, "config": json.loads((d.parent / "config.json").read_text()),
|
|
54
|
+
"steps": len(rows), "last": rows[-1] if rows else None,
|
|
55
|
+
"has_checkpoint": (d.parent / "ckpt.npz").exists()})
|
|
56
|
+
return out
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@app.get("/api/runs/{name}/samples")
|
|
60
|
+
def samples(name: str):
|
|
61
|
+
return _read_jsonl(_run_dir(name) / "samples.jsonl")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@app.get("/api/runs/{name}/stream")
|
|
65
|
+
async def stream(name: str, offset: int = 0):
|
|
66
|
+
"""Server-sent events: every metrics line from byte `offset` on, as the training script appends them."""
|
|
67
|
+
path = _run_dir(name) / "metrics.jsonl"
|
|
68
|
+
|
|
69
|
+
async def lines():
|
|
70
|
+
pos = offset
|
|
71
|
+
while True:
|
|
72
|
+
with path.open() as f:
|
|
73
|
+
f.seek(pos)
|
|
74
|
+
chunk = f.read()
|
|
75
|
+
# a partial line waits for its newline
|
|
76
|
+
end = chunk.rfind("\n") + 1
|
|
77
|
+
for line in chunk[:end].splitlines():
|
|
78
|
+
yield f"data: {json.dumps(_parse(line))}\n\n"
|
|
79
|
+
pos += len(chunk[:end].encode())
|
|
80
|
+
# end idle streams, or finished runs hold the browser's 6 connections per host (GRPO writes every ~47 s)
|
|
81
|
+
if time.time() - path.stat().st_mtime > IDLE_S:
|
|
82
|
+
yield "event: end\ndata: {}\n\n"
|
|
83
|
+
return
|
|
84
|
+
await asyncio.sleep(1.0)
|
|
85
|
+
|
|
86
|
+
return StreamingResponse(lines(), media_type="text/event-stream")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
BINOPS = {ast.Add: operator.add, ast.Sub: operator.sub, ast.Mult: operator.mul, ast.Div: operator.truediv,
|
|
90
|
+
ast.Pow: operator.pow, ast.MatMult: operator.matmul}
|
|
91
|
+
FUNCS = {"exp": Tensor.exp, "log": Tensor.log, "relu": Tensor.relu, "sigmoid": Tensor.sigmoid,
|
|
92
|
+
"tanh": Tensor.tanh, "sum": Tensor.sum, "mean": Tensor.mean}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _eval(node, env):
|
|
96
|
+
# user text: walk an allowlist of AST nodes, never eval it
|
|
97
|
+
if isinstance(node, ast.BinOp) and type(node.op) in BINOPS:
|
|
98
|
+
left, right = _eval(node.left, env), _eval(node.right, env)
|
|
99
|
+
if isinstance(node.op, ast.Pow) and not isinstance(left, Tensor):
|
|
100
|
+
left = Tensor(left) # 2 ** x: a constant base needs the tensor-exponent path
|
|
101
|
+
return BINOPS[type(node.op)](left, right)
|
|
102
|
+
if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub):
|
|
103
|
+
return -_eval(node.operand, env)
|
|
104
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id in FUNCS \
|
|
105
|
+
and len(node.args) == 1 and not node.keywords:
|
|
106
|
+
arg = _eval(node.args[0], env)
|
|
107
|
+
return FUNCS[node.func.id](arg if isinstance(arg, Tensor) else Tensor(arg))
|
|
108
|
+
if isinstance(node, ast.Name) and node.id in env:
|
|
109
|
+
return env[node.id]
|
|
110
|
+
if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)) and not isinstance(node.value, bool):
|
|
111
|
+
return float(node.value) # stays a float so x ** 2 is one pow node, not exp(2 log x)
|
|
112
|
+
raise HTTPException(400, f"unsupported: {ast.unparse(node)}")
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class GraphRequest(BaseModel):
|
|
116
|
+
expr: str = Field(max_length=300)
|
|
117
|
+
variables: dict[str, float | Annotated[list[float], Field(max_length=64)]] = Field(max_length=8)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _small(a):
|
|
121
|
+
if a is None or a.size > 16:
|
|
122
|
+
return None
|
|
123
|
+
a = np.asarray(a, dtype=float)
|
|
124
|
+
return np.where(np.isfinite(a), np.round(a, 6), None).tolist()
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@app.post("/api/graph")
|
|
128
|
+
def graph(req: GraphRequest):
|
|
129
|
+
try:
|
|
130
|
+
with _model_lock:
|
|
131
|
+
env = {k: Tensor(v, requires_grad=True) for k, v in req.variables.items() if k.isidentifier()}
|
|
132
|
+
out = _eval(ast.parse(req.expr, mode="eval").body, env)
|
|
133
|
+
if not isinstance(out, Tensor):
|
|
134
|
+
raise TypeError("the expression must use at least one variable")
|
|
135
|
+
root = out if out.data.size == 1 else out.sum()
|
|
136
|
+
root.backward(keep_grads=True)
|
|
137
|
+
except (SyntaxError, ValueError, TypeError, ArithmeticError, AttributeError) as e: # AttributeError: x @ 2
|
|
138
|
+
raise HTTPException(400, str(e)) from e
|
|
139
|
+
|
|
140
|
+
names = {id(t): k for k, t in env.items()}
|
|
141
|
+
order = _walk(root)
|
|
142
|
+
ids = {id(t): i for i, t in enumerate(order)}
|
|
143
|
+
nodes, edges = [], []
|
|
144
|
+
for t in order:
|
|
145
|
+
label = names.get(id(t)) or (t._op if t._ctx else "const")
|
|
146
|
+
nodes.append({"id": ids[id(t)], "label": label, "op": t._op, "shape": list(t.shape),
|
|
147
|
+
"value": _small(t.data), "grad": _small(t.grad), "leaf": t._ctx is None,
|
|
148
|
+
"requires_grad": t.requires_grad})
|
|
149
|
+
for p in t._ctx.parents if t._ctx else []:
|
|
150
|
+
edges.append({"source": ids[id(p)], "target": ids[id(t)]})
|
|
151
|
+
return {"nodes": nodes, "edges": edges, "root": ids[id(root)]}
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _walk(root):
|
|
155
|
+
seen, order, stack = set(), [], [root]
|
|
156
|
+
while stack:
|
|
157
|
+
t = stack.pop()
|
|
158
|
+
if id(t) not in seen:
|
|
159
|
+
seen.add(id(t))
|
|
160
|
+
order.append(t)
|
|
161
|
+
stack.extend(t._ctx.parents if t._ctx else [])
|
|
162
|
+
return order
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@lru_cache(maxsize=1)
|
|
166
|
+
def _tokenizer():
|
|
167
|
+
bpe = ROOT / "data" / "tinystories_bpe"
|
|
168
|
+
if (bpe / "vocab.json").exists():
|
|
169
|
+
return "tinystories-10k", Tokenizer.from_files(bpe / "vocab.json", bpe / "merges.txt", [EOT])
|
|
170
|
+
from huggingface_hub import hf_hub_download
|
|
171
|
+
return "gpt2", Tokenizer.from_files(hf_hub_download("openai-community/gpt2", "vocab.json"),
|
|
172
|
+
hf_hub_download("openai-community/gpt2", "merges.txt"), [EOT])
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class TextRequest(BaseModel):
|
|
176
|
+
text: str = Field(max_length=2000)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@app.post("/api/bpe")
|
|
180
|
+
def bpe(req: TextRequest):
|
|
181
|
+
name, tok = _tokenizer()
|
|
182
|
+
ids = tok.encode(req.text)
|
|
183
|
+
return {"tokenizer": name, "ids": ids,
|
|
184
|
+
"tokens": [tok.vocab[i].decode("utf-8", errors="replace") for i in ids],
|
|
185
|
+
"trace": tok.trace(req.text)}
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
@app.post("/api/gradcheck")
|
|
189
|
+
def gradcheck():
|
|
190
|
+
sys.path.insert(0, str(ROOT))
|
|
191
|
+
import test_grads
|
|
192
|
+
np.random.seed(0)
|
|
193
|
+
results = []
|
|
194
|
+
with _model_lock:
|
|
195
|
+
for name, fn in test_grads.CHECKS:
|
|
196
|
+
try:
|
|
197
|
+
fn()
|
|
198
|
+
results.append({"name": name, "ok": True})
|
|
199
|
+
except Exception as e: # noqa: BLE001 - a crashing check is a failed check
|
|
200
|
+
results.append({"name": name, "ok": False, "error": repr(e)[:300]})
|
|
201
|
+
return results
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@lru_cache(maxsize=2)
|
|
205
|
+
def _model(name, mtime):
|
|
206
|
+
cfg = json.loads((RUNS / name / "config.json").read_text())
|
|
207
|
+
model = TransformerLM(cfg["vocab_size"], cfg["context"], cfg["d_model"], cfg["layers"], cfg["heads"], cfg["d_ff"],
|
|
208
|
+
norm=cfg.get("norm", "pre"), rope=not cfg.get("no_rope", False), ffn=cfg.get("ffn", "swiglu"))
|
|
209
|
+
return model.load(RUNS / name / "ckpt.npz").eval()
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class GenerateRequest(BaseModel):
|
|
213
|
+
run: str
|
|
214
|
+
prompt: str = Field(max_length=1000)
|
|
215
|
+
max_tokens: int = Field(100, ge=1, le=512)
|
|
216
|
+
temperature: float = Field(0.8, ge=0.0, le=2.0)
|
|
217
|
+
top_p: float = Field(0.95, gt=0.0, le=1.0)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
@app.post("/api/generate")
|
|
221
|
+
def generate(req: GenerateRequest):
|
|
222
|
+
ckpt = _run_dir(req.run) / "ckpt.npz"
|
|
223
|
+
if not ckpt.exists():
|
|
224
|
+
raise HTTPException(404, f"run {req.run!r} has no checkpoint yet")
|
|
225
|
+
model = _model(req.run, ckpt.stat().st_mtime)
|
|
226
|
+
_, tok = _tokenizer()
|
|
227
|
+
prompt = tok.encode(req.prompt) or [tok.ids[EOT.encode()]]
|
|
228
|
+
max_new = min(req.max_tokens, model.context_length - len(prompt))
|
|
229
|
+
if max_new < 1:
|
|
230
|
+
raise HTTPException(400, "prompt fills the whole context")
|
|
231
|
+
with _model_lock:
|
|
232
|
+
ids = model.generate(np.array([prompt]), max_new, req.temperature, top_p=req.top_p, eos_id=tok.ids[EOT.encode()])
|
|
233
|
+
return {"text": tok.decode(ids[0, len(prompt):].tolist()), "prompt_tokens": len(prompt),
|
|
234
|
+
"new_tokens": int(ids.shape[1] - len(prompt))}
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
QWEN = ROOT / "data" / "Qwen3-0.6B"
|
|
238
|
+
_chat = None
|
|
239
|
+
_chat_lock = threading.Lock()
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _qwen():
|
|
243
|
+
global _chat
|
|
244
|
+
with _chat_lock:
|
|
245
|
+
if _chat is None:
|
|
246
|
+
if not QWEN.exists():
|
|
247
|
+
raise HTTPException(503, "Qwen3-0.6B is not downloaded into data/Qwen3-0.6B")
|
|
248
|
+
from stackgrad.qwen3 import load_qwen3, load_tokenizer
|
|
249
|
+
from stackgrad.serve.engine import Engine
|
|
250
|
+
from stackgrad.serve.kv_cache import BlockManager
|
|
251
|
+
from stackgrad.serve.runner import StackgradRunner
|
|
252
|
+
from stackgrad.serve.scheduler import Scheduler
|
|
253
|
+
model = load_qwen3(QWEN).to("metal", "bfloat16")
|
|
254
|
+
_chat = load_tokenizer(QWEN), Engine(Scheduler(StackgradRunner(model), BlockManager(512, 16)), _model_lock)
|
|
255
|
+
return _chat
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class Message(BaseModel):
|
|
259
|
+
role: Literal["system", "user", "assistant", "tool"]
|
|
260
|
+
content: str = Field(max_length=16000) # ~4k tokens: three of SQuAD's longest paragraphs and a question
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class ChatRequest(BaseModel):
|
|
264
|
+
model: str = "qwen3-0.6b"
|
|
265
|
+
messages: list[Message] = Field(min_length=1, max_length=50)
|
|
266
|
+
max_tokens: int = Field(default=512, ge=1, le=2048)
|
|
267
|
+
temperature: float = Field(default=0.7, ge=0, le=2)
|
|
268
|
+
top_p: float = Field(default=0.95, gt=0, le=1)
|
|
269
|
+
stream: bool = False
|
|
270
|
+
# tool calls come back as <tool_call> blocks inside the message text, as Qwen3 writes them
|
|
271
|
+
tools: list[dict] | None = Field(default=None, max_length=16)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _replies(tok, tokens, eos):
|
|
275
|
+
ids, finish = [], "length"
|
|
276
|
+
for t in iter(tokens.get, None):
|
|
277
|
+
if isinstance(t, Exception):
|
|
278
|
+
raise t
|
|
279
|
+
if t == eos:
|
|
280
|
+
finish = "stop"
|
|
281
|
+
continue
|
|
282
|
+
ids.append(t)
|
|
283
|
+
text = tok.decode(ids)
|
|
284
|
+
if not text.endswith("\ufffd"): # a character split across tokens waits for its other bytes
|
|
285
|
+
yield text, None, len(ids)
|
|
286
|
+
yield tok.decode(ids), finish, len(ids)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
@app.post("/v1/chat/completions")
|
|
290
|
+
def chat_completions(req: ChatRequest):
|
|
291
|
+
from stackgrad.qwen3 import chat_prompt
|
|
292
|
+
|
|
293
|
+
tok, engine = _qwen()
|
|
294
|
+
text = chat_prompt([m.model_dump() for m in req.messages], tools=req.tools)
|
|
295
|
+
if len(text) > MAX_PROMPT_CHARS: # before encoding: BPE on 16k characters of one word takes seconds
|
|
296
|
+
raise HTTPException(400, f"the prompt has {len(text):,} characters; the limit is {MAX_PROMPT_CHARS:,}")
|
|
297
|
+
prompt = tok.encode(text)
|
|
298
|
+
eos = tok.ids[b"<|im_end|>"]
|
|
299
|
+
try:
|
|
300
|
+
tokens = engine.submit(prompt, req.max_tokens, req.temperature, req.top_p, eos)
|
|
301
|
+
except ValueError as e:
|
|
302
|
+
raise HTTPException(400, str(e)) from e
|
|
303
|
+
head = {"id": f"chatcmpl-{uuid.uuid4().hex[:12]}", "created": int(time.time()), "model": req.model}
|
|
304
|
+
|
|
305
|
+
def usage(n):
|
|
306
|
+
return {"prompt_tokens": len(prompt), "completion_tokens": n, "total_tokens": len(prompt) + n}
|
|
307
|
+
|
|
308
|
+
if not req.stream:
|
|
309
|
+
text, finish, n = list(_replies(tok, tokens, eos))[-1]
|
|
310
|
+
return head | {"object": "chat.completion", "usage": usage(n),
|
|
311
|
+
"choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": finish}]}
|
|
312
|
+
|
|
313
|
+
def events():
|
|
314
|
+
# shortcut: a disconnected client's request still generates to max_tokens; cancel it in the engine if that costs real time
|
|
315
|
+
def chunk(delta, finish=None, **extra):
|
|
316
|
+
body = head | {"object": "chat.completion.chunk", "choices": [{"index": 0, "delta": delta, "finish_reason": finish}]} | extra
|
|
317
|
+
return f"data: {json.dumps(body)}\n\n"
|
|
318
|
+
|
|
319
|
+
yield chunk({"role": "assistant"})
|
|
320
|
+
sent = ""
|
|
321
|
+
try:
|
|
322
|
+
for text, finish, n in _replies(tok, tokens, eos):
|
|
323
|
+
if text != sent:
|
|
324
|
+
yield chunk({"content": text[len(sent):]})
|
|
325
|
+
sent = text
|
|
326
|
+
if finish:
|
|
327
|
+
yield chunk({}, finish, usage=usage(n))
|
|
328
|
+
except Exception as e: # noqa: BLE001 - the stream has started, so the error travels in it
|
|
329
|
+
yield f"data: {json.dumps({'error': {'message': str(e)}})}\n\n"
|
|
330
|
+
yield "data: [DONE]\n\n"
|
|
331
|
+
|
|
332
|
+
return StreamingResponse(events(), media_type="text/event-stream")
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
class AttentionRequest(BaseModel):
|
|
336
|
+
run: str = "tinystories"
|
|
337
|
+
text: str = Field(max_length=400)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
@app.post("/api/attention")
|
|
341
|
+
def attention_patterns(req: AttentionRequest):
|
|
342
|
+
"""Every head's attention weights for a text: softmax(q k^T / sqrt(d)) with the causal mask, per layer."""
|
|
343
|
+
ckpt = _run_dir(req.run) / "ckpt.npz"
|
|
344
|
+
if not ckpt.exists():
|
|
345
|
+
raise HTTPException(404, f"run {req.run!r} has no checkpoint yet")
|
|
346
|
+
model = _model(req.run, ckpt.stat().st_mtime)
|
|
347
|
+
_, tok = _tokenizer()
|
|
348
|
+
ids = tok.encode(req.text)[:40] or [tok.ids[EOT.encode()]]
|
|
349
|
+
blocks = [b.attn for b in model.blocks]
|
|
350
|
+
with _model_lock, no_grad():
|
|
351
|
+
for a in blocks:
|
|
352
|
+
a.record = True
|
|
353
|
+
try:
|
|
354
|
+
model(np.array([ids]))
|
|
355
|
+
recorded = [a.recorded for a in blocks]
|
|
356
|
+
finally: # left on, every later generate would copy each layer's queries and keys
|
|
357
|
+
for a in blocks:
|
|
358
|
+
a.record, a.recorded = False, None
|
|
359
|
+
layers = []
|
|
360
|
+
for q, k in recorded: # (1, H, T, D), (1, Hkv, T, D)
|
|
361
|
+
k = np.repeat(k, q.shape[1] // k.shape[1], axis=1)
|
|
362
|
+
s = q[0] @ k[0].swapaxes(-1, -2) / np.sqrt(q.shape[-1]) + np.triu(np.full((len(ids),) * 2, -np.inf), k=1)
|
|
363
|
+
p = np.exp(s - s.max(-1, keepdims=True))
|
|
364
|
+
layers.append(np.round(p / p.sum(-1, keepdims=True), 3).tolist())
|
|
365
|
+
return {"tokens": [tok.decode([i]) for i in ids], "layers": layers}
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
class AskRequest(BaseModel):
|
|
369
|
+
question: str = Field(min_length=1, max_length=1000)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _complete(messages, max_tokens, tools=None):
|
|
373
|
+
from stackgrad.qwen3 import chat_prompt
|
|
374
|
+
|
|
375
|
+
tok, engine = _qwen()
|
|
376
|
+
prompt = tok.encode(chat_prompt(messages, tools=tools))
|
|
377
|
+
eos = tok.ids[b"<|im_end|>"]
|
|
378
|
+
start = time.perf_counter()
|
|
379
|
+
try:
|
|
380
|
+
tokens = engine.submit(prompt, max_tokens, 0.0, 1.0, eos)
|
|
381
|
+
except ValueError as e:
|
|
382
|
+
raise HTTPException(400, str(e)) from e
|
|
383
|
+
text, _, n = list(_replies(tok, tokens, eos))[-1]
|
|
384
|
+
return text, len(prompt), n, time.perf_counter() - start
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
@lru_cache(maxsize=1)
|
|
388
|
+
def _retriever():
|
|
389
|
+
# apps sit on top of stackgrad: import them here, not at module top
|
|
390
|
+
from apps.rag import EMBEDDER, INDEX, corpus
|
|
391
|
+
from stackgrad.qwen3 import load_qwen3, load_tokenizer
|
|
392
|
+
|
|
393
|
+
if not INDEX.exists():
|
|
394
|
+
raise HTTPException(503, "no index yet: run uv run python apps/rag.py once to build it")
|
|
395
|
+
return load_tokenizer(EMBEDDER), load_qwen3(EMBEDDER).to("metal"), corpus()[0], np.load(INDEX)
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
@app.post("/api/rag")
|
|
399
|
+
def rag(req: AskRequest):
|
|
400
|
+
"""Embed the question, take the three nearest SQuAD paragraphs, and answer from them."""
|
|
401
|
+
from apps.rag import INSTRUCT, embed, rag_prompt
|
|
402
|
+
|
|
403
|
+
with _model_lock:
|
|
404
|
+
tok, model, paragraphs, index = _retriever()
|
|
405
|
+
start = time.perf_counter()
|
|
406
|
+
query = embed(model, tok, [INSTRUCT + req.question], "metal")[0]
|
|
407
|
+
embed_ms = (time.perf_counter() - start) * 1000
|
|
408
|
+
start = time.perf_counter()
|
|
409
|
+
scores = index @ query
|
|
410
|
+
top = np.argsort(-scores)[:3]
|
|
411
|
+
search_ms = (time.perf_counter() - start) * 1000
|
|
412
|
+
answer, _, _, seconds = _complete([{"role": "user", "content": rag_prompt(req.question, [paragraphs[i][1] for i in top])}], 32)
|
|
413
|
+
return {"passages": [{"title": paragraphs[i][0], "text": paragraphs[i][1], "score": round(float(scores[i]), 4)} for i in top],
|
|
414
|
+
"answer": answer, "embed_ms": round(embed_ms, 1), "search_ms": round(search_ms, 2), "answer_s": round(seconds, 2)}
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
@app.post("/api/agent")
|
|
418
|
+
def agent(req: AskRequest):
|
|
419
|
+
"""apps/agent.py's loop on this server: the model calls the calculator until it answers without a call."""
|
|
420
|
+
from apps.agent import TOOLS, solve
|
|
421
|
+
from bench.gsm8k import final_number
|
|
422
|
+
|
|
423
|
+
def ask(messages):
|
|
424
|
+
return _complete(messages, 320, tools=TOOLS)
|
|
425
|
+
reply, trace = solve(ask, req.question)
|
|
426
|
+
return {"reply": reply, "answer": final_number(reply), "trace": trace}
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
@app.get("/api/bench")
|
|
430
|
+
def bench():
|
|
431
|
+
return {p.stem: json.loads(p.read_text()) for p in sorted((ROOT / "bench" / "results").glob("*.json"))}
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
app.mount("/results", StaticFiles(directory=ROOT / "bench" / "results", check_dir=False), name="results")
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
import queue
|
|
3
|
+
import threading
|
|
4
|
+
|
|
5
|
+
from .scheduler import Request
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class Engine:
|
|
9
|
+
"""A background thread steps the scheduler under `lock`; each request's tokens stream back through a queue."""
|
|
10
|
+
def __init__(self, scheduler, lock):
|
|
11
|
+
self.scheduler, self.lock = scheduler, lock
|
|
12
|
+
self.inbox, self.outputs, self.ids = queue.Queue(), {}, itertools.count()
|
|
13
|
+
threading.Thread(target=self._loop, daemon=True, name="engine").start()
|
|
14
|
+
|
|
15
|
+
def submit(self, prompt, max_tokens, temperature=0.7, top_p=0.95, eos_id=None):
|
|
16
|
+
"""Queue a request. Returns a queue that receives each new token, then None (or an exception if a step failed)."""
|
|
17
|
+
req = Request(next(self.ids), list(prompt), max_tokens, temperature, top_p, eos_id)
|
|
18
|
+
if not self.scheduler.fits(req):
|
|
19
|
+
raise ValueError(f"{len(prompt)} prompt tokens plus {max_tokens} new ones do not fit")
|
|
20
|
+
out = self.outputs[req.id] = queue.Queue()
|
|
21
|
+
self.inbox.put(req)
|
|
22
|
+
return out
|
|
23
|
+
|
|
24
|
+
def _loop(self):
|
|
25
|
+
s = self.scheduler
|
|
26
|
+
while True:
|
|
27
|
+
s.add(self.inbox.get())
|
|
28
|
+
while s.has_work():
|
|
29
|
+
while not self.inbox.empty():
|
|
30
|
+
s.add(self.inbox.get_nowait())
|
|
31
|
+
try:
|
|
32
|
+
with self.lock:
|
|
33
|
+
emitted = s.step()
|
|
34
|
+
except Exception as e: # noqa: BLE001 - every caller must hear about it, or it waits forever
|
|
35
|
+
# self.outputs, not s.running: it also holds a request popped from waiting whose prefill raised
|
|
36
|
+
for rid in list(self.outputs): # snapshot: submit() may add one from another thread
|
|
37
|
+
s.blocks.release(rid)
|
|
38
|
+
s.runner.release(rid)
|
|
39
|
+
self.outputs.pop(rid).put(e)
|
|
40
|
+
s.waiting.clear()
|
|
41
|
+
s.running.clear()
|
|
42
|
+
while not self.inbox.empty(): # submitted during the failed step: tell any caller still listening
|
|
43
|
+
out = self.outputs.pop(self.inbox.get_nowait().id, None)
|
|
44
|
+
if out is not None:
|
|
45
|
+
out.put(e)
|
|
46
|
+
continue
|
|
47
|
+
for req, token in emitted:
|
|
48
|
+
out = self.outputs.get(req.id)
|
|
49
|
+
if out is None: # submitted while a step failed: its caller already has the error
|
|
50
|
+
continue
|
|
51
|
+
out.put(token)
|
|
52
|
+
if req.done:
|
|
53
|
+
self.outputs.pop(req.id).put(None)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
class BlockManager:
|
|
2
|
+
"""Paged KV-cache bookkeeping: which fixed-size blocks of cache memory each sequence owns."""
|
|
3
|
+
def __init__(self, num_blocks, block_size=16):
|
|
4
|
+
self.block_size = block_size
|
|
5
|
+
self.num_blocks = num_blocks
|
|
6
|
+
self.free = list(range(num_blocks - 1, -1, -1)) # pop() hands out block 0 first
|
|
7
|
+
self.tables = {} # seq_id -> [block ids]
|
|
8
|
+
self.lengths = {} # seq_id -> tokens stored
|
|
9
|
+
|
|
10
|
+
def blocks_needed(self, seq_id, n_new):
|
|
11
|
+
"""How many more blocks `seq_id` needs to store `n_new` more tokens."""
|
|
12
|
+
have = len(self.tables.get(seq_id, ()))
|
|
13
|
+
need = -(-(self.lengths.get(seq_id, 0) + n_new) // self.block_size) # ceil
|
|
14
|
+
return max(0, need - have)
|
|
15
|
+
|
|
16
|
+
def can_append(self, seq_id, n_new):
|
|
17
|
+
return self.blocks_needed(seq_id, n_new) <= len(self.free)
|
|
18
|
+
|
|
19
|
+
def append(self, seq_id, n_new):
|
|
20
|
+
"""Record `n_new` more tokens for `seq_id`, taking blocks as needed. Returns its block table."""
|
|
21
|
+
n_blocks = self.blocks_needed(seq_id, n_new)
|
|
22
|
+
if n_blocks > len(self.free):
|
|
23
|
+
raise MemoryError(f"sequence {seq_id} needs {n_blocks} blocks, {len(self.free)} free")
|
|
24
|
+
table = self.tables.setdefault(seq_id, [])
|
|
25
|
+
table.extend(self.free.pop() for _ in range(n_blocks))
|
|
26
|
+
self.lengths[seq_id] = self.lengths.get(seq_id, 0) + n_new
|
|
27
|
+
return table
|
|
28
|
+
|
|
29
|
+
def release(self, seq_id):
|
|
30
|
+
"""Return every block of `seq_id` to the pool."""
|
|
31
|
+
self.free.extend(reversed(self.tables.pop(seq_id, [])))
|
|
32
|
+
self.lengths.pop(seq_id, None)
|
|
33
|
+
|
|
34
|
+
def used(self):
|
|
35
|
+
return self.num_blocks - len(self.free)
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from .. import backend
|
|
4
|
+
from ..attention import KVCache
|
|
5
|
+
from ..tensor import Tensor, no_grad
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class StackgradRunner:
|
|
9
|
+
"""The scheduler's view of the model: prefill one sequence, decode the batch over left-padded KV caches."""
|
|
10
|
+
def __init__(self, model):
|
|
11
|
+
self.model = model
|
|
12
|
+
self.context_length = model.context_length
|
|
13
|
+
self.caches = {} # seq_id -> one KVCache per layer, for sequences outside the batch
|
|
14
|
+
self.batch, self.batched, self.pad = (), None, None # the batch's seq_ids, its padded caches, each row's padding
|
|
15
|
+
|
|
16
|
+
def prefill(self, seq_id, tokens):
|
|
17
|
+
self.caches[seq_id] = [KVCache() for _ in self.model.blocks]
|
|
18
|
+
with no_grad():
|
|
19
|
+
h = self.model.hidden(np.array([tokens]), self.caches[seq_id])
|
|
20
|
+
return self.model.project(h[:, -1])[0].numpy() # last position only: the other logits go unread
|
|
21
|
+
|
|
22
|
+
def decode(self, seq_ids, tokens):
|
|
23
|
+
# shortcut: a membership change copies every live cache twice (split, pad); paged attention over block tables avoids it
|
|
24
|
+
if tuple(seq_ids) != self.batch:
|
|
25
|
+
self._split()
|
|
26
|
+
self._join(seq_ids)
|
|
27
|
+
with no_grad():
|
|
28
|
+
logits = self.model(np.array(tokens)[:, None], self.batched, self.pad if self.pad.any() else None)
|
|
29
|
+
return logits[:, -1].numpy()
|
|
30
|
+
|
|
31
|
+
def release(self, seq_id):
|
|
32
|
+
if seq_id in self.batch:
|
|
33
|
+
self._split()
|
|
34
|
+
self.caches.pop(seq_id, None)
|
|
35
|
+
|
|
36
|
+
def _join(self, seq_ids):
|
|
37
|
+
rows = [self.caches.pop(s) for s in seq_ids]
|
|
38
|
+
lengths = np.array([len(r[0]) for r in rows])
|
|
39
|
+
self.pad = lengths.max() - lengths
|
|
40
|
+
device = rows[0][0].k.device
|
|
41
|
+
xp = backend.get(device)
|
|
42
|
+
|
|
43
|
+
def padded(layer, name):
|
|
44
|
+
parts = [xp.pad(getattr(r[layer], name).data, ((0, 0), (0, 0), (int(p), 0), (0, 0))) for r, p in zip(rows, self.pad)]
|
|
45
|
+
return Tensor(xp.concatenate(parts), device=device)
|
|
46
|
+
|
|
47
|
+
self.batched = []
|
|
48
|
+
for i in range(len(self.model.blocks)):
|
|
49
|
+
cache = KVCache()
|
|
50
|
+
cache.k, cache.v = padded(i, "k"), padded(i, "v")
|
|
51
|
+
self.batched.append(cache)
|
|
52
|
+
self.batch = tuple(seq_ids)
|
|
53
|
+
|
|
54
|
+
def _split(self):
|
|
55
|
+
for b, s in enumerate(self.batch):
|
|
56
|
+
p = int(self.pad[b])
|
|
57
|
+
self.caches[s] = []
|
|
58
|
+
for cache in self.batched:
|
|
59
|
+
own = KVCache()
|
|
60
|
+
own.k = Tensor(cache.k.data[b:b + 1, :, p:], device=cache.k.device)
|
|
61
|
+
own.v = Tensor(cache.v.data[b:b + 1, :, p:], device=cache.v.device)
|
|
62
|
+
self.caches[s].append(own)
|
|
63
|
+
self.batch, self.batched, self.pad = (), None, None
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
from collections import deque
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
|
|
4
|
+
from ..sampling import sample
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(eq=False) # requests compare by identity: two can hold the same tokens
|
|
8
|
+
class Request:
|
|
9
|
+
id: int
|
|
10
|
+
prompt: list
|
|
11
|
+
max_tokens: int
|
|
12
|
+
temperature: float = 1.0
|
|
13
|
+
top_p: float | None = None
|
|
14
|
+
eos_id: int | None = None
|
|
15
|
+
out: list = field(default_factory=list)
|
|
16
|
+
done: bool = False
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Scheduler:
|
|
20
|
+
"""Continuous batching: requests join and leave the running batch between decode steps."""
|
|
21
|
+
def __init__(self, runner, blocks, max_batch=8):
|
|
22
|
+
self.runner, self.blocks, self.max_batch = runner, blocks, max_batch
|
|
23
|
+
self.waiting, self.running = deque(), []
|
|
24
|
+
self.preemptions = 0
|
|
25
|
+
|
|
26
|
+
def fits(self, request):
|
|
27
|
+
"""Whether the request could ever run: its prompt plus all its new tokens within the context and the cache."""
|
|
28
|
+
total = len(request.prompt) + request.max_tokens
|
|
29
|
+
return total <= self.runner.context_length and -(-total // self.blocks.block_size) <= self.blocks.num_blocks
|
|
30
|
+
|
|
31
|
+
def add(self, request):
|
|
32
|
+
if not self.fits(request):
|
|
33
|
+
raise ValueError(f"request {request.id}: {len(request.prompt) + request.max_tokens} tokens can never fit")
|
|
34
|
+
self.waiting.append(request)
|
|
35
|
+
|
|
36
|
+
def has_work(self):
|
|
37
|
+
return bool(self.waiting or self.running)
|
|
38
|
+
|
|
39
|
+
def step(self):
|
|
40
|
+
"""Admit, prefill, decode once. Returns the (request, token) pairs produced this step."""
|
|
41
|
+
emitted = []
|
|
42
|
+
while self.waiting and len(self.running) < self.max_batch:
|
|
43
|
+
req = self.waiting[0]
|
|
44
|
+
tokens = req.prompt + req.out # a preempted request also replays what it already generated
|
|
45
|
+
if not self.blocks.can_append(req.id, len(tokens)):
|
|
46
|
+
break
|
|
47
|
+
self.waiting.popleft()
|
|
48
|
+
self.blocks.append(req.id, len(tokens))
|
|
49
|
+
emitted.append(self._emit(req, self.runner.prefill(req.id, tokens)[None]))
|
|
50
|
+
self.running.append(req)
|
|
51
|
+
self._retire()
|
|
52
|
+
|
|
53
|
+
# requests admitted this step already got their token from prefill
|
|
54
|
+
fresh = {id(r) for r, _ in emitted}
|
|
55
|
+
decode = [r for r in self.running if id(r) not in fresh]
|
|
56
|
+
for req in list(decode):
|
|
57
|
+
# room for the fed-back token: preempt the newest request until it fits; it recomputes on readmission
|
|
58
|
+
while req in self.running and not self.blocks.can_append(req.id, 1):
|
|
59
|
+
victim = self.running[-1]
|
|
60
|
+
self._preempt(victim)
|
|
61
|
+
if victim in decode:
|
|
62
|
+
decode.remove(victim)
|
|
63
|
+
if req in self.running:
|
|
64
|
+
self.blocks.append(req.id, 1)
|
|
65
|
+
if decode:
|
|
66
|
+
logits = self.runner.decode([r.id for r in decode], [r.out[-1] for r in decode])
|
|
67
|
+
emitted += [self._emit(r, logits[i:i + 1]) for i, r in enumerate(decode)]
|
|
68
|
+
self._retire()
|
|
69
|
+
return emitted
|
|
70
|
+
|
|
71
|
+
def _retire(self):
|
|
72
|
+
for req in [r for r in self.running if r.done]:
|
|
73
|
+
self.running.remove(req)
|
|
74
|
+
self.blocks.release(req.id)
|
|
75
|
+
self.runner.release(req.id)
|
|
76
|
+
|
|
77
|
+
def _emit(self, req, logits):
|
|
78
|
+
token = int(sample(logits, req.temperature, top_p=req.top_p)[0])
|
|
79
|
+
req.out.append(token)
|
|
80
|
+
req.done = token == req.eos_id or len(req.out) >= req.max_tokens
|
|
81
|
+
return req, token
|
|
82
|
+
|
|
83
|
+
def _preempt(self, req):
|
|
84
|
+
self.running.remove(req)
|
|
85
|
+
self.blocks.release(req.id)
|
|
86
|
+
self.runner.release(req.id)
|
|
87
|
+
self.waiting.appendleft(req)
|
|
88
|
+
self.preemptions += 1
|