stackgrad 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stackgrad/__init__.py +75 -0
- stackgrad/attention.py +86 -0
- stackgrad/backend/__init__.py +11 -0
- stackgrad/backend/cpu.py +116 -0
- stackgrad/backend/cuda.py +96 -0
- stackgrad/backend/cuda_attention.py +138 -0
- stackgrad/backend/metal.py +106 -0
- stackgrad/backend/metal_attention.py +195 -0
- stackgrad/backend/metal_quant.py +44 -0
- stackgrad/checkpoint.py +38 -0
- stackgrad/conv.py +63 -0
- stackgrad/data.py +28 -0
- stackgrad/functional.py +41 -0
- stackgrad/gpt.py +120 -0
- stackgrad/grad_check.py +55 -0
- stackgrad/layers.py +215 -0
- stackgrad/lora.py +54 -0
- stackgrad/loss.py +28 -0
- stackgrad/mlops.py +352 -0
- stackgrad/module.py +120 -0
- stackgrad/optim.py +121 -0
- stackgrad/profile.py +45 -0
- stackgrad/quant.py +66 -0
- stackgrad/qwen3.py +104 -0
- stackgrad/sampling.py +39 -0
- stackgrad/serve/api.py +434 -0
- stackgrad/serve/engine.py +53 -0
- stackgrad/serve/kv_cache.py +35 -0
- stackgrad/serve/runner.py +63 -0
- stackgrad/serve/scheduler.py +88 -0
- stackgrad/speculative.py +68 -0
- stackgrad/state.py +27 -0
- stackgrad/tensor.py +377 -0
- stackgrad/tokenizer.py +202 -0
- stackgrad-0.2.0.dist-info/METADATA +172 -0
- stackgrad-0.2.0.dist-info/RECORD +38 -0
- stackgrad-0.2.0.dist-info/WHEEL +4 -0
- stackgrad-0.2.0.dist-info/licenses/LICENSE +21 -0
stackgrad/__init__.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from . import functional as F
|
|
2
|
+
from .attention import Attention, KVCache
|
|
3
|
+
from .conv import Conv2d, Flatten, MaxPool2d
|
|
4
|
+
from .data import DataLoader
|
|
5
|
+
from .gpt import TransformerLM
|
|
6
|
+
from .grad_check import check_gradient, numerical_gradient
|
|
7
|
+
from .layers import (
|
|
8
|
+
GELU,
|
|
9
|
+
BatchNorm1d,
|
|
10
|
+
Dropout,
|
|
11
|
+
Embedding,
|
|
12
|
+
GroupNorm,
|
|
13
|
+
LayerNorm,
|
|
14
|
+
Linear,
|
|
15
|
+
ReLU,
|
|
16
|
+
RMSNorm,
|
|
17
|
+
Sigmoid,
|
|
18
|
+
Softmax,
|
|
19
|
+
SwiGLU,
|
|
20
|
+
Tanh,
|
|
21
|
+
)
|
|
22
|
+
from .loss import bce_loss, cross_entropy_loss, mse_loss
|
|
23
|
+
from .module import Module, Sequential
|
|
24
|
+
from .optim import (
|
|
25
|
+
SGD,
|
|
26
|
+
Adam,
|
|
27
|
+
AdamW,
|
|
28
|
+
LossScaler,
|
|
29
|
+
clip_grad_norm,
|
|
30
|
+
cosine_lr,
|
|
31
|
+
)
|
|
32
|
+
from .tensor import Tensor, autocast, cat, no_grad, stack, where
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"GELU",
|
|
36
|
+
"SGD",
|
|
37
|
+
"Adam",
|
|
38
|
+
"AdamW",
|
|
39
|
+
"Attention",
|
|
40
|
+
"BatchNorm1d",
|
|
41
|
+
"Conv2d",
|
|
42
|
+
"DataLoader",
|
|
43
|
+
"Dropout",
|
|
44
|
+
"Embedding",
|
|
45
|
+
"F",
|
|
46
|
+
"Flatten",
|
|
47
|
+
"GroupNorm",
|
|
48
|
+
"KVCache",
|
|
49
|
+
"LayerNorm",
|
|
50
|
+
"Linear",
|
|
51
|
+
"LossScaler",
|
|
52
|
+
"MaxPool2d",
|
|
53
|
+
"Module",
|
|
54
|
+
"RMSNorm",
|
|
55
|
+
"ReLU",
|
|
56
|
+
"Sequential",
|
|
57
|
+
"Sigmoid",
|
|
58
|
+
"Softmax",
|
|
59
|
+
"SwiGLU",
|
|
60
|
+
"Tanh",
|
|
61
|
+
"Tensor",
|
|
62
|
+
"TransformerLM",
|
|
63
|
+
"autocast",
|
|
64
|
+
"bce_loss",
|
|
65
|
+
"cat",
|
|
66
|
+
"check_gradient",
|
|
67
|
+
"clip_grad_norm",
|
|
68
|
+
"cosine_lr",
|
|
69
|
+
"cross_entropy_loss",
|
|
70
|
+
"mse_loss",
|
|
71
|
+
"no_grad",
|
|
72
|
+
"numerical_gradient",
|
|
73
|
+
"stack",
|
|
74
|
+
"where",
|
|
75
|
+
]
|
stackgrad/attention.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from . import mlops
|
|
4
|
+
from .layers import Linear, RMSNorm
|
|
5
|
+
from .module import Module
|
|
6
|
+
from .tensor import _apply, cat
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def apply_rope(x, start, theta):
|
|
10
|
+
"""Rotates each (x[i], x[i + D/2]) pair by its position's angle, in Llama/Qwen's half-split layout."""
|
|
11
|
+
half = x.shape[-1] // 2
|
|
12
|
+
freqs = theta ** (-np.arange(half) / half)
|
|
13
|
+
positions = np.reshape(start, (-1, 1, 1)) + np.arange(x.shape[2])[:, None] # (B or 1, T, 1)
|
|
14
|
+
angles = (positions * freqs)[:, None] # (B or 1, 1, T, D/2)
|
|
15
|
+
cos, sin = np.cos(angles).astype(np.float32), np.sin(angles).astype(np.float32)
|
|
16
|
+
x1, x2 = x[..., :half], x[..., half:]
|
|
17
|
+
return cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def scaled_dot_product_attention(q, k, v, pad=None, causal=True):
|
|
21
|
+
"""q (B, H, T, D) over k, v (B, Hkv, S, D), queries at the last T positions; pad (B,) masks left padding."""
|
|
22
|
+
key_bias = None
|
|
23
|
+
if pad is not None:
|
|
24
|
+
# -1e9, not -inf: a padding query sees only padding, and an all -inf softmax is NaN that leaks via 0 * NaN
|
|
25
|
+
key_bias = np.where(np.arange(k.shape[2]) < np.asarray(pad)[:, None], -1e9, 0.0).astype(np.float32)
|
|
26
|
+
return _apply(mlops.Attention, q, k, v, key_bias=key_bias, causal=causal)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class KVCache:
|
|
30
|
+
"""Keys and values of earlier positions, so a decode step doesn't recompute the prefix."""
|
|
31
|
+
# shortcut: cat copies the whole cache every step (O(T^2) bytes per sequence); M4 swaps in a paged cache
|
|
32
|
+
def __init__(self):
|
|
33
|
+
self.k = self.v = None
|
|
34
|
+
|
|
35
|
+
def __len__(self):
|
|
36
|
+
return 0 if self.k is None else self.k.shape[2]
|
|
37
|
+
|
|
38
|
+
def truncate(self, n):
|
|
39
|
+
"""Drops every position from n on, for speculative decoding's rejected drafts."""
|
|
40
|
+
self.k, self.v = self.k[:, :, :n], self.v[:, :, :n]
|
|
41
|
+
|
|
42
|
+
def update(self, k, v):
|
|
43
|
+
self.k = k if self.k is None else cat([self.k, k], axis=2)
|
|
44
|
+
self.v = v if self.v is None else cat([self.v, v], axis=2)
|
|
45
|
+
return self.k, self.v
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Attention(Module):
|
|
49
|
+
"""Multi-head attention with RoPE and grouped-query heads; qk_norm adds Qwen3's per-head RMSNorm on q, k."""
|
|
50
|
+
def __init__(self, d_model, n_heads, n_kv_heads=None, rope_theta=10000.0, head_dim=None, qk_norm=False, norm_eps=1e-5,
|
|
51
|
+
rope=True):
|
|
52
|
+
super().__init__()
|
|
53
|
+
self.rope = rope # False is the NoPE ablation: causal masking alone orders tokens
|
|
54
|
+
self.n_heads = n_heads
|
|
55
|
+
self.n_kv_heads = n_kv_heads or n_heads
|
|
56
|
+
self.head_dim = head_dim or d_model // n_heads
|
|
57
|
+
self.rope_theta = rope_theta
|
|
58
|
+
self.q_proj = Linear(d_model, n_heads * self.head_dim, bias=False)
|
|
59
|
+
self.k_proj = Linear(d_model, self.n_kv_heads * self.head_dim, bias=False)
|
|
60
|
+
self.v_proj = Linear(d_model, self.n_kv_heads * self.head_dim, bias=False)
|
|
61
|
+
self.o_proj = Linear(n_heads * self.head_dim, d_model, bias=False)
|
|
62
|
+
self.q_norm = RMSNorm(self.head_dim, norm_eps) if qk_norm else None
|
|
63
|
+
self.k_norm = RMSNorm(self.head_dim, norm_eps) if qk_norm else None
|
|
64
|
+
self.record, self.recorded = False, None # record=True keeps the last q and k, for drawing attention maps
|
|
65
|
+
|
|
66
|
+
def forward(self, x, cache=None, pad=None):
|
|
67
|
+
B, T, _ = x.shape
|
|
68
|
+
D = self.head_dim
|
|
69
|
+
start = 0 if cache is None else len(cache)
|
|
70
|
+
if pad is not None:
|
|
71
|
+
start = start - np.asarray(pad) # each row's first real token sits at position 0, after its left padding
|
|
72
|
+
q = self.q_proj(x).reshape(B, T, self.n_heads, D)
|
|
73
|
+
k = self.k_proj(x).reshape(B, T, self.n_kv_heads, D)
|
|
74
|
+
if self.q_norm is not None:
|
|
75
|
+
q, k = self.q_norm(q), self.k_norm(k)
|
|
76
|
+
q, k = q.transpose(1, 2), k.transpose(1, 2) # (B, H, T, D), (B, Hkv, T, D)
|
|
77
|
+
v = self.v_proj(x).reshape(B, T, self.n_kv_heads, D).transpose(1, 2)
|
|
78
|
+
if self.rope:
|
|
79
|
+
q, k = apply_rope(q, start, self.rope_theta), apply_rope(k, start, self.rope_theta)
|
|
80
|
+
if cache is not None:
|
|
81
|
+
k, v = cache.update(k, v)
|
|
82
|
+
if self.record:
|
|
83
|
+
self.recorded = q.numpy(), k.numpy()
|
|
84
|
+
|
|
85
|
+
out = scaled_dot_product_attention(q, k, v, pad) # (B, H, T, D)
|
|
86
|
+
return self.o_proj(out.transpose(1, 2).reshape(B, T, self.n_heads * D))
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import importlib
|
|
3
|
+
|
|
4
|
+
DEVICES = ("cpu", "metal", "cuda")
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@functools.cache
|
|
8
|
+
def get(device):
|
|
9
|
+
if device not in DEVICES:
|
|
10
|
+
raise ValueError(f"unknown device {device!r}, expected one of {DEVICES}")
|
|
11
|
+
return importlib.import_module(f"{__name__}.{device}")
|
stackgrad/backend/cpu.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
from numpy import (
|
|
3
|
+
broadcast_to,
|
|
4
|
+
concatenate,
|
|
5
|
+
cumsum,
|
|
6
|
+
exp,
|
|
7
|
+
expand_dims,
|
|
8
|
+
float32,
|
|
9
|
+
full,
|
|
10
|
+
inf,
|
|
11
|
+
log,
|
|
12
|
+
ones_like,
|
|
13
|
+
pad,
|
|
14
|
+
repeat,
|
|
15
|
+
sign,
|
|
16
|
+
stack,
|
|
17
|
+
triu,
|
|
18
|
+
where,
|
|
19
|
+
zeros_like,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"asarray",
|
|
24
|
+
"broadcast_to",
|
|
25
|
+
"clear_cache",
|
|
26
|
+
"col2im",
|
|
27
|
+
"concatenate",
|
|
28
|
+
"cumsum",
|
|
29
|
+
"dtype",
|
|
30
|
+
"exp",
|
|
31
|
+
"expand_dims",
|
|
32
|
+
"float32",
|
|
33
|
+
"full",
|
|
34
|
+
"im2col",
|
|
35
|
+
"inf",
|
|
36
|
+
"is_float",
|
|
37
|
+
"log",
|
|
38
|
+
"ones_like",
|
|
39
|
+
"pad",
|
|
40
|
+
"realize",
|
|
41
|
+
"repeat",
|
|
42
|
+
"scatter_add",
|
|
43
|
+
"sigmoid",
|
|
44
|
+
"sign",
|
|
45
|
+
"stack",
|
|
46
|
+
"to_numpy",
|
|
47
|
+
"triu",
|
|
48
|
+
"where",
|
|
49
|
+
"zeros_like",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def asarray(x, dtype=None):
|
|
54
|
+
"""Any array, list, or number -> an array of this backend."""
|
|
55
|
+
return np.asarray(x, dtype=dtype)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def to_numpy(x):
|
|
59
|
+
return np.array(x) # a copy, like every other backend's device-to-host transfer
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def dtype(name):
|
|
63
|
+
return np.dtype(name)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def is_float(dt):
|
|
67
|
+
return np.issubdtype(dt, np.floating)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def realize(*arrays):
|
|
71
|
+
"""NumPy runs each op as it is called, so there is nothing to wait for."""
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def clear_cache():
|
|
75
|
+
"""NumPy keeps no memory pool to free."""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def sigmoid(x):
|
|
79
|
+
# exp(-x) overflows to inf for very negative x, and 1 / inf = 0 is correct
|
|
80
|
+
with np.errstate(over="ignore"):
|
|
81
|
+
return 1.0 / (1.0 + np.exp(-x))
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def scatter_add(shape, key, values):
|
|
85
|
+
"""zeros(shape) with `values` added at `key`, summing where an index repeats."""
|
|
86
|
+
out = np.zeros(shape, dtype=values.dtype)
|
|
87
|
+
np.add.at(out, key, values)
|
|
88
|
+
return out
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def im2col(x, kh, kw, stride, padding, xp=np):
|
|
92
|
+
"""(N, C, H, W) -> (N*OH*OW, C*kh*kw), one row per output pixel; no in-place writes, so MLX runs it."""
|
|
93
|
+
N, C = x.shape[:2]
|
|
94
|
+
if padding > 0:
|
|
95
|
+
x = xp.pad(x, ((0, 0), (0, 0), (padding, padding), (padding, padding)))
|
|
96
|
+
OH = (x.shape[2] - kh) // stride + 1
|
|
97
|
+
OW = (x.shape[3] - kw) // stride + 1
|
|
98
|
+
# channels last so the reshape copies nothing: 2.95 GB peak vs 5.37 GB for NCHW (256ch 3x3, 512x512, M4)
|
|
99
|
+
x = x.transpose(0, 2, 3, 1)
|
|
100
|
+
patches = [x[:, y:y + stride * OH:stride, xx:xx + stride * OW:stride, :] for y in range(kh) for xx in range(kw)]
|
|
101
|
+
return xp.stack(patches, axis=-1).reshape(N * OH * OW, C * kh * kw) # (N, OH, OW, C, kh*kw) -> rows
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def col2im(cols, shape, kh, kw, stride, padding):
|
|
105
|
+
"""Inverse of im2col: add every patch row back onto the pixels it came from."""
|
|
106
|
+
N, C, H, W = shape
|
|
107
|
+
H_pad, W_pad = H + 2 * padding, W + 2 * padding
|
|
108
|
+
OH = (H_pad - kh) // stride + 1
|
|
109
|
+
OW = (W_pad - kw) // stride + 1
|
|
110
|
+
|
|
111
|
+
cols = cols.reshape(N, OH, OW, C, kh, kw).transpose(0, 3, 4, 5, 1, 2)
|
|
112
|
+
img = np.zeros((N, C, H_pad, W_pad), dtype=cols.dtype)
|
|
113
|
+
for y in range(kh):
|
|
114
|
+
for xx in range(kw):
|
|
115
|
+
img[:, :, y:y + stride * OH:stride, xx:xx + stride * OW:stride] += cols[:, :, y, xx]
|
|
116
|
+
return img[:, :, padding:H_pad - padding, padding:W_pad - padding]
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import cupy as cp
|
|
2
|
+
import cupyx
|
|
3
|
+
from cupy import (
|
|
4
|
+
broadcast_to,
|
|
5
|
+
concatenate,
|
|
6
|
+
cumsum,
|
|
7
|
+
exp,
|
|
8
|
+
expand_dims,
|
|
9
|
+
float32,
|
|
10
|
+
full,
|
|
11
|
+
inf,
|
|
12
|
+
log,
|
|
13
|
+
ones_like,
|
|
14
|
+
pad,
|
|
15
|
+
repeat,
|
|
16
|
+
sign,
|
|
17
|
+
stack,
|
|
18
|
+
triu,
|
|
19
|
+
where,
|
|
20
|
+
zeros_like,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
from . import cpu, cuda_attention
|
|
24
|
+
from .cpu import dtype, is_float, realize
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"asarray",
|
|
28
|
+
"broadcast_to",
|
|
29
|
+
"clear_cache",
|
|
30
|
+
"col2im",
|
|
31
|
+
"concatenate",
|
|
32
|
+
"cumsum",
|
|
33
|
+
"dtype",
|
|
34
|
+
"exp",
|
|
35
|
+
"expand_dims",
|
|
36
|
+
"float32",
|
|
37
|
+
"full",
|
|
38
|
+
"fused_attention",
|
|
39
|
+
"fused_attention_backward",
|
|
40
|
+
"im2col",
|
|
41
|
+
"inf",
|
|
42
|
+
"is_float",
|
|
43
|
+
"log",
|
|
44
|
+
"ones_like",
|
|
45
|
+
"pad",
|
|
46
|
+
"realize",
|
|
47
|
+
"repeat",
|
|
48
|
+
"scatter_add",
|
|
49
|
+
"sigmoid",
|
|
50
|
+
"sign",
|
|
51
|
+
"stack",
|
|
52
|
+
"to_numpy",
|
|
53
|
+
"triu",
|
|
54
|
+
"where",
|
|
55
|
+
"zeros_like",
|
|
56
|
+
]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def clear_cache():
|
|
60
|
+
cp.get_default_memory_pool().free_all_blocks()
|
|
61
|
+
|
|
62
|
+
def asarray(x, dtype=None):
|
|
63
|
+
return cp.asarray(x, dtype=dtype)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def to_numpy(x):
|
|
67
|
+
return cp.asnumpy(x)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def fused_attention(q, k, v):
|
|
71
|
+
"""Triton FlashAttention forward, or None for shapes the kernels don't cover."""
|
|
72
|
+
return cuda_attention.attention(q, k, v) if cuda_attention.supported(q) else None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def fused_attention_backward(q, k, v, out, lse, grad):
|
|
76
|
+
return cuda_attention.attention_backward(q, k, v, out, lse, grad)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def sigmoid(x):
|
|
80
|
+
return 1.0 / (1.0 + cp.exp(-x))
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def scatter_add(shape, key, values):
|
|
84
|
+
out = cp.zeros(shape, dtype=values.dtype)
|
|
85
|
+
cupyx.scatter_add(out, key, values)
|
|
86
|
+
return out
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
# shortcut: col2im (conv backward) round-trips through the CPU, port it to CuPy if a CNN trains on cuda
|
|
90
|
+
def im2col(x, *args):
|
|
91
|
+
return cpu.im2col(x, *args, xp=cp)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def col2im(cols, *args):
|
|
95
|
+
return cp.asarray(cpu.col2im(cp.asnumpy(cols), *args))
|
|
96
|
+
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import math
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
import cupy as cp
|
|
5
|
+
import triton
|
|
6
|
+
import triton.language as tl
|
|
7
|
+
|
|
8
|
+
# match CuPy's matmuls: tf32 only under CUPY_TF32=1 (training), ieee float32 otherwise (tests)
|
|
9
|
+
_PRECISION = "tf32" if os.environ.get("CUPY_TF32") == "1" else "ieee"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class _Arg:
|
|
13
|
+
"""The data_ptr and dtype Triton reads from a tensor, so CuPy arrays need no torch."""
|
|
14
|
+
def __init__(self, a):
|
|
15
|
+
self.a, self.dtype = a, a.dtype
|
|
16
|
+
|
|
17
|
+
def data_ptr(self):
|
|
18
|
+
return self.a.data.ptr
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@triton.jit
|
|
22
|
+
def _forward(Q, K, V, O, LSE, T, S, H, HKV, scale, D: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, PREC: tl.constexpr):
|
|
23
|
+
qb, bh = tl.program_id(0), tl.program_id(1)
|
|
24
|
+
kv = (bh // H * HKV + bh % H // (H // HKV)) * S * D
|
|
25
|
+
rows = qb * BLOCK_M + tl.arange(0, BLOCK_M)
|
|
26
|
+
dims = tl.arange(0, D)
|
|
27
|
+
q = tl.load(Q + (bh * T + rows[:, None]) * D + dims[None, :], mask=rows[:, None] < T, other=0.0)
|
|
28
|
+
off = S - T
|
|
29
|
+
m = tl.full([BLOCK_M], float("-inf"), tl.float32)
|
|
30
|
+
l = tl.zeros([BLOCK_M], tl.float32)
|
|
31
|
+
acc = tl.zeros([BLOCK_M, D], tl.float32)
|
|
32
|
+
for start in range(0, tl.minimum(off + qb * BLOCK_M + BLOCK_M, S), BLOCK_N): # later keys are hidden from every row here
|
|
33
|
+
cols = start + tl.arange(0, BLOCK_N)
|
|
34
|
+
k = tl.load(K + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
|
|
35
|
+
v = tl.load(V + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
|
|
36
|
+
s = tl.dot(q, tl.trans(k), input_precision=PREC) * scale
|
|
37
|
+
s = tl.where((cols[None, :] <= off + rows[:, None]) & (cols[None, :] < S), s, float("-inf"))
|
|
38
|
+
m_new = tl.maximum(m, tl.max(s, 1))
|
|
39
|
+
p = tl.exp(s - m_new[:, None])
|
|
40
|
+
corr = tl.exp(m - m_new)
|
|
41
|
+
l = l * corr + tl.sum(p, 1)
|
|
42
|
+
acc = acc * corr[:, None] + tl.dot(p, v, input_precision=PREC)
|
|
43
|
+
m = m_new
|
|
44
|
+
tl.store(O + (bh * T + rows[:, None]) * D + dims[None, :], acc / l[:, None], mask=rows[:, None] < T)
|
|
45
|
+
tl.store(LSE + bh * T + rows, m + tl.log(l), mask=rows < T)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@triton.jit
|
|
49
|
+
def _backward_kv(Q, K, V, DO, LSE, DELTA, DK, DV, T, S, H, HKV, scale,
|
|
50
|
+
D: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, PREC: tl.constexpr):
|
|
51
|
+
kb, bh = tl.program_id(0), tl.program_id(1)
|
|
52
|
+
kv = (bh // H * HKV + bh % H // (H // HKV)) * S * D
|
|
53
|
+
cols = kb * BLOCK_N + tl.arange(0, BLOCK_N)
|
|
54
|
+
dims = tl.arange(0, D)
|
|
55
|
+
k = tl.load(K + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
|
|
56
|
+
v = tl.load(V + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
|
|
57
|
+
off = S - T
|
|
58
|
+
dk = tl.zeros([BLOCK_N, D], tl.float32)
|
|
59
|
+
dv = tl.zeros([BLOCK_N, D], tl.float32)
|
|
60
|
+
first = tl.maximum(kb * BLOCK_N - off, 0) // BLOCK_M * BLOCK_M # earlier query rows see none of these keys
|
|
61
|
+
for start in range(first, T, BLOCK_M):
|
|
62
|
+
rows = start + tl.arange(0, BLOCK_M)
|
|
63
|
+
q = tl.load(Q + (bh * T + rows[:, None]) * D + dims[None, :], mask=rows[:, None] < T, other=0.0)
|
|
64
|
+
do = tl.load(DO + (bh * T + rows[:, None]) * D + dims[None, :], mask=rows[:, None] < T, other=0.0)
|
|
65
|
+
lse = tl.load(LSE + bh * T + rows, mask=rows < T, other=0.0)
|
|
66
|
+
delta = tl.load(DELTA + bh * T + rows, mask=rows < T, other=0.0)
|
|
67
|
+
visible = (cols[None, :] <= off + rows[:, None]) & (rows[:, None] < T) & (cols[None, :] < S)
|
|
68
|
+
p = tl.where(visible, tl.exp(tl.dot(q, tl.trans(k), input_precision=PREC) * scale - lse[:, None]), 0.0)
|
|
69
|
+
dv += tl.dot(tl.trans(p), do, input_precision=PREC)
|
|
70
|
+
ds = p * (tl.dot(do, tl.trans(v), input_precision=PREC) - delta[:, None])
|
|
71
|
+
dk += tl.dot(tl.trans(ds), q, input_precision=PREC) * scale
|
|
72
|
+
# per query head; the caller sums each group into its kv head
|
|
73
|
+
tl.store(DK + (bh * S + cols[:, None]) * D + dims[None, :], dk, mask=cols[:, None] < S)
|
|
74
|
+
tl.store(DV + (bh * S + cols[:, None]) * D + dims[None, :], dv, mask=cols[:, None] < S)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@triton.jit
|
|
78
|
+
def _backward_q(Q, K, V, DO, LSE, DELTA, DQ, T, S, H, HKV, scale, D: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, PREC: tl.constexpr):
|
|
79
|
+
qb, bh = tl.program_id(0), tl.program_id(1)
|
|
80
|
+
kv = (bh // H * HKV + bh % H // (H // HKV)) * S * D
|
|
81
|
+
rows = qb * BLOCK_M + tl.arange(0, BLOCK_M)
|
|
82
|
+
dims = tl.arange(0, D)
|
|
83
|
+
q = tl.load(Q + (bh * T + rows[:, None]) * D + dims[None, :], mask=rows[:, None] < T, other=0.0)
|
|
84
|
+
do = tl.load(DO + (bh * T + rows[:, None]) * D + dims[None, :], mask=rows[:, None] < T, other=0.0)
|
|
85
|
+
lse = tl.load(LSE + bh * T + rows, mask=rows < T, other=0.0)
|
|
86
|
+
delta = tl.load(DELTA + bh * T + rows, mask=rows < T, other=0.0)
|
|
87
|
+
off = S - T
|
|
88
|
+
dq = tl.zeros([BLOCK_M, D], tl.float32)
|
|
89
|
+
for start in range(0, tl.minimum(off + qb * BLOCK_M + BLOCK_M, S), BLOCK_N):
|
|
90
|
+
cols = start + tl.arange(0, BLOCK_N)
|
|
91
|
+
k = tl.load(K + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
|
|
92
|
+
v = tl.load(V + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
|
|
93
|
+
visible = (cols[None, :] <= off + rows[:, None]) & (rows[:, None] < T) & (cols[None, :] < S)
|
|
94
|
+
p = tl.where(visible, tl.exp(tl.dot(q, tl.trans(k), input_precision=PREC) * scale - lse[:, None]), 0.0)
|
|
95
|
+
ds = p * (tl.dot(do, tl.trans(v), input_precision=PREC) - delta[:, None])
|
|
96
|
+
dq += tl.dot(ds, k, input_precision=PREC) * scale
|
|
97
|
+
tl.store(DQ + (bh * T + rows[:, None]) * D + dims[None, :], dq, mask=rows[:, None] < T)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _blocks(D):
|
|
101
|
+
# D=128 at 64x32 blocks needs 114,688 B of shared memory; the 3090 has 101,376
|
|
102
|
+
return (64, 64) if D <= 64 else (32, 32)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def supported(q):
|
|
106
|
+
D = q.shape[-1]
|
|
107
|
+
return q.dtype == cp.float32 and 16 <= D <= 128 and D & (D - 1) == 0 # tl.arange needs a power of two
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def attention(q, k, v):
|
|
111
|
+
"""Causal FlashAttention-2: q (B, H, T, D), k, v (B, Hkv, S, D) -> out (B, H, T, D), lse (B, H, T)."""
|
|
112
|
+
B, H, T, D = q.shape
|
|
113
|
+
S, Hkv = k.shape[2], k.shape[1]
|
|
114
|
+
q, k, v = (cp.ascontiguousarray(x) for x in (q, k, v))
|
|
115
|
+
out, lse = cp.empty_like(q), cp.empty((B, H, T), cp.float32)
|
|
116
|
+
bm, bn = _blocks(D)
|
|
117
|
+
_forward[(triton.cdiv(T, bm), B * H)](_Arg(q), _Arg(k), _Arg(v), _Arg(out), _Arg(lse), T, S, H, Hkv, 1 / math.sqrt(D),
|
|
118
|
+
D=D, BLOCK_M=bm, BLOCK_N=bn, PREC=_PRECISION)
|
|
119
|
+
return out, lse
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def attention_backward(q, k, v, out, lse, dout):
|
|
123
|
+
"""(dq, dk, dv) of attention(q, k, v)[0] given dout, rebuilding the softmax from out and lse."""
|
|
124
|
+
B, H, T, D = q.shape
|
|
125
|
+
S, Hkv = k.shape[2], k.shape[1]
|
|
126
|
+
q, k, v, out, dout = (cp.ascontiguousarray(x) for x in (q, k, v, out, dout))
|
|
127
|
+
delta = (dout * out).sum(axis=-1) # rowsum(dO * O), the softmax backward's correction term
|
|
128
|
+
dq = cp.empty_like(q)
|
|
129
|
+
dk, dv = cp.empty((B, H, S, D), cp.float32), cp.empty((B, H, S, D), cp.float32)
|
|
130
|
+
bm, bn = _blocks(D)
|
|
131
|
+
args = (T, S, H, Hkv, 1 / math.sqrt(D))
|
|
132
|
+
_backward_kv[(triton.cdiv(S, bn), B * H)](*map(_Arg, (q, k, v, dout, lse, delta, dk, dv)), *args, D=D, BLOCK_M=bm, BLOCK_N=bn, PREC=_PRECISION,
|
|
133
|
+
num_stages=1)
|
|
134
|
+
# one pipeline stage: the backward holds more tiles, and three stages overflow shared memory
|
|
135
|
+
_backward_q[(triton.cdiv(T, bm), B * H)](*map(_Arg, (q, k, v, dout, lse, delta, dq)), *args, D=D, BLOCK_M=bm, BLOCK_N=bn, PREC=_PRECISION,
|
|
136
|
+
num_stages=1)
|
|
137
|
+
rep = H // Hkv
|
|
138
|
+
return dq, dk.reshape(B, Hkv, rep, S, D).sum(axis=2), dv.reshape(B, Hkv, rep, S, D).sum(axis=2)
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import mlx.core as mx
|
|
2
|
+
import numpy as np
|
|
3
|
+
from mlx.core import (
|
|
4
|
+
broadcast_to,
|
|
5
|
+
concatenate,
|
|
6
|
+
cumsum,
|
|
7
|
+
exp,
|
|
8
|
+
expand_dims,
|
|
9
|
+
float32,
|
|
10
|
+
full,
|
|
11
|
+
inf,
|
|
12
|
+
log,
|
|
13
|
+
ones_like,
|
|
14
|
+
pad,
|
|
15
|
+
repeat,
|
|
16
|
+
sigmoid,
|
|
17
|
+
sign,
|
|
18
|
+
stack,
|
|
19
|
+
triu,
|
|
20
|
+
where,
|
|
21
|
+
zeros_like,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
from . import cpu, metal_attention, metal_quant
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"asarray",
|
|
28
|
+
"broadcast_to",
|
|
29
|
+
"clear_cache",
|
|
30
|
+
"col2im",
|
|
31
|
+
"concatenate",
|
|
32
|
+
"cumsum",
|
|
33
|
+
"dtype",
|
|
34
|
+
"exp",
|
|
35
|
+
"expand_dims",
|
|
36
|
+
"float32",
|
|
37
|
+
"full",
|
|
38
|
+
"fused_attention",
|
|
39
|
+
"im2col",
|
|
40
|
+
"inf",
|
|
41
|
+
"is_float",
|
|
42
|
+
"log",
|
|
43
|
+
"ones_like",
|
|
44
|
+
"pad",
|
|
45
|
+
"quant_matmul",
|
|
46
|
+
"realize",
|
|
47
|
+
"repeat",
|
|
48
|
+
"scatter_add",
|
|
49
|
+
"sigmoid",
|
|
50
|
+
"sign",
|
|
51
|
+
"stack",
|
|
52
|
+
"to_numpy",
|
|
53
|
+
"triu",
|
|
54
|
+
"where",
|
|
55
|
+
"zeros_like",
|
|
56
|
+
]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def asarray(x, dtype=None):
|
|
60
|
+
if isinstance(x, mx.array):
|
|
61
|
+
return x if dtype is None or x.dtype == dtype else x.astype(dtype)
|
|
62
|
+
return mx.array(x) if dtype is None else mx.array(x, dtype=dtype)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def to_numpy(x):
|
|
66
|
+
# numpy has no bfloat16; widening to float32 is exact
|
|
67
|
+
return np.array(x.astype(mx.float32) if x.dtype == mx.bfloat16 else x)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def dtype(name):
|
|
71
|
+
return getattr(mx, name)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def is_float(dt):
|
|
75
|
+
return mx.issubdtype(dt, mx.floating)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
realize = mx.eval
|
|
79
|
+
clear_cache = mx.clear_cache # mlx keeps freed buffers cached for reuse until this
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def scatter_add(shape, key, values):
|
|
83
|
+
return mx.zeros(shape, dtype=values.dtype).at[key].add(values)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# shortcut: col2im (conv backward) round-trips through the CPU, port it to MLX if a CNN trains on metal
|
|
87
|
+
def im2col(x, *args):
|
|
88
|
+
return cpu.im2col(x, *args, xp=mx)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def col2im(cols, *args):
|
|
92
|
+
return mx.array(cpu.col2im(to_numpy(cols), *args))
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def fused_attention(q, k, v):
|
|
96
|
+
"""Metal attention forward, or None for shapes the kernels don't cover."""
|
|
97
|
+
D = q.shape[-1]
|
|
98
|
+
if D % 8 or D > 128:
|
|
99
|
+
return None
|
|
100
|
+
# one query row (decode) leaves v3's 32-row threadgroups mostly idle; v1 measured faster there
|
|
101
|
+
return metal_attention.attention(q, k, v, "v1" if q.shape[2] == 1 else "v3")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def quant_matmul(x, packed, scales, biases, bits, group):
|
|
105
|
+
"""x (M, K) @ dequantized W (N, K).T as one Metal kernel, for decode-sized M."""
|
|
106
|
+
return metal_quant.quant_matmul(x, packed, scales, biases, bits, group)
|