stackgrad 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stackgrad/__init__.py ADDED
@@ -0,0 +1,75 @@
1
+ from . import functional as F
2
+ from .attention import Attention, KVCache
3
+ from .conv import Conv2d, Flatten, MaxPool2d
4
+ from .data import DataLoader
5
+ from .gpt import TransformerLM
6
+ from .grad_check import check_gradient, numerical_gradient
7
+ from .layers import (
8
+ GELU,
9
+ BatchNorm1d,
10
+ Dropout,
11
+ Embedding,
12
+ GroupNorm,
13
+ LayerNorm,
14
+ Linear,
15
+ ReLU,
16
+ RMSNorm,
17
+ Sigmoid,
18
+ Softmax,
19
+ SwiGLU,
20
+ Tanh,
21
+ )
22
+ from .loss import bce_loss, cross_entropy_loss, mse_loss
23
+ from .module import Module, Sequential
24
+ from .optim import (
25
+ SGD,
26
+ Adam,
27
+ AdamW,
28
+ LossScaler,
29
+ clip_grad_norm,
30
+ cosine_lr,
31
+ )
32
+ from .tensor import Tensor, autocast, cat, no_grad, stack, where
33
+
34
+ __all__ = [
35
+ "GELU",
36
+ "SGD",
37
+ "Adam",
38
+ "AdamW",
39
+ "Attention",
40
+ "BatchNorm1d",
41
+ "Conv2d",
42
+ "DataLoader",
43
+ "Dropout",
44
+ "Embedding",
45
+ "F",
46
+ "Flatten",
47
+ "GroupNorm",
48
+ "KVCache",
49
+ "LayerNorm",
50
+ "Linear",
51
+ "LossScaler",
52
+ "MaxPool2d",
53
+ "Module",
54
+ "RMSNorm",
55
+ "ReLU",
56
+ "Sequential",
57
+ "Sigmoid",
58
+ "Softmax",
59
+ "SwiGLU",
60
+ "Tanh",
61
+ "Tensor",
62
+ "TransformerLM",
63
+ "autocast",
64
+ "bce_loss",
65
+ "cat",
66
+ "check_gradient",
67
+ "clip_grad_norm",
68
+ "cosine_lr",
69
+ "cross_entropy_loss",
70
+ "mse_loss",
71
+ "no_grad",
72
+ "numerical_gradient",
73
+ "stack",
74
+ "where",
75
+ ]
stackgrad/attention.py ADDED
@@ -0,0 +1,86 @@
1
+ import numpy as np
2
+
3
+ from . import mlops
4
+ from .layers import Linear, RMSNorm
5
+ from .module import Module
6
+ from .tensor import _apply, cat
7
+
8
+
9
+ def apply_rope(x, start, theta):
10
+ """Rotates each (x[i], x[i + D/2]) pair by its position's angle, in Llama/Qwen's half-split layout."""
11
+ half = x.shape[-1] // 2
12
+ freqs = theta ** (-np.arange(half) / half)
13
+ positions = np.reshape(start, (-1, 1, 1)) + np.arange(x.shape[2])[:, None] # (B or 1, T, 1)
14
+ angles = (positions * freqs)[:, None] # (B or 1, 1, T, D/2)
15
+ cos, sin = np.cos(angles).astype(np.float32), np.sin(angles).astype(np.float32)
16
+ x1, x2 = x[..., :half], x[..., half:]
17
+ return cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1)
18
+
19
+
20
+ def scaled_dot_product_attention(q, k, v, pad=None, causal=True):
21
+ """q (B, H, T, D) over k, v (B, Hkv, S, D), queries at the last T positions; pad (B,) masks left padding."""
22
+ key_bias = None
23
+ if pad is not None:
24
+ # -1e9, not -inf: a padding query sees only padding, and an all -inf softmax is NaN that leaks via 0 * NaN
25
+ key_bias = np.where(np.arange(k.shape[2]) < np.asarray(pad)[:, None], -1e9, 0.0).astype(np.float32)
26
+ return _apply(mlops.Attention, q, k, v, key_bias=key_bias, causal=causal)
27
+
28
+
29
+ class KVCache:
30
+ """Keys and values of earlier positions, so a decode step doesn't recompute the prefix."""
31
+ # shortcut: cat copies the whole cache every step (O(T^2) bytes per sequence); M4 swaps in a paged cache
32
+ def __init__(self):
33
+ self.k = self.v = None
34
+
35
+ def __len__(self):
36
+ return 0 if self.k is None else self.k.shape[2]
37
+
38
+ def truncate(self, n):
39
+ """Drops every position from n on, for speculative decoding's rejected drafts."""
40
+ self.k, self.v = self.k[:, :, :n], self.v[:, :, :n]
41
+
42
+ def update(self, k, v):
43
+ self.k = k if self.k is None else cat([self.k, k], axis=2)
44
+ self.v = v if self.v is None else cat([self.v, v], axis=2)
45
+ return self.k, self.v
46
+
47
+
48
+ class Attention(Module):
49
+ """Multi-head attention with RoPE and grouped-query heads; qk_norm adds Qwen3's per-head RMSNorm on q, k."""
50
+ def __init__(self, d_model, n_heads, n_kv_heads=None, rope_theta=10000.0, head_dim=None, qk_norm=False, norm_eps=1e-5,
51
+ rope=True):
52
+ super().__init__()
53
+ self.rope = rope # False is the NoPE ablation: causal masking alone orders tokens
54
+ self.n_heads = n_heads
55
+ self.n_kv_heads = n_kv_heads or n_heads
56
+ self.head_dim = head_dim or d_model // n_heads
57
+ self.rope_theta = rope_theta
58
+ self.q_proj = Linear(d_model, n_heads * self.head_dim, bias=False)
59
+ self.k_proj = Linear(d_model, self.n_kv_heads * self.head_dim, bias=False)
60
+ self.v_proj = Linear(d_model, self.n_kv_heads * self.head_dim, bias=False)
61
+ self.o_proj = Linear(n_heads * self.head_dim, d_model, bias=False)
62
+ self.q_norm = RMSNorm(self.head_dim, norm_eps) if qk_norm else None
63
+ self.k_norm = RMSNorm(self.head_dim, norm_eps) if qk_norm else None
64
+ self.record, self.recorded = False, None # record=True keeps the last q and k, for drawing attention maps
65
+
66
+ def forward(self, x, cache=None, pad=None):
67
+ B, T, _ = x.shape
68
+ D = self.head_dim
69
+ start = 0 if cache is None else len(cache)
70
+ if pad is not None:
71
+ start = start - np.asarray(pad) # each row's first real token sits at position 0, after its left padding
72
+ q = self.q_proj(x).reshape(B, T, self.n_heads, D)
73
+ k = self.k_proj(x).reshape(B, T, self.n_kv_heads, D)
74
+ if self.q_norm is not None:
75
+ q, k = self.q_norm(q), self.k_norm(k)
76
+ q, k = q.transpose(1, 2), k.transpose(1, 2) # (B, H, T, D), (B, Hkv, T, D)
77
+ v = self.v_proj(x).reshape(B, T, self.n_kv_heads, D).transpose(1, 2)
78
+ if self.rope:
79
+ q, k = apply_rope(q, start, self.rope_theta), apply_rope(k, start, self.rope_theta)
80
+ if cache is not None:
81
+ k, v = cache.update(k, v)
82
+ if self.record:
83
+ self.recorded = q.numpy(), k.numpy()
84
+
85
+ out = scaled_dot_product_attention(q, k, v, pad) # (B, H, T, D)
86
+ return self.o_proj(out.transpose(1, 2).reshape(B, T, self.n_heads * D))
@@ -0,0 +1,11 @@
1
+ import functools
2
+ import importlib
3
+
4
+ DEVICES = ("cpu", "metal", "cuda")
5
+
6
+
7
+ @functools.cache
8
+ def get(device):
9
+ if device not in DEVICES:
10
+ raise ValueError(f"unknown device {device!r}, expected one of {DEVICES}")
11
+ return importlib.import_module(f"{__name__}.{device}")
@@ -0,0 +1,116 @@
1
+ import numpy as np
2
+ from numpy import (
3
+ broadcast_to,
4
+ concatenate,
5
+ cumsum,
6
+ exp,
7
+ expand_dims,
8
+ float32,
9
+ full,
10
+ inf,
11
+ log,
12
+ ones_like,
13
+ pad,
14
+ repeat,
15
+ sign,
16
+ stack,
17
+ triu,
18
+ where,
19
+ zeros_like,
20
+ )
21
+
22
+ __all__ = [
23
+ "asarray",
24
+ "broadcast_to",
25
+ "clear_cache",
26
+ "col2im",
27
+ "concatenate",
28
+ "cumsum",
29
+ "dtype",
30
+ "exp",
31
+ "expand_dims",
32
+ "float32",
33
+ "full",
34
+ "im2col",
35
+ "inf",
36
+ "is_float",
37
+ "log",
38
+ "ones_like",
39
+ "pad",
40
+ "realize",
41
+ "repeat",
42
+ "scatter_add",
43
+ "sigmoid",
44
+ "sign",
45
+ "stack",
46
+ "to_numpy",
47
+ "triu",
48
+ "where",
49
+ "zeros_like",
50
+ ]
51
+
52
+
53
+ def asarray(x, dtype=None):
54
+ """Any array, list, or number -> an array of this backend."""
55
+ return np.asarray(x, dtype=dtype)
56
+
57
+
58
+ def to_numpy(x):
59
+ return np.array(x) # a copy, like every other backend's device-to-host transfer
60
+
61
+
62
+ def dtype(name):
63
+ return np.dtype(name)
64
+
65
+
66
+ def is_float(dt):
67
+ return np.issubdtype(dt, np.floating)
68
+
69
+
70
+ def realize(*arrays):
71
+ """NumPy runs each op as it is called, so there is nothing to wait for."""
72
+
73
+
74
+ def clear_cache():
75
+ """NumPy keeps no memory pool to free."""
76
+
77
+
78
+ def sigmoid(x):
79
+ # exp(-x) overflows to inf for very negative x, and 1 / inf = 0 is correct
80
+ with np.errstate(over="ignore"):
81
+ return 1.0 / (1.0 + np.exp(-x))
82
+
83
+
84
+ def scatter_add(shape, key, values):
85
+ """zeros(shape) with `values` added at `key`, summing where an index repeats."""
86
+ out = np.zeros(shape, dtype=values.dtype)
87
+ np.add.at(out, key, values)
88
+ return out
89
+
90
+
91
+ def im2col(x, kh, kw, stride, padding, xp=np):
92
+ """(N, C, H, W) -> (N*OH*OW, C*kh*kw), one row per output pixel; no in-place writes, so MLX runs it."""
93
+ N, C = x.shape[:2]
94
+ if padding > 0:
95
+ x = xp.pad(x, ((0, 0), (0, 0), (padding, padding), (padding, padding)))
96
+ OH = (x.shape[2] - kh) // stride + 1
97
+ OW = (x.shape[3] - kw) // stride + 1
98
+ # channels last so the reshape copies nothing: 2.95 GB peak vs 5.37 GB for NCHW (256ch 3x3, 512x512, M4)
99
+ x = x.transpose(0, 2, 3, 1)
100
+ patches = [x[:, y:y + stride * OH:stride, xx:xx + stride * OW:stride, :] for y in range(kh) for xx in range(kw)]
101
+ return xp.stack(patches, axis=-1).reshape(N * OH * OW, C * kh * kw) # (N, OH, OW, C, kh*kw) -> rows
102
+
103
+
104
+ def col2im(cols, shape, kh, kw, stride, padding):
105
+ """Inverse of im2col: add every patch row back onto the pixels it came from."""
106
+ N, C, H, W = shape
107
+ H_pad, W_pad = H + 2 * padding, W + 2 * padding
108
+ OH = (H_pad - kh) // stride + 1
109
+ OW = (W_pad - kw) // stride + 1
110
+
111
+ cols = cols.reshape(N, OH, OW, C, kh, kw).transpose(0, 3, 4, 5, 1, 2)
112
+ img = np.zeros((N, C, H_pad, W_pad), dtype=cols.dtype)
113
+ for y in range(kh):
114
+ for xx in range(kw):
115
+ img[:, :, y:y + stride * OH:stride, xx:xx + stride * OW:stride] += cols[:, :, y, xx]
116
+ return img[:, :, padding:H_pad - padding, padding:W_pad - padding]
@@ -0,0 +1,96 @@
1
+ import cupy as cp
2
+ import cupyx
3
+ from cupy import (
4
+ broadcast_to,
5
+ concatenate,
6
+ cumsum,
7
+ exp,
8
+ expand_dims,
9
+ float32,
10
+ full,
11
+ inf,
12
+ log,
13
+ ones_like,
14
+ pad,
15
+ repeat,
16
+ sign,
17
+ stack,
18
+ triu,
19
+ where,
20
+ zeros_like,
21
+ )
22
+
23
+ from . import cpu, cuda_attention
24
+ from .cpu import dtype, is_float, realize
25
+
26
+ __all__ = [
27
+ "asarray",
28
+ "broadcast_to",
29
+ "clear_cache",
30
+ "col2im",
31
+ "concatenate",
32
+ "cumsum",
33
+ "dtype",
34
+ "exp",
35
+ "expand_dims",
36
+ "float32",
37
+ "full",
38
+ "fused_attention",
39
+ "fused_attention_backward",
40
+ "im2col",
41
+ "inf",
42
+ "is_float",
43
+ "log",
44
+ "ones_like",
45
+ "pad",
46
+ "realize",
47
+ "repeat",
48
+ "scatter_add",
49
+ "sigmoid",
50
+ "sign",
51
+ "stack",
52
+ "to_numpy",
53
+ "triu",
54
+ "where",
55
+ "zeros_like",
56
+ ]
57
+
58
+
59
+ def clear_cache():
60
+ cp.get_default_memory_pool().free_all_blocks()
61
+
62
+ def asarray(x, dtype=None):
63
+ return cp.asarray(x, dtype=dtype)
64
+
65
+
66
+ def to_numpy(x):
67
+ return cp.asnumpy(x)
68
+
69
+
70
+ def fused_attention(q, k, v):
71
+ """Triton FlashAttention forward, or None for shapes the kernels don't cover."""
72
+ return cuda_attention.attention(q, k, v) if cuda_attention.supported(q) else None
73
+
74
+
75
+ def fused_attention_backward(q, k, v, out, lse, grad):
76
+ return cuda_attention.attention_backward(q, k, v, out, lse, grad)
77
+
78
+
79
+ def sigmoid(x):
80
+ return 1.0 / (1.0 + cp.exp(-x))
81
+
82
+
83
+ def scatter_add(shape, key, values):
84
+ out = cp.zeros(shape, dtype=values.dtype)
85
+ cupyx.scatter_add(out, key, values)
86
+ return out
87
+
88
+
89
+ # shortcut: col2im (conv backward) round-trips through the CPU, port it to CuPy if a CNN trains on cuda
90
+ def im2col(x, *args):
91
+ return cpu.im2col(x, *args, xp=cp)
92
+
93
+
94
+ def col2im(cols, *args):
95
+ return cp.asarray(cpu.col2im(cp.asnumpy(cols), *args))
96
+
@@ -0,0 +1,138 @@
1
+ import math
2
+ import os
3
+
4
+ import cupy as cp
5
+ import triton
6
+ import triton.language as tl
7
+
8
+ # match CuPy's matmuls: tf32 only under CUPY_TF32=1 (training), ieee float32 otherwise (tests)
9
+ _PRECISION = "tf32" if os.environ.get("CUPY_TF32") == "1" else "ieee"
10
+
11
+
12
+ class _Arg:
13
+ """The data_ptr and dtype Triton reads from a tensor, so CuPy arrays need no torch."""
14
+ def __init__(self, a):
15
+ self.a, self.dtype = a, a.dtype
16
+
17
+ def data_ptr(self):
18
+ return self.a.data.ptr
19
+
20
+
21
+ @triton.jit
22
+ def _forward(Q, K, V, O, LSE, T, S, H, HKV, scale, D: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, PREC: tl.constexpr):
23
+ qb, bh = tl.program_id(0), tl.program_id(1)
24
+ kv = (bh // H * HKV + bh % H // (H // HKV)) * S * D
25
+ rows = qb * BLOCK_M + tl.arange(0, BLOCK_M)
26
+ dims = tl.arange(0, D)
27
+ q = tl.load(Q + (bh * T + rows[:, None]) * D + dims[None, :], mask=rows[:, None] < T, other=0.0)
28
+ off = S - T
29
+ m = tl.full([BLOCK_M], float("-inf"), tl.float32)
30
+ l = tl.zeros([BLOCK_M], tl.float32)
31
+ acc = tl.zeros([BLOCK_M, D], tl.float32)
32
+ for start in range(0, tl.minimum(off + qb * BLOCK_M + BLOCK_M, S), BLOCK_N): # later keys are hidden from every row here
33
+ cols = start + tl.arange(0, BLOCK_N)
34
+ k = tl.load(K + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
35
+ v = tl.load(V + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
36
+ s = tl.dot(q, tl.trans(k), input_precision=PREC) * scale
37
+ s = tl.where((cols[None, :] <= off + rows[:, None]) & (cols[None, :] < S), s, float("-inf"))
38
+ m_new = tl.maximum(m, tl.max(s, 1))
39
+ p = tl.exp(s - m_new[:, None])
40
+ corr = tl.exp(m - m_new)
41
+ l = l * corr + tl.sum(p, 1)
42
+ acc = acc * corr[:, None] + tl.dot(p, v, input_precision=PREC)
43
+ m = m_new
44
+ tl.store(O + (bh * T + rows[:, None]) * D + dims[None, :], acc / l[:, None], mask=rows[:, None] < T)
45
+ tl.store(LSE + bh * T + rows, m + tl.log(l), mask=rows < T)
46
+
47
+
48
+ @triton.jit
49
+ def _backward_kv(Q, K, V, DO, LSE, DELTA, DK, DV, T, S, H, HKV, scale,
50
+ D: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, PREC: tl.constexpr):
51
+ kb, bh = tl.program_id(0), tl.program_id(1)
52
+ kv = (bh // H * HKV + bh % H // (H // HKV)) * S * D
53
+ cols = kb * BLOCK_N + tl.arange(0, BLOCK_N)
54
+ dims = tl.arange(0, D)
55
+ k = tl.load(K + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
56
+ v = tl.load(V + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
57
+ off = S - T
58
+ dk = tl.zeros([BLOCK_N, D], tl.float32)
59
+ dv = tl.zeros([BLOCK_N, D], tl.float32)
60
+ first = tl.maximum(kb * BLOCK_N - off, 0) // BLOCK_M * BLOCK_M # earlier query rows see none of these keys
61
+ for start in range(first, T, BLOCK_M):
62
+ rows = start + tl.arange(0, BLOCK_M)
63
+ q = tl.load(Q + (bh * T + rows[:, None]) * D + dims[None, :], mask=rows[:, None] < T, other=0.0)
64
+ do = tl.load(DO + (bh * T + rows[:, None]) * D + dims[None, :], mask=rows[:, None] < T, other=0.0)
65
+ lse = tl.load(LSE + bh * T + rows, mask=rows < T, other=0.0)
66
+ delta = tl.load(DELTA + bh * T + rows, mask=rows < T, other=0.0)
67
+ visible = (cols[None, :] <= off + rows[:, None]) & (rows[:, None] < T) & (cols[None, :] < S)
68
+ p = tl.where(visible, tl.exp(tl.dot(q, tl.trans(k), input_precision=PREC) * scale - lse[:, None]), 0.0)
69
+ dv += tl.dot(tl.trans(p), do, input_precision=PREC)
70
+ ds = p * (tl.dot(do, tl.trans(v), input_precision=PREC) - delta[:, None])
71
+ dk += tl.dot(tl.trans(ds), q, input_precision=PREC) * scale
72
+ # per query head; the caller sums each group into its kv head
73
+ tl.store(DK + (bh * S + cols[:, None]) * D + dims[None, :], dk, mask=cols[:, None] < S)
74
+ tl.store(DV + (bh * S + cols[:, None]) * D + dims[None, :], dv, mask=cols[:, None] < S)
75
+
76
+
77
+ @triton.jit
78
+ def _backward_q(Q, K, V, DO, LSE, DELTA, DQ, T, S, H, HKV, scale, D: tl.constexpr, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr, PREC: tl.constexpr):
79
+ qb, bh = tl.program_id(0), tl.program_id(1)
80
+ kv = (bh // H * HKV + bh % H // (H // HKV)) * S * D
81
+ rows = qb * BLOCK_M + tl.arange(0, BLOCK_M)
82
+ dims = tl.arange(0, D)
83
+ q = tl.load(Q + (bh * T + rows[:, None]) * D + dims[None, :], mask=rows[:, None] < T, other=0.0)
84
+ do = tl.load(DO + (bh * T + rows[:, None]) * D + dims[None, :], mask=rows[:, None] < T, other=0.0)
85
+ lse = tl.load(LSE + bh * T + rows, mask=rows < T, other=0.0)
86
+ delta = tl.load(DELTA + bh * T + rows, mask=rows < T, other=0.0)
87
+ off = S - T
88
+ dq = tl.zeros([BLOCK_M, D], tl.float32)
89
+ for start in range(0, tl.minimum(off + qb * BLOCK_M + BLOCK_M, S), BLOCK_N):
90
+ cols = start + tl.arange(0, BLOCK_N)
91
+ k = tl.load(K + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
92
+ v = tl.load(V + kv + cols[:, None] * D + dims[None, :], mask=cols[:, None] < S, other=0.0)
93
+ visible = (cols[None, :] <= off + rows[:, None]) & (rows[:, None] < T) & (cols[None, :] < S)
94
+ p = tl.where(visible, tl.exp(tl.dot(q, tl.trans(k), input_precision=PREC) * scale - lse[:, None]), 0.0)
95
+ ds = p * (tl.dot(do, tl.trans(v), input_precision=PREC) - delta[:, None])
96
+ dq += tl.dot(ds, k, input_precision=PREC) * scale
97
+ tl.store(DQ + (bh * T + rows[:, None]) * D + dims[None, :], dq, mask=rows[:, None] < T)
98
+
99
+
100
+ def _blocks(D):
101
+ # D=128 at 64x32 blocks needs 114,688 B of shared memory; the 3090 has 101,376
102
+ return (64, 64) if D <= 64 else (32, 32)
103
+
104
+
105
+ def supported(q):
106
+ D = q.shape[-1]
107
+ return q.dtype == cp.float32 and 16 <= D <= 128 and D & (D - 1) == 0 # tl.arange needs a power of two
108
+
109
+
110
+ def attention(q, k, v):
111
+ """Causal FlashAttention-2: q (B, H, T, D), k, v (B, Hkv, S, D) -> out (B, H, T, D), lse (B, H, T)."""
112
+ B, H, T, D = q.shape
113
+ S, Hkv = k.shape[2], k.shape[1]
114
+ q, k, v = (cp.ascontiguousarray(x) for x in (q, k, v))
115
+ out, lse = cp.empty_like(q), cp.empty((B, H, T), cp.float32)
116
+ bm, bn = _blocks(D)
117
+ _forward[(triton.cdiv(T, bm), B * H)](_Arg(q), _Arg(k), _Arg(v), _Arg(out), _Arg(lse), T, S, H, Hkv, 1 / math.sqrt(D),
118
+ D=D, BLOCK_M=bm, BLOCK_N=bn, PREC=_PRECISION)
119
+ return out, lse
120
+
121
+
122
+ def attention_backward(q, k, v, out, lse, dout):
123
+ """(dq, dk, dv) of attention(q, k, v)[0] given dout, rebuilding the softmax from out and lse."""
124
+ B, H, T, D = q.shape
125
+ S, Hkv = k.shape[2], k.shape[1]
126
+ q, k, v, out, dout = (cp.ascontiguousarray(x) for x in (q, k, v, out, dout))
127
+ delta = (dout * out).sum(axis=-1) # rowsum(dO * O), the softmax backward's correction term
128
+ dq = cp.empty_like(q)
129
+ dk, dv = cp.empty((B, H, S, D), cp.float32), cp.empty((B, H, S, D), cp.float32)
130
+ bm, bn = _blocks(D)
131
+ args = (T, S, H, Hkv, 1 / math.sqrt(D))
132
+ _backward_kv[(triton.cdiv(S, bn), B * H)](*map(_Arg, (q, k, v, dout, lse, delta, dk, dv)), *args, D=D, BLOCK_M=bm, BLOCK_N=bn, PREC=_PRECISION,
133
+ num_stages=1)
134
+ # one pipeline stage: the backward holds more tiles, and three stages overflow shared memory
135
+ _backward_q[(triton.cdiv(T, bm), B * H)](*map(_Arg, (q, k, v, dout, lse, delta, dq)), *args, D=D, BLOCK_M=bm, BLOCK_N=bn, PREC=_PRECISION,
136
+ num_stages=1)
137
+ rep = H // Hkv
138
+ return dq, dk.reshape(B, Hkv, rep, S, D).sum(axis=2), dv.reshape(B, Hkv, rep, S, D).sum(axis=2)
@@ -0,0 +1,106 @@
1
+ import mlx.core as mx
2
+ import numpy as np
3
+ from mlx.core import (
4
+ broadcast_to,
5
+ concatenate,
6
+ cumsum,
7
+ exp,
8
+ expand_dims,
9
+ float32,
10
+ full,
11
+ inf,
12
+ log,
13
+ ones_like,
14
+ pad,
15
+ repeat,
16
+ sigmoid,
17
+ sign,
18
+ stack,
19
+ triu,
20
+ where,
21
+ zeros_like,
22
+ )
23
+
24
+ from . import cpu, metal_attention, metal_quant
25
+
26
+ __all__ = [
27
+ "asarray",
28
+ "broadcast_to",
29
+ "clear_cache",
30
+ "col2im",
31
+ "concatenate",
32
+ "cumsum",
33
+ "dtype",
34
+ "exp",
35
+ "expand_dims",
36
+ "float32",
37
+ "full",
38
+ "fused_attention",
39
+ "im2col",
40
+ "inf",
41
+ "is_float",
42
+ "log",
43
+ "ones_like",
44
+ "pad",
45
+ "quant_matmul",
46
+ "realize",
47
+ "repeat",
48
+ "scatter_add",
49
+ "sigmoid",
50
+ "sign",
51
+ "stack",
52
+ "to_numpy",
53
+ "triu",
54
+ "where",
55
+ "zeros_like",
56
+ ]
57
+
58
+
59
+ def asarray(x, dtype=None):
60
+ if isinstance(x, mx.array):
61
+ return x if dtype is None or x.dtype == dtype else x.astype(dtype)
62
+ return mx.array(x) if dtype is None else mx.array(x, dtype=dtype)
63
+
64
+
65
+ def to_numpy(x):
66
+ # numpy has no bfloat16; widening to float32 is exact
67
+ return np.array(x.astype(mx.float32) if x.dtype == mx.bfloat16 else x)
68
+
69
+
70
+ def dtype(name):
71
+ return getattr(mx, name)
72
+
73
+
74
+ def is_float(dt):
75
+ return mx.issubdtype(dt, mx.floating)
76
+
77
+
78
+ realize = mx.eval
79
+ clear_cache = mx.clear_cache # mlx keeps freed buffers cached for reuse until this
80
+
81
+
82
+ def scatter_add(shape, key, values):
83
+ return mx.zeros(shape, dtype=values.dtype).at[key].add(values)
84
+
85
+
86
+ # shortcut: col2im (conv backward) round-trips through the CPU, port it to MLX if a CNN trains on metal
87
+ def im2col(x, *args):
88
+ return cpu.im2col(x, *args, xp=mx)
89
+
90
+
91
+ def col2im(cols, *args):
92
+ return mx.array(cpu.col2im(to_numpy(cols), *args))
93
+
94
+
95
+ def fused_attention(q, k, v):
96
+ """Metal attention forward, or None for shapes the kernels don't cover."""
97
+ D = q.shape[-1]
98
+ if D % 8 or D > 128:
99
+ return None
100
+ # one query row (decode) leaves v3's 32-row threadgroups mostly idle; v1 measured faster there
101
+ return metal_attention.attention(q, k, v, "v1" if q.shape[2] == 1 else "v3")
102
+
103
+
104
+ def quant_matmul(x, packed, scales, biases, bits, group):
105
+ """x (M, K) @ dequantized W (N, K).T as one Metal kernel, for decode-sized M."""
106
+ return metal_quant.quant_matmul(x, packed, scales, biases, bits, group)