stackgrad 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stackgrad/layers.py ADDED
@@ -0,0 +1,215 @@
1
+ import numpy as np
2
+
3
+ from . import functional as F
4
+ from . import tensor as T
5
+ from .module import Module
6
+ from .tensor import Tensor
7
+
8
+
9
+ class Linear(Module):
10
+ """Fully-connected layer `y = x @ W + b` over the last dim, Kaiming init by default (`init='xavier'`)."""
11
+ def __init__(self, in_features, out_features, bias=True, init='kaiming'):
12
+ super().__init__()
13
+ assert in_features > 0 and out_features > 0, "features must be positive"
14
+ if init == 'xavier':
15
+ scale = np.sqrt(2.0 / (in_features + out_features))
16
+ else:
17
+ scale = np.sqrt(2.0 / in_features)
18
+ self.weight = Tensor(
19
+ (np.random.randn(in_features, out_features) * scale).astype(np.float32),
20
+ requires_grad=True
21
+ )
22
+ self.bias = Tensor(
23
+ np.zeros(out_features, dtype=np.float32),
24
+ requires_grad=True
25
+ ) if bias else None
26
+
27
+ def forward(self, x):
28
+ assert x.shape[-1] == self.weight.shape[0], f"Linear expects last dim {self.weight.shape[0]}, got {x.shape[-1]}"
29
+ # flatten to 2-d: one GEMM, and the weight grad needs no (batch, in, out) stack to sum
30
+ rows = x.reshape(-1, x.shape[-1])
31
+ if T._autocast is None:
32
+ y = rows @ self.weight
33
+ else:
34
+ y = (rows.cast(T._autocast) @ self.weight.cast(T._autocast)).cast("float32")
35
+ y = y.reshape(*x.shape[:-1], self.weight.shape[1])
36
+ if self.bias is not None:
37
+ y = y + self.bias
38
+ return y
39
+
40
+
41
+ class ReLU(Module):
42
+ """Rectified linear unit, `max(0, x)`."""
43
+ def forward(self, x):
44
+ return F.relu(x)
45
+
46
+
47
+ class Sigmoid(Module):
48
+ """Logistic sigmoid, `1 / (1 + e**-x)`."""
49
+ def forward(self, x):
50
+ return F.sigmoid(x)
51
+
52
+
53
+ class Tanh(Module):
54
+ """Hyperbolic tangent."""
55
+ def forward(self, x):
56
+ return F.tanh(x)
57
+
58
+
59
+ class Softmax(Module):
60
+ """Softmax over `axis`, turning logits into a probability distribution."""
61
+ def __init__(self, axis=-1):
62
+ super().__init__()
63
+ self.axis = axis
64
+
65
+ def forward(self, x):
66
+ return F.softmax(x, axis=self.axis)
67
+
68
+
69
+ class GELU(Module):
70
+ def forward(self, x):
71
+ return F.gelu(x)
72
+
73
+
74
+ class LayerNorm(Module):
75
+ """Normalize over the last dimension of any (..., num_features) input."""
76
+ def __init__(self, num_features, eps=1e-5):
77
+ super().__init__()
78
+ assert num_features > 0, "num_features must be positive"
79
+ self.gamma = Tensor(np.ones(num_features, dtype=np.float32), requires_grad=True)
80
+ self.beta = Tensor(np.zeros(num_features, dtype=np.float32), requires_grad=True)
81
+ self.eps = eps
82
+
83
+ def forward(self, x):
84
+ assert x.shape[-1] == self.gamma.shape[0], f"LayerNorm expects last dim {self.gamma.shape[0]}, got {x.shape[-1]}"
85
+ mean = x.mean(axis=-1, keepdims=True)
86
+ diff = x - mean
87
+ var = (diff ** 2).mean(axis=-1, keepdims=True)
88
+ x_hat = diff / (var + self.eps) ** 0.5
89
+ return x_hat * self.gamma + self.beta
90
+
91
+
92
+ class GroupNorm(Module):
93
+ """Normalize each group of channels of an (N, C, ...) input, then scale and shift each channel."""
94
+ def __init__(self, num_groups, num_channels, eps=1e-5):
95
+ super().__init__()
96
+ assert num_channels % num_groups == 0, "channels must split evenly into groups"
97
+ self.num_groups, self.eps = num_groups, eps
98
+ self.weight = Tensor(np.ones(num_channels, dtype=np.float32), requires_grad=True)
99
+ self.bias = Tensor(np.zeros(num_channels, dtype=np.float32), requires_grad=True)
100
+
101
+ def forward(self, x):
102
+ g = x.reshape(x.shape[0], self.num_groups, -1)
103
+ mean = g.mean(axis=-1, keepdims=True)
104
+ diff = g - mean
105
+ y = (diff / ((diff ** 2).mean(axis=-1, keepdims=True) + self.eps) ** 0.5).reshape(*x.shape)
106
+ per_channel = (1, x.shape[1]) + (1,) * (len(x.shape) - 2)
107
+ return y * self.weight.reshape(*per_channel) + self.bias.reshape(*per_channel)
108
+
109
+
110
+ class Embedding(Module):
111
+ """Lookup table mapping integer ids to dense vectors."""
112
+ def __init__(self, num_embeddings, embed_dim):
113
+ super().__init__()
114
+ assert num_embeddings > 0 and embed_dim > 0, "sizes must be positive"
115
+ self.weight = Tensor(
116
+ (np.random.randn(num_embeddings, embed_dim) * 0.02).astype(np.float32),
117
+ requires_grad=True
118
+ )
119
+
120
+ def forward(self, idx):
121
+ return self.weight[np.asarray(idx)]
122
+
123
+
124
+ class Dropout(Module):
125
+ """Zero each activation with probability `p` during training, scaling the rest; a no-op in eval."""
126
+ def __init__(self, p=0.5):
127
+ super().__init__()
128
+ assert 0.0 <= p < 1.0, "dropout probability must be in [0, 1)"
129
+ self.p = p
130
+
131
+ def forward(self, x):
132
+ if not self._training or self.p == 0.0:
133
+ return x
134
+ mask = (np.random.rand(*x.shape) > self.p).astype(np.float32) / (1.0 - self.p)
135
+ return x * x._const(mask)
136
+
137
+
138
+ class BatchNorm1d(Module):
139
+ """Normalize a `(batch, features)` input per feature, tracking running stats for eval."""
140
+ def __init__(self, num_features, eps=1e-5, momentum=0.1):
141
+ super().__init__()
142
+ assert num_features > 0, "num_features must be positive"
143
+ self.gamma = Tensor(np.ones(num_features, dtype=np.float32), requires_grad=True)
144
+ self.beta = Tensor(np.zeros(num_features, dtype=np.float32), requires_grad=True)
145
+ self.eps = eps
146
+ self.momentum = momentum
147
+ self.running_mean = np.zeros(num_features, dtype=np.float32)
148
+ self.running_var = np.ones(num_features, dtype=np.float32)
149
+
150
+ def forward(self, x):
151
+ assert x.ndim == 2 and x.shape[1] == self.gamma.shape[0], \
152
+ f"Expected input shape (batch, {self.gamma.shape[0]}), got {x.shape}"
153
+
154
+ if self._training:
155
+ mean = x.mean(axis=0, keepdims=True)
156
+ diff = x - mean
157
+ var = (diff ** 2).mean(axis=0, keepdims=True)
158
+ x_hat = diff / (var + self.eps) ** 0.5
159
+
160
+ # running stats stay host NumPy arrays on any device, so state_dict saves them as buffers
161
+ self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean.numpy().flatten()
162
+ self.running_var = (1 - self.momentum) * self.running_var + self.momentum * var.numpy().flatten()
163
+ else:
164
+ rm = x._const(self.running_mean.reshape(1, -1))
165
+ rv = x._const(self.running_var.reshape(1, -1))
166
+ x_hat = (x - rm) / (rv + self.eps) ** 0.5
167
+
168
+ return x_hat * self.gamma + self.beta
169
+
170
+
171
+ class RMSNorm(Module):
172
+ """Divide each vector by its root-mean-square, then scale. LayerNorm without the mean and bias."""
173
+ def __init__(self, num_features, eps=1e-5):
174
+ super().__init__()
175
+ self.weight = Tensor(np.ones(num_features, dtype=np.float32), requires_grad=True)
176
+ self.eps = eps
177
+
178
+ def forward(self, x):
179
+ return x * ((x * x).mean(axis=-1, keepdims=True) + self.eps) ** -0.5 * self.weight
180
+
181
+
182
+ class DepthAttention(Module):
183
+ """Attention Residuals: a layer's input is a learned softmax mix of earlier layers' outputs, not their sum."""
184
+ def __init__(self, d_model, eps=1e-5):
185
+ super().__init__()
186
+ self.query = Tensor(np.zeros(d_model, dtype=np.float32), requires_grad=True)
187
+ self.norm = RMSNorm(d_model, eps)
188
+
189
+ def forward(self, sources):
190
+ v = T.stack(sources) # (N, B, T, D)
191
+ scores = (self.norm(v) * self.query).sum(axis=-1, keepdims=True) # (N, B, T, 1): keys normalized, values raw
192
+ return (F.softmax(scores, axis=0) * v).sum(axis=0)
193
+
194
+
195
+ class SiLUMLP(Module):
196
+ """Ungated feed-forward w2(silu(w1 x)), the baseline for the CS336 SwiGLU ablation."""
197
+ def __init__(self, d_model, d_ff):
198
+ super().__init__()
199
+ self.w1 = Linear(d_model, d_ff, bias=False)
200
+ self.w2 = Linear(d_ff, d_model, bias=False)
201
+
202
+ def forward(self, x):
203
+ return self.w2(F.silu(self.w1(x)))
204
+
205
+
206
+ class SwiGLU(Module):
207
+ """Gated feed-forward from Llama and Qwen: w2(silu(w1 x) * w3 x), no biases."""
208
+ def __init__(self, d_model, d_ff):
209
+ super().__init__()
210
+ self.w1 = Linear(d_model, d_ff, bias=False)
211
+ self.w2 = Linear(d_ff, d_model, bias=False)
212
+ self.w3 = Linear(d_model, d_ff, bias=False)
213
+
214
+ def forward(self, x):
215
+ return self.w2(F.silu(self.w1(x)) * self.w3(x))
stackgrad/lora.py ADDED
@@ -0,0 +1,54 @@
1
+ import numpy as np
2
+
3
+ from .layers import Linear
4
+ from .module import Module
5
+ from .tensor import Tensor
6
+
7
+ TARGETS = ("q_proj", "k_proj", "v_proj", "o_proj", "w1", "w2", "w3") # every matmul in a TransformerLM block
8
+
9
+
10
+ class LoRALinear(Module):
11
+ """A frozen Linear plus a trained low-rank update: y = x W + (alpha / r) (x A) B, B starting at zero."""
12
+ def __init__(self, base, r, alpha):
13
+ super().__init__()
14
+ n_in, n_out = base.weight.shape
15
+ self.base = base
16
+ # cast after dividing: numpy 2 promotes float32 / np.sqrt's float64 scalar to float64
17
+ init = (np.random.randn(n_in, r) / np.sqrt(n_in)).astype(np.float32)
18
+ self.A = Tensor(init, requires_grad=True, device=base.weight.device)
19
+ self.B = Tensor(np.zeros((r, n_out), np.float32), requires_grad=True, device=base.weight.device)
20
+ self.scale = alpha / r
21
+
22
+ def forward(self, x):
23
+ flat = x.reshape(-1, x.shape[-1])
24
+ delta = (flat @ self.A * self.scale) @ self.B # scaling the rank-r middle, not the full-width output, saves a copy
25
+ return self.base(x) + delta.reshape(*x.shape[:-1], delta.shape[-1])
26
+
27
+
28
+ def apply_lora(model, r=16, alpha=32):
29
+ """Freezes a TransformerLM and wraps each target Linear in a LoRALinear, so only A and B train."""
30
+ for _, p in model.named_parameters():
31
+ p.requires_grad = False
32
+ for block in model.blocks:
33
+ for parent in (block.attn, block.ffn):
34
+ for name in TARGETS:
35
+ if isinstance(getattr(parent, name, None), Linear):
36
+ setattr(parent, name, LoRALinear(getattr(parent, name), r, alpha))
37
+ return model
38
+
39
+
40
+ def save_adapter(model, path, r, alpha):
41
+ np.savez(path, r=r, alpha=alpha, **{name: p.numpy() for name, p in model.named_parameters()})
42
+
43
+
44
+ def merge_adapter(model, path):
45
+ """Add a saved adapter's (alpha / r) A B into the matching weights of a plain (un-wrapped) model."""
46
+ adapter = np.load(path)
47
+ scale = float(adapter["alpha"]) / float(adapter["r"])
48
+ params = dict(model.named_parameters())
49
+ layers = [k[:-len(".A")] for k in adapter.files if k.endswith(".A")]
50
+ for layer in layers:
51
+ w = params[f"{layer}.weight"] # a KeyError here means the adapter was trained on another model
52
+ assert w.device == "cpu", "merge the adapter before moving the model to a device"
53
+ w.data = w.data + (scale * (adapter[f"{layer}.A"] @ adapter[f"{layer}.B"])).astype(w.dtype)
54
+ return len(layers)
stackgrad/loss.py ADDED
@@ -0,0 +1,28 @@
1
+ import math
2
+
3
+ import numpy as np
4
+
5
+ from . import functional as F
6
+ from .tensor import Tensor
7
+
8
+
9
+ def mse_loss(input, target):
10
+ """Mean squared error between predictions and targets."""
11
+ return ((input - target) ** 2).mean()
12
+
13
+
14
+ def bce_loss(input, target):
15
+ """Binary cross-entropy on probabilities, not logits."""
16
+ p = input.clamp(1e-7, 1 - 1e-7)
17
+ return -(target * p.log() + (1.0 - target) * (1.0 - p).log()).mean()
18
+
19
+
20
+ def cross_entropy_loss(input, target):
21
+ """Softmax cross-entropy over the last axis; `target` holds class indices or one-hot rows."""
22
+ log_probs = F.log_softmax(input, axis=-1)
23
+ labels = target.numpy() if isinstance(target, Tensor) else np.asarray(target)
24
+ if labels.shape == input.shape:
25
+ return -(log_probs * target).sum() / (math.prod(input.shape) // input.shape[-1])
26
+ labels = labels.reshape(-1).astype(np.int64)
27
+ log_probs = log_probs.reshape(-1, input.shape[-1])
28
+ return -log_probs[np.arange(len(labels)), labels].mean()
stackgrad/mlops.py ADDED
@@ -0,0 +1,352 @@
1
+ import math
2
+
3
+ import numpy as np
4
+
5
+
6
+ class Function:
7
+ differentiable = True
8
+
9
+ def __init__(self, xp, *parents):
10
+ self.xp = xp
11
+ self.parents = parents
12
+ self.needs_grad = [p.requires_grad for p in parents]
13
+
14
+ def forward(self, *args, **kwargs):
15
+ raise NotImplementedError
16
+
17
+ def backward(self, grad):
18
+ raise NotImplementedError
19
+
20
+ def out_shape(self, *shapes, **kwargs):
21
+ return np.broadcast_shapes(*shapes)
22
+
23
+ def out_dtype(self, *dtypes, **kwargs):
24
+ """Promote the input dtypes as the backend does, by adding 0-d probes."""
25
+ out = dtypes[0]
26
+ for d in dtypes[1:]:
27
+ if d != out:
28
+ out = (self.xp.asarray(0, dtype=out) + self.xp.asarray(0, dtype=d)).dtype
29
+ return out
30
+
31
+
32
+ def _reduced(shape, axis, keepdims):
33
+ axes = range(len(shape)) if axis is None else [a % len(shape) for a in (axis if isinstance(axis, (tuple, list)) else (axis,))]
34
+ return tuple(1 if i in axes else d for i, d in enumerate(shape) if keepdims or i not in axes)
35
+
36
+
37
+ def unbroadcast(grad, shape):
38
+ while grad.ndim > len(shape):
39
+ grad = grad.sum(axis=0)
40
+ for i, size in enumerate(shape):
41
+ if size == 1 and grad.shape[i] != 1:
42
+ grad = grad.sum(axis=i, keepdims=True)
43
+ return grad
44
+
45
+
46
+ class Recip(Function):
47
+ def forward(self, x):
48
+ self.out = 1.0 / x
49
+ return self.out
50
+
51
+ def backward(self, grad):
52
+ return (-grad * self.out * self.out,)
53
+
54
+
55
+ class Exp(Function):
56
+ def forward(self, x):
57
+ self.out = self.xp.exp(x)
58
+ return self.out
59
+
60
+ def backward(self, grad):
61
+ return (grad * self.out,)
62
+
63
+
64
+ class Log(Function):
65
+ def forward(self, x):
66
+ self.x = x
67
+ return self.xp.log(x)
68
+
69
+ def backward(self, grad):
70
+ return (grad / self.x,)
71
+
72
+
73
+ class Sigmoid(Function):
74
+ # a primitive: composed from exp/recip, a very negative x gives 0 * inf = nan in backward
75
+ def forward(self, x):
76
+ self.out = self.xp.sigmoid(x)
77
+ return self.out
78
+
79
+ def backward(self, grad):
80
+ return (grad * self.out * (1.0 - self.out),)
81
+
82
+
83
+ class Pow(Function):
84
+ def forward(self, x, c):
85
+ self.x, self.c = x, c
86
+ return x ** c
87
+
88
+ def backward(self, grad):
89
+ return (grad * self.c * self.x ** (self.c - 1),)
90
+
91
+
92
+ class Cast(Function):
93
+ def forward(self, x, dtype):
94
+ self.dtype = x.dtype
95
+ return x.astype(dtype)
96
+
97
+ def backward(self, grad):
98
+ return (grad.astype(self.dtype),)
99
+
100
+ def out_dtype(self, x, dtype):
101
+ return dtype
102
+
103
+
104
+ # binary ops: inputs that need a gradient arrive already expanded to the output shape
105
+
106
+ class Add(Function):
107
+ def forward(self, x, y):
108
+ return x + y
109
+
110
+ def backward(self, grad):
111
+ return grad, grad
112
+
113
+
114
+ class Mul(Function):
115
+ def forward(self, x, y):
116
+ self.x, self.y = x, y
117
+ return x * y
118
+
119
+ def backward(self, grad):
120
+ return (grad * self.y if self.needs_grad[0] else None,
121
+ grad * self.x if self.needs_grad[1] else None)
122
+
123
+
124
+ class Where(Function):
125
+ def forward(self, x, y, cond):
126
+ self.cond = cond
127
+ return self.xp.where(cond, x, y)
128
+
129
+ def backward(self, grad):
130
+ zero = self.xp.zeros_like(grad)
131
+ return self.xp.where(self.cond, grad, zero), self.xp.where(self.cond, zero, grad)
132
+
133
+ def out_dtype(self, x, y, cond):
134
+ return super().out_dtype(x, y)
135
+
136
+
137
+ class Less(Function):
138
+ differentiable = False
139
+
140
+ def forward(self, x, y):
141
+ return (x < y).astype(self.xp.float32)
142
+
143
+ def out_dtype(self, x, y):
144
+ return self.xp.float32
145
+
146
+
147
+ class Sum(Function):
148
+ def forward(self, x, axis, keepdims):
149
+ self.shape, self.axis, self.keepdims = x.shape, axis, keepdims
150
+ return x.sum(axis=axis, keepdims=keepdims)
151
+
152
+ def out_shape(self, x, axis, keepdims):
153
+ return _reduced(x, axis, keepdims)
154
+
155
+ def backward(self, grad):
156
+ if self.axis is not None and not self.keepdims:
157
+ grad = self.xp.expand_dims(grad, self.axis)
158
+ return (self.xp.broadcast_to(grad, self.shape),)
159
+
160
+
161
+ class Max(Function):
162
+ def forward(self, x, axis, keepdims):
163
+ self.x, self.axis, self.keepdims = x, axis, keepdims
164
+ self.out = x.max(axis=axis, keepdims=True)
165
+ return self.out if keepdims else x.max(axis=axis)
166
+
167
+ def out_shape(self, x, axis, keepdims):
168
+ return _reduced(x, axis, keepdims)
169
+
170
+ def backward(self, grad):
171
+ # ties split the gradient evenly
172
+ mask = (self.x == self.out).astype(self.x.dtype)
173
+ mask = mask / mask.sum(axis=self.axis, keepdims=True)
174
+ if self.axis is not None and not self.keepdims:
175
+ grad = self.xp.expand_dims(grad, self.axis)
176
+ return (mask * grad,)
177
+
178
+
179
+ class Reshape(Function):
180
+ def forward(self, x, shape):
181
+ self.shape = x.shape
182
+ return x.reshape(shape)
183
+
184
+ def backward(self, grad):
185
+ return (grad.reshape(self.shape),)
186
+
187
+ def out_shape(self, x, shape):
188
+ size = math.prod(x)
189
+ known = math.prod(d for d in shape if d != -1)
190
+ out = tuple(size // known if d == -1 else d for d in shape)
191
+ assert math.prod(out) == size, f"cannot reshape {x} into {shape}"
192
+ return out
193
+
194
+
195
+ class Permute(Function):
196
+ def forward(self, x, order):
197
+ self.order = order
198
+ return x.transpose(order)
199
+
200
+ def backward(self, grad):
201
+ inverse = [self.order.index(i) for i in range(len(self.order))]
202
+ return (grad.transpose(inverse),)
203
+
204
+ def out_shape(self, x, order):
205
+ return tuple(x[i] for i in order)
206
+
207
+
208
+ class Expand(Function):
209
+ """x already has the target rank; Tensor.expand adds the leading dims."""
210
+ def forward(self, x, shape):
211
+ self.shape = x.shape
212
+ return self.xp.broadcast_to(x, shape)
213
+
214
+ def backward(self, grad):
215
+ return (unbroadcast(grad, self.shape),)
216
+
217
+ def out_shape(self, x, shape):
218
+ return tuple(shape)
219
+
220
+
221
+ class Index(Function):
222
+ def forward(self, x, key):
223
+ self.shape, self.key = x.shape, key
224
+ return x[key]
225
+
226
+ def backward(self, grad):
227
+ # scatter_add: an index picked twice gets both gradients
228
+ return (self.xp.scatter_add(self.shape, self.key, grad),)
229
+
230
+ def out_shape(self, x, key):
231
+ """Index a zero-stride stand-in of shape x; index arrays become zeros, bool masks stay (their count matters)."""
232
+ def stand_in(k):
233
+ if not hasattr(k, "dtype"):
234
+ return k
235
+ if "bool" in str(k.dtype):
236
+ return self.xp.to_numpy(k)
237
+ return np.zeros(tuple(k.shape), dtype=np.intp)
238
+ key = tuple(map(stand_in, key)) if isinstance(key, tuple) else stand_in(key)
239
+ # shortcut: a gather allocates one byte per result element here, compute the shape if it shows in a profile
240
+ return np.lib.stride_tricks.as_strided(np.zeros(1, np.uint8), x, (0,) * len(x))[key].shape
241
+
242
+
243
+ class Cat(Function):
244
+ def forward(self, *xs, axis):
245
+ self.axis = axis
246
+ self.bounds = []
247
+ end = 0
248
+ for x in xs:
249
+ self.bounds.append((end, end + x.shape[axis]))
250
+ end += x.shape[axis]
251
+ return self.xp.concatenate(xs, axis=axis)
252
+
253
+ def backward(self, grad):
254
+ index = [slice(None)] * grad.ndim
255
+ grads = []
256
+ for start, end in self.bounds:
257
+ index[self.axis] = slice(start, end)
258
+ grads.append(grad[tuple(index)])
259
+ return tuple(grads)
260
+
261
+ def out_shape(self, *shapes, axis):
262
+ axis %= len(shapes[0])
263
+ return shapes[0][:axis] + (sum(s[axis] for s in shapes),) + shapes[0][axis + 1:]
264
+
265
+
266
+ class Im2col(Function):
267
+ def forward(self, x, kh, kw, stride, padding):
268
+ self.args = (x.shape, kh, kw, stride, padding)
269
+ return self.xp.im2col(x, kh, kw, stride, padding)
270
+
271
+ def backward(self, grad):
272
+ return (self.xp.col2im(grad, *self.args),)
273
+
274
+ def out_shape(self, x, kh, kw, stride, padding):
275
+ n, c, h, w = x
276
+ oh, ow = (h + 2 * padding - kh) // stride + 1, (w + 2 * padding - kw) // stride + 1
277
+ return (n * oh * ow, c * kh * kw)
278
+
279
+
280
+ # a primitive: as broadcast-multiply-sum it would lose BLAS
281
+
282
+ class Matmul(Function):
283
+ def forward(self, x, y):
284
+ self.x, self.y = x, y
285
+ return x @ y
286
+
287
+ def backward(self, grad):
288
+ return (unbroadcast(grad @ self.y.swapaxes(-1, -2), self.x.shape) if self.needs_grad[0] else None,
289
+ unbroadcast(self.x.swapaxes(-1, -2) @ grad, self.y.shape) if self.needs_grad[1] else None)
290
+
291
+ def out_shape(self, x, y):
292
+ """NumPy's matmul shape rule, 1-d operands included."""
293
+ x1, y1 = len(x) == 1, len(y) == 1
294
+ x, y = (1,) + x if x1 else x, y + (1,) if y1 else y
295
+ assert x[-1] == y[-2], f"cannot multiply {x} by {y}"
296
+ out = np.broadcast_shapes(x[:-2], y[:-2]) + (x[-2], y[-1])
297
+ out = out[:-2] + out[-1:] if x1 else out
298
+ return out[:-1] if y1 else out
299
+
300
+
301
+ class Attention(Function):
302
+ """Causal GQA attention, a primitive so a backend can fuse it; backward rebuilds p from the saved lse."""
303
+ def scores(self, q, k):
304
+ xp = self.xp
305
+ T, S = q.shape[2], k.shape[2]
306
+ k = xp.repeat(k, q.shape[1] // k.shape[1], axis=1)
307
+ s = q @ k.swapaxes(-1, -2) * q.shape[-1] ** -0.5
308
+ if self.causal: # query i sits at position S - T + i (kv cache)
309
+ s = s + xp.triu(xp.full((T, S), -xp.inf, dtype=q.dtype), k=S - T + 1)
310
+ if self.key_bias is not None:
311
+ s = s + xp.asarray(self.key_bias, dtype=q.dtype)[:, None, None, :]
312
+ return s
313
+
314
+ def forward(self, q, k, v, key_bias=None, causal=True):
315
+ """key_bias: optional (B, S) array added to each key's score (left-padding mask)."""
316
+ self.key_bias, self.causal = key_bias, causal
317
+ fused = getattr(self.xp, "fused_attention", None) if key_bias is None and causal else None # the kernels take no mask
318
+ out = fused(q, k, v) if fused else None
319
+ self.p = None
320
+ if out is None:
321
+ s = self.scores(q, k)
322
+ m = s.max(axis=-1, keepdims=True)
323
+ p = self.xp.exp(s - m)
324
+ l = p.sum(axis=-1, keepdims=True)
325
+ self.p = p / l
326
+ out = self.p @ self.xp.repeat(v, q.shape[1] // k.shape[1], axis=1), (m + self.xp.log(l))[..., 0]
327
+ self.saved = q, k, v, out[0], out[1].astype(q.dtype)
328
+ return out[0]
329
+
330
+ def backward(self, grad):
331
+ q, k, v, out, lse = self.saved
332
+ xp = self.xp
333
+ fused = getattr(xp, "fused_attention_backward", None)
334
+ if fused is not None and self.p is None: # p is None only after a fused forward
335
+ return fused(q, k, v, out, lse, grad)
336
+ B, Hkv, S, D = k.shape
337
+ rep = q.shape[1] // Hkv
338
+ p = self.p if self.p is not None else xp.exp(self.scores(q, k) - lse[..., None])
339
+ dv = p.swapaxes(-1, -2) @ grad
340
+ dp = grad @ xp.repeat(v, rep, axis=1).swapaxes(-1, -2)
341
+ # softmax backward: ds = p * (dp - rowsum(p * dp)), and rowsum(p * dp) = rowsum(grad * out)
342
+ ds = p * (dp - (grad * out).sum(axis=-1, keepdims=True)) * D ** -0.5
343
+ dq = ds @ xp.repeat(k, rep, axis=1)
344
+ dk = ds.swapaxes(-1, -2) @ q
345
+ # each kv head sums the grads of the `rep` query heads it served
346
+ return dq, dk.reshape(B, Hkv, rep, S, D).sum(axis=2), dv.reshape(B, Hkv, rep, S, D).sum(axis=2)
347
+
348
+ def out_shape(self, q, k, v, key_bias=None, causal=True):
349
+ return q[:-1] + v[-1:]
350
+
351
+ def out_dtype(self, q, k, v, key_bias=None, causal=True):
352
+ return q