stackgrad 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stackgrad/__init__.py +75 -0
- stackgrad/attention.py +86 -0
- stackgrad/backend/__init__.py +11 -0
- stackgrad/backend/cpu.py +116 -0
- stackgrad/backend/cuda.py +96 -0
- stackgrad/backend/cuda_attention.py +138 -0
- stackgrad/backend/metal.py +106 -0
- stackgrad/backend/metal_attention.py +195 -0
- stackgrad/backend/metal_quant.py +44 -0
- stackgrad/checkpoint.py +38 -0
- stackgrad/conv.py +63 -0
- stackgrad/data.py +28 -0
- stackgrad/functional.py +41 -0
- stackgrad/gpt.py +120 -0
- stackgrad/grad_check.py +55 -0
- stackgrad/layers.py +215 -0
- stackgrad/lora.py +54 -0
- stackgrad/loss.py +28 -0
- stackgrad/mlops.py +352 -0
- stackgrad/module.py +120 -0
- stackgrad/optim.py +121 -0
- stackgrad/profile.py +45 -0
- stackgrad/quant.py +66 -0
- stackgrad/qwen3.py +104 -0
- stackgrad/sampling.py +39 -0
- stackgrad/serve/api.py +434 -0
- stackgrad/serve/engine.py +53 -0
- stackgrad/serve/kv_cache.py +35 -0
- stackgrad/serve/runner.py +63 -0
- stackgrad/serve/scheduler.py +88 -0
- stackgrad/speculative.py +68 -0
- stackgrad/state.py +27 -0
- stackgrad/tensor.py +377 -0
- stackgrad/tokenizer.py +202 -0
- stackgrad-0.2.0.dist-info/METADATA +172 -0
- stackgrad-0.2.0.dist-info/RECORD +38 -0
- stackgrad-0.2.0.dist-info/WHEEL +4 -0
- stackgrad-0.2.0.dist-info/licenses/LICENSE +21 -0
stackgrad/layers.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from . import functional as F
|
|
4
|
+
from . import tensor as T
|
|
5
|
+
from .module import Module
|
|
6
|
+
from .tensor import Tensor
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Linear(Module):
|
|
10
|
+
"""Fully-connected layer `y = x @ W + b` over the last dim, Kaiming init by default (`init='xavier'`)."""
|
|
11
|
+
def __init__(self, in_features, out_features, bias=True, init='kaiming'):
|
|
12
|
+
super().__init__()
|
|
13
|
+
assert in_features > 0 and out_features > 0, "features must be positive"
|
|
14
|
+
if init == 'xavier':
|
|
15
|
+
scale = np.sqrt(2.0 / (in_features + out_features))
|
|
16
|
+
else:
|
|
17
|
+
scale = np.sqrt(2.0 / in_features)
|
|
18
|
+
self.weight = Tensor(
|
|
19
|
+
(np.random.randn(in_features, out_features) * scale).astype(np.float32),
|
|
20
|
+
requires_grad=True
|
|
21
|
+
)
|
|
22
|
+
self.bias = Tensor(
|
|
23
|
+
np.zeros(out_features, dtype=np.float32),
|
|
24
|
+
requires_grad=True
|
|
25
|
+
) if bias else None
|
|
26
|
+
|
|
27
|
+
def forward(self, x):
|
|
28
|
+
assert x.shape[-1] == self.weight.shape[0], f"Linear expects last dim {self.weight.shape[0]}, got {x.shape[-1]}"
|
|
29
|
+
# flatten to 2-d: one GEMM, and the weight grad needs no (batch, in, out) stack to sum
|
|
30
|
+
rows = x.reshape(-1, x.shape[-1])
|
|
31
|
+
if T._autocast is None:
|
|
32
|
+
y = rows @ self.weight
|
|
33
|
+
else:
|
|
34
|
+
y = (rows.cast(T._autocast) @ self.weight.cast(T._autocast)).cast("float32")
|
|
35
|
+
y = y.reshape(*x.shape[:-1], self.weight.shape[1])
|
|
36
|
+
if self.bias is not None:
|
|
37
|
+
y = y + self.bias
|
|
38
|
+
return y
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ReLU(Module):
|
|
42
|
+
"""Rectified linear unit, `max(0, x)`."""
|
|
43
|
+
def forward(self, x):
|
|
44
|
+
return F.relu(x)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Sigmoid(Module):
|
|
48
|
+
"""Logistic sigmoid, `1 / (1 + e**-x)`."""
|
|
49
|
+
def forward(self, x):
|
|
50
|
+
return F.sigmoid(x)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class Tanh(Module):
|
|
54
|
+
"""Hyperbolic tangent."""
|
|
55
|
+
def forward(self, x):
|
|
56
|
+
return F.tanh(x)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Softmax(Module):
|
|
60
|
+
"""Softmax over `axis`, turning logits into a probability distribution."""
|
|
61
|
+
def __init__(self, axis=-1):
|
|
62
|
+
super().__init__()
|
|
63
|
+
self.axis = axis
|
|
64
|
+
|
|
65
|
+
def forward(self, x):
|
|
66
|
+
return F.softmax(x, axis=self.axis)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class GELU(Module):
|
|
70
|
+
def forward(self, x):
|
|
71
|
+
return F.gelu(x)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class LayerNorm(Module):
|
|
75
|
+
"""Normalize over the last dimension of any (..., num_features) input."""
|
|
76
|
+
def __init__(self, num_features, eps=1e-5):
|
|
77
|
+
super().__init__()
|
|
78
|
+
assert num_features > 0, "num_features must be positive"
|
|
79
|
+
self.gamma = Tensor(np.ones(num_features, dtype=np.float32), requires_grad=True)
|
|
80
|
+
self.beta = Tensor(np.zeros(num_features, dtype=np.float32), requires_grad=True)
|
|
81
|
+
self.eps = eps
|
|
82
|
+
|
|
83
|
+
def forward(self, x):
|
|
84
|
+
assert x.shape[-1] == self.gamma.shape[0], f"LayerNorm expects last dim {self.gamma.shape[0]}, got {x.shape[-1]}"
|
|
85
|
+
mean = x.mean(axis=-1, keepdims=True)
|
|
86
|
+
diff = x - mean
|
|
87
|
+
var = (diff ** 2).mean(axis=-1, keepdims=True)
|
|
88
|
+
x_hat = diff / (var + self.eps) ** 0.5
|
|
89
|
+
return x_hat * self.gamma + self.beta
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class GroupNorm(Module):
|
|
93
|
+
"""Normalize each group of channels of an (N, C, ...) input, then scale and shift each channel."""
|
|
94
|
+
def __init__(self, num_groups, num_channels, eps=1e-5):
|
|
95
|
+
super().__init__()
|
|
96
|
+
assert num_channels % num_groups == 0, "channels must split evenly into groups"
|
|
97
|
+
self.num_groups, self.eps = num_groups, eps
|
|
98
|
+
self.weight = Tensor(np.ones(num_channels, dtype=np.float32), requires_grad=True)
|
|
99
|
+
self.bias = Tensor(np.zeros(num_channels, dtype=np.float32), requires_grad=True)
|
|
100
|
+
|
|
101
|
+
def forward(self, x):
|
|
102
|
+
g = x.reshape(x.shape[0], self.num_groups, -1)
|
|
103
|
+
mean = g.mean(axis=-1, keepdims=True)
|
|
104
|
+
diff = g - mean
|
|
105
|
+
y = (diff / ((diff ** 2).mean(axis=-1, keepdims=True) + self.eps) ** 0.5).reshape(*x.shape)
|
|
106
|
+
per_channel = (1, x.shape[1]) + (1,) * (len(x.shape) - 2)
|
|
107
|
+
return y * self.weight.reshape(*per_channel) + self.bias.reshape(*per_channel)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class Embedding(Module):
|
|
111
|
+
"""Lookup table mapping integer ids to dense vectors."""
|
|
112
|
+
def __init__(self, num_embeddings, embed_dim):
|
|
113
|
+
super().__init__()
|
|
114
|
+
assert num_embeddings > 0 and embed_dim > 0, "sizes must be positive"
|
|
115
|
+
self.weight = Tensor(
|
|
116
|
+
(np.random.randn(num_embeddings, embed_dim) * 0.02).astype(np.float32),
|
|
117
|
+
requires_grad=True
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
def forward(self, idx):
|
|
121
|
+
return self.weight[np.asarray(idx)]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class Dropout(Module):
|
|
125
|
+
"""Zero each activation with probability `p` during training, scaling the rest; a no-op in eval."""
|
|
126
|
+
def __init__(self, p=0.5):
|
|
127
|
+
super().__init__()
|
|
128
|
+
assert 0.0 <= p < 1.0, "dropout probability must be in [0, 1)"
|
|
129
|
+
self.p = p
|
|
130
|
+
|
|
131
|
+
def forward(self, x):
|
|
132
|
+
if not self._training or self.p == 0.0:
|
|
133
|
+
return x
|
|
134
|
+
mask = (np.random.rand(*x.shape) > self.p).astype(np.float32) / (1.0 - self.p)
|
|
135
|
+
return x * x._const(mask)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class BatchNorm1d(Module):
|
|
139
|
+
"""Normalize a `(batch, features)` input per feature, tracking running stats for eval."""
|
|
140
|
+
def __init__(self, num_features, eps=1e-5, momentum=0.1):
|
|
141
|
+
super().__init__()
|
|
142
|
+
assert num_features > 0, "num_features must be positive"
|
|
143
|
+
self.gamma = Tensor(np.ones(num_features, dtype=np.float32), requires_grad=True)
|
|
144
|
+
self.beta = Tensor(np.zeros(num_features, dtype=np.float32), requires_grad=True)
|
|
145
|
+
self.eps = eps
|
|
146
|
+
self.momentum = momentum
|
|
147
|
+
self.running_mean = np.zeros(num_features, dtype=np.float32)
|
|
148
|
+
self.running_var = np.ones(num_features, dtype=np.float32)
|
|
149
|
+
|
|
150
|
+
def forward(self, x):
|
|
151
|
+
assert x.ndim == 2 and x.shape[1] == self.gamma.shape[0], \
|
|
152
|
+
f"Expected input shape (batch, {self.gamma.shape[0]}), got {x.shape}"
|
|
153
|
+
|
|
154
|
+
if self._training:
|
|
155
|
+
mean = x.mean(axis=0, keepdims=True)
|
|
156
|
+
diff = x - mean
|
|
157
|
+
var = (diff ** 2).mean(axis=0, keepdims=True)
|
|
158
|
+
x_hat = diff / (var + self.eps) ** 0.5
|
|
159
|
+
|
|
160
|
+
# running stats stay host NumPy arrays on any device, so state_dict saves them as buffers
|
|
161
|
+
self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean.numpy().flatten()
|
|
162
|
+
self.running_var = (1 - self.momentum) * self.running_var + self.momentum * var.numpy().flatten()
|
|
163
|
+
else:
|
|
164
|
+
rm = x._const(self.running_mean.reshape(1, -1))
|
|
165
|
+
rv = x._const(self.running_var.reshape(1, -1))
|
|
166
|
+
x_hat = (x - rm) / (rv + self.eps) ** 0.5
|
|
167
|
+
|
|
168
|
+
return x_hat * self.gamma + self.beta
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class RMSNorm(Module):
|
|
172
|
+
"""Divide each vector by its root-mean-square, then scale. LayerNorm without the mean and bias."""
|
|
173
|
+
def __init__(self, num_features, eps=1e-5):
|
|
174
|
+
super().__init__()
|
|
175
|
+
self.weight = Tensor(np.ones(num_features, dtype=np.float32), requires_grad=True)
|
|
176
|
+
self.eps = eps
|
|
177
|
+
|
|
178
|
+
def forward(self, x):
|
|
179
|
+
return x * ((x * x).mean(axis=-1, keepdims=True) + self.eps) ** -0.5 * self.weight
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class DepthAttention(Module):
|
|
183
|
+
"""Attention Residuals: a layer's input is a learned softmax mix of earlier layers' outputs, not their sum."""
|
|
184
|
+
def __init__(self, d_model, eps=1e-5):
|
|
185
|
+
super().__init__()
|
|
186
|
+
self.query = Tensor(np.zeros(d_model, dtype=np.float32), requires_grad=True)
|
|
187
|
+
self.norm = RMSNorm(d_model, eps)
|
|
188
|
+
|
|
189
|
+
def forward(self, sources):
|
|
190
|
+
v = T.stack(sources) # (N, B, T, D)
|
|
191
|
+
scores = (self.norm(v) * self.query).sum(axis=-1, keepdims=True) # (N, B, T, 1): keys normalized, values raw
|
|
192
|
+
return (F.softmax(scores, axis=0) * v).sum(axis=0)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class SiLUMLP(Module):
|
|
196
|
+
"""Ungated feed-forward w2(silu(w1 x)), the baseline for the CS336 SwiGLU ablation."""
|
|
197
|
+
def __init__(self, d_model, d_ff):
|
|
198
|
+
super().__init__()
|
|
199
|
+
self.w1 = Linear(d_model, d_ff, bias=False)
|
|
200
|
+
self.w2 = Linear(d_ff, d_model, bias=False)
|
|
201
|
+
|
|
202
|
+
def forward(self, x):
|
|
203
|
+
return self.w2(F.silu(self.w1(x)))
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
class SwiGLU(Module):
|
|
207
|
+
"""Gated feed-forward from Llama and Qwen: w2(silu(w1 x) * w3 x), no biases."""
|
|
208
|
+
def __init__(self, d_model, d_ff):
|
|
209
|
+
super().__init__()
|
|
210
|
+
self.w1 = Linear(d_model, d_ff, bias=False)
|
|
211
|
+
self.w2 = Linear(d_ff, d_model, bias=False)
|
|
212
|
+
self.w3 = Linear(d_model, d_ff, bias=False)
|
|
213
|
+
|
|
214
|
+
def forward(self, x):
|
|
215
|
+
return self.w2(F.silu(self.w1(x)) * self.w3(x))
|
stackgrad/lora.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from .layers import Linear
|
|
4
|
+
from .module import Module
|
|
5
|
+
from .tensor import Tensor
|
|
6
|
+
|
|
7
|
+
TARGETS = ("q_proj", "k_proj", "v_proj", "o_proj", "w1", "w2", "w3") # every matmul in a TransformerLM block
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class LoRALinear(Module):
|
|
11
|
+
"""A frozen Linear plus a trained low-rank update: y = x W + (alpha / r) (x A) B, B starting at zero."""
|
|
12
|
+
def __init__(self, base, r, alpha):
|
|
13
|
+
super().__init__()
|
|
14
|
+
n_in, n_out = base.weight.shape
|
|
15
|
+
self.base = base
|
|
16
|
+
# cast after dividing: numpy 2 promotes float32 / np.sqrt's float64 scalar to float64
|
|
17
|
+
init = (np.random.randn(n_in, r) / np.sqrt(n_in)).astype(np.float32)
|
|
18
|
+
self.A = Tensor(init, requires_grad=True, device=base.weight.device)
|
|
19
|
+
self.B = Tensor(np.zeros((r, n_out), np.float32), requires_grad=True, device=base.weight.device)
|
|
20
|
+
self.scale = alpha / r
|
|
21
|
+
|
|
22
|
+
def forward(self, x):
|
|
23
|
+
flat = x.reshape(-1, x.shape[-1])
|
|
24
|
+
delta = (flat @ self.A * self.scale) @ self.B # scaling the rank-r middle, not the full-width output, saves a copy
|
|
25
|
+
return self.base(x) + delta.reshape(*x.shape[:-1], delta.shape[-1])
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def apply_lora(model, r=16, alpha=32):
|
|
29
|
+
"""Freezes a TransformerLM and wraps each target Linear in a LoRALinear, so only A and B train."""
|
|
30
|
+
for _, p in model.named_parameters():
|
|
31
|
+
p.requires_grad = False
|
|
32
|
+
for block in model.blocks:
|
|
33
|
+
for parent in (block.attn, block.ffn):
|
|
34
|
+
for name in TARGETS:
|
|
35
|
+
if isinstance(getattr(parent, name, None), Linear):
|
|
36
|
+
setattr(parent, name, LoRALinear(getattr(parent, name), r, alpha))
|
|
37
|
+
return model
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def save_adapter(model, path, r, alpha):
|
|
41
|
+
np.savez(path, r=r, alpha=alpha, **{name: p.numpy() for name, p in model.named_parameters()})
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def merge_adapter(model, path):
|
|
45
|
+
"""Add a saved adapter's (alpha / r) A B into the matching weights of a plain (un-wrapped) model."""
|
|
46
|
+
adapter = np.load(path)
|
|
47
|
+
scale = float(adapter["alpha"]) / float(adapter["r"])
|
|
48
|
+
params = dict(model.named_parameters())
|
|
49
|
+
layers = [k[:-len(".A")] for k in adapter.files if k.endswith(".A")]
|
|
50
|
+
for layer in layers:
|
|
51
|
+
w = params[f"{layer}.weight"] # a KeyError here means the adapter was trained on another model
|
|
52
|
+
assert w.device == "cpu", "merge the adapter before moving the model to a device"
|
|
53
|
+
w.data = w.data + (scale * (adapter[f"{layer}.A"] @ adapter[f"{layer}.B"])).astype(w.dtype)
|
|
54
|
+
return len(layers)
|
stackgrad/loss.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import math
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
from . import functional as F
|
|
6
|
+
from .tensor import Tensor
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def mse_loss(input, target):
|
|
10
|
+
"""Mean squared error between predictions and targets."""
|
|
11
|
+
return ((input - target) ** 2).mean()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def bce_loss(input, target):
|
|
15
|
+
"""Binary cross-entropy on probabilities, not logits."""
|
|
16
|
+
p = input.clamp(1e-7, 1 - 1e-7)
|
|
17
|
+
return -(target * p.log() + (1.0 - target) * (1.0 - p).log()).mean()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def cross_entropy_loss(input, target):
|
|
21
|
+
"""Softmax cross-entropy over the last axis; `target` holds class indices or one-hot rows."""
|
|
22
|
+
log_probs = F.log_softmax(input, axis=-1)
|
|
23
|
+
labels = target.numpy() if isinstance(target, Tensor) else np.asarray(target)
|
|
24
|
+
if labels.shape == input.shape:
|
|
25
|
+
return -(log_probs * target).sum() / (math.prod(input.shape) // input.shape[-1])
|
|
26
|
+
labels = labels.reshape(-1).astype(np.int64)
|
|
27
|
+
log_probs = log_probs.reshape(-1, input.shape[-1])
|
|
28
|
+
return -log_probs[np.arange(len(labels)), labels].mean()
|
stackgrad/mlops.py
ADDED
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
import math
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Function:
|
|
7
|
+
differentiable = True
|
|
8
|
+
|
|
9
|
+
def __init__(self, xp, *parents):
|
|
10
|
+
self.xp = xp
|
|
11
|
+
self.parents = parents
|
|
12
|
+
self.needs_grad = [p.requires_grad for p in parents]
|
|
13
|
+
|
|
14
|
+
def forward(self, *args, **kwargs):
|
|
15
|
+
raise NotImplementedError
|
|
16
|
+
|
|
17
|
+
def backward(self, grad):
|
|
18
|
+
raise NotImplementedError
|
|
19
|
+
|
|
20
|
+
def out_shape(self, *shapes, **kwargs):
|
|
21
|
+
return np.broadcast_shapes(*shapes)
|
|
22
|
+
|
|
23
|
+
def out_dtype(self, *dtypes, **kwargs):
|
|
24
|
+
"""Promote the input dtypes as the backend does, by adding 0-d probes."""
|
|
25
|
+
out = dtypes[0]
|
|
26
|
+
for d in dtypes[1:]:
|
|
27
|
+
if d != out:
|
|
28
|
+
out = (self.xp.asarray(0, dtype=out) + self.xp.asarray(0, dtype=d)).dtype
|
|
29
|
+
return out
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _reduced(shape, axis, keepdims):
|
|
33
|
+
axes = range(len(shape)) if axis is None else [a % len(shape) for a in (axis if isinstance(axis, (tuple, list)) else (axis,))]
|
|
34
|
+
return tuple(1 if i in axes else d for i, d in enumerate(shape) if keepdims or i not in axes)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def unbroadcast(grad, shape):
|
|
38
|
+
while grad.ndim > len(shape):
|
|
39
|
+
grad = grad.sum(axis=0)
|
|
40
|
+
for i, size in enumerate(shape):
|
|
41
|
+
if size == 1 and grad.shape[i] != 1:
|
|
42
|
+
grad = grad.sum(axis=i, keepdims=True)
|
|
43
|
+
return grad
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class Recip(Function):
|
|
47
|
+
def forward(self, x):
|
|
48
|
+
self.out = 1.0 / x
|
|
49
|
+
return self.out
|
|
50
|
+
|
|
51
|
+
def backward(self, grad):
|
|
52
|
+
return (-grad * self.out * self.out,)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class Exp(Function):
|
|
56
|
+
def forward(self, x):
|
|
57
|
+
self.out = self.xp.exp(x)
|
|
58
|
+
return self.out
|
|
59
|
+
|
|
60
|
+
def backward(self, grad):
|
|
61
|
+
return (grad * self.out,)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class Log(Function):
|
|
65
|
+
def forward(self, x):
|
|
66
|
+
self.x = x
|
|
67
|
+
return self.xp.log(x)
|
|
68
|
+
|
|
69
|
+
def backward(self, grad):
|
|
70
|
+
return (grad / self.x,)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class Sigmoid(Function):
|
|
74
|
+
# a primitive: composed from exp/recip, a very negative x gives 0 * inf = nan in backward
|
|
75
|
+
def forward(self, x):
|
|
76
|
+
self.out = self.xp.sigmoid(x)
|
|
77
|
+
return self.out
|
|
78
|
+
|
|
79
|
+
def backward(self, grad):
|
|
80
|
+
return (grad * self.out * (1.0 - self.out),)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class Pow(Function):
|
|
84
|
+
def forward(self, x, c):
|
|
85
|
+
self.x, self.c = x, c
|
|
86
|
+
return x ** c
|
|
87
|
+
|
|
88
|
+
def backward(self, grad):
|
|
89
|
+
return (grad * self.c * self.x ** (self.c - 1),)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class Cast(Function):
|
|
93
|
+
def forward(self, x, dtype):
|
|
94
|
+
self.dtype = x.dtype
|
|
95
|
+
return x.astype(dtype)
|
|
96
|
+
|
|
97
|
+
def backward(self, grad):
|
|
98
|
+
return (grad.astype(self.dtype),)
|
|
99
|
+
|
|
100
|
+
def out_dtype(self, x, dtype):
|
|
101
|
+
return dtype
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# binary ops: inputs that need a gradient arrive already expanded to the output shape
|
|
105
|
+
|
|
106
|
+
class Add(Function):
|
|
107
|
+
def forward(self, x, y):
|
|
108
|
+
return x + y
|
|
109
|
+
|
|
110
|
+
def backward(self, grad):
|
|
111
|
+
return grad, grad
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class Mul(Function):
|
|
115
|
+
def forward(self, x, y):
|
|
116
|
+
self.x, self.y = x, y
|
|
117
|
+
return x * y
|
|
118
|
+
|
|
119
|
+
def backward(self, grad):
|
|
120
|
+
return (grad * self.y if self.needs_grad[0] else None,
|
|
121
|
+
grad * self.x if self.needs_grad[1] else None)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class Where(Function):
|
|
125
|
+
def forward(self, x, y, cond):
|
|
126
|
+
self.cond = cond
|
|
127
|
+
return self.xp.where(cond, x, y)
|
|
128
|
+
|
|
129
|
+
def backward(self, grad):
|
|
130
|
+
zero = self.xp.zeros_like(grad)
|
|
131
|
+
return self.xp.where(self.cond, grad, zero), self.xp.where(self.cond, zero, grad)
|
|
132
|
+
|
|
133
|
+
def out_dtype(self, x, y, cond):
|
|
134
|
+
return super().out_dtype(x, y)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class Less(Function):
|
|
138
|
+
differentiable = False
|
|
139
|
+
|
|
140
|
+
def forward(self, x, y):
|
|
141
|
+
return (x < y).astype(self.xp.float32)
|
|
142
|
+
|
|
143
|
+
def out_dtype(self, x, y):
|
|
144
|
+
return self.xp.float32
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class Sum(Function):
|
|
148
|
+
def forward(self, x, axis, keepdims):
|
|
149
|
+
self.shape, self.axis, self.keepdims = x.shape, axis, keepdims
|
|
150
|
+
return x.sum(axis=axis, keepdims=keepdims)
|
|
151
|
+
|
|
152
|
+
def out_shape(self, x, axis, keepdims):
|
|
153
|
+
return _reduced(x, axis, keepdims)
|
|
154
|
+
|
|
155
|
+
def backward(self, grad):
|
|
156
|
+
if self.axis is not None and not self.keepdims:
|
|
157
|
+
grad = self.xp.expand_dims(grad, self.axis)
|
|
158
|
+
return (self.xp.broadcast_to(grad, self.shape),)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class Max(Function):
|
|
162
|
+
def forward(self, x, axis, keepdims):
|
|
163
|
+
self.x, self.axis, self.keepdims = x, axis, keepdims
|
|
164
|
+
self.out = x.max(axis=axis, keepdims=True)
|
|
165
|
+
return self.out if keepdims else x.max(axis=axis)
|
|
166
|
+
|
|
167
|
+
def out_shape(self, x, axis, keepdims):
|
|
168
|
+
return _reduced(x, axis, keepdims)
|
|
169
|
+
|
|
170
|
+
def backward(self, grad):
|
|
171
|
+
# ties split the gradient evenly
|
|
172
|
+
mask = (self.x == self.out).astype(self.x.dtype)
|
|
173
|
+
mask = mask / mask.sum(axis=self.axis, keepdims=True)
|
|
174
|
+
if self.axis is not None and not self.keepdims:
|
|
175
|
+
grad = self.xp.expand_dims(grad, self.axis)
|
|
176
|
+
return (mask * grad,)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class Reshape(Function):
|
|
180
|
+
def forward(self, x, shape):
|
|
181
|
+
self.shape = x.shape
|
|
182
|
+
return x.reshape(shape)
|
|
183
|
+
|
|
184
|
+
def backward(self, grad):
|
|
185
|
+
return (grad.reshape(self.shape),)
|
|
186
|
+
|
|
187
|
+
def out_shape(self, x, shape):
|
|
188
|
+
size = math.prod(x)
|
|
189
|
+
known = math.prod(d for d in shape if d != -1)
|
|
190
|
+
out = tuple(size // known if d == -1 else d for d in shape)
|
|
191
|
+
assert math.prod(out) == size, f"cannot reshape {x} into {shape}"
|
|
192
|
+
return out
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
class Permute(Function):
|
|
196
|
+
def forward(self, x, order):
|
|
197
|
+
self.order = order
|
|
198
|
+
return x.transpose(order)
|
|
199
|
+
|
|
200
|
+
def backward(self, grad):
|
|
201
|
+
inverse = [self.order.index(i) for i in range(len(self.order))]
|
|
202
|
+
return (grad.transpose(inverse),)
|
|
203
|
+
|
|
204
|
+
def out_shape(self, x, order):
|
|
205
|
+
return tuple(x[i] for i in order)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class Expand(Function):
|
|
209
|
+
"""x already has the target rank; Tensor.expand adds the leading dims."""
|
|
210
|
+
def forward(self, x, shape):
|
|
211
|
+
self.shape = x.shape
|
|
212
|
+
return self.xp.broadcast_to(x, shape)
|
|
213
|
+
|
|
214
|
+
def backward(self, grad):
|
|
215
|
+
return (unbroadcast(grad, self.shape),)
|
|
216
|
+
|
|
217
|
+
def out_shape(self, x, shape):
|
|
218
|
+
return tuple(shape)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
class Index(Function):
|
|
222
|
+
def forward(self, x, key):
|
|
223
|
+
self.shape, self.key = x.shape, key
|
|
224
|
+
return x[key]
|
|
225
|
+
|
|
226
|
+
def backward(self, grad):
|
|
227
|
+
# scatter_add: an index picked twice gets both gradients
|
|
228
|
+
return (self.xp.scatter_add(self.shape, self.key, grad),)
|
|
229
|
+
|
|
230
|
+
def out_shape(self, x, key):
|
|
231
|
+
"""Index a zero-stride stand-in of shape x; index arrays become zeros, bool masks stay (their count matters)."""
|
|
232
|
+
def stand_in(k):
|
|
233
|
+
if not hasattr(k, "dtype"):
|
|
234
|
+
return k
|
|
235
|
+
if "bool" in str(k.dtype):
|
|
236
|
+
return self.xp.to_numpy(k)
|
|
237
|
+
return np.zeros(tuple(k.shape), dtype=np.intp)
|
|
238
|
+
key = tuple(map(stand_in, key)) if isinstance(key, tuple) else stand_in(key)
|
|
239
|
+
# shortcut: a gather allocates one byte per result element here, compute the shape if it shows in a profile
|
|
240
|
+
return np.lib.stride_tricks.as_strided(np.zeros(1, np.uint8), x, (0,) * len(x))[key].shape
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class Cat(Function):
|
|
244
|
+
def forward(self, *xs, axis):
|
|
245
|
+
self.axis = axis
|
|
246
|
+
self.bounds = []
|
|
247
|
+
end = 0
|
|
248
|
+
for x in xs:
|
|
249
|
+
self.bounds.append((end, end + x.shape[axis]))
|
|
250
|
+
end += x.shape[axis]
|
|
251
|
+
return self.xp.concatenate(xs, axis=axis)
|
|
252
|
+
|
|
253
|
+
def backward(self, grad):
|
|
254
|
+
index = [slice(None)] * grad.ndim
|
|
255
|
+
grads = []
|
|
256
|
+
for start, end in self.bounds:
|
|
257
|
+
index[self.axis] = slice(start, end)
|
|
258
|
+
grads.append(grad[tuple(index)])
|
|
259
|
+
return tuple(grads)
|
|
260
|
+
|
|
261
|
+
def out_shape(self, *shapes, axis):
|
|
262
|
+
axis %= len(shapes[0])
|
|
263
|
+
return shapes[0][:axis] + (sum(s[axis] for s in shapes),) + shapes[0][axis + 1:]
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
class Im2col(Function):
|
|
267
|
+
def forward(self, x, kh, kw, stride, padding):
|
|
268
|
+
self.args = (x.shape, kh, kw, stride, padding)
|
|
269
|
+
return self.xp.im2col(x, kh, kw, stride, padding)
|
|
270
|
+
|
|
271
|
+
def backward(self, grad):
|
|
272
|
+
return (self.xp.col2im(grad, *self.args),)
|
|
273
|
+
|
|
274
|
+
def out_shape(self, x, kh, kw, stride, padding):
|
|
275
|
+
n, c, h, w = x
|
|
276
|
+
oh, ow = (h + 2 * padding - kh) // stride + 1, (w + 2 * padding - kw) // stride + 1
|
|
277
|
+
return (n * oh * ow, c * kh * kw)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
# a primitive: as broadcast-multiply-sum it would lose BLAS
|
|
281
|
+
|
|
282
|
+
class Matmul(Function):
|
|
283
|
+
def forward(self, x, y):
|
|
284
|
+
self.x, self.y = x, y
|
|
285
|
+
return x @ y
|
|
286
|
+
|
|
287
|
+
def backward(self, grad):
|
|
288
|
+
return (unbroadcast(grad @ self.y.swapaxes(-1, -2), self.x.shape) if self.needs_grad[0] else None,
|
|
289
|
+
unbroadcast(self.x.swapaxes(-1, -2) @ grad, self.y.shape) if self.needs_grad[1] else None)
|
|
290
|
+
|
|
291
|
+
def out_shape(self, x, y):
|
|
292
|
+
"""NumPy's matmul shape rule, 1-d operands included."""
|
|
293
|
+
x1, y1 = len(x) == 1, len(y) == 1
|
|
294
|
+
x, y = (1,) + x if x1 else x, y + (1,) if y1 else y
|
|
295
|
+
assert x[-1] == y[-2], f"cannot multiply {x} by {y}"
|
|
296
|
+
out = np.broadcast_shapes(x[:-2], y[:-2]) + (x[-2], y[-1])
|
|
297
|
+
out = out[:-2] + out[-1:] if x1 else out
|
|
298
|
+
return out[:-1] if y1 else out
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
class Attention(Function):
|
|
302
|
+
"""Causal GQA attention, a primitive so a backend can fuse it; backward rebuilds p from the saved lse."""
|
|
303
|
+
def scores(self, q, k):
|
|
304
|
+
xp = self.xp
|
|
305
|
+
T, S = q.shape[2], k.shape[2]
|
|
306
|
+
k = xp.repeat(k, q.shape[1] // k.shape[1], axis=1)
|
|
307
|
+
s = q @ k.swapaxes(-1, -2) * q.shape[-1] ** -0.5
|
|
308
|
+
if self.causal: # query i sits at position S - T + i (kv cache)
|
|
309
|
+
s = s + xp.triu(xp.full((T, S), -xp.inf, dtype=q.dtype), k=S - T + 1)
|
|
310
|
+
if self.key_bias is not None:
|
|
311
|
+
s = s + xp.asarray(self.key_bias, dtype=q.dtype)[:, None, None, :]
|
|
312
|
+
return s
|
|
313
|
+
|
|
314
|
+
def forward(self, q, k, v, key_bias=None, causal=True):
|
|
315
|
+
"""key_bias: optional (B, S) array added to each key's score (left-padding mask)."""
|
|
316
|
+
self.key_bias, self.causal = key_bias, causal
|
|
317
|
+
fused = getattr(self.xp, "fused_attention", None) if key_bias is None and causal else None # the kernels take no mask
|
|
318
|
+
out = fused(q, k, v) if fused else None
|
|
319
|
+
self.p = None
|
|
320
|
+
if out is None:
|
|
321
|
+
s = self.scores(q, k)
|
|
322
|
+
m = s.max(axis=-1, keepdims=True)
|
|
323
|
+
p = self.xp.exp(s - m)
|
|
324
|
+
l = p.sum(axis=-1, keepdims=True)
|
|
325
|
+
self.p = p / l
|
|
326
|
+
out = self.p @ self.xp.repeat(v, q.shape[1] // k.shape[1], axis=1), (m + self.xp.log(l))[..., 0]
|
|
327
|
+
self.saved = q, k, v, out[0], out[1].astype(q.dtype)
|
|
328
|
+
return out[0]
|
|
329
|
+
|
|
330
|
+
def backward(self, grad):
|
|
331
|
+
q, k, v, out, lse = self.saved
|
|
332
|
+
xp = self.xp
|
|
333
|
+
fused = getattr(xp, "fused_attention_backward", None)
|
|
334
|
+
if fused is not None and self.p is None: # p is None only after a fused forward
|
|
335
|
+
return fused(q, k, v, out, lse, grad)
|
|
336
|
+
B, Hkv, S, D = k.shape
|
|
337
|
+
rep = q.shape[1] // Hkv
|
|
338
|
+
p = self.p if self.p is not None else xp.exp(self.scores(q, k) - lse[..., None])
|
|
339
|
+
dv = p.swapaxes(-1, -2) @ grad
|
|
340
|
+
dp = grad @ xp.repeat(v, rep, axis=1).swapaxes(-1, -2)
|
|
341
|
+
# softmax backward: ds = p * (dp - rowsum(p * dp)), and rowsum(p * dp) = rowsum(grad * out)
|
|
342
|
+
ds = p * (dp - (grad * out).sum(axis=-1, keepdims=True)) * D ** -0.5
|
|
343
|
+
dq = ds @ xp.repeat(k, rep, axis=1)
|
|
344
|
+
dk = ds.swapaxes(-1, -2) @ q
|
|
345
|
+
# each kv head sums the grads of the `rep` query heads it served
|
|
346
|
+
return dq, dk.reshape(B, Hkv, rep, S, D).sum(axis=2), dv.reshape(B, Hkv, rep, S, D).sum(axis=2)
|
|
347
|
+
|
|
348
|
+
def out_shape(self, q, k, v, key_bias=None, causal=True):
|
|
349
|
+
return q[:-1] + v[-1:]
|
|
350
|
+
|
|
351
|
+
def out_dtype(self, q, k, v, key_bias=None, causal=True):
|
|
352
|
+
return q
|