stackgrad 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,195 @@
1
+ import mlx.core as mx
2
+
3
+ _SHAPES = """
4
+ const int T = q_shape[2], S = k_shape[2], H = q_shape[1], Hkv = k_shape[1];
5
+ const float scale = rsqrt(float(D));
6
+ """
7
+
8
+ _V1 = _SHAPES + """
9
+ const uint row = thread_position_in_grid.x; // over B * H * T query rows
10
+ if (row >= uint(q_shape[0] * H * T)) return;
11
+ const int i = row % T, bh = row / T, b = bh / H, h = bh % H;
12
+ const int kv = (b * Hkv + h / (H / Hkv)) * S * D;
13
+
14
+ float qr[D], acc[D];
15
+ for (int d = 0; d < D; d++) { qr[d] = float(q[row * D + d]) * scale; acc[d] = 0.0f; }
16
+ float m = -INFINITY, l = 0.0f;
17
+ for (int j = 0; j <= S - T + i; j++) {
18
+ float s = 0.0f;
19
+ for (int d = 0; d < D; d++) s += qr[d] * float(k[kv + j * D + d]);
20
+ const float m_new = max(m, s), corr = exp(m - m_new), p = exp(s - m_new);
21
+ l = l * corr + p;
22
+ for (int d = 0; d < D; d++) acc[d] = acc[d] * corr + p * float(v[kv + j * D + d]);
23
+ m = m_new;
24
+ }
25
+ for (int d = 0; d < D; d++) out[row * D + d] = InT(acc[d] / l);
26
+ lse[row] = m + log(l);
27
+ """
28
+
29
+ _V2 = _SHAPES + """
30
+ constexpr int BQ = 32; // query rows (= threads) per threadgroup
31
+ threadgroup float Ks[BK * D], Vs[BK * D];
32
+ const int tid = thread_position_in_threadgroup.x;
33
+ const int n_qblocks = (T + BQ - 1) / BQ;
34
+ const int bh = threadgroup_position_in_grid.x / n_qblocks, qb = threadgroup_position_in_grid.x % n_qblocks;
35
+ const int b = bh / H, h = bh % H, kv = (b * Hkv + h / (H / Hkv)) * S * D;
36
+ const int i = qb * BQ + tid, last = S - T + i; // this row sees keys 0 .. last
37
+ const int group_last = min(S - T + qb * BQ + BQ - 1, S - 1);
38
+
39
+ float qr[D], acc[D];
40
+ for (int d = 0; d < D; d++) { qr[d] = i < T ? float(q[(bh * T + i) * D + d]) * scale : 0.0f; acc[d] = 0.0f; }
41
+ float m = -INFINITY, l = 0.0f;
42
+ for (int j0 = 0; j0 <= group_last; j0 += BK) {
43
+ for (int e = tid; e < BK * D; e += BQ) { // the 32 threads load the tile together
44
+ const bool in = j0 + e / D < S;
45
+ Ks[e] = in ? float(k[kv + j0 * D + e]) : 0.0f;
46
+ Vs[e] = in ? float(v[kv + j0 * D + e]) : 0.0f;
47
+ }
48
+ threadgroup_barrier(mem_flags::mem_threadgroup);
49
+ float s[BK], m_tile = -INFINITY;
50
+ for (int jj = 0; jj < BK; jj++) {
51
+ float dot = 0.0f;
52
+ for (int d = 0; d < D; d++) dot += qr[d] * Ks[jj * D + d];
53
+ s[jj] = j0 + jj <= last ? dot : -INFINITY;
54
+ m_tile = max(m_tile, s[jj]);
55
+ }
56
+ const float m_new = max(m, m_tile), corr = exp(m - m_new);
57
+ l *= corr;
58
+ for (int d = 0; d < D; d++) acc[d] *= corr;
59
+ for (int jj = 0; jj < BK; jj++) {
60
+ const float p = exp(s[jj] - m_new);
61
+ l += p;
62
+ for (int d = 0; d < D; d++) acc[d] += p * Vs[jj * D + d];
63
+ }
64
+ m = m_new;
65
+ threadgroup_barrier(mem_flags::mem_threadgroup); // everyone is done with this tile
66
+ }
67
+ if (i < T) {
68
+ for (int d = 0; d < D; d++) out[(bh * T + i) * D + d] = InT(acc[d] / l);
69
+ lse[bh * T + i] = m + log(l);
70
+ }
71
+ """
72
+
73
+ # v3: 4 simdgroups x 8 query rows; lanes 4r..4r+3 share softmax row r, a quarter of the columns each
74
+ _V3 = _SHAPES + """
75
+ constexpr int BQ = 32, TILE = (32 * D > 2 * BK * D) ? 32 * D : 2 * BK * D;
76
+ threadgroup float smem[TILE]; // Q staging, then K/V tiles, then O staging
77
+ threadgroup float Ps[4][8 * BK]; // each simdgroup's scores, then probabilities
78
+ threadgroup float Dg[4][64]; // each simdgroup's diag(corr), for rescaling O
79
+ threadgroup float* Ks = smem;
80
+ threadgroup float* Vs = smem + BK * D;
81
+ const int tid = thread_index_in_threadgroup, sg = simdgroup_index_in_threadgroup, lane = thread_index_in_simdgroup;
82
+ const int n_qblocks = (T + BQ - 1) / BQ;
83
+ const int bh = threadgroup_position_in_grid.x / n_qblocks, qb = threadgroup_position_in_grid.x % n_qblocks;
84
+ const int b = bh / H, h = bh % H, kv = (b * Hkv + h / (H / Hkv)) * S * D;
85
+ const int group_last = min(S - T + qb * BQ + BQ - 1, S - 1);
86
+ const int r = lane / 4, part = lane % 4; // this lane's softmax row and column quarter
87
+ const int i = qb * BQ + sg * 8 + r, last = S - T + i;
88
+
89
+ // stage this block's Q (scaled) and load each simdgroup's 8 rows into matrix registers
90
+ for (int e = tid; e < BQ * D; e += 128) {
91
+ const int qi = qb * BQ + e / D;
92
+ smem[e] = qi < T ? float(q[(bh * T + qb * BQ) * D + e]) * scale : 0.0f;
93
+ }
94
+ for (int e = lane; e < 64; e += 32) Dg[sg][e] = 0.0f;
95
+ threadgroup_barrier(mem_flags::mem_threadgroup);
96
+ simdgroup_float8x8 Q[D / 8], O[D / 8];
97
+ for (int dc = 0; dc < D / 8; dc++) {
98
+ simdgroup_load(Q[dc], smem + sg * 8 * D + dc * 8, D);
99
+ O[dc] = simdgroup_float8x8(0.0f);
100
+ }
101
+ threadgroup_barrier(mem_flags::mem_threadgroup);
102
+
103
+ float m = -INFINITY, l = 0.0f;
104
+ for (int j0 = 0; j0 <= group_last; j0 += BK) {
105
+ for (int e = tid; e < BK * D; e += 128) {
106
+ const bool in = j0 + e / D < S;
107
+ Ks[e] = in ? float(k[kv + j0 * D + e]) : 0.0f;
108
+ Vs[e] = in ? float(v[kv + j0 * D + e]) : 0.0f;
109
+ }
110
+ threadgroup_barrier(mem_flags::mem_threadgroup);
111
+
112
+ // scores: (8 x D) @ (D x BK), one 8x8 output block per 8 keys
113
+ for (int kb = 0; kb < BK / 8; kb++) {
114
+ simdgroup_float8x8 acc(0.0f), Kt;
115
+ for (int dc = 0; dc < D / 8; dc++) {
116
+ simdgroup_load(Kt, Ks + kb * 8 * D + dc * 8, D, ulong2(0, 0), true);
117
+ simdgroup_multiply_accumulate(acc, Q[dc], Kt, acc);
118
+ }
119
+ simdgroup_store(acc, Ps[sg] + kb * 8, BK);
120
+ }
121
+ simdgroup_barrier(mem_flags::mem_threadgroup);
122
+
123
+ // online softmax on row r, columns part*BK/4 .. (part+1)*BK/4, reduced across the row's 4 lanes
124
+ constexpr int C = BK / 4;
125
+ float m_tile = -INFINITY;
126
+ for (int c = part * C; c < (part + 1) * C; c++) {
127
+ const float s = j0 + c <= last ? Ps[sg][r * BK + c] : -INFINITY;
128
+ Ps[sg][r * BK + c] = s;
129
+ m_tile = max(m_tile, s);
130
+ }
131
+ m_tile = max(m_tile, simd_shuffle_xor(m_tile, 1));
132
+ m_tile = max(m_tile, simd_shuffle_xor(m_tile, 2));
133
+ const float m_new = max(m, m_tile), corr = exp(m - m_new);
134
+ float psum = 0.0f;
135
+ for (int c = part * C; c < (part + 1) * C; c++) {
136
+ const float p = exp(Ps[sg][r * BK + c] - m_new);
137
+ Ps[sg][r * BK + c] = p;
138
+ psum += p;
139
+ }
140
+ psum += simd_shuffle_xor(psum, 1);
141
+ psum += simd_shuffle_xor(psum, 2);
142
+ l = l * corr + psum;
143
+ m = m_new;
144
+ if (part == 0) Dg[sg][r * 8 + r] = corr;
145
+ simdgroup_barrier(mem_flags::mem_threadgroup);
146
+
147
+ // O = diag(corr) @ O + P @ V_tile
148
+ simdgroup_float8x8 Corr, P, Vm;
149
+ simdgroup_load(Corr, Dg[sg], 8);
150
+ for (int dc = 0; dc < D / 8; dc++) {
151
+ simdgroup_multiply(O[dc], Corr, O[dc]);
152
+ for (int kb = 0; kb < BK / 8; kb++) {
153
+ simdgroup_load(P, Ps[sg] + kb * 8, BK);
154
+ simdgroup_load(Vm, Vs + kb * 8 * D + dc * 8, D);
155
+ simdgroup_multiply_accumulate(O[dc], P, Vm, O[dc]);
156
+ }
157
+ }
158
+ threadgroup_barrier(mem_flags::mem_threadgroup); // everyone is done with this tile
159
+ }
160
+
161
+ // stage O through threadgroup memory, then each lane writes its row's quarter divided by l
162
+ for (int dc = 0; dc < D / 8; dc++) simdgroup_store(O[dc], smem + sg * 8 * D + dc * 8, D);
163
+ simdgroup_barrier(mem_flags::mem_threadgroup);
164
+ if (i < T) {
165
+ for (int d = part * (D / 4); d < (part + 1) * (D / 4); d++)
166
+ out[(bh * T + i) * D + d] = InT(smem[(sg * 8 + r) * D + d] / l);
167
+ if (part == 0) lse[bh * T + i] = m + log(l);
168
+ }
169
+ """
170
+
171
+ _KERNELS = {
172
+ name: mx.fast.metal_kernel(name=f"stackgrad_attention_{name}", input_names=["q", "k", "v"],
173
+ output_names=["out", "lse"], source=src)
174
+ for name, src in [("v1", _V1), ("v2", _V2), ("v3", _V3)]
175
+ }
176
+
177
+
178
+ def attention(q, k, v, version="v3"):
179
+ """Causal attention: q (B, H, T, D), k, v (B, Hkv, S, D) -> out (B, H, T, D), lse (B, H, T)."""
180
+ B, H, T, D = q.shape
181
+ assert k.shape[1] and H % k.shape[1] == 0 and D % 8 == 0 and D <= 128, (q.shape, k.shape)
182
+ bk = 32 if D <= 64 else 16 # keeps the K/V tiles within 32 KB of threadgroup memory
183
+ if version == "v1":
184
+ grid, group = (B * H * T, 1, 1), (min(256, B * H * T), 1, 1)
185
+ else:
186
+ threads = 32 if version == "v2" else 128
187
+ grid, group = (B * H * -(-T // 32) * threads, 1, 1), (threads, 1, 1)
188
+ return _KERNELS[version](
189
+ inputs=[q, k, v],
190
+ template=[("InT", q.dtype), ("D", D), ("BK", bk)],
191
+ grid=grid,
192
+ threadgroup=group,
193
+ output_shapes=[q.shape, (B, H, T)],
194
+ output_dtypes=[q.dtype, mx.float32],
195
+ )
@@ -0,0 +1,44 @@
1
+ import mlx.core as mx
2
+
3
+ SIMDGROUPS = 8 # output columns per threadgroup
4
+
5
+ # one simdgroup per output column; s * sum(x q) + b * sum(x) applies scale and bias once per word
6
+ _SOURCE = """
7
+ constexpr int PER_WORD = 32 / BITS, MASK = (1 << BITS) - 1;
8
+ const int M = x_shape[0], K = x_shape[1], N = w_shape[0];
9
+ const int words = K / PER_WORD, groups = K / GROUP;
10
+ const int n = threadgroup_position_in_grid.x * SG + simdgroup_index_in_threadgroup;
11
+ const int lane = thread_index_in_simdgroup;
12
+ if (n >= N) return; // the whole simdgroup leaves together
13
+ for (int m = 0; m < M; m++) {
14
+ float acc = 0.0f;
15
+ for (int wi = lane; wi < words; wi += 32) {
16
+ const uint word = w[n * words + wi];
17
+ const int k0 = wi * PER_WORD, g = k0 / GROUP;
18
+ float xq = 0.0f, xs = 0.0f;
19
+ for (int j = 0; j < PER_WORD; j++) {
20
+ const float xv = float(x[m * K + k0 + j]);
21
+ xq += xv * float((word >> (j * BITS)) & MASK);
22
+ xs += xv;
23
+ }
24
+ acc += float(scales[n * groups + g]) * xq + float(biases[n * groups + g]) * xs;
25
+ }
26
+ acc = simd_sum(acc);
27
+ if (lane == 0) out[m * N + n] = InT(acc);
28
+ }
29
+ """
30
+
31
+ _KERNEL = mx.fast.metal_kernel(name="stackgrad_quant_matmul", input_names=["x", "w", "scales", "biases"],
32
+ output_names=["out"], source=_SOURCE)
33
+
34
+
35
+ def quant_matmul(x, packed, scales, biases, bits, group):
36
+ M, N = x.shape[0], packed.shape[0]
37
+ return _KERNEL(
38
+ inputs=[x, packed, scales, biases],
39
+ template=[("InT", x.dtype), ("BITS", bits), ("GROUP", group), ("SG", SIMDGROUPS)],
40
+ grid=(-(-N // SIMDGROUPS) * SIMDGROUPS * 32, 1, 1),
41
+ threadgroup=(SIMDGROUPS * 32, 1, 1),
42
+ output_shapes=[(M, N)],
43
+ output_dtypes=[x.dtype],
44
+ )[0]
@@ -0,0 +1,38 @@
1
+ from . import backend
2
+ from . import tensor as T
3
+ from .mlops import Function
4
+
5
+
6
+ class _Checkpoint(Function):
7
+ def forward(self, *arrays, fn):
8
+ self.fn, self.arrays, self.cast = fn, arrays, T._autocast
9
+ device = self.parents[0].device
10
+ with T.no_grad():
11
+ out = fn(*[T.Tensor(a, device=device) for a in arrays]).data
12
+ # realize now, or MLX runs it later inside one big eval that holds every activation
13
+ self.xp.realize(out)
14
+ return out
15
+
16
+ def backward(self, grad):
17
+ device = self.parents[0].device
18
+ leaves = [T.Tensor(a, requires_grad=needs, device=device) for a, needs in zip(self.arrays, self.needs_grad)]
19
+ with T.autocast(self.cast): # forward's precision: backward usually runs outside autocast
20
+ out = self.fn(*leaves)
21
+ # backward of sum(out * grad) pushes grad through fn; weights inside fn collect .grad here
22
+ (out * T.Tensor(grad, device=device)).sum().backward()
23
+ # realize now, so this block's activations free before the next block recomputes
24
+ weights = self.fn.parameters() if hasattr(self.fn, "parameters") else []
25
+ self.xp.realize(*[t.grad for t in leaves + list(weights) if t.grad is not None])
26
+ return tuple(leaf.grad for leaf in leaves)
27
+
28
+
29
+ def checkpoint(fn, *inputs):
30
+ """fn(*inputs), recomputed during backward instead of storing its activations. fn must be deterministic."""
31
+ if not T.grad_enabled():
32
+ return fn(*inputs)
33
+ device = inputs[0].device
34
+ ctx = _Checkpoint(backend.get(device), *inputs)
35
+ out = T.Tensor(ctx.forward(*[x.data for x in inputs], fn=fn), device=device)
36
+ # always recorded: weights inside fn may need grads (LoRA adapters in frozen layers)
37
+ out.requires_grad, out._ctx = True, ctx
38
+ return out
stackgrad/conv.py ADDED
@@ -0,0 +1,63 @@
1
+ import numpy as np
2
+
3
+ from .module import Module
4
+ from .tensor import Tensor
5
+
6
+
7
+ class Conv2d(Module):
8
+ """2D convolution over `(N, C, H, W)` input, implemented with im2col."""
9
+ def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):
10
+ super().__init__()
11
+ assert in_channels > 0 and out_channels > 0, "channels must be positive"
12
+ self.in_channels = in_channels
13
+ self.out_channels = out_channels
14
+ self.kernel_size = kernel_size if isinstance(kernel_size, tuple) else (kernel_size, kernel_size)
15
+ self.stride = stride
16
+ self.padding = padding
17
+
18
+ kh, kw = self.kernel_size
19
+ assert kh > 0 and kw > 0, "kernel_size must be positive"
20
+ fan_in = in_channels * kh * kw
21
+ scale = np.sqrt(2.0 / fan_in)
22
+ self.weight = Tensor(
23
+ (np.random.randn(out_channels, in_channels * kh * kw) * scale).astype(np.float32),
24
+ requires_grad=True
25
+ )
26
+ self.bias = Tensor(np.zeros(out_channels, dtype=np.float32), requires_grad=True)
27
+
28
+ def forward(self, x):
29
+ assert x.ndim == 4, f"Conv2d expects 4D input (N,C,H,W), got {x.ndim}D"
30
+ N, _, H, W = x.shape
31
+ kh, kw = self.kernel_size
32
+ OH = (H + 2 * self.padding - kh) // self.stride + 1
33
+ OW = (W + 2 * self.padding - kw) // self.stride + 1
34
+
35
+ cols = x.im2col(kh, kw, self.stride, self.padding) # (N*OH*OW, C*kh*kw)
36
+ out = cols @ self.weight.T + self.bias # (N*OH*OW, out_channels)
37
+ return out.reshape(N, OH, OW, self.out_channels).permute(0, 3, 1, 2)
38
+
39
+
40
+ class MaxPool2d(Module):
41
+ """Max pooling over `(N, C, H, W)` input: im2col, then max over each patch."""
42
+ def __init__(self, kernel_size, stride=None):
43
+ super().__init__()
44
+ self.kernel_size = kernel_size if isinstance(kernel_size, tuple) else (kernel_size, kernel_size)
45
+ self.stride = stride if stride is not None else self.kernel_size[0]
46
+
47
+ def forward(self, x):
48
+ assert x.ndim == 4, f"MaxPool2d expects 4D input (N,C,H,W), got {x.ndim}D"
49
+ N, C, H, W = x.shape
50
+ kh, kw = self.kernel_size
51
+ OH = (H - kh) // self.stride + 1
52
+ OW = (W - kw) // self.stride + 1
53
+
54
+ cols = x.im2col(kh, kw, self.stride, 0) # (N*OH*OW, C*kh*kw)
55
+ out = cols.reshape(N, OH, OW, C, kh * kw).max(axis=-1)
56
+ return out.permute(0, 3, 1, 2)
57
+
58
+
59
+ class Flatten(Module):
60
+ """Flatten all dims except the batch dim into one."""
61
+ def forward(self, x):
62
+ batch_size = x.shape[0]
63
+ return x.reshape(batch_size, -1)
stackgrad/data.py ADDED
@@ -0,0 +1,28 @@
1
+ import numpy as np
2
+
3
+
4
+ class DataLoader:
5
+ """Iterate over `(x, y)` in mini-batches, optionally shuffling each epoch."""
6
+ def __init__(self, x, y, batch_size=32, shuffle=True, drop_last=False):
7
+ self.x = x
8
+ self.y = y
9
+ self.batch_size = batch_size
10
+ self.shuffle = shuffle
11
+ self.drop_last = drop_last
12
+ self.n = x.shape[0]
13
+
14
+ def __iter__(self):
15
+ indices = np.arange(self.n)
16
+ if self.shuffle:
17
+ np.random.shuffle(indices)
18
+ for start in range(0, self.n, self.batch_size):
19
+ end = min(start + self.batch_size, self.n)
20
+ if self.drop_last and (end - start) < self.batch_size:
21
+ break
22
+ idx = indices[start:end]
23
+ yield self.x[idx], self.y[idx]
24
+
25
+ def __len__(self):
26
+ if self.drop_last:
27
+ return self.n // self.batch_size
28
+ return (self.n + self.batch_size - 1) // self.batch_size
@@ -0,0 +1,41 @@
1
+ import math
2
+
3
+
4
+ def relu(x):
5
+ """Rectified linear unit, `max(0, x)`."""
6
+ return x.relu()
7
+
8
+
9
+ def sigmoid(x):
10
+ """Logistic sigmoid, `1 / (1 + e**-x)`."""
11
+ return x.sigmoid()
12
+
13
+
14
+ def tanh(x):
15
+ """Hyperbolic tangent."""
16
+ return x.tanh()
17
+
18
+
19
+ def gelu(x):
20
+ """Gaussian error linear unit (tanh approximation, as in GPT-2)."""
21
+ c = math.sqrt(2.0 / math.pi)
22
+ inner = (x + x ** 3 * 0.044715) * c
23
+ return x * 0.5 * (tanh(inner) + 1.0)
24
+
25
+
26
+ def softmax(x, axis=-1):
27
+ """Softmax over `axis`, max-subtracted so `exp` cannot overflow."""
28
+ # softmax is shift-invariant, so the subtracted max needs no gradient
29
+ e = (x - x.max(axis=axis, keepdims=True).detach()).exp()
30
+ return e / e.sum(axis=axis, keepdims=True)
31
+
32
+
33
+ def log_softmax(x, axis=-1):
34
+ """Log of softmax over `axis`. More stable than `softmax(x).log()`."""
35
+ shifted = x - x.max(axis=axis, keepdims=True).detach()
36
+ return shifted - shifted.exp().sum(axis=axis, keepdims=True).log()
37
+
38
+
39
+ def silu(x):
40
+ """x * sigmoid(x), the gate activation in SwiGLU."""
41
+ return x * x.sigmoid()
stackgrad/gpt.py ADDED
@@ -0,0 +1,120 @@
1
+ import numpy as np
2
+
3
+ from .attention import Attention, KVCache
4
+ from .checkpoint import checkpoint
5
+ from .layers import DepthAttention, Embedding, Linear, RMSNorm, SiLUMLP, SwiGLU
6
+ from .module import Module
7
+ from .sampling import sample, sample_on_device
8
+ from .tensor import no_grad
9
+
10
+
11
+ def trunc_normal(shape, std):
12
+ """N(0, std^2) resampled until every value lies within 3 std."""
13
+ x = np.random.randn(*shape)
14
+ while (bad := np.abs(x) > 3).any():
15
+ x[bad] = np.random.randn(bad.sum())
16
+ return (x * std).astype(np.float32)
17
+
18
+
19
+ class Block(Module):
20
+ """norm, rope, and ffn set the CS336 ablations; the defaults are the modern block."""
21
+ def __init__(self, d_model, n_heads, n_kv_heads, d_ff, rope_theta, head_dim=None, qk_norm=False, norm_eps=1e-5,
22
+ norm="pre", rope=True, ffn="swiglu"):
23
+ super().__init__()
24
+ self.norm = norm
25
+ self.attn_norm = None if norm == "none" else RMSNorm(d_model, norm_eps)
26
+ self.attn = Attention(d_model, n_heads, n_kv_heads, rope_theta, head_dim, qk_norm, norm_eps, rope)
27
+ self.ffn_norm = None if norm == "none" else RMSNorm(d_model, norm_eps)
28
+ self.ffn = SwiGLU(d_model, d_ff) if ffn == "swiglu" else SiLUMLP(d_model, d_ff)
29
+
30
+ def forward(self, x, cache=None, pad=None):
31
+ if self.norm == "post": # the original transformer's order
32
+ x = self.attn_norm(x + self.attn(x, cache, pad))
33
+ return self.ffn_norm(x + self.ffn(x))
34
+ if self.norm == "none":
35
+ x = x + self.attn(x, cache, pad)
36
+ return x + self.ffn(x)
37
+ x = x + self.attn(self.attn_norm(x), cache, pad)
38
+ return x + self.ffn(self.ffn_norm(x))
39
+
40
+ def sublayers(self, cache=None, pad=None):
41
+ """Attention and feed-forward as functions of their input, without forward()'s residual adds."""
42
+ return [lambda x: self.attn(self.attn_norm(x), cache, pad), lambda x: self.ffn(self.ffn_norm(x))]
43
+
44
+
45
+ class TransformerLM(Module):
46
+ """Decoder-only transformer LM: the CS336 model by default, Qwen3 with the options qwen3.py sets."""
47
+ def __init__(self, vocab_size, context_length, d_model, n_layers, n_heads, d_ff,
48
+ n_kv_heads=None, rope_theta=10000.0, head_dim=None, qk_norm=False, norm_eps=1e-5, tie_embeddings=False,
49
+ norm="pre", rope=True, ffn="swiglu", attn_res=False, attn_res_block=8):
50
+ super().__init__()
51
+ assert not attn_res or norm == "pre", "Attention Residuals is defined on pre-norm blocks"
52
+ self.context_length = context_length
53
+ self.attn_res_block = attn_res_block
54
+ self.depth = [DepthAttention(d_model, norm_eps) for _ in range(2 * n_layers + 1)] if attn_res else None
55
+ self.embed = Embedding(vocab_size, d_model)
56
+ self.blocks = [Block(d_model, n_heads, n_kv_heads, d_ff, rope_theta, head_dim, qk_norm, norm_eps, norm, rope, ffn)
57
+ for _ in range(n_layers)]
58
+ self.norm = None if norm == "none" else RMSNorm(d_model, norm_eps) # "none" removes every RMSNorm, the final one too
59
+ self.head = None if tie_embeddings else Linear(d_model, vocab_size, bias=False)
60
+ self.checkpoint_blocks = False # recompute each block in backward instead of storing its activations
61
+
62
+ # CS336 init
63
+ for p in self.parameters():
64
+ if p.ndim == 2:
65
+ p.data = trunc_normal(p.shape, np.sqrt(2.0 / sum(p.shape)))
66
+ self.embed.weight.data = trunc_normal(self.embed.weight.shape, 1.0)
67
+
68
+ def forward(self, idx, caches=None, pad=None):
69
+ """idx (B, T) ints -> logits (B, T, vocab); caches is one KVCache per block, pad (B,) the left padding."""
70
+ return self.project(self.hidden(idx, caches, pad))
71
+
72
+ def hidden(self, idx, caches=None, pad=None):
73
+ """Final normalized hidden states (B, T, d_model): the model without the output projection."""
74
+ x = self.embed(idx)
75
+ if self.depth is not None:
76
+ x = self.attn_res(x, caches, pad)
77
+ return x if self.norm is None else self.norm(x)
78
+ for i, block in enumerate(self.blocks):
79
+ if self.checkpoint_blocks and caches is None and pad is None:
80
+ x = checkpoint(block, x)
81
+ else:
82
+ x = block(x, None if caches is None else caches[i], pad)
83
+ return x if self.norm is None else self.norm(x)
84
+
85
+ def attn_res(self, x, caches=None, pad=None):
86
+ """Block Attention Residuals over the embedding x: the hidden states before the final norm."""
87
+ # shortcut: no activation checkpointing here; add it per block if attn_res ever trains a model too big to fit
88
+ blocks, partial = [x], None # finished block sums (the embedding first), and the running sum of this block
89
+ layers = [f for i, b in enumerate(self.blocks) for f in b.sublayers(None if caches is None else caches[i], pad)]
90
+ for i, f in enumerate(layers):
91
+ out = f(self.depth[i](blocks if partial is None else blocks + [partial]))
92
+ partial = out if partial is None else partial + out
93
+ if (i + 1) % self.attn_res_block == 0:
94
+ blocks, partial = blocks + [partial], None
95
+ return self.depth[-1](blocks if partial is None else blocks + [partial])
96
+
97
+ def project(self, h):
98
+ """Hidden states -> logits, through the head or the tied embedding table."""
99
+ return self.head(h) if self.head is not None else h @ self.embed.weight.T
100
+
101
+ def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None, top_p=None, eos_id=None, pad=None):
102
+ """Extends idx (B, T0) by up to max_new_tokens until every row emits eos_id; pad (B,) is left padding."""
103
+ idx = np.asarray(idx)
104
+ assert idx.shape[1] + max_new_tokens <= self.context_length, "prompt + new tokens exceed context_length"
105
+ caches = [KVCache() for _ in self.blocks]
106
+ done = np.zeros(len(idx), dtype=bool)
107
+ with no_grad():
108
+ logits = self.project(self.hidden(idx, caches, pad)[:, -1])
109
+ for _ in range(max_new_tokens):
110
+ if top_k is None and top_p is None:
111
+ next_id = sample_on_device(logits, temperature)
112
+ else:
113
+ next_id = sample(logits.numpy(), temperature, top_k, top_p)
114
+ idx = np.concatenate([idx, next_id[:, None]], axis=1)
115
+ if eos_id is not None:
116
+ done |= next_id == eos_id
117
+ if done.all():
118
+ break
119
+ logits = self.project(self.hidden(next_id[:, None], caches, pad)[:, -1])
120
+ return idx
@@ -0,0 +1,55 @@
1
+ import numpy as np
2
+
3
+
4
+ def numerical_gradient(f, inputs, eps=1e-5):
5
+ """Estimate gradients of `f` w.r.t. each input by central differences."""
6
+ grads = []
7
+ # perturb in float64 for precision
8
+ orig_data = [inp.data.copy() for inp in inputs]
9
+ for inp in inputs:
10
+ inp.data = inp.data.astype(np.float64)
11
+ f64_data = [inp.data.copy() for inp in inputs]
12
+
13
+ for k, inp in enumerate(inputs):
14
+ grad = np.zeros(inp.data.shape, dtype=np.float64)
15
+ it = np.nditer(f64_data[k], flags=['multi_index'])
16
+ while not it.finished:
17
+ idx = it.multi_index
18
+ old_val = f64_data[k][idx]
19
+
20
+ inp.data = f64_data[k].copy()
21
+ inp.data[idx] = old_val + eps
22
+ loss_plus = float(f().data)
23
+
24
+ inp.data = f64_data[k].copy()
25
+ inp.data[idx] = old_val - eps
26
+ loss_minus = float(f().data)
27
+
28
+ grad[idx] = (loss_plus - loss_minus) / (2 * eps)
29
+ it.iternext()
30
+ inp.data = f64_data[k].copy()
31
+ grads.append(grad.astype(np.float32))
32
+
33
+ for inp, od in zip(inputs, orig_data):
34
+ inp.data = od
35
+ return grads
36
+
37
+
38
+ def check_gradient(f, inputs, eps=1e-5, atol=1e-4, rtol=1e-3):
39
+ """Compare backward() gradients against numerical ones, raising AssertionError past the tolerance."""
40
+ for inp in inputs:
41
+ inp.zero_grad()
42
+ loss = f()
43
+ loss.backward()
44
+ analytic = [inp.grad.copy() for inp in inputs]
45
+
46
+ numerical = numerical_gradient(f, inputs, eps)
47
+
48
+ for i, (a, n) in enumerate(zip(analytic, numerical)):
49
+ if not np.allclose(a, n, atol=atol, rtol=rtol):
50
+ max_diff = np.max(np.abs(a - n))
51
+ raise AssertionError(
52
+ f"Gradient check failed for input {i}: max diff = {max_diff}\n"
53
+ f"Analytic:\n{a}\nNumerical:\n{n}"
54
+ )
55
+ return True