stackgrad 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stackgrad/__init__.py +75 -0
- stackgrad/attention.py +86 -0
- stackgrad/backend/__init__.py +11 -0
- stackgrad/backend/cpu.py +116 -0
- stackgrad/backend/cuda.py +96 -0
- stackgrad/backend/cuda_attention.py +138 -0
- stackgrad/backend/metal.py +106 -0
- stackgrad/backend/metal_attention.py +195 -0
- stackgrad/backend/metal_quant.py +44 -0
- stackgrad/checkpoint.py +38 -0
- stackgrad/conv.py +63 -0
- stackgrad/data.py +28 -0
- stackgrad/functional.py +41 -0
- stackgrad/gpt.py +120 -0
- stackgrad/grad_check.py +55 -0
- stackgrad/layers.py +215 -0
- stackgrad/lora.py +54 -0
- stackgrad/loss.py +28 -0
- stackgrad/mlops.py +352 -0
- stackgrad/module.py +120 -0
- stackgrad/optim.py +121 -0
- stackgrad/profile.py +45 -0
- stackgrad/quant.py +66 -0
- stackgrad/qwen3.py +104 -0
- stackgrad/sampling.py +39 -0
- stackgrad/serve/api.py +434 -0
- stackgrad/serve/engine.py +53 -0
- stackgrad/serve/kv_cache.py +35 -0
- stackgrad/serve/runner.py +63 -0
- stackgrad/serve/scheduler.py +88 -0
- stackgrad/speculative.py +68 -0
- stackgrad/state.py +27 -0
- stackgrad/tensor.py +377 -0
- stackgrad/tokenizer.py +202 -0
- stackgrad-0.2.0.dist-info/METADATA +172 -0
- stackgrad-0.2.0.dist-info/RECORD +38 -0
- stackgrad-0.2.0.dist-info/WHEEL +4 -0
- stackgrad-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import mlx.core as mx
|
|
2
|
+
|
|
3
|
+
_SHAPES = """
|
|
4
|
+
const int T = q_shape[2], S = k_shape[2], H = q_shape[1], Hkv = k_shape[1];
|
|
5
|
+
const float scale = rsqrt(float(D));
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
_V1 = _SHAPES + """
|
|
9
|
+
const uint row = thread_position_in_grid.x; // over B * H * T query rows
|
|
10
|
+
if (row >= uint(q_shape[0] * H * T)) return;
|
|
11
|
+
const int i = row % T, bh = row / T, b = bh / H, h = bh % H;
|
|
12
|
+
const int kv = (b * Hkv + h / (H / Hkv)) * S * D;
|
|
13
|
+
|
|
14
|
+
float qr[D], acc[D];
|
|
15
|
+
for (int d = 0; d < D; d++) { qr[d] = float(q[row * D + d]) * scale; acc[d] = 0.0f; }
|
|
16
|
+
float m = -INFINITY, l = 0.0f;
|
|
17
|
+
for (int j = 0; j <= S - T + i; j++) {
|
|
18
|
+
float s = 0.0f;
|
|
19
|
+
for (int d = 0; d < D; d++) s += qr[d] * float(k[kv + j * D + d]);
|
|
20
|
+
const float m_new = max(m, s), corr = exp(m - m_new), p = exp(s - m_new);
|
|
21
|
+
l = l * corr + p;
|
|
22
|
+
for (int d = 0; d < D; d++) acc[d] = acc[d] * corr + p * float(v[kv + j * D + d]);
|
|
23
|
+
m = m_new;
|
|
24
|
+
}
|
|
25
|
+
for (int d = 0; d < D; d++) out[row * D + d] = InT(acc[d] / l);
|
|
26
|
+
lse[row] = m + log(l);
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
_V2 = _SHAPES + """
|
|
30
|
+
constexpr int BQ = 32; // query rows (= threads) per threadgroup
|
|
31
|
+
threadgroup float Ks[BK * D], Vs[BK * D];
|
|
32
|
+
const int tid = thread_position_in_threadgroup.x;
|
|
33
|
+
const int n_qblocks = (T + BQ - 1) / BQ;
|
|
34
|
+
const int bh = threadgroup_position_in_grid.x / n_qblocks, qb = threadgroup_position_in_grid.x % n_qblocks;
|
|
35
|
+
const int b = bh / H, h = bh % H, kv = (b * Hkv + h / (H / Hkv)) * S * D;
|
|
36
|
+
const int i = qb * BQ + tid, last = S - T + i; // this row sees keys 0 .. last
|
|
37
|
+
const int group_last = min(S - T + qb * BQ + BQ - 1, S - 1);
|
|
38
|
+
|
|
39
|
+
float qr[D], acc[D];
|
|
40
|
+
for (int d = 0; d < D; d++) { qr[d] = i < T ? float(q[(bh * T + i) * D + d]) * scale : 0.0f; acc[d] = 0.0f; }
|
|
41
|
+
float m = -INFINITY, l = 0.0f;
|
|
42
|
+
for (int j0 = 0; j0 <= group_last; j0 += BK) {
|
|
43
|
+
for (int e = tid; e < BK * D; e += BQ) { // the 32 threads load the tile together
|
|
44
|
+
const bool in = j0 + e / D < S;
|
|
45
|
+
Ks[e] = in ? float(k[kv + j0 * D + e]) : 0.0f;
|
|
46
|
+
Vs[e] = in ? float(v[kv + j0 * D + e]) : 0.0f;
|
|
47
|
+
}
|
|
48
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
49
|
+
float s[BK], m_tile = -INFINITY;
|
|
50
|
+
for (int jj = 0; jj < BK; jj++) {
|
|
51
|
+
float dot = 0.0f;
|
|
52
|
+
for (int d = 0; d < D; d++) dot += qr[d] * Ks[jj * D + d];
|
|
53
|
+
s[jj] = j0 + jj <= last ? dot : -INFINITY;
|
|
54
|
+
m_tile = max(m_tile, s[jj]);
|
|
55
|
+
}
|
|
56
|
+
const float m_new = max(m, m_tile), corr = exp(m - m_new);
|
|
57
|
+
l *= corr;
|
|
58
|
+
for (int d = 0; d < D; d++) acc[d] *= corr;
|
|
59
|
+
for (int jj = 0; jj < BK; jj++) {
|
|
60
|
+
const float p = exp(s[jj] - m_new);
|
|
61
|
+
l += p;
|
|
62
|
+
for (int d = 0; d < D; d++) acc[d] += p * Vs[jj * D + d];
|
|
63
|
+
}
|
|
64
|
+
m = m_new;
|
|
65
|
+
threadgroup_barrier(mem_flags::mem_threadgroup); // everyone is done with this tile
|
|
66
|
+
}
|
|
67
|
+
if (i < T) {
|
|
68
|
+
for (int d = 0; d < D; d++) out[(bh * T + i) * D + d] = InT(acc[d] / l);
|
|
69
|
+
lse[bh * T + i] = m + log(l);
|
|
70
|
+
}
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
# v3: 4 simdgroups x 8 query rows; lanes 4r..4r+3 share softmax row r, a quarter of the columns each
|
|
74
|
+
_V3 = _SHAPES + """
|
|
75
|
+
constexpr int BQ = 32, TILE = (32 * D > 2 * BK * D) ? 32 * D : 2 * BK * D;
|
|
76
|
+
threadgroup float smem[TILE]; // Q staging, then K/V tiles, then O staging
|
|
77
|
+
threadgroup float Ps[4][8 * BK]; // each simdgroup's scores, then probabilities
|
|
78
|
+
threadgroup float Dg[4][64]; // each simdgroup's diag(corr), for rescaling O
|
|
79
|
+
threadgroup float* Ks = smem;
|
|
80
|
+
threadgroup float* Vs = smem + BK * D;
|
|
81
|
+
const int tid = thread_index_in_threadgroup, sg = simdgroup_index_in_threadgroup, lane = thread_index_in_simdgroup;
|
|
82
|
+
const int n_qblocks = (T + BQ - 1) / BQ;
|
|
83
|
+
const int bh = threadgroup_position_in_grid.x / n_qblocks, qb = threadgroup_position_in_grid.x % n_qblocks;
|
|
84
|
+
const int b = bh / H, h = bh % H, kv = (b * Hkv + h / (H / Hkv)) * S * D;
|
|
85
|
+
const int group_last = min(S - T + qb * BQ + BQ - 1, S - 1);
|
|
86
|
+
const int r = lane / 4, part = lane % 4; // this lane's softmax row and column quarter
|
|
87
|
+
const int i = qb * BQ + sg * 8 + r, last = S - T + i;
|
|
88
|
+
|
|
89
|
+
// stage this block's Q (scaled) and load each simdgroup's 8 rows into matrix registers
|
|
90
|
+
for (int e = tid; e < BQ * D; e += 128) {
|
|
91
|
+
const int qi = qb * BQ + e / D;
|
|
92
|
+
smem[e] = qi < T ? float(q[(bh * T + qb * BQ) * D + e]) * scale : 0.0f;
|
|
93
|
+
}
|
|
94
|
+
for (int e = lane; e < 64; e += 32) Dg[sg][e] = 0.0f;
|
|
95
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
96
|
+
simdgroup_float8x8 Q[D / 8], O[D / 8];
|
|
97
|
+
for (int dc = 0; dc < D / 8; dc++) {
|
|
98
|
+
simdgroup_load(Q[dc], smem + sg * 8 * D + dc * 8, D);
|
|
99
|
+
O[dc] = simdgroup_float8x8(0.0f);
|
|
100
|
+
}
|
|
101
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
102
|
+
|
|
103
|
+
float m = -INFINITY, l = 0.0f;
|
|
104
|
+
for (int j0 = 0; j0 <= group_last; j0 += BK) {
|
|
105
|
+
for (int e = tid; e < BK * D; e += 128) {
|
|
106
|
+
const bool in = j0 + e / D < S;
|
|
107
|
+
Ks[e] = in ? float(k[kv + j0 * D + e]) : 0.0f;
|
|
108
|
+
Vs[e] = in ? float(v[kv + j0 * D + e]) : 0.0f;
|
|
109
|
+
}
|
|
110
|
+
threadgroup_barrier(mem_flags::mem_threadgroup);
|
|
111
|
+
|
|
112
|
+
// scores: (8 x D) @ (D x BK), one 8x8 output block per 8 keys
|
|
113
|
+
for (int kb = 0; kb < BK / 8; kb++) {
|
|
114
|
+
simdgroup_float8x8 acc(0.0f), Kt;
|
|
115
|
+
for (int dc = 0; dc < D / 8; dc++) {
|
|
116
|
+
simdgroup_load(Kt, Ks + kb * 8 * D + dc * 8, D, ulong2(0, 0), true);
|
|
117
|
+
simdgroup_multiply_accumulate(acc, Q[dc], Kt, acc);
|
|
118
|
+
}
|
|
119
|
+
simdgroup_store(acc, Ps[sg] + kb * 8, BK);
|
|
120
|
+
}
|
|
121
|
+
simdgroup_barrier(mem_flags::mem_threadgroup);
|
|
122
|
+
|
|
123
|
+
// online softmax on row r, columns part*BK/4 .. (part+1)*BK/4, reduced across the row's 4 lanes
|
|
124
|
+
constexpr int C = BK / 4;
|
|
125
|
+
float m_tile = -INFINITY;
|
|
126
|
+
for (int c = part * C; c < (part + 1) * C; c++) {
|
|
127
|
+
const float s = j0 + c <= last ? Ps[sg][r * BK + c] : -INFINITY;
|
|
128
|
+
Ps[sg][r * BK + c] = s;
|
|
129
|
+
m_tile = max(m_tile, s);
|
|
130
|
+
}
|
|
131
|
+
m_tile = max(m_tile, simd_shuffle_xor(m_tile, 1));
|
|
132
|
+
m_tile = max(m_tile, simd_shuffle_xor(m_tile, 2));
|
|
133
|
+
const float m_new = max(m, m_tile), corr = exp(m - m_new);
|
|
134
|
+
float psum = 0.0f;
|
|
135
|
+
for (int c = part * C; c < (part + 1) * C; c++) {
|
|
136
|
+
const float p = exp(Ps[sg][r * BK + c] - m_new);
|
|
137
|
+
Ps[sg][r * BK + c] = p;
|
|
138
|
+
psum += p;
|
|
139
|
+
}
|
|
140
|
+
psum += simd_shuffle_xor(psum, 1);
|
|
141
|
+
psum += simd_shuffle_xor(psum, 2);
|
|
142
|
+
l = l * corr + psum;
|
|
143
|
+
m = m_new;
|
|
144
|
+
if (part == 0) Dg[sg][r * 8 + r] = corr;
|
|
145
|
+
simdgroup_barrier(mem_flags::mem_threadgroup);
|
|
146
|
+
|
|
147
|
+
// O = diag(corr) @ O + P @ V_tile
|
|
148
|
+
simdgroup_float8x8 Corr, P, Vm;
|
|
149
|
+
simdgroup_load(Corr, Dg[sg], 8);
|
|
150
|
+
for (int dc = 0; dc < D / 8; dc++) {
|
|
151
|
+
simdgroup_multiply(O[dc], Corr, O[dc]);
|
|
152
|
+
for (int kb = 0; kb < BK / 8; kb++) {
|
|
153
|
+
simdgroup_load(P, Ps[sg] + kb * 8, BK);
|
|
154
|
+
simdgroup_load(Vm, Vs + kb * 8 * D + dc * 8, D);
|
|
155
|
+
simdgroup_multiply_accumulate(O[dc], P, Vm, O[dc]);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
threadgroup_barrier(mem_flags::mem_threadgroup); // everyone is done with this tile
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// stage O through threadgroup memory, then each lane writes its row's quarter divided by l
|
|
162
|
+
for (int dc = 0; dc < D / 8; dc++) simdgroup_store(O[dc], smem + sg * 8 * D + dc * 8, D);
|
|
163
|
+
simdgroup_barrier(mem_flags::mem_threadgroup);
|
|
164
|
+
if (i < T) {
|
|
165
|
+
for (int d = part * (D / 4); d < (part + 1) * (D / 4); d++)
|
|
166
|
+
out[(bh * T + i) * D + d] = InT(smem[(sg * 8 + r) * D + d] / l);
|
|
167
|
+
if (part == 0) lse[bh * T + i] = m + log(l);
|
|
168
|
+
}
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
_KERNELS = {
|
|
172
|
+
name: mx.fast.metal_kernel(name=f"stackgrad_attention_{name}", input_names=["q", "k", "v"],
|
|
173
|
+
output_names=["out", "lse"], source=src)
|
|
174
|
+
for name, src in [("v1", _V1), ("v2", _V2), ("v3", _V3)]
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def attention(q, k, v, version="v3"):
|
|
179
|
+
"""Causal attention: q (B, H, T, D), k, v (B, Hkv, S, D) -> out (B, H, T, D), lse (B, H, T)."""
|
|
180
|
+
B, H, T, D = q.shape
|
|
181
|
+
assert k.shape[1] and H % k.shape[1] == 0 and D % 8 == 0 and D <= 128, (q.shape, k.shape)
|
|
182
|
+
bk = 32 if D <= 64 else 16 # keeps the K/V tiles within 32 KB of threadgroup memory
|
|
183
|
+
if version == "v1":
|
|
184
|
+
grid, group = (B * H * T, 1, 1), (min(256, B * H * T), 1, 1)
|
|
185
|
+
else:
|
|
186
|
+
threads = 32 if version == "v2" else 128
|
|
187
|
+
grid, group = (B * H * -(-T // 32) * threads, 1, 1), (threads, 1, 1)
|
|
188
|
+
return _KERNELS[version](
|
|
189
|
+
inputs=[q, k, v],
|
|
190
|
+
template=[("InT", q.dtype), ("D", D), ("BK", bk)],
|
|
191
|
+
grid=grid,
|
|
192
|
+
threadgroup=group,
|
|
193
|
+
output_shapes=[q.shape, (B, H, T)],
|
|
194
|
+
output_dtypes=[q.dtype, mx.float32],
|
|
195
|
+
)
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import mlx.core as mx
|
|
2
|
+
|
|
3
|
+
SIMDGROUPS = 8 # output columns per threadgroup
|
|
4
|
+
|
|
5
|
+
# one simdgroup per output column; s * sum(x q) + b * sum(x) applies scale and bias once per word
|
|
6
|
+
_SOURCE = """
|
|
7
|
+
constexpr int PER_WORD = 32 / BITS, MASK = (1 << BITS) - 1;
|
|
8
|
+
const int M = x_shape[0], K = x_shape[1], N = w_shape[0];
|
|
9
|
+
const int words = K / PER_WORD, groups = K / GROUP;
|
|
10
|
+
const int n = threadgroup_position_in_grid.x * SG + simdgroup_index_in_threadgroup;
|
|
11
|
+
const int lane = thread_index_in_simdgroup;
|
|
12
|
+
if (n >= N) return; // the whole simdgroup leaves together
|
|
13
|
+
for (int m = 0; m < M; m++) {
|
|
14
|
+
float acc = 0.0f;
|
|
15
|
+
for (int wi = lane; wi < words; wi += 32) {
|
|
16
|
+
const uint word = w[n * words + wi];
|
|
17
|
+
const int k0 = wi * PER_WORD, g = k0 / GROUP;
|
|
18
|
+
float xq = 0.0f, xs = 0.0f;
|
|
19
|
+
for (int j = 0; j < PER_WORD; j++) {
|
|
20
|
+
const float xv = float(x[m * K + k0 + j]);
|
|
21
|
+
xq += xv * float((word >> (j * BITS)) & MASK);
|
|
22
|
+
xs += xv;
|
|
23
|
+
}
|
|
24
|
+
acc += float(scales[n * groups + g]) * xq + float(biases[n * groups + g]) * xs;
|
|
25
|
+
}
|
|
26
|
+
acc = simd_sum(acc);
|
|
27
|
+
if (lane == 0) out[m * N + n] = InT(acc);
|
|
28
|
+
}
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
_KERNEL = mx.fast.metal_kernel(name="stackgrad_quant_matmul", input_names=["x", "w", "scales", "biases"],
|
|
32
|
+
output_names=["out"], source=_SOURCE)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def quant_matmul(x, packed, scales, biases, bits, group):
|
|
36
|
+
M, N = x.shape[0], packed.shape[0]
|
|
37
|
+
return _KERNEL(
|
|
38
|
+
inputs=[x, packed, scales, biases],
|
|
39
|
+
template=[("InT", x.dtype), ("BITS", bits), ("GROUP", group), ("SG", SIMDGROUPS)],
|
|
40
|
+
grid=(-(-N // SIMDGROUPS) * SIMDGROUPS * 32, 1, 1),
|
|
41
|
+
threadgroup=(SIMDGROUPS * 32, 1, 1),
|
|
42
|
+
output_shapes=[(M, N)],
|
|
43
|
+
output_dtypes=[x.dtype],
|
|
44
|
+
)[0]
|
stackgrad/checkpoint.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from . import backend
|
|
2
|
+
from . import tensor as T
|
|
3
|
+
from .mlops import Function
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class _Checkpoint(Function):
|
|
7
|
+
def forward(self, *arrays, fn):
|
|
8
|
+
self.fn, self.arrays, self.cast = fn, arrays, T._autocast
|
|
9
|
+
device = self.parents[0].device
|
|
10
|
+
with T.no_grad():
|
|
11
|
+
out = fn(*[T.Tensor(a, device=device) for a in arrays]).data
|
|
12
|
+
# realize now, or MLX runs it later inside one big eval that holds every activation
|
|
13
|
+
self.xp.realize(out)
|
|
14
|
+
return out
|
|
15
|
+
|
|
16
|
+
def backward(self, grad):
|
|
17
|
+
device = self.parents[0].device
|
|
18
|
+
leaves = [T.Tensor(a, requires_grad=needs, device=device) for a, needs in zip(self.arrays, self.needs_grad)]
|
|
19
|
+
with T.autocast(self.cast): # forward's precision: backward usually runs outside autocast
|
|
20
|
+
out = self.fn(*leaves)
|
|
21
|
+
# backward of sum(out * grad) pushes grad through fn; weights inside fn collect .grad here
|
|
22
|
+
(out * T.Tensor(grad, device=device)).sum().backward()
|
|
23
|
+
# realize now, so this block's activations free before the next block recomputes
|
|
24
|
+
weights = self.fn.parameters() if hasattr(self.fn, "parameters") else []
|
|
25
|
+
self.xp.realize(*[t.grad for t in leaves + list(weights) if t.grad is not None])
|
|
26
|
+
return tuple(leaf.grad for leaf in leaves)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def checkpoint(fn, *inputs):
|
|
30
|
+
"""fn(*inputs), recomputed during backward instead of storing its activations. fn must be deterministic."""
|
|
31
|
+
if not T.grad_enabled():
|
|
32
|
+
return fn(*inputs)
|
|
33
|
+
device = inputs[0].device
|
|
34
|
+
ctx = _Checkpoint(backend.get(device), *inputs)
|
|
35
|
+
out = T.Tensor(ctx.forward(*[x.data for x in inputs], fn=fn), device=device)
|
|
36
|
+
# always recorded: weights inside fn may need grads (LoRA adapters in frozen layers)
|
|
37
|
+
out.requires_grad, out._ctx = True, ctx
|
|
38
|
+
return out
|
stackgrad/conv.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from .module import Module
|
|
4
|
+
from .tensor import Tensor
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class Conv2d(Module):
|
|
8
|
+
"""2D convolution over `(N, C, H, W)` input, implemented with im2col."""
|
|
9
|
+
def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):
|
|
10
|
+
super().__init__()
|
|
11
|
+
assert in_channels > 0 and out_channels > 0, "channels must be positive"
|
|
12
|
+
self.in_channels = in_channels
|
|
13
|
+
self.out_channels = out_channels
|
|
14
|
+
self.kernel_size = kernel_size if isinstance(kernel_size, tuple) else (kernel_size, kernel_size)
|
|
15
|
+
self.stride = stride
|
|
16
|
+
self.padding = padding
|
|
17
|
+
|
|
18
|
+
kh, kw = self.kernel_size
|
|
19
|
+
assert kh > 0 and kw > 0, "kernel_size must be positive"
|
|
20
|
+
fan_in = in_channels * kh * kw
|
|
21
|
+
scale = np.sqrt(2.0 / fan_in)
|
|
22
|
+
self.weight = Tensor(
|
|
23
|
+
(np.random.randn(out_channels, in_channels * kh * kw) * scale).astype(np.float32),
|
|
24
|
+
requires_grad=True
|
|
25
|
+
)
|
|
26
|
+
self.bias = Tensor(np.zeros(out_channels, dtype=np.float32), requires_grad=True)
|
|
27
|
+
|
|
28
|
+
def forward(self, x):
|
|
29
|
+
assert x.ndim == 4, f"Conv2d expects 4D input (N,C,H,W), got {x.ndim}D"
|
|
30
|
+
N, _, H, W = x.shape
|
|
31
|
+
kh, kw = self.kernel_size
|
|
32
|
+
OH = (H + 2 * self.padding - kh) // self.stride + 1
|
|
33
|
+
OW = (W + 2 * self.padding - kw) // self.stride + 1
|
|
34
|
+
|
|
35
|
+
cols = x.im2col(kh, kw, self.stride, self.padding) # (N*OH*OW, C*kh*kw)
|
|
36
|
+
out = cols @ self.weight.T + self.bias # (N*OH*OW, out_channels)
|
|
37
|
+
return out.reshape(N, OH, OW, self.out_channels).permute(0, 3, 1, 2)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class MaxPool2d(Module):
|
|
41
|
+
"""Max pooling over `(N, C, H, W)` input: im2col, then max over each patch."""
|
|
42
|
+
def __init__(self, kernel_size, stride=None):
|
|
43
|
+
super().__init__()
|
|
44
|
+
self.kernel_size = kernel_size if isinstance(kernel_size, tuple) else (kernel_size, kernel_size)
|
|
45
|
+
self.stride = stride if stride is not None else self.kernel_size[0]
|
|
46
|
+
|
|
47
|
+
def forward(self, x):
|
|
48
|
+
assert x.ndim == 4, f"MaxPool2d expects 4D input (N,C,H,W), got {x.ndim}D"
|
|
49
|
+
N, C, H, W = x.shape
|
|
50
|
+
kh, kw = self.kernel_size
|
|
51
|
+
OH = (H - kh) // self.stride + 1
|
|
52
|
+
OW = (W - kw) // self.stride + 1
|
|
53
|
+
|
|
54
|
+
cols = x.im2col(kh, kw, self.stride, 0) # (N*OH*OW, C*kh*kw)
|
|
55
|
+
out = cols.reshape(N, OH, OW, C, kh * kw).max(axis=-1)
|
|
56
|
+
return out.permute(0, 3, 1, 2)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class Flatten(Module):
|
|
60
|
+
"""Flatten all dims except the batch dim into one."""
|
|
61
|
+
def forward(self, x):
|
|
62
|
+
batch_size = x.shape[0]
|
|
63
|
+
return x.reshape(batch_size, -1)
|
stackgrad/data.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class DataLoader:
|
|
5
|
+
"""Iterate over `(x, y)` in mini-batches, optionally shuffling each epoch."""
|
|
6
|
+
def __init__(self, x, y, batch_size=32, shuffle=True, drop_last=False):
|
|
7
|
+
self.x = x
|
|
8
|
+
self.y = y
|
|
9
|
+
self.batch_size = batch_size
|
|
10
|
+
self.shuffle = shuffle
|
|
11
|
+
self.drop_last = drop_last
|
|
12
|
+
self.n = x.shape[0]
|
|
13
|
+
|
|
14
|
+
def __iter__(self):
|
|
15
|
+
indices = np.arange(self.n)
|
|
16
|
+
if self.shuffle:
|
|
17
|
+
np.random.shuffle(indices)
|
|
18
|
+
for start in range(0, self.n, self.batch_size):
|
|
19
|
+
end = min(start + self.batch_size, self.n)
|
|
20
|
+
if self.drop_last and (end - start) < self.batch_size:
|
|
21
|
+
break
|
|
22
|
+
idx = indices[start:end]
|
|
23
|
+
yield self.x[idx], self.y[idx]
|
|
24
|
+
|
|
25
|
+
def __len__(self):
|
|
26
|
+
if self.drop_last:
|
|
27
|
+
return self.n // self.batch_size
|
|
28
|
+
return (self.n + self.batch_size - 1) // self.batch_size
|
stackgrad/functional.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import math
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def relu(x):
|
|
5
|
+
"""Rectified linear unit, `max(0, x)`."""
|
|
6
|
+
return x.relu()
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def sigmoid(x):
|
|
10
|
+
"""Logistic sigmoid, `1 / (1 + e**-x)`."""
|
|
11
|
+
return x.sigmoid()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def tanh(x):
|
|
15
|
+
"""Hyperbolic tangent."""
|
|
16
|
+
return x.tanh()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def gelu(x):
|
|
20
|
+
"""Gaussian error linear unit (tanh approximation, as in GPT-2)."""
|
|
21
|
+
c = math.sqrt(2.0 / math.pi)
|
|
22
|
+
inner = (x + x ** 3 * 0.044715) * c
|
|
23
|
+
return x * 0.5 * (tanh(inner) + 1.0)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def softmax(x, axis=-1):
|
|
27
|
+
"""Softmax over `axis`, max-subtracted so `exp` cannot overflow."""
|
|
28
|
+
# softmax is shift-invariant, so the subtracted max needs no gradient
|
|
29
|
+
e = (x - x.max(axis=axis, keepdims=True).detach()).exp()
|
|
30
|
+
return e / e.sum(axis=axis, keepdims=True)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def log_softmax(x, axis=-1):
|
|
34
|
+
"""Log of softmax over `axis`. More stable than `softmax(x).log()`."""
|
|
35
|
+
shifted = x - x.max(axis=axis, keepdims=True).detach()
|
|
36
|
+
return shifted - shifted.exp().sum(axis=axis, keepdims=True).log()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def silu(x):
|
|
40
|
+
"""x * sigmoid(x), the gate activation in SwiGLU."""
|
|
41
|
+
return x * x.sigmoid()
|
stackgrad/gpt.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from .attention import Attention, KVCache
|
|
4
|
+
from .checkpoint import checkpoint
|
|
5
|
+
from .layers import DepthAttention, Embedding, Linear, RMSNorm, SiLUMLP, SwiGLU
|
|
6
|
+
from .module import Module
|
|
7
|
+
from .sampling import sample, sample_on_device
|
|
8
|
+
from .tensor import no_grad
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def trunc_normal(shape, std):
|
|
12
|
+
"""N(0, std^2) resampled until every value lies within 3 std."""
|
|
13
|
+
x = np.random.randn(*shape)
|
|
14
|
+
while (bad := np.abs(x) > 3).any():
|
|
15
|
+
x[bad] = np.random.randn(bad.sum())
|
|
16
|
+
return (x * std).astype(np.float32)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class Block(Module):
|
|
20
|
+
"""norm, rope, and ffn set the CS336 ablations; the defaults are the modern block."""
|
|
21
|
+
def __init__(self, d_model, n_heads, n_kv_heads, d_ff, rope_theta, head_dim=None, qk_norm=False, norm_eps=1e-5,
|
|
22
|
+
norm="pre", rope=True, ffn="swiglu"):
|
|
23
|
+
super().__init__()
|
|
24
|
+
self.norm = norm
|
|
25
|
+
self.attn_norm = None if norm == "none" else RMSNorm(d_model, norm_eps)
|
|
26
|
+
self.attn = Attention(d_model, n_heads, n_kv_heads, rope_theta, head_dim, qk_norm, norm_eps, rope)
|
|
27
|
+
self.ffn_norm = None if norm == "none" else RMSNorm(d_model, norm_eps)
|
|
28
|
+
self.ffn = SwiGLU(d_model, d_ff) if ffn == "swiglu" else SiLUMLP(d_model, d_ff)
|
|
29
|
+
|
|
30
|
+
def forward(self, x, cache=None, pad=None):
|
|
31
|
+
if self.norm == "post": # the original transformer's order
|
|
32
|
+
x = self.attn_norm(x + self.attn(x, cache, pad))
|
|
33
|
+
return self.ffn_norm(x + self.ffn(x))
|
|
34
|
+
if self.norm == "none":
|
|
35
|
+
x = x + self.attn(x, cache, pad)
|
|
36
|
+
return x + self.ffn(x)
|
|
37
|
+
x = x + self.attn(self.attn_norm(x), cache, pad)
|
|
38
|
+
return x + self.ffn(self.ffn_norm(x))
|
|
39
|
+
|
|
40
|
+
def sublayers(self, cache=None, pad=None):
|
|
41
|
+
"""Attention and feed-forward as functions of their input, without forward()'s residual adds."""
|
|
42
|
+
return [lambda x: self.attn(self.attn_norm(x), cache, pad), lambda x: self.ffn(self.ffn_norm(x))]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class TransformerLM(Module):
|
|
46
|
+
"""Decoder-only transformer LM: the CS336 model by default, Qwen3 with the options qwen3.py sets."""
|
|
47
|
+
def __init__(self, vocab_size, context_length, d_model, n_layers, n_heads, d_ff,
|
|
48
|
+
n_kv_heads=None, rope_theta=10000.0, head_dim=None, qk_norm=False, norm_eps=1e-5, tie_embeddings=False,
|
|
49
|
+
norm="pre", rope=True, ffn="swiglu", attn_res=False, attn_res_block=8):
|
|
50
|
+
super().__init__()
|
|
51
|
+
assert not attn_res or norm == "pre", "Attention Residuals is defined on pre-norm blocks"
|
|
52
|
+
self.context_length = context_length
|
|
53
|
+
self.attn_res_block = attn_res_block
|
|
54
|
+
self.depth = [DepthAttention(d_model, norm_eps) for _ in range(2 * n_layers + 1)] if attn_res else None
|
|
55
|
+
self.embed = Embedding(vocab_size, d_model)
|
|
56
|
+
self.blocks = [Block(d_model, n_heads, n_kv_heads, d_ff, rope_theta, head_dim, qk_norm, norm_eps, norm, rope, ffn)
|
|
57
|
+
for _ in range(n_layers)]
|
|
58
|
+
self.norm = None if norm == "none" else RMSNorm(d_model, norm_eps) # "none" removes every RMSNorm, the final one too
|
|
59
|
+
self.head = None if tie_embeddings else Linear(d_model, vocab_size, bias=False)
|
|
60
|
+
self.checkpoint_blocks = False # recompute each block in backward instead of storing its activations
|
|
61
|
+
|
|
62
|
+
# CS336 init
|
|
63
|
+
for p in self.parameters():
|
|
64
|
+
if p.ndim == 2:
|
|
65
|
+
p.data = trunc_normal(p.shape, np.sqrt(2.0 / sum(p.shape)))
|
|
66
|
+
self.embed.weight.data = trunc_normal(self.embed.weight.shape, 1.0)
|
|
67
|
+
|
|
68
|
+
def forward(self, idx, caches=None, pad=None):
|
|
69
|
+
"""idx (B, T) ints -> logits (B, T, vocab); caches is one KVCache per block, pad (B,) the left padding."""
|
|
70
|
+
return self.project(self.hidden(idx, caches, pad))
|
|
71
|
+
|
|
72
|
+
def hidden(self, idx, caches=None, pad=None):
|
|
73
|
+
"""Final normalized hidden states (B, T, d_model): the model without the output projection."""
|
|
74
|
+
x = self.embed(idx)
|
|
75
|
+
if self.depth is not None:
|
|
76
|
+
x = self.attn_res(x, caches, pad)
|
|
77
|
+
return x if self.norm is None else self.norm(x)
|
|
78
|
+
for i, block in enumerate(self.blocks):
|
|
79
|
+
if self.checkpoint_blocks and caches is None and pad is None:
|
|
80
|
+
x = checkpoint(block, x)
|
|
81
|
+
else:
|
|
82
|
+
x = block(x, None if caches is None else caches[i], pad)
|
|
83
|
+
return x if self.norm is None else self.norm(x)
|
|
84
|
+
|
|
85
|
+
def attn_res(self, x, caches=None, pad=None):
|
|
86
|
+
"""Block Attention Residuals over the embedding x: the hidden states before the final norm."""
|
|
87
|
+
# shortcut: no activation checkpointing here; add it per block if attn_res ever trains a model too big to fit
|
|
88
|
+
blocks, partial = [x], None # finished block sums (the embedding first), and the running sum of this block
|
|
89
|
+
layers = [f for i, b in enumerate(self.blocks) for f in b.sublayers(None if caches is None else caches[i], pad)]
|
|
90
|
+
for i, f in enumerate(layers):
|
|
91
|
+
out = f(self.depth[i](blocks if partial is None else blocks + [partial]))
|
|
92
|
+
partial = out if partial is None else partial + out
|
|
93
|
+
if (i + 1) % self.attn_res_block == 0:
|
|
94
|
+
blocks, partial = blocks + [partial], None
|
|
95
|
+
return self.depth[-1](blocks if partial is None else blocks + [partial])
|
|
96
|
+
|
|
97
|
+
def project(self, h):
|
|
98
|
+
"""Hidden states -> logits, through the head or the tied embedding table."""
|
|
99
|
+
return self.head(h) if self.head is not None else h @ self.embed.weight.T
|
|
100
|
+
|
|
101
|
+
def generate(self, idx, max_new_tokens, temperature=1.0, top_k=None, top_p=None, eos_id=None, pad=None):
|
|
102
|
+
"""Extends idx (B, T0) by up to max_new_tokens until every row emits eos_id; pad (B,) is left padding."""
|
|
103
|
+
idx = np.asarray(idx)
|
|
104
|
+
assert idx.shape[1] + max_new_tokens <= self.context_length, "prompt + new tokens exceed context_length"
|
|
105
|
+
caches = [KVCache() for _ in self.blocks]
|
|
106
|
+
done = np.zeros(len(idx), dtype=bool)
|
|
107
|
+
with no_grad():
|
|
108
|
+
logits = self.project(self.hidden(idx, caches, pad)[:, -1])
|
|
109
|
+
for _ in range(max_new_tokens):
|
|
110
|
+
if top_k is None and top_p is None:
|
|
111
|
+
next_id = sample_on_device(logits, temperature)
|
|
112
|
+
else:
|
|
113
|
+
next_id = sample(logits.numpy(), temperature, top_k, top_p)
|
|
114
|
+
idx = np.concatenate([idx, next_id[:, None]], axis=1)
|
|
115
|
+
if eos_id is not None:
|
|
116
|
+
done |= next_id == eos_id
|
|
117
|
+
if done.all():
|
|
118
|
+
break
|
|
119
|
+
logits = self.project(self.hidden(next_id[:, None], caches, pad)[:, -1])
|
|
120
|
+
return idx
|
stackgrad/grad_check.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def numerical_gradient(f, inputs, eps=1e-5):
|
|
5
|
+
"""Estimate gradients of `f` w.r.t. each input by central differences."""
|
|
6
|
+
grads = []
|
|
7
|
+
# perturb in float64 for precision
|
|
8
|
+
orig_data = [inp.data.copy() for inp in inputs]
|
|
9
|
+
for inp in inputs:
|
|
10
|
+
inp.data = inp.data.astype(np.float64)
|
|
11
|
+
f64_data = [inp.data.copy() for inp in inputs]
|
|
12
|
+
|
|
13
|
+
for k, inp in enumerate(inputs):
|
|
14
|
+
grad = np.zeros(inp.data.shape, dtype=np.float64)
|
|
15
|
+
it = np.nditer(f64_data[k], flags=['multi_index'])
|
|
16
|
+
while not it.finished:
|
|
17
|
+
idx = it.multi_index
|
|
18
|
+
old_val = f64_data[k][idx]
|
|
19
|
+
|
|
20
|
+
inp.data = f64_data[k].copy()
|
|
21
|
+
inp.data[idx] = old_val + eps
|
|
22
|
+
loss_plus = float(f().data)
|
|
23
|
+
|
|
24
|
+
inp.data = f64_data[k].copy()
|
|
25
|
+
inp.data[idx] = old_val - eps
|
|
26
|
+
loss_minus = float(f().data)
|
|
27
|
+
|
|
28
|
+
grad[idx] = (loss_plus - loss_minus) / (2 * eps)
|
|
29
|
+
it.iternext()
|
|
30
|
+
inp.data = f64_data[k].copy()
|
|
31
|
+
grads.append(grad.astype(np.float32))
|
|
32
|
+
|
|
33
|
+
for inp, od in zip(inputs, orig_data):
|
|
34
|
+
inp.data = od
|
|
35
|
+
return grads
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def check_gradient(f, inputs, eps=1e-5, atol=1e-4, rtol=1e-3):
|
|
39
|
+
"""Compare backward() gradients against numerical ones, raising AssertionError past the tolerance."""
|
|
40
|
+
for inp in inputs:
|
|
41
|
+
inp.zero_grad()
|
|
42
|
+
loss = f()
|
|
43
|
+
loss.backward()
|
|
44
|
+
analytic = [inp.grad.copy() for inp in inputs]
|
|
45
|
+
|
|
46
|
+
numerical = numerical_gradient(f, inputs, eps)
|
|
47
|
+
|
|
48
|
+
for i, (a, n) in enumerate(zip(analytic, numerical)):
|
|
49
|
+
if not np.allclose(a, n, atol=atol, rtol=rtol):
|
|
50
|
+
max_diff = np.max(np.abs(a - n))
|
|
51
|
+
raise AssertionError(
|
|
52
|
+
f"Gradient check failed for input {i}: max diff = {max_diff}\n"
|
|
53
|
+
f"Analytic:\n{a}\nNumerical:\n{n}"
|
|
54
|
+
)
|
|
55
|
+
return True
|