stackgrad 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. stackgrad-0.2.0/.gitignore +49 -0
  2. stackgrad-0.2.0/LICENSE +21 -0
  3. stackgrad-0.2.0/PKG-INFO +172 -0
  4. stackgrad-0.2.0/README.md +147 -0
  5. stackgrad-0.2.0/pyproject.toml +58 -0
  6. stackgrad-0.2.0/stackgrad/__init__.py +75 -0
  7. stackgrad-0.2.0/stackgrad/attention.py +86 -0
  8. stackgrad-0.2.0/stackgrad/backend/__init__.py +11 -0
  9. stackgrad-0.2.0/stackgrad/backend/cpu.py +116 -0
  10. stackgrad-0.2.0/stackgrad/backend/cuda.py +96 -0
  11. stackgrad-0.2.0/stackgrad/backend/cuda_attention.py +138 -0
  12. stackgrad-0.2.0/stackgrad/backend/metal.py +106 -0
  13. stackgrad-0.2.0/stackgrad/backend/metal_attention.py +195 -0
  14. stackgrad-0.2.0/stackgrad/backend/metal_quant.py +44 -0
  15. stackgrad-0.2.0/stackgrad/checkpoint.py +38 -0
  16. stackgrad-0.2.0/stackgrad/conv.py +63 -0
  17. stackgrad-0.2.0/stackgrad/data.py +28 -0
  18. stackgrad-0.2.0/stackgrad/functional.py +41 -0
  19. stackgrad-0.2.0/stackgrad/gpt.py +120 -0
  20. stackgrad-0.2.0/stackgrad/grad_check.py +55 -0
  21. stackgrad-0.2.0/stackgrad/layers.py +215 -0
  22. stackgrad-0.2.0/stackgrad/lora.py +54 -0
  23. stackgrad-0.2.0/stackgrad/loss.py +28 -0
  24. stackgrad-0.2.0/stackgrad/mlops.py +352 -0
  25. stackgrad-0.2.0/stackgrad/module.py +120 -0
  26. stackgrad-0.2.0/stackgrad/optim.py +121 -0
  27. stackgrad-0.2.0/stackgrad/profile.py +45 -0
  28. stackgrad-0.2.0/stackgrad/quant.py +66 -0
  29. stackgrad-0.2.0/stackgrad/qwen3.py +104 -0
  30. stackgrad-0.2.0/stackgrad/sampling.py +39 -0
  31. stackgrad-0.2.0/stackgrad/serve/api.py +434 -0
  32. stackgrad-0.2.0/stackgrad/serve/engine.py +53 -0
  33. stackgrad-0.2.0/stackgrad/serve/kv_cache.py +35 -0
  34. stackgrad-0.2.0/stackgrad/serve/runner.py +63 -0
  35. stackgrad-0.2.0/stackgrad/serve/scheduler.py +88 -0
  36. stackgrad-0.2.0/stackgrad/speculative.py +68 -0
  37. stackgrad-0.2.0/stackgrad/state.py +27 -0
  38. stackgrad-0.2.0/stackgrad/tensor.py +377 -0
  39. stackgrad-0.2.0/stackgrad/tokenizer.py +202 -0
@@ -0,0 +1,49 @@
1
+ # Build artifacts
2
+ site/
3
+ dist/
4
+ build/
5
+ *.egg-info/
6
+
7
+ # Python
8
+ __pycache__/
9
+ *.pyc
10
+ *.pyo
11
+
12
+ # Virtual environments
13
+ .venv/
14
+ venv/
15
+
16
+ # IDE
17
+ .idea/
18
+ .vscode/
19
+ *.swp
20
+ *.swo
21
+
22
+ # OS
23
+ .DS_Store
24
+ Thumbs.db
25
+
26
+ # Data
27
+ *.npz
28
+ *.pkl
29
+
30
+ # Logs
31
+ *.log
32
+
33
+ # datasets, training runs, downloaded weights
34
+ data/
35
+ runs/
36
+ *.safetensors
37
+ drafts/
38
+
39
+ # AI assistant files: kept locally, not published
40
+ CLAUDE.md
41
+ CLAUDE.local.md
42
+ AGENTS.md
43
+ GEMINI.md
44
+ SPEC.md
45
+ .claude/
46
+ .cursor/
47
+ .cursorrules
48
+ .aider*
49
+ .mcp.json
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Samit Mohan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,172 @@
1
+ Metadata-Version: 2.5
2
+ Name: stackgrad
3
+ Version: 0.2.0
4
+ Summary: A deep learning framework written from scratch, up to a served Qwen3 with retrieval and a tool-using agent
5
+ Project-URL: Homepage, https://github.com/samitmohan/stackgrad
6
+ Project-URL: Documentation, https://samitmohan.github.io/stackgrad/
7
+ Author: Samit Mohan
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Requires-Python: >=3.10
11
+ Requires-Dist: numpy
12
+ Requires-Dist: regex
13
+ Provides-Extra: cuda
14
+ Requires-Dist: cupy-cuda13x[ctk]; extra == 'cuda'
15
+ Requires-Dist: triton; (sys_platform == 'linux') and extra == 'cuda'
16
+ Provides-Extra: docs
17
+ Requires-Dist: mkdocs-material; extra == 'docs'
18
+ Requires-Dist: mkdocstrings[python]; extra == 'docs'
19
+ Provides-Extra: metal
20
+ Requires-Dist: mlx; extra == 'metal'
21
+ Provides-Extra: serve
22
+ Requires-Dist: fastapi; extra == 'serve'
23
+ Requires-Dist: uvicorn; extra == 'serve'
24
+ Description-Content-Type: text/markdown
25
+
26
+ <div align="center">
27
+
28
+ <picture>
29
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/samitmohan/stackgrad/master/docs/logo_dark.svg">
30
+ <img alt="stackgrad" src="https://raw.githubusercontent.com/samitmohan/stackgrad/master/docs/logo_light.svg" width="50%" height="50%">
31
+ </picture>
32
+
33
+ stackgrad: A deep learning framework written from scratch, up to a served Qwen3 with retrieval and a tool-using agent.
34
+
35
+ <h3>
36
+
37
+ [Homepage](https://github.com/samitmohan/stackgrad) | [Documentation](https://samitmohan.github.io/stackgrad/) | [Write-up](https://samitmohan.in/blog/minitorch)
38
+
39
+ </h3>
40
+
41
+ [![CI](https://github.com/samitmohan/stackgrad/actions/workflows/ci.yml/badge.svg)](https://github.com/samitmohan/stackgrad/actions/workflows/ci.yml)
42
+
43
+ </div>
44
+
45
+ ---
46
+
47
+ ![Qwen3-0.6B streaming a worked answer through stackgrad's chat endpoint](https://raw.githubusercontent.com/samitmohan/stackgrad/master/assets/chat.gif)
48
+
49
+ stackgrad is an end-to-end deep learning stack:
50
+
51
+ - a tensor library with lazy evaluation and autograd, every gradient in [one file](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/mlops.py)
52
+ - three backends (NumPy, MLX, CuPy) and attention kernels written in Metal and Triton
53
+ - layers, optimizers, a BPE tokenizer, and a transformer trained from nothing
54
+ - Qwen3-0.6B loaded, served behind an OpenAI-style endpoint, and fine-tuned with LoRA and GRPO
55
+ - retrieval and a tool-using agent on that server
56
+
57
+ The `stackgrad/` package is under 3,000 lines, and CI keeps it there.
58
+
59
+ PyTorch, tiktoken, and transformers appear only in the tests, as the references each piece is checked against.
60
+
61
+ ---
62
+
63
+ ### Laziness
64
+
65
+ Try a matmul. Nothing runs until you ask for the result.
66
+
67
+ ```python
68
+ import numpy as np
69
+ from stackgrad import Tensor
70
+ from stackgrad.profile import profile
71
+
72
+ N = 1024
73
+ a, b = Tensor(np.random.randn(N, N)), Tensor(np.random.randn(N, N))
74
+ with profile() as p:
75
+ c = (a @ b).relu().sum() # records four ops, runs none
76
+ print(c.shape, dict(p.calls)) # () {}
77
+ c.realize() # Matmul, Less, Where, Sum run, once each
78
+ print(p.table())
79
+ ```
80
+
81
+ `relu` shows up as `Less` and `Where`: it composes primitives, so it never defines a gradient of its own.
82
+
83
+ ### Neural networks
84
+
85
+ ```python
86
+ import numpy as np
87
+ from stackgrad import Tensor, Module, Linear, Adam, cross_entropy_loss
88
+
89
+ class LinearNet(Module):
90
+ def __init__(self):
91
+ super().__init__()
92
+ self.l1, self.l2 = Linear(784, 128), Linear(128, 10)
93
+
94
+ def forward(self, x):
95
+ return self.l2(self.l1(x.reshape(x.shape[0], -1)).relu())
96
+
97
+ model = LinearNet()
98
+ opt = Adam(model.parameters(), lr=0.001)
99
+
100
+ x, y = Tensor(np.random.rand(4, 1, 28, 28)), np.array([2, 4, 3, 7]) # replace with real mnist
101
+
102
+ for i in range(10):
103
+ loss = cross_entropy_loss(model(x), y)
104
+ opt.zero_grad()
105
+ loss.backward()
106
+ opt.step()
107
+ print(i, loss.item())
108
+ ```
109
+
110
+ See [train_gpt.py](https://github.com/samitmohan/stackgrad/blob/master/train_gpt.py) for the full version: a 22.7M-parameter transformer that trains on TinyStories to a
111
+ validation loss of 1.399 in 71 minutes on one RTX 3090.
112
+
113
+ ## Accelerators
114
+
115
+ - [x] [CPU](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cpu.py), NumPy
116
+ - [x] [METAL](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/metal.py), MLX, with attention and quantized-matmul kernels in Metal
117
+ - [x] [CUDA](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cuda.py), CuPy, with FlashAttention-2 in Triton
118
+
119
+ A new backend is one module that exports the 27 names [cpu.py](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cpu.py) exports.
120
+
121
+ ## Installation
122
+
123
+ ```sh
124
+ git clone https://github.com/samitmohan/stackgrad.git
125
+ cd stackgrad
126
+ uv sync --extra metal --extra serve # Apple silicon; NVIDIA: uv sync --extra cuda --no-dev
127
+ ```
128
+
129
+ ## Documentation
130
+
131
+ The [docs website](https://samitmohan.github.io/stackgrad/) has a quickstart, the API reference, and a
132
+ [showcase](https://samitmohan.github.io/stackgrad/showcase/) of what stackgrad runs, with every measured number and the
133
+ command that produced it. It builds from the [docs/](docs) directory. The write-up,
134
+ [Modern AI: Autograd to Agents](https://samitmohan.in/blog/minitorch), explains each piece from first principles.
135
+
136
+ ### Quick example comparing to PyTorch
137
+
138
+ ```python
139
+ import numpy as np
140
+ from stackgrad import Tensor
141
+
142
+ x = Tensor(np.eye(3), requires_grad=True)
143
+ y = Tensor([[2.0, 0, -2.0]], requires_grad=True)
144
+ z = (y @ x).sum()
145
+ z.backward()
146
+
147
+ print(x.grad.tolist()) # dz/dx
148
+ print(y.grad.tolist()) # dz/dy
149
+ ```
150
+
151
+ The same thing but in PyTorch:
152
+
153
+ ```python
154
+ import torch
155
+
156
+ x = torch.eye(3, requires_grad=True)
157
+ y = torch.tensor([[2.0, 0, -2.0]], requires_grad=True)
158
+ z = y.matmul(x).sum()
159
+ z.backward()
160
+
161
+ print(x.grad.tolist()) # dz/dx
162
+ print(y.grad.tolist()) # dz/dy
163
+ ```
164
+
165
+ ## Running tests
166
+
167
+ ```sh
168
+ uv run python test_grads.py # every gradient against finite differences
169
+ uv run python tests/parity.py # every op and layer against PyTorch
170
+ ```
171
+
172
+ The [CI workflow](.github/workflows/ci.yml) runs the rest.
@@ -0,0 +1,147 @@
1
+ <div align="center">
2
+
3
+ <picture>
4
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/samitmohan/stackgrad/master/docs/logo_dark.svg">
5
+ <img alt="stackgrad" src="https://raw.githubusercontent.com/samitmohan/stackgrad/master/docs/logo_light.svg" width="50%" height="50%">
6
+ </picture>
7
+
8
+ stackgrad: A deep learning framework written from scratch, up to a served Qwen3 with retrieval and a tool-using agent.
9
+
10
+ <h3>
11
+
12
+ [Homepage](https://github.com/samitmohan/stackgrad) | [Documentation](https://samitmohan.github.io/stackgrad/) | [Write-up](https://samitmohan.in/blog/minitorch)
13
+
14
+ </h3>
15
+
16
+ [![CI](https://github.com/samitmohan/stackgrad/actions/workflows/ci.yml/badge.svg)](https://github.com/samitmohan/stackgrad/actions/workflows/ci.yml)
17
+
18
+ </div>
19
+
20
+ ---
21
+
22
+ ![Qwen3-0.6B streaming a worked answer through stackgrad's chat endpoint](https://raw.githubusercontent.com/samitmohan/stackgrad/master/assets/chat.gif)
23
+
24
+ stackgrad is an end-to-end deep learning stack:
25
+
26
+ - a tensor library with lazy evaluation and autograd, every gradient in [one file](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/mlops.py)
27
+ - three backends (NumPy, MLX, CuPy) and attention kernels written in Metal and Triton
28
+ - layers, optimizers, a BPE tokenizer, and a transformer trained from nothing
29
+ - Qwen3-0.6B loaded, served behind an OpenAI-style endpoint, and fine-tuned with LoRA and GRPO
30
+ - retrieval and a tool-using agent on that server
31
+
32
+ The `stackgrad/` package is under 3,000 lines, and CI keeps it there.
33
+
34
+ PyTorch, tiktoken, and transformers appear only in the tests, as the references each piece is checked against.
35
+
36
+ ---
37
+
38
+ ### Laziness
39
+
40
+ Try a matmul. Nothing runs until you ask for the result.
41
+
42
+ ```python
43
+ import numpy as np
44
+ from stackgrad import Tensor
45
+ from stackgrad.profile import profile
46
+
47
+ N = 1024
48
+ a, b = Tensor(np.random.randn(N, N)), Tensor(np.random.randn(N, N))
49
+ with profile() as p:
50
+ c = (a @ b).relu().sum() # records four ops, runs none
51
+ print(c.shape, dict(p.calls)) # () {}
52
+ c.realize() # Matmul, Less, Where, Sum run, once each
53
+ print(p.table())
54
+ ```
55
+
56
+ `relu` shows up as `Less` and `Where`: it composes primitives, so it never defines a gradient of its own.
57
+
58
+ ### Neural networks
59
+
60
+ ```python
61
+ import numpy as np
62
+ from stackgrad import Tensor, Module, Linear, Adam, cross_entropy_loss
63
+
64
+ class LinearNet(Module):
65
+ def __init__(self):
66
+ super().__init__()
67
+ self.l1, self.l2 = Linear(784, 128), Linear(128, 10)
68
+
69
+ def forward(self, x):
70
+ return self.l2(self.l1(x.reshape(x.shape[0], -1)).relu())
71
+
72
+ model = LinearNet()
73
+ opt = Adam(model.parameters(), lr=0.001)
74
+
75
+ x, y = Tensor(np.random.rand(4, 1, 28, 28)), np.array([2, 4, 3, 7]) # replace with real mnist
76
+
77
+ for i in range(10):
78
+ loss = cross_entropy_loss(model(x), y)
79
+ opt.zero_grad()
80
+ loss.backward()
81
+ opt.step()
82
+ print(i, loss.item())
83
+ ```
84
+
85
+ See [train_gpt.py](https://github.com/samitmohan/stackgrad/blob/master/train_gpt.py) for the full version: a 22.7M-parameter transformer that trains on TinyStories to a
86
+ validation loss of 1.399 in 71 minutes on one RTX 3090.
87
+
88
+ ## Accelerators
89
+
90
+ - [x] [CPU](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cpu.py), NumPy
91
+ - [x] [METAL](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/metal.py), MLX, with attention and quantized-matmul kernels in Metal
92
+ - [x] [CUDA](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cuda.py), CuPy, with FlashAttention-2 in Triton
93
+
94
+ A new backend is one module that exports the 27 names [cpu.py](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cpu.py) exports.
95
+
96
+ ## Installation
97
+
98
+ ```sh
99
+ git clone https://github.com/samitmohan/stackgrad.git
100
+ cd stackgrad
101
+ uv sync --extra metal --extra serve # Apple silicon; NVIDIA: uv sync --extra cuda --no-dev
102
+ ```
103
+
104
+ ## Documentation
105
+
106
+ The [docs website](https://samitmohan.github.io/stackgrad/) has a quickstart, the API reference, and a
107
+ [showcase](https://samitmohan.github.io/stackgrad/showcase/) of what stackgrad runs, with every measured number and the
108
+ command that produced it. It builds from the [docs/](docs) directory. The write-up,
109
+ [Modern AI: Autograd to Agents](https://samitmohan.in/blog/minitorch), explains each piece from first principles.
110
+
111
+ ### Quick example comparing to PyTorch
112
+
113
+ ```python
114
+ import numpy as np
115
+ from stackgrad import Tensor
116
+
117
+ x = Tensor(np.eye(3), requires_grad=True)
118
+ y = Tensor([[2.0, 0, -2.0]], requires_grad=True)
119
+ z = (y @ x).sum()
120
+ z.backward()
121
+
122
+ print(x.grad.tolist()) # dz/dx
123
+ print(y.grad.tolist()) # dz/dy
124
+ ```
125
+
126
+ The same thing but in PyTorch:
127
+
128
+ ```python
129
+ import torch
130
+
131
+ x = torch.eye(3, requires_grad=True)
132
+ y = torch.tensor([[2.0, 0, -2.0]], requires_grad=True)
133
+ z = y.matmul(x).sum()
134
+ z.backward()
135
+
136
+ print(x.grad.tolist()) # dz/dx
137
+ print(y.grad.tolist()) # dz/dy
138
+ ```
139
+
140
+ ## Running tests
141
+
142
+ ```sh
143
+ uv run python test_grads.py # every gradient against finite differences
144
+ uv run python tests/parity.py # every op and layer against PyTorch
145
+ ```
146
+
147
+ The [CI workflow](.github/workflows/ci.yml) runs the rest.
@@ -0,0 +1,58 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "stackgrad"
7
+ version = "0.2.0"
8
+ description = "A deep learning framework written from scratch, up to a served Qwen3 with retrieval and a tool-using agent"
9
+ authors = [{name = "Samit Mohan"}]
10
+ readme = "README.md"
11
+ license = "MIT"
12
+ requires-python = ">=3.10"
13
+ dependencies = [
14
+ "numpy",
15
+ "regex",
16
+ ]
17
+
18
+ [project.urls]
19
+ Homepage = "https://github.com/samitmohan/stackgrad"
20
+ Documentation = "https://samitmohan.github.io/stackgrad/"
21
+
22
+ [project.optional-dependencies]
23
+ metal = ["mlx"]
24
+ cuda = ["cupy-cuda13x[ctk]", "triton; sys_platform == 'linux'"]
25
+ serve = ["fastapi", "uvicorn"]
26
+ docs = [
27
+ "mkdocs-material",
28
+ "mkdocstrings[python]",
29
+ ]
30
+
31
+ [tool.hatch.build.targets.sdist]
32
+ include = ["stackgrad", "README.md", "LICENSE"]
33
+
34
+ [tool.ruff]
35
+ line-length = 150
36
+ indent-width = 2
37
+
38
+ [tool.ruff.lint]
39
+ extend-select = ["TID251", "E501", "E701", "E702", "E703", "E731"] # 150 columns; no code golf: one statement per line, no assigned lambdas
40
+
41
+ [dependency-groups]
42
+ dev = [
43
+ "torch",
44
+ "tiktoken",
45
+ "transformers",
46
+ "huggingface_hub",
47
+ "ruff",
48
+ "diffusers>=0.41.0",
49
+ "matplotlib",
50
+ ]
51
+
52
+ [tool.ruff.lint.flake8-tidy-imports.banned-api]
53
+ # CuPy is an array backend for training throughput. Custom kernels are Metal (mx.fast.metal_kernel)
54
+ # on the Mac and Triton on NVIDIA, never CuPy's raw kernel APIs.
55
+ "cupy.RawKernel".msg = "Custom kernels are Metal or Triton: use mx.fast.metal_kernel or a @triton.jit kernel."
56
+ "cupy.RawModule".msg = "Custom kernels are Metal or Triton: use mx.fast.metal_kernel or a @triton.jit kernel."
57
+ "cupy.ElementwiseKernel".msg = "Custom kernels are Metal or Triton: use mx.fast.metal_kernel or a @triton.jit kernel."
58
+ "cupy.ReductionKernel".msg = "Custom kernels are Metal or Triton: use mx.fast.metal_kernel or a @triton.jit kernel."
@@ -0,0 +1,75 @@
1
+ from . import functional as F
2
+ from .attention import Attention, KVCache
3
+ from .conv import Conv2d, Flatten, MaxPool2d
4
+ from .data import DataLoader
5
+ from .gpt import TransformerLM
6
+ from .grad_check import check_gradient, numerical_gradient
7
+ from .layers import (
8
+ GELU,
9
+ BatchNorm1d,
10
+ Dropout,
11
+ Embedding,
12
+ GroupNorm,
13
+ LayerNorm,
14
+ Linear,
15
+ ReLU,
16
+ RMSNorm,
17
+ Sigmoid,
18
+ Softmax,
19
+ SwiGLU,
20
+ Tanh,
21
+ )
22
+ from .loss import bce_loss, cross_entropy_loss, mse_loss
23
+ from .module import Module, Sequential
24
+ from .optim import (
25
+ SGD,
26
+ Adam,
27
+ AdamW,
28
+ LossScaler,
29
+ clip_grad_norm,
30
+ cosine_lr,
31
+ )
32
+ from .tensor import Tensor, autocast, cat, no_grad, stack, where
33
+
34
+ __all__ = [
35
+ "GELU",
36
+ "SGD",
37
+ "Adam",
38
+ "AdamW",
39
+ "Attention",
40
+ "BatchNorm1d",
41
+ "Conv2d",
42
+ "DataLoader",
43
+ "Dropout",
44
+ "Embedding",
45
+ "F",
46
+ "Flatten",
47
+ "GroupNorm",
48
+ "KVCache",
49
+ "LayerNorm",
50
+ "Linear",
51
+ "LossScaler",
52
+ "MaxPool2d",
53
+ "Module",
54
+ "RMSNorm",
55
+ "ReLU",
56
+ "Sequential",
57
+ "Sigmoid",
58
+ "Softmax",
59
+ "SwiGLU",
60
+ "Tanh",
61
+ "Tensor",
62
+ "TransformerLM",
63
+ "autocast",
64
+ "bce_loss",
65
+ "cat",
66
+ "check_gradient",
67
+ "clip_grad_norm",
68
+ "cosine_lr",
69
+ "cross_entropy_loss",
70
+ "mse_loss",
71
+ "no_grad",
72
+ "numerical_gradient",
73
+ "stack",
74
+ "where",
75
+ ]
@@ -0,0 +1,86 @@
1
+ import numpy as np
2
+
3
+ from . import mlops
4
+ from .layers import Linear, RMSNorm
5
+ from .module import Module
6
+ from .tensor import _apply, cat
7
+
8
+
9
+ def apply_rope(x, start, theta):
10
+ """Rotates each (x[i], x[i + D/2]) pair by its position's angle, in Llama/Qwen's half-split layout."""
11
+ half = x.shape[-1] // 2
12
+ freqs = theta ** (-np.arange(half) / half)
13
+ positions = np.reshape(start, (-1, 1, 1)) + np.arange(x.shape[2])[:, None] # (B or 1, T, 1)
14
+ angles = (positions * freqs)[:, None] # (B or 1, 1, T, D/2)
15
+ cos, sin = np.cos(angles).astype(np.float32), np.sin(angles).astype(np.float32)
16
+ x1, x2 = x[..., :half], x[..., half:]
17
+ return cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1)
18
+
19
+
20
+ def scaled_dot_product_attention(q, k, v, pad=None, causal=True):
21
+ """q (B, H, T, D) over k, v (B, Hkv, S, D), queries at the last T positions; pad (B,) masks left padding."""
22
+ key_bias = None
23
+ if pad is not None:
24
+ # -1e9, not -inf: a padding query sees only padding, and an all -inf softmax is NaN that leaks via 0 * NaN
25
+ key_bias = np.where(np.arange(k.shape[2]) < np.asarray(pad)[:, None], -1e9, 0.0).astype(np.float32)
26
+ return _apply(mlops.Attention, q, k, v, key_bias=key_bias, causal=causal)
27
+
28
+
29
+ class KVCache:
30
+ """Keys and values of earlier positions, so a decode step doesn't recompute the prefix."""
31
+ # shortcut: cat copies the whole cache every step (O(T^2) bytes per sequence); M4 swaps in a paged cache
32
+ def __init__(self):
33
+ self.k = self.v = None
34
+
35
+ def __len__(self):
36
+ return 0 if self.k is None else self.k.shape[2]
37
+
38
+ def truncate(self, n):
39
+ """Drops every position from n on, for speculative decoding's rejected drafts."""
40
+ self.k, self.v = self.k[:, :, :n], self.v[:, :, :n]
41
+
42
+ def update(self, k, v):
43
+ self.k = k if self.k is None else cat([self.k, k], axis=2)
44
+ self.v = v if self.v is None else cat([self.v, v], axis=2)
45
+ return self.k, self.v
46
+
47
+
48
+ class Attention(Module):
49
+ """Multi-head attention with RoPE and grouped-query heads; qk_norm adds Qwen3's per-head RMSNorm on q, k."""
50
+ def __init__(self, d_model, n_heads, n_kv_heads=None, rope_theta=10000.0, head_dim=None, qk_norm=False, norm_eps=1e-5,
51
+ rope=True):
52
+ super().__init__()
53
+ self.rope = rope # False is the NoPE ablation: causal masking alone orders tokens
54
+ self.n_heads = n_heads
55
+ self.n_kv_heads = n_kv_heads or n_heads
56
+ self.head_dim = head_dim or d_model // n_heads
57
+ self.rope_theta = rope_theta
58
+ self.q_proj = Linear(d_model, n_heads * self.head_dim, bias=False)
59
+ self.k_proj = Linear(d_model, self.n_kv_heads * self.head_dim, bias=False)
60
+ self.v_proj = Linear(d_model, self.n_kv_heads * self.head_dim, bias=False)
61
+ self.o_proj = Linear(n_heads * self.head_dim, d_model, bias=False)
62
+ self.q_norm = RMSNorm(self.head_dim, norm_eps) if qk_norm else None
63
+ self.k_norm = RMSNorm(self.head_dim, norm_eps) if qk_norm else None
64
+ self.record, self.recorded = False, None # record=True keeps the last q and k, for drawing attention maps
65
+
66
+ def forward(self, x, cache=None, pad=None):
67
+ B, T, _ = x.shape
68
+ D = self.head_dim
69
+ start = 0 if cache is None else len(cache)
70
+ if pad is not None:
71
+ start = start - np.asarray(pad) # each row's first real token sits at position 0, after its left padding
72
+ q = self.q_proj(x).reshape(B, T, self.n_heads, D)
73
+ k = self.k_proj(x).reshape(B, T, self.n_kv_heads, D)
74
+ if self.q_norm is not None:
75
+ q, k = self.q_norm(q), self.k_norm(k)
76
+ q, k = q.transpose(1, 2), k.transpose(1, 2) # (B, H, T, D), (B, Hkv, T, D)
77
+ v = self.v_proj(x).reshape(B, T, self.n_kv_heads, D).transpose(1, 2)
78
+ if self.rope:
79
+ q, k = apply_rope(q, start, self.rope_theta), apply_rope(k, start, self.rope_theta)
80
+ if cache is not None:
81
+ k, v = cache.update(k, v)
82
+ if self.record:
83
+ self.recorded = q.numpy(), k.numpy()
84
+
85
+ out = scaled_dot_product_attention(q, k, v, pad) # (B, H, T, D)
86
+ return self.o_proj(out.transpose(1, 2).reshape(B, T, self.n_heads * D))
@@ -0,0 +1,11 @@
1
+ import functools
2
+ import importlib
3
+
4
+ DEVICES = ("cpu", "metal", "cuda")
5
+
6
+
7
+ @functools.cache
8
+ def get(device):
9
+ if device not in DEVICES:
10
+ raise ValueError(f"unknown device {device!r}, expected one of {DEVICES}")
11
+ return importlib.import_module(f"{__name__}.{device}")