stackgrad 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stackgrad-0.2.0/.gitignore +49 -0
- stackgrad-0.2.0/LICENSE +21 -0
- stackgrad-0.2.0/PKG-INFO +172 -0
- stackgrad-0.2.0/README.md +147 -0
- stackgrad-0.2.0/pyproject.toml +58 -0
- stackgrad-0.2.0/stackgrad/__init__.py +75 -0
- stackgrad-0.2.0/stackgrad/attention.py +86 -0
- stackgrad-0.2.0/stackgrad/backend/__init__.py +11 -0
- stackgrad-0.2.0/stackgrad/backend/cpu.py +116 -0
- stackgrad-0.2.0/stackgrad/backend/cuda.py +96 -0
- stackgrad-0.2.0/stackgrad/backend/cuda_attention.py +138 -0
- stackgrad-0.2.0/stackgrad/backend/metal.py +106 -0
- stackgrad-0.2.0/stackgrad/backend/metal_attention.py +195 -0
- stackgrad-0.2.0/stackgrad/backend/metal_quant.py +44 -0
- stackgrad-0.2.0/stackgrad/checkpoint.py +38 -0
- stackgrad-0.2.0/stackgrad/conv.py +63 -0
- stackgrad-0.2.0/stackgrad/data.py +28 -0
- stackgrad-0.2.0/stackgrad/functional.py +41 -0
- stackgrad-0.2.0/stackgrad/gpt.py +120 -0
- stackgrad-0.2.0/stackgrad/grad_check.py +55 -0
- stackgrad-0.2.0/stackgrad/layers.py +215 -0
- stackgrad-0.2.0/stackgrad/lora.py +54 -0
- stackgrad-0.2.0/stackgrad/loss.py +28 -0
- stackgrad-0.2.0/stackgrad/mlops.py +352 -0
- stackgrad-0.2.0/stackgrad/module.py +120 -0
- stackgrad-0.2.0/stackgrad/optim.py +121 -0
- stackgrad-0.2.0/stackgrad/profile.py +45 -0
- stackgrad-0.2.0/stackgrad/quant.py +66 -0
- stackgrad-0.2.0/stackgrad/qwen3.py +104 -0
- stackgrad-0.2.0/stackgrad/sampling.py +39 -0
- stackgrad-0.2.0/stackgrad/serve/api.py +434 -0
- stackgrad-0.2.0/stackgrad/serve/engine.py +53 -0
- stackgrad-0.2.0/stackgrad/serve/kv_cache.py +35 -0
- stackgrad-0.2.0/stackgrad/serve/runner.py +63 -0
- stackgrad-0.2.0/stackgrad/serve/scheduler.py +88 -0
- stackgrad-0.2.0/stackgrad/speculative.py +68 -0
- stackgrad-0.2.0/stackgrad/state.py +27 -0
- stackgrad-0.2.0/stackgrad/tensor.py +377 -0
- stackgrad-0.2.0/stackgrad/tokenizer.py +202 -0
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Build artifacts
|
|
2
|
+
site/
|
|
3
|
+
dist/
|
|
4
|
+
build/
|
|
5
|
+
*.egg-info/
|
|
6
|
+
|
|
7
|
+
# Python
|
|
8
|
+
__pycache__/
|
|
9
|
+
*.pyc
|
|
10
|
+
*.pyo
|
|
11
|
+
|
|
12
|
+
# Virtual environments
|
|
13
|
+
.venv/
|
|
14
|
+
venv/
|
|
15
|
+
|
|
16
|
+
# IDE
|
|
17
|
+
.idea/
|
|
18
|
+
.vscode/
|
|
19
|
+
*.swp
|
|
20
|
+
*.swo
|
|
21
|
+
|
|
22
|
+
# OS
|
|
23
|
+
.DS_Store
|
|
24
|
+
Thumbs.db
|
|
25
|
+
|
|
26
|
+
# Data
|
|
27
|
+
*.npz
|
|
28
|
+
*.pkl
|
|
29
|
+
|
|
30
|
+
# Logs
|
|
31
|
+
*.log
|
|
32
|
+
|
|
33
|
+
# datasets, training runs, downloaded weights
|
|
34
|
+
data/
|
|
35
|
+
runs/
|
|
36
|
+
*.safetensors
|
|
37
|
+
drafts/
|
|
38
|
+
|
|
39
|
+
# AI assistant files: kept locally, not published
|
|
40
|
+
CLAUDE.md
|
|
41
|
+
CLAUDE.local.md
|
|
42
|
+
AGENTS.md
|
|
43
|
+
GEMINI.md
|
|
44
|
+
SPEC.md
|
|
45
|
+
.claude/
|
|
46
|
+
.cursor/
|
|
47
|
+
.cursorrules
|
|
48
|
+
.aider*
|
|
49
|
+
.mcp.json
|
stackgrad-0.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Samit Mohan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
stackgrad-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: stackgrad
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: A deep learning framework written from scratch, up to a served Qwen3 with retrieval and a tool-using agent
|
|
5
|
+
Project-URL: Homepage, https://github.com/samitmohan/stackgrad
|
|
6
|
+
Project-URL: Documentation, https://samitmohan.github.io/stackgrad/
|
|
7
|
+
Author: Samit Mohan
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Requires-Dist: numpy
|
|
12
|
+
Requires-Dist: regex
|
|
13
|
+
Provides-Extra: cuda
|
|
14
|
+
Requires-Dist: cupy-cuda13x[ctk]; extra == 'cuda'
|
|
15
|
+
Requires-Dist: triton; (sys_platform == 'linux') and extra == 'cuda'
|
|
16
|
+
Provides-Extra: docs
|
|
17
|
+
Requires-Dist: mkdocs-material; extra == 'docs'
|
|
18
|
+
Requires-Dist: mkdocstrings[python]; extra == 'docs'
|
|
19
|
+
Provides-Extra: metal
|
|
20
|
+
Requires-Dist: mlx; extra == 'metal'
|
|
21
|
+
Provides-Extra: serve
|
|
22
|
+
Requires-Dist: fastapi; extra == 'serve'
|
|
23
|
+
Requires-Dist: uvicorn; extra == 'serve'
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
<div align="center">
|
|
27
|
+
|
|
28
|
+
<picture>
|
|
29
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/samitmohan/stackgrad/master/docs/logo_dark.svg">
|
|
30
|
+
<img alt="stackgrad" src="https://raw.githubusercontent.com/samitmohan/stackgrad/master/docs/logo_light.svg" width="50%" height="50%">
|
|
31
|
+
</picture>
|
|
32
|
+
|
|
33
|
+
stackgrad: A deep learning framework written from scratch, up to a served Qwen3 with retrieval and a tool-using agent.
|
|
34
|
+
|
|
35
|
+
<h3>
|
|
36
|
+
|
|
37
|
+
[Homepage](https://github.com/samitmohan/stackgrad) | [Documentation](https://samitmohan.github.io/stackgrad/) | [Write-up](https://samitmohan.in/blog/minitorch)
|
|
38
|
+
|
|
39
|
+
</h3>
|
|
40
|
+
|
|
41
|
+
[](https://github.com/samitmohan/stackgrad/actions/workflows/ci.yml)
|
|
42
|
+
|
|
43
|
+
</div>
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+

|
|
48
|
+
|
|
49
|
+
stackgrad is an end-to-end deep learning stack:
|
|
50
|
+
|
|
51
|
+
- a tensor library with lazy evaluation and autograd, every gradient in [one file](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/mlops.py)
|
|
52
|
+
- three backends (NumPy, MLX, CuPy) and attention kernels written in Metal and Triton
|
|
53
|
+
- layers, optimizers, a BPE tokenizer, and a transformer trained from nothing
|
|
54
|
+
- Qwen3-0.6B loaded, served behind an OpenAI-style endpoint, and fine-tuned with LoRA and GRPO
|
|
55
|
+
- retrieval and a tool-using agent on that server
|
|
56
|
+
|
|
57
|
+
The `stackgrad/` package is under 3,000 lines, and CI keeps it there.
|
|
58
|
+
|
|
59
|
+
PyTorch, tiktoken, and transformers appear only in the tests, as the references each piece is checked against.
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
### Laziness
|
|
64
|
+
|
|
65
|
+
Try a matmul. Nothing runs until you ask for the result.
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
import numpy as np
|
|
69
|
+
from stackgrad import Tensor
|
|
70
|
+
from stackgrad.profile import profile
|
|
71
|
+
|
|
72
|
+
N = 1024
|
|
73
|
+
a, b = Tensor(np.random.randn(N, N)), Tensor(np.random.randn(N, N))
|
|
74
|
+
with profile() as p:
|
|
75
|
+
c = (a @ b).relu().sum() # records four ops, runs none
|
|
76
|
+
print(c.shape, dict(p.calls)) # () {}
|
|
77
|
+
c.realize() # Matmul, Less, Where, Sum run, once each
|
|
78
|
+
print(p.table())
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`relu` shows up as `Less` and `Where`: it composes primitives, so it never defines a gradient of its own.
|
|
82
|
+
|
|
83
|
+
### Neural networks
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
import numpy as np
|
|
87
|
+
from stackgrad import Tensor, Module, Linear, Adam, cross_entropy_loss
|
|
88
|
+
|
|
89
|
+
class LinearNet(Module):
|
|
90
|
+
def __init__(self):
|
|
91
|
+
super().__init__()
|
|
92
|
+
self.l1, self.l2 = Linear(784, 128), Linear(128, 10)
|
|
93
|
+
|
|
94
|
+
def forward(self, x):
|
|
95
|
+
return self.l2(self.l1(x.reshape(x.shape[0], -1)).relu())
|
|
96
|
+
|
|
97
|
+
model = LinearNet()
|
|
98
|
+
opt = Adam(model.parameters(), lr=0.001)
|
|
99
|
+
|
|
100
|
+
x, y = Tensor(np.random.rand(4, 1, 28, 28)), np.array([2, 4, 3, 7]) # replace with real mnist
|
|
101
|
+
|
|
102
|
+
for i in range(10):
|
|
103
|
+
loss = cross_entropy_loss(model(x), y)
|
|
104
|
+
opt.zero_grad()
|
|
105
|
+
loss.backward()
|
|
106
|
+
opt.step()
|
|
107
|
+
print(i, loss.item())
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
See [train_gpt.py](https://github.com/samitmohan/stackgrad/blob/master/train_gpt.py) for the full version: a 22.7M-parameter transformer that trains on TinyStories to a
|
|
111
|
+
validation loss of 1.399 in 71 minutes on one RTX 3090.
|
|
112
|
+
|
|
113
|
+
## Accelerators
|
|
114
|
+
|
|
115
|
+
- [x] [CPU](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cpu.py), NumPy
|
|
116
|
+
- [x] [METAL](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/metal.py), MLX, with attention and quantized-matmul kernels in Metal
|
|
117
|
+
- [x] [CUDA](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cuda.py), CuPy, with FlashAttention-2 in Triton
|
|
118
|
+
|
|
119
|
+
A new backend is one module that exports the 27 names [cpu.py](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cpu.py) exports.
|
|
120
|
+
|
|
121
|
+
## Installation
|
|
122
|
+
|
|
123
|
+
```sh
|
|
124
|
+
git clone https://github.com/samitmohan/stackgrad.git
|
|
125
|
+
cd stackgrad
|
|
126
|
+
uv sync --extra metal --extra serve # Apple silicon; NVIDIA: uv sync --extra cuda --no-dev
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
## Documentation
|
|
130
|
+
|
|
131
|
+
The [docs website](https://samitmohan.github.io/stackgrad/) has a quickstart, the API reference, and a
|
|
132
|
+
[showcase](https://samitmohan.github.io/stackgrad/showcase/) of what stackgrad runs, with every measured number and the
|
|
133
|
+
command that produced it. It builds from the [docs/](docs) directory. The write-up,
|
|
134
|
+
[Modern AI: Autograd to Agents](https://samitmohan.in/blog/minitorch), explains each piece from first principles.
|
|
135
|
+
|
|
136
|
+
### Quick example comparing to PyTorch
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
import numpy as np
|
|
140
|
+
from stackgrad import Tensor
|
|
141
|
+
|
|
142
|
+
x = Tensor(np.eye(3), requires_grad=True)
|
|
143
|
+
y = Tensor([[2.0, 0, -2.0]], requires_grad=True)
|
|
144
|
+
z = (y @ x).sum()
|
|
145
|
+
z.backward()
|
|
146
|
+
|
|
147
|
+
print(x.grad.tolist()) # dz/dx
|
|
148
|
+
print(y.grad.tolist()) # dz/dy
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
The same thing but in PyTorch:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
import torch
|
|
155
|
+
|
|
156
|
+
x = torch.eye(3, requires_grad=True)
|
|
157
|
+
y = torch.tensor([[2.0, 0, -2.0]], requires_grad=True)
|
|
158
|
+
z = y.matmul(x).sum()
|
|
159
|
+
z.backward()
|
|
160
|
+
|
|
161
|
+
print(x.grad.tolist()) # dz/dx
|
|
162
|
+
print(y.grad.tolist()) # dz/dy
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## Running tests
|
|
166
|
+
|
|
167
|
+
```sh
|
|
168
|
+
uv run python test_grads.py # every gradient against finite differences
|
|
169
|
+
uv run python tests/parity.py # every op and layer against PyTorch
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
The [CI workflow](.github/workflows/ci.yml) runs the rest.
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<picture>
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/samitmohan/stackgrad/master/docs/logo_dark.svg">
|
|
5
|
+
<img alt="stackgrad" src="https://raw.githubusercontent.com/samitmohan/stackgrad/master/docs/logo_light.svg" width="50%" height="50%">
|
|
6
|
+
</picture>
|
|
7
|
+
|
|
8
|
+
stackgrad: A deep learning framework written from scratch, up to a served Qwen3 with retrieval and a tool-using agent.
|
|
9
|
+
|
|
10
|
+
<h3>
|
|
11
|
+
|
|
12
|
+
[Homepage](https://github.com/samitmohan/stackgrad) | [Documentation](https://samitmohan.github.io/stackgrad/) | [Write-up](https://samitmohan.in/blog/minitorch)
|
|
13
|
+
|
|
14
|
+
</h3>
|
|
15
|
+
|
|
16
|
+
[](https://github.com/samitmohan/stackgrad/actions/workflows/ci.yml)
|
|
17
|
+
|
|
18
|
+
</div>
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+

|
|
23
|
+
|
|
24
|
+
stackgrad is an end-to-end deep learning stack:
|
|
25
|
+
|
|
26
|
+
- a tensor library with lazy evaluation and autograd, every gradient in [one file](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/mlops.py)
|
|
27
|
+
- three backends (NumPy, MLX, CuPy) and attention kernels written in Metal and Triton
|
|
28
|
+
- layers, optimizers, a BPE tokenizer, and a transformer trained from nothing
|
|
29
|
+
- Qwen3-0.6B loaded, served behind an OpenAI-style endpoint, and fine-tuned with LoRA and GRPO
|
|
30
|
+
- retrieval and a tool-using agent on that server
|
|
31
|
+
|
|
32
|
+
The `stackgrad/` package is under 3,000 lines, and CI keeps it there.
|
|
33
|
+
|
|
34
|
+
PyTorch, tiktoken, and transformers appear only in the tests, as the references each piece is checked against.
|
|
35
|
+
|
|
36
|
+
---
|
|
37
|
+
|
|
38
|
+
### Laziness
|
|
39
|
+
|
|
40
|
+
Try a matmul. Nothing runs until you ask for the result.
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
import numpy as np
|
|
44
|
+
from stackgrad import Tensor
|
|
45
|
+
from stackgrad.profile import profile
|
|
46
|
+
|
|
47
|
+
N = 1024
|
|
48
|
+
a, b = Tensor(np.random.randn(N, N)), Tensor(np.random.randn(N, N))
|
|
49
|
+
with profile() as p:
|
|
50
|
+
c = (a @ b).relu().sum() # records four ops, runs none
|
|
51
|
+
print(c.shape, dict(p.calls)) # () {}
|
|
52
|
+
c.realize() # Matmul, Less, Where, Sum run, once each
|
|
53
|
+
print(p.table())
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
`relu` shows up as `Less` and `Where`: it composes primitives, so it never defines a gradient of its own.
|
|
57
|
+
|
|
58
|
+
### Neural networks
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
import numpy as np
|
|
62
|
+
from stackgrad import Tensor, Module, Linear, Adam, cross_entropy_loss
|
|
63
|
+
|
|
64
|
+
class LinearNet(Module):
|
|
65
|
+
def __init__(self):
|
|
66
|
+
super().__init__()
|
|
67
|
+
self.l1, self.l2 = Linear(784, 128), Linear(128, 10)
|
|
68
|
+
|
|
69
|
+
def forward(self, x):
|
|
70
|
+
return self.l2(self.l1(x.reshape(x.shape[0], -1)).relu())
|
|
71
|
+
|
|
72
|
+
model = LinearNet()
|
|
73
|
+
opt = Adam(model.parameters(), lr=0.001)
|
|
74
|
+
|
|
75
|
+
x, y = Tensor(np.random.rand(4, 1, 28, 28)), np.array([2, 4, 3, 7]) # replace with real mnist
|
|
76
|
+
|
|
77
|
+
for i in range(10):
|
|
78
|
+
loss = cross_entropy_loss(model(x), y)
|
|
79
|
+
opt.zero_grad()
|
|
80
|
+
loss.backward()
|
|
81
|
+
opt.step()
|
|
82
|
+
print(i, loss.item())
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
See [train_gpt.py](https://github.com/samitmohan/stackgrad/blob/master/train_gpt.py) for the full version: a 22.7M-parameter transformer that trains on TinyStories to a
|
|
86
|
+
validation loss of 1.399 in 71 minutes on one RTX 3090.
|
|
87
|
+
|
|
88
|
+
## Accelerators
|
|
89
|
+
|
|
90
|
+
- [x] [CPU](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cpu.py), NumPy
|
|
91
|
+
- [x] [METAL](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/metal.py), MLX, with attention and quantized-matmul kernels in Metal
|
|
92
|
+
- [x] [CUDA](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cuda.py), CuPy, with FlashAttention-2 in Triton
|
|
93
|
+
|
|
94
|
+
A new backend is one module that exports the 27 names [cpu.py](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cpu.py) exports.
|
|
95
|
+
|
|
96
|
+
## Installation
|
|
97
|
+
|
|
98
|
+
```sh
|
|
99
|
+
git clone https://github.com/samitmohan/stackgrad.git
|
|
100
|
+
cd stackgrad
|
|
101
|
+
uv sync --extra metal --extra serve # Apple silicon; NVIDIA: uv sync --extra cuda --no-dev
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Documentation
|
|
105
|
+
|
|
106
|
+
The [docs website](https://samitmohan.github.io/stackgrad/) has a quickstart, the API reference, and a
|
|
107
|
+
[showcase](https://samitmohan.github.io/stackgrad/showcase/) of what stackgrad runs, with every measured number and the
|
|
108
|
+
command that produced it. It builds from the [docs/](docs) directory. The write-up,
|
|
109
|
+
[Modern AI: Autograd to Agents](https://samitmohan.in/blog/minitorch), explains each piece from first principles.
|
|
110
|
+
|
|
111
|
+
### Quick example comparing to PyTorch
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
import numpy as np
|
|
115
|
+
from stackgrad import Tensor
|
|
116
|
+
|
|
117
|
+
x = Tensor(np.eye(3), requires_grad=True)
|
|
118
|
+
y = Tensor([[2.0, 0, -2.0]], requires_grad=True)
|
|
119
|
+
z = (y @ x).sum()
|
|
120
|
+
z.backward()
|
|
121
|
+
|
|
122
|
+
print(x.grad.tolist()) # dz/dx
|
|
123
|
+
print(y.grad.tolist()) # dz/dy
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
The same thing but in PyTorch:
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
import torch
|
|
130
|
+
|
|
131
|
+
x = torch.eye(3, requires_grad=True)
|
|
132
|
+
y = torch.tensor([[2.0, 0, -2.0]], requires_grad=True)
|
|
133
|
+
z = y.matmul(x).sum()
|
|
134
|
+
z.backward()
|
|
135
|
+
|
|
136
|
+
print(x.grad.tolist()) # dz/dx
|
|
137
|
+
print(y.grad.tolist()) # dz/dy
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Running tests
|
|
141
|
+
|
|
142
|
+
```sh
|
|
143
|
+
uv run python test_grads.py # every gradient against finite differences
|
|
144
|
+
uv run python tests/parity.py # every op and layer against PyTorch
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
The [CI workflow](.github/workflows/ci.yml) runs the rest.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "stackgrad"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "A deep learning framework written from scratch, up to a served Qwen3 with retrieval and a tool-using agent"
|
|
9
|
+
authors = [{name = "Samit Mohan"}]
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
dependencies = [
|
|
14
|
+
"numpy",
|
|
15
|
+
"regex",
|
|
16
|
+
]
|
|
17
|
+
|
|
18
|
+
[project.urls]
|
|
19
|
+
Homepage = "https://github.com/samitmohan/stackgrad"
|
|
20
|
+
Documentation = "https://samitmohan.github.io/stackgrad/"
|
|
21
|
+
|
|
22
|
+
[project.optional-dependencies]
|
|
23
|
+
metal = ["mlx"]
|
|
24
|
+
cuda = ["cupy-cuda13x[ctk]", "triton; sys_platform == 'linux'"]
|
|
25
|
+
serve = ["fastapi", "uvicorn"]
|
|
26
|
+
docs = [
|
|
27
|
+
"mkdocs-material",
|
|
28
|
+
"mkdocstrings[python]",
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
[tool.hatch.build.targets.sdist]
|
|
32
|
+
include = ["stackgrad", "README.md", "LICENSE"]
|
|
33
|
+
|
|
34
|
+
[tool.ruff]
|
|
35
|
+
line-length = 150
|
|
36
|
+
indent-width = 2
|
|
37
|
+
|
|
38
|
+
[tool.ruff.lint]
|
|
39
|
+
extend-select = ["TID251", "E501", "E701", "E702", "E703", "E731"] # 150 columns; no code golf: one statement per line, no assigned lambdas
|
|
40
|
+
|
|
41
|
+
[dependency-groups]
|
|
42
|
+
dev = [
|
|
43
|
+
"torch",
|
|
44
|
+
"tiktoken",
|
|
45
|
+
"transformers",
|
|
46
|
+
"huggingface_hub",
|
|
47
|
+
"ruff",
|
|
48
|
+
"diffusers>=0.41.0",
|
|
49
|
+
"matplotlib",
|
|
50
|
+
]
|
|
51
|
+
|
|
52
|
+
[tool.ruff.lint.flake8-tidy-imports.banned-api]
|
|
53
|
+
# CuPy is an array backend for training throughput. Custom kernels are Metal (mx.fast.metal_kernel)
|
|
54
|
+
# on the Mac and Triton on NVIDIA, never CuPy's raw kernel APIs.
|
|
55
|
+
"cupy.RawKernel".msg = "Custom kernels are Metal or Triton: use mx.fast.metal_kernel or a @triton.jit kernel."
|
|
56
|
+
"cupy.RawModule".msg = "Custom kernels are Metal or Triton: use mx.fast.metal_kernel or a @triton.jit kernel."
|
|
57
|
+
"cupy.ElementwiseKernel".msg = "Custom kernels are Metal or Triton: use mx.fast.metal_kernel or a @triton.jit kernel."
|
|
58
|
+
"cupy.ReductionKernel".msg = "Custom kernels are Metal or Triton: use mx.fast.metal_kernel or a @triton.jit kernel."
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
from . import functional as F
|
|
2
|
+
from .attention import Attention, KVCache
|
|
3
|
+
from .conv import Conv2d, Flatten, MaxPool2d
|
|
4
|
+
from .data import DataLoader
|
|
5
|
+
from .gpt import TransformerLM
|
|
6
|
+
from .grad_check import check_gradient, numerical_gradient
|
|
7
|
+
from .layers import (
|
|
8
|
+
GELU,
|
|
9
|
+
BatchNorm1d,
|
|
10
|
+
Dropout,
|
|
11
|
+
Embedding,
|
|
12
|
+
GroupNorm,
|
|
13
|
+
LayerNorm,
|
|
14
|
+
Linear,
|
|
15
|
+
ReLU,
|
|
16
|
+
RMSNorm,
|
|
17
|
+
Sigmoid,
|
|
18
|
+
Softmax,
|
|
19
|
+
SwiGLU,
|
|
20
|
+
Tanh,
|
|
21
|
+
)
|
|
22
|
+
from .loss import bce_loss, cross_entropy_loss, mse_loss
|
|
23
|
+
from .module import Module, Sequential
|
|
24
|
+
from .optim import (
|
|
25
|
+
SGD,
|
|
26
|
+
Adam,
|
|
27
|
+
AdamW,
|
|
28
|
+
LossScaler,
|
|
29
|
+
clip_grad_norm,
|
|
30
|
+
cosine_lr,
|
|
31
|
+
)
|
|
32
|
+
from .tensor import Tensor, autocast, cat, no_grad, stack, where
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"GELU",
|
|
36
|
+
"SGD",
|
|
37
|
+
"Adam",
|
|
38
|
+
"AdamW",
|
|
39
|
+
"Attention",
|
|
40
|
+
"BatchNorm1d",
|
|
41
|
+
"Conv2d",
|
|
42
|
+
"DataLoader",
|
|
43
|
+
"Dropout",
|
|
44
|
+
"Embedding",
|
|
45
|
+
"F",
|
|
46
|
+
"Flatten",
|
|
47
|
+
"GroupNorm",
|
|
48
|
+
"KVCache",
|
|
49
|
+
"LayerNorm",
|
|
50
|
+
"Linear",
|
|
51
|
+
"LossScaler",
|
|
52
|
+
"MaxPool2d",
|
|
53
|
+
"Module",
|
|
54
|
+
"RMSNorm",
|
|
55
|
+
"ReLU",
|
|
56
|
+
"Sequential",
|
|
57
|
+
"Sigmoid",
|
|
58
|
+
"Softmax",
|
|
59
|
+
"SwiGLU",
|
|
60
|
+
"Tanh",
|
|
61
|
+
"Tensor",
|
|
62
|
+
"TransformerLM",
|
|
63
|
+
"autocast",
|
|
64
|
+
"bce_loss",
|
|
65
|
+
"cat",
|
|
66
|
+
"check_gradient",
|
|
67
|
+
"clip_grad_norm",
|
|
68
|
+
"cosine_lr",
|
|
69
|
+
"cross_entropy_loss",
|
|
70
|
+
"mse_loss",
|
|
71
|
+
"no_grad",
|
|
72
|
+
"numerical_gradient",
|
|
73
|
+
"stack",
|
|
74
|
+
"where",
|
|
75
|
+
]
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from . import mlops
|
|
4
|
+
from .layers import Linear, RMSNorm
|
|
5
|
+
from .module import Module
|
|
6
|
+
from .tensor import _apply, cat
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def apply_rope(x, start, theta):
|
|
10
|
+
"""Rotates each (x[i], x[i + D/2]) pair by its position's angle, in Llama/Qwen's half-split layout."""
|
|
11
|
+
half = x.shape[-1] // 2
|
|
12
|
+
freqs = theta ** (-np.arange(half) / half)
|
|
13
|
+
positions = np.reshape(start, (-1, 1, 1)) + np.arange(x.shape[2])[:, None] # (B or 1, T, 1)
|
|
14
|
+
angles = (positions * freqs)[:, None] # (B or 1, 1, T, D/2)
|
|
15
|
+
cos, sin = np.cos(angles).astype(np.float32), np.sin(angles).astype(np.float32)
|
|
16
|
+
x1, x2 = x[..., :half], x[..., half:]
|
|
17
|
+
return cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def scaled_dot_product_attention(q, k, v, pad=None, causal=True):
|
|
21
|
+
"""q (B, H, T, D) over k, v (B, Hkv, S, D), queries at the last T positions; pad (B,) masks left padding."""
|
|
22
|
+
key_bias = None
|
|
23
|
+
if pad is not None:
|
|
24
|
+
# -1e9, not -inf: a padding query sees only padding, and an all -inf softmax is NaN that leaks via 0 * NaN
|
|
25
|
+
key_bias = np.where(np.arange(k.shape[2]) < np.asarray(pad)[:, None], -1e9, 0.0).astype(np.float32)
|
|
26
|
+
return _apply(mlops.Attention, q, k, v, key_bias=key_bias, causal=causal)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class KVCache:
|
|
30
|
+
"""Keys and values of earlier positions, so a decode step doesn't recompute the prefix."""
|
|
31
|
+
# shortcut: cat copies the whole cache every step (O(T^2) bytes per sequence); M4 swaps in a paged cache
|
|
32
|
+
def __init__(self):
|
|
33
|
+
self.k = self.v = None
|
|
34
|
+
|
|
35
|
+
def __len__(self):
|
|
36
|
+
return 0 if self.k is None else self.k.shape[2]
|
|
37
|
+
|
|
38
|
+
def truncate(self, n):
|
|
39
|
+
"""Drops every position from n on, for speculative decoding's rejected drafts."""
|
|
40
|
+
self.k, self.v = self.k[:, :, :n], self.v[:, :, :n]
|
|
41
|
+
|
|
42
|
+
def update(self, k, v):
|
|
43
|
+
self.k = k if self.k is None else cat([self.k, k], axis=2)
|
|
44
|
+
self.v = v if self.v is None else cat([self.v, v], axis=2)
|
|
45
|
+
return self.k, self.v
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Attention(Module):
|
|
49
|
+
"""Multi-head attention with RoPE and grouped-query heads; qk_norm adds Qwen3's per-head RMSNorm on q, k."""
|
|
50
|
+
def __init__(self, d_model, n_heads, n_kv_heads=None, rope_theta=10000.0, head_dim=None, qk_norm=False, norm_eps=1e-5,
|
|
51
|
+
rope=True):
|
|
52
|
+
super().__init__()
|
|
53
|
+
self.rope = rope # False is the NoPE ablation: causal masking alone orders tokens
|
|
54
|
+
self.n_heads = n_heads
|
|
55
|
+
self.n_kv_heads = n_kv_heads or n_heads
|
|
56
|
+
self.head_dim = head_dim or d_model // n_heads
|
|
57
|
+
self.rope_theta = rope_theta
|
|
58
|
+
self.q_proj = Linear(d_model, n_heads * self.head_dim, bias=False)
|
|
59
|
+
self.k_proj = Linear(d_model, self.n_kv_heads * self.head_dim, bias=False)
|
|
60
|
+
self.v_proj = Linear(d_model, self.n_kv_heads * self.head_dim, bias=False)
|
|
61
|
+
self.o_proj = Linear(n_heads * self.head_dim, d_model, bias=False)
|
|
62
|
+
self.q_norm = RMSNorm(self.head_dim, norm_eps) if qk_norm else None
|
|
63
|
+
self.k_norm = RMSNorm(self.head_dim, norm_eps) if qk_norm else None
|
|
64
|
+
self.record, self.recorded = False, None # record=True keeps the last q and k, for drawing attention maps
|
|
65
|
+
|
|
66
|
+
def forward(self, x, cache=None, pad=None):
|
|
67
|
+
B, T, _ = x.shape
|
|
68
|
+
D = self.head_dim
|
|
69
|
+
start = 0 if cache is None else len(cache)
|
|
70
|
+
if pad is not None:
|
|
71
|
+
start = start - np.asarray(pad) # each row's first real token sits at position 0, after its left padding
|
|
72
|
+
q = self.q_proj(x).reshape(B, T, self.n_heads, D)
|
|
73
|
+
k = self.k_proj(x).reshape(B, T, self.n_kv_heads, D)
|
|
74
|
+
if self.q_norm is not None:
|
|
75
|
+
q, k = self.q_norm(q), self.k_norm(k)
|
|
76
|
+
q, k = q.transpose(1, 2), k.transpose(1, 2) # (B, H, T, D), (B, Hkv, T, D)
|
|
77
|
+
v = self.v_proj(x).reshape(B, T, self.n_kv_heads, D).transpose(1, 2)
|
|
78
|
+
if self.rope:
|
|
79
|
+
q, k = apply_rope(q, start, self.rope_theta), apply_rope(k, start, self.rope_theta)
|
|
80
|
+
if cache is not None:
|
|
81
|
+
k, v = cache.update(k, v)
|
|
82
|
+
if self.record:
|
|
83
|
+
self.recorded = q.numpy(), k.numpy()
|
|
84
|
+
|
|
85
|
+
out = scaled_dot_product_attention(q, k, v, pad) # (B, H, T, D)
|
|
86
|
+
return self.o_proj(out.transpose(1, 2).reshape(B, T, self.n_heads * D))
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import functools
|
|
2
|
+
import importlib
|
|
3
|
+
|
|
4
|
+
DEVICES = ("cpu", "metal", "cuda")
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@functools.cache
|
|
8
|
+
def get(device):
|
|
9
|
+
if device not in DEVICES:
|
|
10
|
+
raise ValueError(f"unknown device {device!r}, expected one of {DEVICES}")
|
|
11
|
+
return importlib.import_module(f"{__name__}.{device}")
|