stackgrad 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stackgrad/__init__.py +75 -0
- stackgrad/attention.py +86 -0
- stackgrad/backend/__init__.py +11 -0
- stackgrad/backend/cpu.py +116 -0
- stackgrad/backend/cuda.py +96 -0
- stackgrad/backend/cuda_attention.py +138 -0
- stackgrad/backend/metal.py +106 -0
- stackgrad/backend/metal_attention.py +195 -0
- stackgrad/backend/metal_quant.py +44 -0
- stackgrad/checkpoint.py +38 -0
- stackgrad/conv.py +63 -0
- stackgrad/data.py +28 -0
- stackgrad/functional.py +41 -0
- stackgrad/gpt.py +120 -0
- stackgrad/grad_check.py +55 -0
- stackgrad/layers.py +215 -0
- stackgrad/lora.py +54 -0
- stackgrad/loss.py +28 -0
- stackgrad/mlops.py +352 -0
- stackgrad/module.py +120 -0
- stackgrad/optim.py +121 -0
- stackgrad/profile.py +45 -0
- stackgrad/quant.py +66 -0
- stackgrad/qwen3.py +104 -0
- stackgrad/sampling.py +39 -0
- stackgrad/serve/api.py +434 -0
- stackgrad/serve/engine.py +53 -0
- stackgrad/serve/kv_cache.py +35 -0
- stackgrad/serve/runner.py +63 -0
- stackgrad/serve/scheduler.py +88 -0
- stackgrad/speculative.py +68 -0
- stackgrad/state.py +27 -0
- stackgrad/tensor.py +377 -0
- stackgrad/tokenizer.py +202 -0
- stackgrad-0.2.0.dist-info/METADATA +172 -0
- stackgrad-0.2.0.dist-info/RECORD +38 -0
- stackgrad-0.2.0.dist-info/WHEEL +4 -0
- stackgrad-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: stackgrad
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: A deep learning framework written from scratch, up to a served Qwen3 with retrieval and a tool-using agent
|
|
5
|
+
Project-URL: Homepage, https://github.com/samitmohan/stackgrad
|
|
6
|
+
Project-URL: Documentation, https://samitmohan.github.io/stackgrad/
|
|
7
|
+
Author: Samit Mohan
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Python: >=3.10
|
|
11
|
+
Requires-Dist: numpy
|
|
12
|
+
Requires-Dist: regex
|
|
13
|
+
Provides-Extra: cuda
|
|
14
|
+
Requires-Dist: cupy-cuda13x[ctk]; extra == 'cuda'
|
|
15
|
+
Requires-Dist: triton; (sys_platform == 'linux') and extra == 'cuda'
|
|
16
|
+
Provides-Extra: docs
|
|
17
|
+
Requires-Dist: mkdocs-material; extra == 'docs'
|
|
18
|
+
Requires-Dist: mkdocstrings[python]; extra == 'docs'
|
|
19
|
+
Provides-Extra: metal
|
|
20
|
+
Requires-Dist: mlx; extra == 'metal'
|
|
21
|
+
Provides-Extra: serve
|
|
22
|
+
Requires-Dist: fastapi; extra == 'serve'
|
|
23
|
+
Requires-Dist: uvicorn; extra == 'serve'
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
<div align="center">
|
|
27
|
+
|
|
28
|
+
<picture>
|
|
29
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/samitmohan/stackgrad/master/docs/logo_dark.svg">
|
|
30
|
+
<img alt="stackgrad" src="https://raw.githubusercontent.com/samitmohan/stackgrad/master/docs/logo_light.svg" width="50%" height="50%">
|
|
31
|
+
</picture>
|
|
32
|
+
|
|
33
|
+
stackgrad: A deep learning framework written from scratch, up to a served Qwen3 with retrieval and a tool-using agent.
|
|
34
|
+
|
|
35
|
+
<h3>
|
|
36
|
+
|
|
37
|
+
[Homepage](https://github.com/samitmohan/stackgrad) | [Documentation](https://samitmohan.github.io/stackgrad/) | [Write-up](https://samitmohan.in/blog/minitorch)
|
|
38
|
+
|
|
39
|
+
</h3>
|
|
40
|
+
|
|
41
|
+
[](https://github.com/samitmohan/stackgrad/actions/workflows/ci.yml)
|
|
42
|
+
|
|
43
|
+
</div>
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+

|
|
48
|
+
|
|
49
|
+
stackgrad is an end-to-end deep learning stack:
|
|
50
|
+
|
|
51
|
+
- a tensor library with lazy evaluation and autograd, every gradient in [one file](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/mlops.py)
|
|
52
|
+
- three backends (NumPy, MLX, CuPy) and attention kernels written in Metal and Triton
|
|
53
|
+
- layers, optimizers, a BPE tokenizer, and a transformer trained from nothing
|
|
54
|
+
- Qwen3-0.6B loaded, served behind an OpenAI-style endpoint, and fine-tuned with LoRA and GRPO
|
|
55
|
+
- retrieval and a tool-using agent on that server
|
|
56
|
+
|
|
57
|
+
The `stackgrad/` package is under 3,000 lines, and CI keeps it there.
|
|
58
|
+
|
|
59
|
+
PyTorch, tiktoken, and transformers appear only in the tests, as the references each piece is checked against.
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
### Laziness
|
|
64
|
+
|
|
65
|
+
Try a matmul. Nothing runs until you ask for the result.
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
import numpy as np
|
|
69
|
+
from stackgrad import Tensor
|
|
70
|
+
from stackgrad.profile import profile
|
|
71
|
+
|
|
72
|
+
N = 1024
|
|
73
|
+
a, b = Tensor(np.random.randn(N, N)), Tensor(np.random.randn(N, N))
|
|
74
|
+
with profile() as p:
|
|
75
|
+
c = (a @ b).relu().sum() # records four ops, runs none
|
|
76
|
+
print(c.shape, dict(p.calls)) # () {}
|
|
77
|
+
c.realize() # Matmul, Less, Where, Sum run, once each
|
|
78
|
+
print(p.table())
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`relu` shows up as `Less` and `Where`: it composes primitives, so it never defines a gradient of its own.
|
|
82
|
+
|
|
83
|
+
### Neural networks
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
import numpy as np
|
|
87
|
+
from stackgrad import Tensor, Module, Linear, Adam, cross_entropy_loss
|
|
88
|
+
|
|
89
|
+
class LinearNet(Module):
|
|
90
|
+
def __init__(self):
|
|
91
|
+
super().__init__()
|
|
92
|
+
self.l1, self.l2 = Linear(784, 128), Linear(128, 10)
|
|
93
|
+
|
|
94
|
+
def forward(self, x):
|
|
95
|
+
return self.l2(self.l1(x.reshape(x.shape[0], -1)).relu())
|
|
96
|
+
|
|
97
|
+
model = LinearNet()
|
|
98
|
+
opt = Adam(model.parameters(), lr=0.001)
|
|
99
|
+
|
|
100
|
+
x, y = Tensor(np.random.rand(4, 1, 28, 28)), np.array([2, 4, 3, 7]) # replace with real mnist
|
|
101
|
+
|
|
102
|
+
for i in range(10):
|
|
103
|
+
loss = cross_entropy_loss(model(x), y)
|
|
104
|
+
opt.zero_grad()
|
|
105
|
+
loss.backward()
|
|
106
|
+
opt.step()
|
|
107
|
+
print(i, loss.item())
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
See [train_gpt.py](https://github.com/samitmohan/stackgrad/blob/master/train_gpt.py) for the full version: a 22.7M-parameter transformer that trains on TinyStories to a
|
|
111
|
+
validation loss of 1.399 in 71 minutes on one RTX 3090.
|
|
112
|
+
|
|
113
|
+
## Accelerators
|
|
114
|
+
|
|
115
|
+
- [x] [CPU](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cpu.py), NumPy
|
|
116
|
+
- [x] [METAL](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/metal.py), MLX, with attention and quantized-matmul kernels in Metal
|
|
117
|
+
- [x] [CUDA](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cuda.py), CuPy, with FlashAttention-2 in Triton
|
|
118
|
+
|
|
119
|
+
A new backend is one module that exports the 27 names [cpu.py](https://github.com/samitmohan/stackgrad/blob/master/stackgrad/backend/cpu.py) exports.
|
|
120
|
+
|
|
121
|
+
## Installation
|
|
122
|
+
|
|
123
|
+
```sh
|
|
124
|
+
git clone https://github.com/samitmohan/stackgrad.git
|
|
125
|
+
cd stackgrad
|
|
126
|
+
uv sync --extra metal --extra serve # Apple silicon; NVIDIA: uv sync --extra cuda --no-dev
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
## Documentation
|
|
130
|
+
|
|
131
|
+
The [docs website](https://samitmohan.github.io/stackgrad/) has a quickstart, the API reference, and a
|
|
132
|
+
[showcase](https://samitmohan.github.io/stackgrad/showcase/) of what stackgrad runs, with every measured number and the
|
|
133
|
+
command that produced it. It builds from the [docs/](docs) directory. The write-up,
|
|
134
|
+
[Modern AI: Autograd to Agents](https://samitmohan.in/blog/minitorch), explains each piece from first principles.
|
|
135
|
+
|
|
136
|
+
### Quick example comparing to PyTorch
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
import numpy as np
|
|
140
|
+
from stackgrad import Tensor
|
|
141
|
+
|
|
142
|
+
x = Tensor(np.eye(3), requires_grad=True)
|
|
143
|
+
y = Tensor([[2.0, 0, -2.0]], requires_grad=True)
|
|
144
|
+
z = (y @ x).sum()
|
|
145
|
+
z.backward()
|
|
146
|
+
|
|
147
|
+
print(x.grad.tolist()) # dz/dx
|
|
148
|
+
print(y.grad.tolist()) # dz/dy
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
The same thing but in PyTorch:
|
|
152
|
+
|
|
153
|
+
```python
|
|
154
|
+
import torch
|
|
155
|
+
|
|
156
|
+
x = torch.eye(3, requires_grad=True)
|
|
157
|
+
y = torch.tensor([[2.0, 0, -2.0]], requires_grad=True)
|
|
158
|
+
z = y.matmul(x).sum()
|
|
159
|
+
z.backward()
|
|
160
|
+
|
|
161
|
+
print(x.grad.tolist()) # dz/dx
|
|
162
|
+
print(y.grad.tolist()) # dz/dy
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## Running tests
|
|
166
|
+
|
|
167
|
+
```sh
|
|
168
|
+
uv run python test_grads.py # every gradient against finite differences
|
|
169
|
+
uv run python tests/parity.py # every op and layer against PyTorch
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
The [CI workflow](.github/workflows/ci.yml) runs the rest.
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
stackgrad/__init__.py,sha256=k7sDFIBeK9rOOjnpVY0KCDApmRIU4X6nk0GoWA11wP8,1341
|
|
2
|
+
stackgrad/attention.py,sha256=ffCqfTwIGrlLg7dqWmu-39KWmCF-IAJuwqX5LVPyZIg,4077
|
|
3
|
+
stackgrad/checkpoint.py,sha256=HLZScbk7C8yaLFKqUZh6aQXQxAOxD2tpc-KOJvOWKuc,1696
|
|
4
|
+
stackgrad/conv.py,sha256=-trhC-91gq8S4hhnxLCRtdvplnISlYY3ey3hKPfaxSA,2409
|
|
5
|
+
stackgrad/data.py,sha256=XzejjDTSkiMGfX4WcDXPp1sVw9ZRIooZ2TGpe4p-Zwg,836
|
|
6
|
+
stackgrad/functional.py,sha256=bpkBtRgDjdaQ6DxdwlCjd3omAg7TKNtl5w55N2-Jg9A,1025
|
|
7
|
+
stackgrad/gpt.py,sha256=Yb3fFeKXB8ZyO45qLMd83P_Dj4V35hh9duJGRzFLBi0,6127
|
|
8
|
+
stackgrad/grad_check.py,sha256=91wcJRGlsV82AraJmBkFiymkgIivWZSC0XgIyTzQ4t8,1686
|
|
9
|
+
stackgrad/layers.py,sha256=7fa8xu2zZ1WQ41dghKKMCxAv6RSc62TK7TihizAuyOc,7787
|
|
10
|
+
stackgrad/lora.py,sha256=L86G3z8NafEHdmdNuCVNROljjh_9Wga_MI0cqh5FdVM,2342
|
|
11
|
+
stackgrad/loss.py,sha256=l_KXsIw59mBmsb3JvF2iNPQqyknogNRwUfe2fczkn_E,955
|
|
12
|
+
stackgrad/mlops.py,sha256=zwyIqXuolSty1Mmu4_z4TD4lGgIgmoVf0iimGiFy8vk,10583
|
|
13
|
+
stackgrad/module.py,sha256=uPk6uhxfUG64xM6kqJ5whkK9UBc6fPy7eeeROmR6aA8,4343
|
|
14
|
+
stackgrad/optim.py,sha256=-PFONi9BgPnvhvPPz4UxhFLaXdge9XWdRuA2oomg2aA,4386
|
|
15
|
+
stackgrad/profile.py,sha256=TCdl7ryPo-LLW-EgOYzz664u4K0e4yvXFfa_dxW6CGg,1435
|
|
16
|
+
stackgrad/quant.py,sha256=HJda8FFgg-yjmhc_yA-ltoVoLtqwKOvEhBegtueJmUc,3273
|
|
17
|
+
stackgrad/qwen3.py,sha256=m1_aR35tyAZfcTNKh4E_3EcPuppUx2tpFVf4EV2zXDI,5844
|
|
18
|
+
stackgrad/sampling.py,sha256=Tik7tWHufVz26m2h7hQms98I-3SifiIbqKro3UdEyrE,1731
|
|
19
|
+
stackgrad/speculative.py,sha256=431qsNTLitFsUmf_JqMl0dcb3QWmQV0xwcJCDI-eObU,2666
|
|
20
|
+
stackgrad/state.py,sha256=rFbYBYNQChicUCMV_PwrBV3O-U7lm1U1Iy8weR1IM2s,868
|
|
21
|
+
stackgrad/tensor.py,sha256=Id4f8I-KNy3cgS8FP3SlTb86kVVdz9gvoUcJPycG_Zg,12837
|
|
22
|
+
stackgrad/tokenizer.py,sha256=yiIo3iudwMpTE_cohy3eBt2eVNeD9B8bokOplF-RAqo,7700
|
|
23
|
+
stackgrad/backend/__init__.py,sha256=gitzScRqUf0qY8rzdMa0UTEK4p-zwRvHSRX5yALdy2M,269
|
|
24
|
+
stackgrad/backend/cpu.py,sha256=YT7DGRiFa-3xNMILit92e3uSRtrII9ZOiczyMK3jR78,2851
|
|
25
|
+
stackgrad/backend/cuda.py,sha256=NbAlzwajZa_uH13q1IswMxdBDhEQTe_IYwoyfJERv94,1742
|
|
26
|
+
stackgrad/backend/cuda_attention.py,sha256=0emR5Hd2Lf5PqyJRMbdPVqR9Awu5hgdaMmPtGDyXkzI,7203
|
|
27
|
+
stackgrad/backend/metal.py,sha256=OcCS1HpT_I1fee3qIKnCoXTL0S8UVw2Lcz8aZpE0qI4,2257
|
|
28
|
+
stackgrad/backend/metal_attention.py,sha256=Z5K7RYgiqMD4KXLk0UNIhEGcRJ41BXy9OiUzyQjPg3Q,8897
|
|
29
|
+
stackgrad/backend/metal_quant.py,sha256=MG8-DW5JuH2YdFqrbT6G6u6OfUOnUxmchjDixWnubRU,1853
|
|
30
|
+
stackgrad/serve/api.py,sha256=k8xt5jbtv862VA2lT6sPbSIyrsU2z6IyCnkcFYbeCHc,16393
|
|
31
|
+
stackgrad/serve/engine.py,sha256=VECTdjAcPZXgEy4fWBPJRDjRQsWsB5ndlE_dWLf8bRs,2226
|
|
32
|
+
stackgrad/serve/kv_cache.py,sha256=P4i-fkvB-FuW4ckFdCWvMPn9DVL4sJWckzt2YpA7olI,1530
|
|
33
|
+
stackgrad/serve/runner.py,sha256=4_odE8tWrtOIuNiN2SB6iT7So-oYjkX9n1LSKx623D8,2432
|
|
34
|
+
stackgrad/serve/scheduler.py,sha256=DeoGG7eF-W_kDfCWvOGhpwMu5ShhOAYTRMHovfwuMVw,3247
|
|
35
|
+
stackgrad-0.2.0.dist-info/METADATA,sha256=lIeh0n-Ta8EsJbYi5w3EUkORlk-RyDyzhZQ63oOTAgs,5919
|
|
36
|
+
stackgrad-0.2.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
37
|
+
stackgrad-0.2.0.dist-info/licenses/LICENSE,sha256=NSh8S4nSADzAuq3S0DGuVcvYg9iBfS3rGgGlOlUwo4w,1068
|
|
38
|
+
stackgrad-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Samit Mohan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|