stackgrad 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stackgrad/__init__.py +75 -0
- stackgrad/attention.py +86 -0
- stackgrad/backend/__init__.py +11 -0
- stackgrad/backend/cpu.py +116 -0
- stackgrad/backend/cuda.py +96 -0
- stackgrad/backend/cuda_attention.py +138 -0
- stackgrad/backend/metal.py +106 -0
- stackgrad/backend/metal_attention.py +195 -0
- stackgrad/backend/metal_quant.py +44 -0
- stackgrad/checkpoint.py +38 -0
- stackgrad/conv.py +63 -0
- stackgrad/data.py +28 -0
- stackgrad/functional.py +41 -0
- stackgrad/gpt.py +120 -0
- stackgrad/grad_check.py +55 -0
- stackgrad/layers.py +215 -0
- stackgrad/lora.py +54 -0
- stackgrad/loss.py +28 -0
- stackgrad/mlops.py +352 -0
- stackgrad/module.py +120 -0
- stackgrad/optim.py +121 -0
- stackgrad/profile.py +45 -0
- stackgrad/quant.py +66 -0
- stackgrad/qwen3.py +104 -0
- stackgrad/sampling.py +39 -0
- stackgrad/serve/api.py +434 -0
- stackgrad/serve/engine.py +53 -0
- stackgrad/serve/kv_cache.py +35 -0
- stackgrad/serve/runner.py +63 -0
- stackgrad/serve/scheduler.py +88 -0
- stackgrad/speculative.py +68 -0
- stackgrad/state.py +27 -0
- stackgrad/tensor.py +377 -0
- stackgrad/tokenizer.py +202 -0
- stackgrad-0.2.0.dist-info/METADATA +172 -0
- stackgrad-0.2.0.dist-info/RECORD +38 -0
- stackgrad-0.2.0.dist-info/WHEEL +4 -0
- stackgrad-0.2.0.dist-info/licenses/LICENSE +21 -0
stackgrad/speculative.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from .attention import KVCache
|
|
4
|
+
from .tensor import no_grad
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _probs(logits, temperature):
|
|
8
|
+
if temperature == 0:
|
|
9
|
+
p = np.zeros_like(logits)
|
|
10
|
+
p[np.arange(len(logits)), logits.argmax(axis=-1)] = 1.0
|
|
11
|
+
return p
|
|
12
|
+
z = logits / temperature
|
|
13
|
+
p = np.exp(z - z.max(axis=-1, keepdims=True))
|
|
14
|
+
return p / p.sum(axis=-1, keepdims=True)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _draw(p):
|
|
18
|
+
return min(int(np.searchsorted(np.cumsum(p), np.random.rand() * p.sum(), side="right")), len(p) - 1)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _scores(model, caches, tokens, last):
|
|
22
|
+
h = model.hidden(np.array([tokens]), caches)
|
|
23
|
+
return model.project(h[:, -last:])[0].numpy().astype(np.float64)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def speculative_generate(target, draft, ids, max_new, k=4, temperature=0.0, eos_id=None):
|
|
27
|
+
"""Up to max_new target tokens after ids, draft proposing k at a time; returns (tokens, proposed, accepted)."""
|
|
28
|
+
assert len(ids) + max_new + k + 1 <= target.context_length, "prompt + new tokens exceed context_length"
|
|
29
|
+
t_caches = [KVCache() for _ in target.blocks]
|
|
30
|
+
d_caches = [KVCache() for _ in draft.blocks]
|
|
31
|
+
out, proposed, accepted = [], 0, 0
|
|
32
|
+
with no_grad():
|
|
33
|
+
# each cache holds every token except the model's pending ones
|
|
34
|
+
if len(ids) > 1:
|
|
35
|
+
_scores(target, t_caches, ids[:-1], 1)
|
|
36
|
+
_scores(draft, d_caches, ids[:-1], 1)
|
|
37
|
+
pending_t, pending_d = [ids[-1]], [ids[-1]]
|
|
38
|
+
while len(out) < max_new:
|
|
39
|
+
drafts, qs = [], []
|
|
40
|
+
q = _probs(_scores(draft, d_caches, pending_d, 1), temperature)[0]
|
|
41
|
+
for i in range(k):
|
|
42
|
+
drafts.append(_draw(q))
|
|
43
|
+
qs.append(q)
|
|
44
|
+
if i < k - 1:
|
|
45
|
+
q = _probs(_scores(draft, d_caches, [drafts[-1]], 1), temperature)[0]
|
|
46
|
+
# one target pass over its pending token and every draft: k + 1 distributions
|
|
47
|
+
p = _probs(_scores(target, t_caches, pending_t + drafts, k + 1), temperature)
|
|
48
|
+
# keep draft x with prob min(1, p/q), else resample max(0, p - q): exact target distribution
|
|
49
|
+
n = 0
|
|
50
|
+
while n < k and np.random.rand() < min(1.0, p[n, drafts[n]] / qs[n][drafts[n]]):
|
|
51
|
+
n += 1
|
|
52
|
+
last = _draw(np.maximum(p[n] - qs[n], 0.0)) if n < k else _draw(p[k])
|
|
53
|
+
proposed, accepted = proposed + k, accepted + n
|
|
54
|
+
|
|
55
|
+
# roll both caches back to the kept tokens: the target saw every draft, the draft all but the last
|
|
56
|
+
for c in t_caches:
|
|
57
|
+
c.truncate(len(c) - (k - n))
|
|
58
|
+
for c in d_caches:
|
|
59
|
+
c.truncate(len(c) - (k - 1 - min(n, k - 1)))
|
|
60
|
+
pending_t = [last]
|
|
61
|
+
pending_d = ([drafts[-1]] if n == k else []) + [last]
|
|
62
|
+
|
|
63
|
+
new = drafts[:n] + [last]
|
|
64
|
+
if eos_id is not None and eos_id in new:
|
|
65
|
+
out += new[:new.index(eos_id)]
|
|
66
|
+
break
|
|
67
|
+
out += new
|
|
68
|
+
return out[:max_new], proposed, accepted
|
stackgrad/state.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import numpy as np
|
|
4
|
+
|
|
5
|
+
_DTYPES = {"F32": np.float32, "F16": np.float16, "F64": np.float64, "I64": np.int64, "I32": np.int32, "U8": np.uint8}
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def load_safetensors(path):
|
|
9
|
+
return dict(iter_safetensors(path))
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def iter_safetensors(path):
|
|
13
|
+
with open(path, "rb") as f:
|
|
14
|
+
n = int.from_bytes(f.read(8), "little")
|
|
15
|
+
header = json.loads(f.read(n))
|
|
16
|
+
raw = np.memmap(path, dtype=np.uint8, mode="r", offset=8 + n)
|
|
17
|
+
for name, info in header.items():
|
|
18
|
+
if name == "__metadata__":
|
|
19
|
+
continue
|
|
20
|
+
start, end = info["data_offsets"]
|
|
21
|
+
b = raw[start:end]
|
|
22
|
+
if info["dtype"] == "BF16":
|
|
23
|
+
# bf16 is the top half of an f32, so shifting its bits up 16 rebuilds the f32 exactly
|
|
24
|
+
arr = (b.view(np.uint16).astype(np.uint32) << 16).view(np.float32)
|
|
25
|
+
else:
|
|
26
|
+
arr = np.array(b.view(_DTYPES[info["dtype"]]))
|
|
27
|
+
yield name, arr.reshape(info["shape"])
|
stackgrad/tensor.py
ADDED
|
@@ -0,0 +1,377 @@
|
|
|
1
|
+
import math
|
|
2
|
+
import threading
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
from . import backend, mlops
|
|
7
|
+
|
|
8
|
+
_local = threading.local() # thread-local: the serving engine's no_grad() must not freeze other threads' graphs
|
|
9
|
+
_profiler = None
|
|
10
|
+
_autocast = None
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def grad_enabled():
|
|
14
|
+
return getattr(_local, "grad_enabled", True)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class no_grad:
|
|
18
|
+
"""Context manager to disable gradient tracking during inference."""
|
|
19
|
+
def __enter__(self):
|
|
20
|
+
self._prev = grad_enabled()
|
|
21
|
+
_local.grad_enabled = False
|
|
22
|
+
return self
|
|
23
|
+
|
|
24
|
+
def __exit__(self, *args):
|
|
25
|
+
_local.grad_enabled = self._prev
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class autocast:
|
|
29
|
+
"""Mixed precision: every Linear multiplies in `dtype` ("bfloat16" or "float16"), all else stays float32."""
|
|
30
|
+
def __init__(self, dtype):
|
|
31
|
+
self.dtype = dtype
|
|
32
|
+
|
|
33
|
+
def __enter__(self):
|
|
34
|
+
global _autocast
|
|
35
|
+
self._prev, _autocast = _autocast, self.dtype
|
|
36
|
+
return self
|
|
37
|
+
|
|
38
|
+
def __exit__(self, *args):
|
|
39
|
+
global _autocast
|
|
40
|
+
_autocast = self._prev
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _to_array(data, device):
|
|
44
|
+
xp = backend.get(device)
|
|
45
|
+
if not hasattr(data, "dtype"):
|
|
46
|
+
data = np.asarray(data, dtype=np.float32)
|
|
47
|
+
data = xp.asarray(data)
|
|
48
|
+
return data if xp.is_float(data.dtype) else data.astype(xp.float32)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _shape(args):
|
|
52
|
+
return tuple(args[0]) if len(args) == 1 and isinstance(args[0], (tuple, list)) else tuple(args)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _apply(fn, *parents, **kwargs):
|
|
56
|
+
"""Records `fn` without running it; a realized parent is held by its array, so a later .data swap can't alter it."""
|
|
57
|
+
device = parents[0].device
|
|
58
|
+
assert all(p.device == device for p in parents), f"tensors on different devices: {[p.device for p in parents]}"
|
|
59
|
+
xp = backend.get(device)
|
|
60
|
+
ctx = fn(xp, *parents)
|
|
61
|
+
dtype = ctx.out_dtype(*[p.dtype for p in parents], **kwargs)
|
|
62
|
+
out = Tensor._recorded((ctx, kwargs, [p if p._buf is None else p._buf for p in parents]),
|
|
63
|
+
tuple(ctx.out_shape(*[tuple(p.shape) for p in parents], **kwargs)),
|
|
64
|
+
dtype if xp.is_float(dtype) else xp.float32, device) # Tensor() makes ints and bools float32
|
|
65
|
+
if grad_enabled() and fn.differentiable and any(ctx.needs_grad):
|
|
66
|
+
out.requires_grad = True
|
|
67
|
+
out._ctx = ctx
|
|
68
|
+
return out
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _broadcast(tensors, shape=()):
|
|
72
|
+
"""Expands only tensors that need a gradient, since Expand's backward sums it back; constants broadcast free."""
|
|
73
|
+
shape = np.broadcast_shapes(shape, *(t.shape for t in tensors))
|
|
74
|
+
return [t.expand(shape) if grad_enabled() and t.requires_grad and t.shape != shape else t
|
|
75
|
+
for t in tensors]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Tensor:
|
|
79
|
+
"""An n-dimensional array on a device ("cpu", "metal", "cuda") that records its ops for reverse-mode autodiff."""
|
|
80
|
+
__array_ufunc__ = None # makes np.float32(2) * t call t.__rmul__, not build an object array that drops the graph
|
|
81
|
+
|
|
82
|
+
def __init__(self, data, *, requires_grad=False, device="cpu"):
|
|
83
|
+
self.data = _to_array(data, device)
|
|
84
|
+
self.device = device
|
|
85
|
+
self.requires_grad = requires_grad and grad_enabled()
|
|
86
|
+
self.grad = None
|
|
87
|
+
self._ctx = None
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def _recorded(cls, lazy, shape, dtype, device):
|
|
91
|
+
"""`lazy` is (ctx, kwargs, inputs)."""
|
|
92
|
+
out = cls.__new__(cls)
|
|
93
|
+
out._buf, out._lazy, out._shape, out._dtype = None, lazy, shape, dtype
|
|
94
|
+
out.device, out.requires_grad, out.grad, out._ctx = device, False, None, None
|
|
95
|
+
return out
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def data(self):
|
|
99
|
+
"""The backend array; reading it realizes the tensor."""
|
|
100
|
+
if self._buf is None:
|
|
101
|
+
self.realize()
|
|
102
|
+
return self._buf
|
|
103
|
+
|
|
104
|
+
@data.setter
|
|
105
|
+
def data(self, array):
|
|
106
|
+
self._buf, self._lazy = array, None
|
|
107
|
+
|
|
108
|
+
def realize(self):
|
|
109
|
+
"""Computes this tensor by running every recorded op it depends on, once each. Returns self."""
|
|
110
|
+
order, seen, stack = [], set(), [(self, False)]
|
|
111
|
+
while stack: # iterative, so a deep graph cannot hit the recursion limit
|
|
112
|
+
node, inputs_done = stack.pop()
|
|
113
|
+
if inputs_done:
|
|
114
|
+
order.append(node)
|
|
115
|
+
elif node._buf is None and node not in seen:
|
|
116
|
+
seen.add(node)
|
|
117
|
+
stack.append((node, True))
|
|
118
|
+
stack.extend((x, False) for x in node._lazy[2] if isinstance(x, Tensor))
|
|
119
|
+
for node in order:
|
|
120
|
+
ctx, kwargs, inputs = node._lazy
|
|
121
|
+
args = [x._buf if isinstance(x, Tensor) else x for x in inputs]
|
|
122
|
+
if _profiler is None:
|
|
123
|
+
out = ctx.forward(*args, **kwargs)
|
|
124
|
+
else:
|
|
125
|
+
out = _profiler.time(type(ctx).__name__, node.device, ctx.forward, *args, **kwargs)
|
|
126
|
+
if out.dtype != node._dtype:
|
|
127
|
+
out = _to_array(out, node.device) # ints and bools become float32, as in Tensor()
|
|
128
|
+
# later ops were built on the recorded shape and dtype
|
|
129
|
+
assert tuple(out.shape) == node._shape and out.dtype == node._dtype, \
|
|
130
|
+
f"{type(ctx).__name__} recorded {node._shape} {node._dtype} but computed {tuple(out.shape)} {out.dtype}"
|
|
131
|
+
node.data = out # drops _lazy, so inputs nothing else holds can be freed
|
|
132
|
+
return self
|
|
133
|
+
|
|
134
|
+
@property
|
|
135
|
+
def shape(self):
|
|
136
|
+
return self._shape if self._buf is None else self._buf.shape
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def ndim(self):
|
|
140
|
+
return len(self.shape)
|
|
141
|
+
|
|
142
|
+
@property
|
|
143
|
+
def dtype(self):
|
|
144
|
+
return self._dtype if self._buf is None else self._buf.dtype
|
|
145
|
+
|
|
146
|
+
@property
|
|
147
|
+
def T(self):
|
|
148
|
+
return self.transpose()
|
|
149
|
+
|
|
150
|
+
# read by the graph explorer (serve/api.py)
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def _op(self):
|
|
154
|
+
return type(self._ctx).__name__.lower() if self._ctx else ''
|
|
155
|
+
|
|
156
|
+
def _const(self, x):
|
|
157
|
+
return x if isinstance(x, Tensor) else Tensor(backend.get(self.device).asarray(x, dtype=self.dtype), device=self.device)
|
|
158
|
+
|
|
159
|
+
def __add__(self, other):
|
|
160
|
+
return _apply(mlops.Add, *_broadcast([self, self._const(other)]))
|
|
161
|
+
|
|
162
|
+
def __radd__(self, other):
|
|
163
|
+
return self + other
|
|
164
|
+
|
|
165
|
+
def __mul__(self, other):
|
|
166
|
+
return _apply(mlops.Mul, *_broadcast([self, self._const(other)]))
|
|
167
|
+
|
|
168
|
+
def __rmul__(self, other):
|
|
169
|
+
return self * other
|
|
170
|
+
|
|
171
|
+
def __neg__(self):
|
|
172
|
+
return self * -1.0
|
|
173
|
+
|
|
174
|
+
def __sub__(self, other):
|
|
175
|
+
return self + -self._const(other)
|
|
176
|
+
|
|
177
|
+
def __rsub__(self, other):
|
|
178
|
+
return -self + other
|
|
179
|
+
|
|
180
|
+
def __truediv__(self, other):
|
|
181
|
+
return self * self._const(other).recip()
|
|
182
|
+
|
|
183
|
+
def __rtruediv__(self, other):
|
|
184
|
+
return self.recip() * other
|
|
185
|
+
|
|
186
|
+
def __pow__(self, exponent):
|
|
187
|
+
if isinstance(exponent, Tensor):
|
|
188
|
+
# a**b = exp(b * ln a), so the base must be positive
|
|
189
|
+
return (exponent * self.log()).exp()
|
|
190
|
+
return _apply(mlops.Pow, self, c=exponent)
|
|
191
|
+
|
|
192
|
+
def __matmul__(self, other):
|
|
193
|
+
return _apply(mlops.Matmul, self, other)
|
|
194
|
+
|
|
195
|
+
def __lt__(self, other):
|
|
196
|
+
return _apply(mlops.Less, self, self._const(other))
|
|
197
|
+
|
|
198
|
+
def __gt__(self, other):
|
|
199
|
+
return _apply(mlops.Less, self._const(other), self)
|
|
200
|
+
|
|
201
|
+
def sum(self, axis=None, keepdims=False):
|
|
202
|
+
"""Sum over `axis`, or over every element when `axis` is None."""
|
|
203
|
+
return _apply(mlops.Sum, self, axis=axis, keepdims=keepdims)
|
|
204
|
+
|
|
205
|
+
def mean(self, axis=None, keepdims=False):
|
|
206
|
+
"""Mean over `axis`, or over every element when `axis` is None."""
|
|
207
|
+
out = self.sum(axis=axis, keepdims=keepdims)
|
|
208
|
+
# from shapes, not .data, so the graph stays lazy
|
|
209
|
+
return out * (math.prod(out.shape) / math.prod(self.shape))
|
|
210
|
+
|
|
211
|
+
def max(self, axis=None, keepdims=False):
|
|
212
|
+
return _apply(mlops.Max, self, axis=axis, keepdims=keepdims)
|
|
213
|
+
|
|
214
|
+
def min(self, axis=None, keepdims=False):
|
|
215
|
+
return -((-self).max(axis=axis, keepdims=keepdims))
|
|
216
|
+
|
|
217
|
+
def reshape(self, *shape):
|
|
218
|
+
"""Same data in a new shape; one dim may be -1."""
|
|
219
|
+
return _apply(mlops.Reshape, self, shape=_shape(shape))
|
|
220
|
+
|
|
221
|
+
def permute(self, *order):
|
|
222
|
+
return _apply(mlops.Permute, self, order=tuple(d % self.ndim for d in _shape(order)))
|
|
223
|
+
|
|
224
|
+
def transpose(self, dim0=-2, dim1=-1):
|
|
225
|
+
"""Swap two axes, the last two by default."""
|
|
226
|
+
order = list(range(self.ndim))
|
|
227
|
+
order[dim0], order[dim1] = order[dim1], order[dim0]
|
|
228
|
+
return self.permute(order)
|
|
229
|
+
|
|
230
|
+
def expand(self, *shape):
|
|
231
|
+
"""Broadcast to `shape`, adding leading dims if needed."""
|
|
232
|
+
shape = _shape(shape)
|
|
233
|
+
x = self.reshape((1,) * (len(shape) - self.ndim) + self.shape) if self.ndim < len(shape) else self
|
|
234
|
+
return _apply(mlops.Expand, x, shape=shape)
|
|
235
|
+
|
|
236
|
+
def squeeze(self, axis=None):
|
|
237
|
+
axes = range(self.ndim) if axis is None else [a % self.ndim for a in np.atleast_1d(axis)]
|
|
238
|
+
return self.reshape(tuple(d for i, d in enumerate(self.shape) if not (i in axes and d == 1)))
|
|
239
|
+
|
|
240
|
+
def unsqueeze(self, axis):
|
|
241
|
+
axis = axis if axis >= 0 else self.ndim + axis + 1
|
|
242
|
+
return self.reshape(self.shape[:axis] + (1,) + self.shape[axis:])
|
|
243
|
+
|
|
244
|
+
def __getitem__(self, key):
|
|
245
|
+
# numpy index arrays must move to the tensor's device
|
|
246
|
+
xp = backend.get(self.device)
|
|
247
|
+
|
|
248
|
+
def on_device(k):
|
|
249
|
+
return xp.asarray(k) if isinstance(k, np.ndarray) else k
|
|
250
|
+
key = tuple(map(on_device, key)) if isinstance(key, tuple) else on_device(key)
|
|
251
|
+
return _apply(mlops.Index, self, key=key)
|
|
252
|
+
|
|
253
|
+
def im2col(self, kh, kw, stride=1, padding=0):
|
|
254
|
+
"""(N, C, H, W) -> (N*OH*OW, C*kh*kw) patch rows, the input to a conv-as-matmul."""
|
|
255
|
+
return _apply(mlops.Im2col, self, kh=kh, kw=kw, stride=stride, padding=padding)
|
|
256
|
+
|
|
257
|
+
def cast(self, dtype):
|
|
258
|
+
"""Convert to `dtype` ("float32", "bfloat16", ...); the gradient comes back in the old dtype."""
|
|
259
|
+
return _apply(mlops.Cast, self, dtype=backend.get(self.device).dtype(dtype))
|
|
260
|
+
|
|
261
|
+
def exp(self):
|
|
262
|
+
"""Elementwise `e ** x`."""
|
|
263
|
+
return _apply(mlops.Exp, self)
|
|
264
|
+
|
|
265
|
+
def log(self):
|
|
266
|
+
return _apply(mlops.Log, self)
|
|
267
|
+
|
|
268
|
+
def recip(self):
|
|
269
|
+
return _apply(mlops.Recip, self)
|
|
270
|
+
|
|
271
|
+
def sigmoid(self):
|
|
272
|
+
return _apply(mlops.Sigmoid, self)
|
|
273
|
+
|
|
274
|
+
def relu(self):
|
|
275
|
+
return where(self > 0, self, 0.0)
|
|
276
|
+
|
|
277
|
+
def tanh(self):
|
|
278
|
+
return 2.0 * (2.0 * self).sigmoid() - 1.0
|
|
279
|
+
|
|
280
|
+
def abs(self):
|
|
281
|
+
return where(self < 0, -self, self)
|
|
282
|
+
|
|
283
|
+
def clamp(self, min_val=None, max_val=None):
|
|
284
|
+
out = self
|
|
285
|
+
if min_val is not None:
|
|
286
|
+
out = where(out < min_val, min_val, out)
|
|
287
|
+
if max_val is not None:
|
|
288
|
+
out = where(out > max_val, max_val, out)
|
|
289
|
+
return out
|
|
290
|
+
|
|
291
|
+
def backward(self, keep_grads=False):
|
|
292
|
+
"""Backpropagate from this scalar, filling `.grad` on every leaf (and intermediates if `keep_grads`)."""
|
|
293
|
+
assert self.data.size == 1, "backward() only works on scalar tensors - call .sum() or .mean() first"
|
|
294
|
+
|
|
295
|
+
# iterative, so a deep graph cannot hit the recursion limit
|
|
296
|
+
topo = []
|
|
297
|
+
visited = set()
|
|
298
|
+
stack = [(self, False)]
|
|
299
|
+
while stack:
|
|
300
|
+
node, processed = stack.pop()
|
|
301
|
+
if processed:
|
|
302
|
+
topo.append(node)
|
|
303
|
+
continue
|
|
304
|
+
if node in visited:
|
|
305
|
+
continue
|
|
306
|
+
visited.add(node)
|
|
307
|
+
stack.append((node, True))
|
|
308
|
+
if node._ctx:
|
|
309
|
+
for parent in node._ctx.parents:
|
|
310
|
+
if parent.requires_grad and parent not in visited:
|
|
311
|
+
stack.append((parent, False))
|
|
312
|
+
|
|
313
|
+
self.grad = backend.get(self.device).ones_like(self.data)
|
|
314
|
+
for node in reversed(topo):
|
|
315
|
+
if node._ctx is None or node.grad is None:
|
|
316
|
+
continue
|
|
317
|
+
if _profiler is None:
|
|
318
|
+
grads = node._ctx.backward(node.grad)
|
|
319
|
+
else:
|
|
320
|
+
grads = _profiler.time(type(node._ctx).__name__ + " backward", node.device, node._ctx.backward, node.grad)
|
|
321
|
+
# free intermediate grads once passed on, so a second backward() starts clean
|
|
322
|
+
if not keep_grads:
|
|
323
|
+
node.grad = None
|
|
324
|
+
for parent, grad in zip(node._ctx.parents, grads):
|
|
325
|
+
if grad is not None and parent.requires_grad:
|
|
326
|
+
# never +=: Add hands one grad array to both parents
|
|
327
|
+
parent.grad = grad if parent.grad is None else parent.grad + grad
|
|
328
|
+
|
|
329
|
+
def zero_grad(self):
|
|
330
|
+
self.grad = None
|
|
331
|
+
|
|
332
|
+
def detach(self):
|
|
333
|
+
"""Return a tensor sharing this data but cut out of the graph."""
|
|
334
|
+
return Tensor(self.data, device=self.device)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def to(self, device=None, dtype=None):
|
|
338
|
+
"""This tensor outside the graph, on `device` in `dtype` ("bfloat16", ...); None keeps the current one."""
|
|
339
|
+
device = device or self.device
|
|
340
|
+
data = self.data if device == self.device else backend.get(device).asarray(self.numpy())
|
|
341
|
+
if dtype is not None:
|
|
342
|
+
data = data.astype(backend.get(device).dtype(dtype))
|
|
343
|
+
return Tensor(data, device=device, requires_grad=self.requires_grad)
|
|
344
|
+
|
|
345
|
+
def numpy(self):
|
|
346
|
+
"""The data as a NumPy array on the host (a copy)."""
|
|
347
|
+
return backend.get(self.device).to_numpy(self.data)
|
|
348
|
+
|
|
349
|
+
def item(self):
|
|
350
|
+
return self.data.item()
|
|
351
|
+
|
|
352
|
+
def __repr__(self):
|
|
353
|
+
return f"Tensor(data={self.data}, grad={self.grad})"
|
|
354
|
+
|
|
355
|
+
def __len__(self):
|
|
356
|
+
return self.shape[0]
|
|
357
|
+
|
|
358
|
+
def __float__(self):
|
|
359
|
+
return float(self.data.item())
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def where(cond, x, y):
|
|
363
|
+
"""Elementwise `x if cond else y`. `cond` is a mask and gets no gradient."""
|
|
364
|
+
ref = x if isinstance(x, Tensor) else y
|
|
365
|
+
cond = cond if isinstance(cond, Tensor) else Tensor(backend.get(ref.device).asarray(cond), device=ref.device)
|
|
366
|
+
x, y = _broadcast([ref._const(x), ref._const(y)], cond.shape)
|
|
367
|
+
return _apply(mlops.Where, x, y, cond)
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def cat(tensors, axis=0):
|
|
371
|
+
"""Concatenate tensors along an existing axis."""
|
|
372
|
+
return _apply(mlops.Cat, *tensors, axis=axis)
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def stack(tensors, axis=0):
|
|
376
|
+
"""Stack tensors along a new axis."""
|
|
377
|
+
return cat([t.unsqueeze(axis) for t in tensors], axis=axis)
|
stackgrad/tokenizer.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import itertools
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import unicodedata
|
|
5
|
+
from collections import Counter, defaultdict
|
|
6
|
+
from multiprocessing import Pool
|
|
7
|
+
|
|
8
|
+
import regex as re
|
|
9
|
+
|
|
10
|
+
PAT = re.compile(r"""'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _special_splitter(special_tokens):
|
|
14
|
+
# longest first, so "<|eot|><|eot|>" registered as one token beats two "<|eot|>"
|
|
15
|
+
alternatives = "|".join(re.escape(s) for s in sorted(special_tokens, key=len, reverse=True))
|
|
16
|
+
return re.compile(f"({alternatives})") if special_tokens else None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def chunk_bounds(path, n_chunks, boundary=b"<|endoftext|>"):
|
|
20
|
+
"""Byte offsets that cut the file into ~n_chunks pieces, each cut moved forward to the next `boundary`."""
|
|
21
|
+
size = os.path.getsize(path)
|
|
22
|
+
bounds = [size * i // n_chunks for i in range(n_chunks)] + [size]
|
|
23
|
+
with open(path, "rb") as f:
|
|
24
|
+
for i in range(1, n_chunks):
|
|
25
|
+
f.seek(bounds[i])
|
|
26
|
+
# shortcut: assumes a boundary occurs within 1 MiB of every cut, true for TinyStories; scan further if not
|
|
27
|
+
found = f.read(1 << 20).find(boundary)
|
|
28
|
+
bounds[i] = size if found == -1 else bounds[i] + found
|
|
29
|
+
return sorted(set(bounds))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _count_pretokens(args):
|
|
33
|
+
path, start, end, special_tokens = args
|
|
34
|
+
with open(path, "rb") as f:
|
|
35
|
+
f.seek(start)
|
|
36
|
+
text = f.read(end - start).decode("utf-8", errors="ignore")
|
|
37
|
+
splitter = _special_splitter(special_tokens)
|
|
38
|
+
pieces = splitter.split(text)[::2] if splitter else [text] # even slots are the text between specials
|
|
39
|
+
return Counter(m.group() for piece in pieces for m in PAT.finditer(piece))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def train_bpe(input_path, vocab_size, special_tokens, num_workers=None):
|
|
43
|
+
"""Learn merges until the vocab (256 bytes + specials + merges) reaches vocab_size."""
|
|
44
|
+
num_workers = num_workers or os.cpu_count()
|
|
45
|
+
bounds = chunk_bounds(input_path, num_workers * 4)
|
|
46
|
+
jobs = [(input_path, s, e, special_tokens) for s, e in itertools.pairwise(bounds)]
|
|
47
|
+
counts = Counter()
|
|
48
|
+
with Pool(num_workers) as pool:
|
|
49
|
+
for c in pool.imap_unordered(_count_pretokens, jobs):
|
|
50
|
+
counts.update(c)
|
|
51
|
+
|
|
52
|
+
words = [[bytes([b]) for b in w.encode("utf-8")] for w in counts]
|
|
53
|
+
freqs = list(counts.values())
|
|
54
|
+
pair_counts = Counter()
|
|
55
|
+
where = defaultdict(set) # pair -> ids of words containing it
|
|
56
|
+
for i, w in enumerate(words):
|
|
57
|
+
for pair in itertools.pairwise(w):
|
|
58
|
+
pair_counts[pair] += freqs[i]
|
|
59
|
+
where[pair].add(i)
|
|
60
|
+
|
|
61
|
+
vocab = {i: bytes([i]) for i in range(256)}
|
|
62
|
+
for s in special_tokens:
|
|
63
|
+
vocab[len(vocab)] = s.encode("utf-8")
|
|
64
|
+
merges = []
|
|
65
|
+
while len(vocab) < vocab_size and pair_counts:
|
|
66
|
+
# ties go to the lexicographically greater pair
|
|
67
|
+
# shortcut: O(distinct pairs) scan per merge; switch to a lazy max-heap if training time matters
|
|
68
|
+
best = max(pair_counts, key=lambda p: (pair_counts[p], p))
|
|
69
|
+
a, b = best
|
|
70
|
+
merges.append(best)
|
|
71
|
+
vocab[len(vocab)] = a + b
|
|
72
|
+
for i in list(where[best]):
|
|
73
|
+
w, f = words[i], freqs[i]
|
|
74
|
+
for pair in itertools.pairwise(w):
|
|
75
|
+
pair_counts[pair] -= f
|
|
76
|
+
if pair_counts[pair] == 0:
|
|
77
|
+
del pair_counts[pair]
|
|
78
|
+
words[i] = w = _merge(w, a, b)
|
|
79
|
+
for pair in itertools.pairwise(w):
|
|
80
|
+
pair_counts[pair] += f
|
|
81
|
+
where[pair].add(i)
|
|
82
|
+
del where[best]
|
|
83
|
+
return vocab, merges
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _merge(word, a, b):
|
|
87
|
+
out, i = [], 0
|
|
88
|
+
while i < len(word):
|
|
89
|
+
if i + 1 < len(word) and word[i] == a and word[i + 1] == b:
|
|
90
|
+
out.append(a + b)
|
|
91
|
+
i += 2
|
|
92
|
+
else:
|
|
93
|
+
out.append(word[i])
|
|
94
|
+
i += 1
|
|
95
|
+
return out
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class Tokenizer:
|
|
99
|
+
"""Byte-level BPE; pattern splits text into pre-tokens, nfc normalizes to Unicode NFC first (as Qwen does)."""
|
|
100
|
+
def __init__(self, vocab, merges, special_tokens=None, pattern=PAT, nfc=False):
|
|
101
|
+
self.pattern, self.nfc = pattern, nfc
|
|
102
|
+
self.vocab = dict(vocab)
|
|
103
|
+
self.special_tokens = list(special_tokens or [])
|
|
104
|
+
for s in self.special_tokens:
|
|
105
|
+
if s.encode("utf-8") not in self.vocab.values():
|
|
106
|
+
self.vocab[len(self.vocab)] = s.encode("utf-8")
|
|
107
|
+
self.ids = {tok: i for i, tok in self.vocab.items()}
|
|
108
|
+
self.ranks = {pair: r for r, pair in enumerate(merges)}
|
|
109
|
+
self.splitter = _special_splitter(self.special_tokens)
|
|
110
|
+
self.cache = {}
|
|
111
|
+
|
|
112
|
+
@classmethod
|
|
113
|
+
def from_files(cls, vocab_path, merges_path, special_tokens=None, **options):
|
|
114
|
+
"""Load GPT-2 format files: vocab.json and merges.txt, with bytes spelled via bytes_to_unicode."""
|
|
115
|
+
decode = {c: b for b, c in bytes_to_unicode().items()}
|
|
116
|
+
with open(vocab_path, encoding="utf-8") as f:
|
|
117
|
+
vocab = {i: bytes(decode[c] for c in s) for s, i in json.load(f).items()}
|
|
118
|
+
with open(merges_path, encoding="utf-8") as f:
|
|
119
|
+
lines = [ln.rstrip("\n") for ln in f if ln.strip() and not ln.startswith("#version")]
|
|
120
|
+
merges = [tuple(bytes(decode[c] for c in part) for part in ln.split(" ")) for ln in lines]
|
|
121
|
+
return cls(vocab, merges, special_tokens, **options)
|
|
122
|
+
|
|
123
|
+
def _bpe(self, token, steps=None):
|
|
124
|
+
word = [bytes([b]) for b in token]
|
|
125
|
+
while True:
|
|
126
|
+
if steps is not None:
|
|
127
|
+
steps.append(word)
|
|
128
|
+
pair = min(itertools.pairwise(word), key=lambda p: self.ranks.get(p, float("inf")), default=None)
|
|
129
|
+
if pair not in self.ranks:
|
|
130
|
+
return word
|
|
131
|
+
word = _merge(word, *pair)
|
|
132
|
+
|
|
133
|
+
def _encode_pretoken(self, s):
|
|
134
|
+
ids = self.cache.get(s)
|
|
135
|
+
if ids is None:
|
|
136
|
+
ids = self.cache[s] = [self.ids[t] for t in self._bpe(s.encode("utf-8"))]
|
|
137
|
+
return ids
|
|
138
|
+
|
|
139
|
+
def encode(self, text):
|
|
140
|
+
if self.nfc:
|
|
141
|
+
text = unicodedata.normalize("NFC", text)
|
|
142
|
+
ids = []
|
|
143
|
+
pieces = self.splitter.split(text) if self.splitter else [text]
|
|
144
|
+
for k, piece in enumerate(pieces):
|
|
145
|
+
if k % 2: # odd slots are the special tokens the splitter captured
|
|
146
|
+
ids.append(self.ids[piece.encode("utf-8")])
|
|
147
|
+
else:
|
|
148
|
+
for m in self.pattern.finditer(piece):
|
|
149
|
+
ids.extend(self._encode_pretoken(m.group()))
|
|
150
|
+
return ids
|
|
151
|
+
|
|
152
|
+
def encode_iterable(self, iterable):
|
|
153
|
+
"""Encodes in constant memory, holding back each string's last pre-token, which more text can resplit."""
|
|
154
|
+
carry = ""
|
|
155
|
+
for text in iterable:
|
|
156
|
+
text = carry + text
|
|
157
|
+
tail = self.splitter.split(text)[-1] if self.splitter else text
|
|
158
|
+
starts = [m.start() for m in self.pattern.finditer(tail)]
|
|
159
|
+
cut = len(text) - len(tail) + starts[-1] if starts else len(text)
|
|
160
|
+
yield from self.encode(text[:cut])
|
|
161
|
+
carry = text[cut:]
|
|
162
|
+
yield from self.encode(carry)
|
|
163
|
+
|
|
164
|
+
def decode(self, ids):
|
|
165
|
+
return b"".join(self.vocab[i] for i in ids).decode("utf-8", errors="replace")
|
|
166
|
+
|
|
167
|
+
def trace(self, text):
|
|
168
|
+
"""Every merge step for every pre-token, for showing BPE at work."""
|
|
169
|
+
out = []
|
|
170
|
+
for m in self.pattern.finditer(text):
|
|
171
|
+
steps = []
|
|
172
|
+
self._bpe(m.group().encode("utf-8"), steps)
|
|
173
|
+
out.append({"pretoken": m.group(),
|
|
174
|
+
"steps": [[t.decode("utf-8", errors="replace") for t in w] for w in steps]})
|
|
175
|
+
return out
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def bytes_to_unicode():
|
|
179
|
+
"""GPT-2's map from each byte to a printable character, so vocab files are plain text."""
|
|
180
|
+
printable = [*range(ord("!"), ord("~") + 1), *range(ord("¡"), ord("¬") + 1), *range(ord("®"), ord("ÿ") + 1)]
|
|
181
|
+
chars, n = {}, 0
|
|
182
|
+
for b in range(256):
|
|
183
|
+
if b in printable:
|
|
184
|
+
chars[b] = chr(b)
|
|
185
|
+
else:
|
|
186
|
+
chars[b] = chr(256 + n)
|
|
187
|
+
n += 1
|
|
188
|
+
return chars
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def save(vocab, merges, vocab_path, merges_path):
|
|
192
|
+
"""Write vocab.json and merges.txt in the GPT-2 format that `Tokenizer.from_files` reads."""
|
|
193
|
+
enc = bytes_to_unicode()
|
|
194
|
+
|
|
195
|
+
def spell(token):
|
|
196
|
+
return "".join(enc[b] for b in token)
|
|
197
|
+
|
|
198
|
+
with open(vocab_path, "w", encoding="utf-8") as f:
|
|
199
|
+
json.dump({spell(t): i for i, t in vocab.items()}, f, ensure_ascii=False)
|
|
200
|
+
with open(merges_path, "w", encoding="utf-8") as f:
|
|
201
|
+
f.write("#version: 0.2\n")
|
|
202
|
+
f.writelines(f"{spell(a)} {spell(b)}\n" for a, b in merges)
|