stackgrad 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stackgrad/__init__.py +75 -0
- stackgrad/attention.py +86 -0
- stackgrad/backend/__init__.py +11 -0
- stackgrad/backend/cpu.py +116 -0
- stackgrad/backend/cuda.py +96 -0
- stackgrad/backend/cuda_attention.py +138 -0
- stackgrad/backend/metal.py +106 -0
- stackgrad/backend/metal_attention.py +195 -0
- stackgrad/backend/metal_quant.py +44 -0
- stackgrad/checkpoint.py +38 -0
- stackgrad/conv.py +63 -0
- stackgrad/data.py +28 -0
- stackgrad/functional.py +41 -0
- stackgrad/gpt.py +120 -0
- stackgrad/grad_check.py +55 -0
- stackgrad/layers.py +215 -0
- stackgrad/lora.py +54 -0
- stackgrad/loss.py +28 -0
- stackgrad/mlops.py +352 -0
- stackgrad/module.py +120 -0
- stackgrad/optim.py +121 -0
- stackgrad/profile.py +45 -0
- stackgrad/quant.py +66 -0
- stackgrad/qwen3.py +104 -0
- stackgrad/sampling.py +39 -0
- stackgrad/serve/api.py +434 -0
- stackgrad/serve/engine.py +53 -0
- stackgrad/serve/kv_cache.py +35 -0
- stackgrad/serve/runner.py +63 -0
- stackgrad/serve/scheduler.py +88 -0
- stackgrad/speculative.py +68 -0
- stackgrad/state.py +27 -0
- stackgrad/tensor.py +377 -0
- stackgrad/tokenizer.py +202 -0
- stackgrad-0.2.0.dist-info/METADATA +172 -0
- stackgrad-0.2.0.dist-info/RECORD +38 -0
- stackgrad-0.2.0.dist-info/WHEEL +4 -0
- stackgrad-0.2.0.dist-info/licenses/LICENSE +21 -0
stackgrad/module.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
from . import backend
|
|
7
|
+
from .tensor import Tensor
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Module:
|
|
11
|
+
"""Base class for layers and models; Tensor and Module attributes are found by parameters() and state_dict()."""
|
|
12
|
+
def __init__(self):
|
|
13
|
+
self._training = True
|
|
14
|
+
|
|
15
|
+
def forward(self, *args, **kwargs):
|
|
16
|
+
raise NotImplementedError
|
|
17
|
+
|
|
18
|
+
def __call__(self, *args, **kwargs):
|
|
19
|
+
return self.forward(*args, **kwargs)
|
|
20
|
+
|
|
21
|
+
def _attrs(self):
|
|
22
|
+
for name, val in self.__dict__.items():
|
|
23
|
+
if isinstance(val, (list, tuple)):
|
|
24
|
+
yield from ((f"{name}.{i}", m) for i, m in enumerate(val) if isinstance(m, Module))
|
|
25
|
+
else:
|
|
26
|
+
yield name, val
|
|
27
|
+
|
|
28
|
+
def parameters(self):
|
|
29
|
+
"""Collect every trainable `Tensor` in this module and its children."""
|
|
30
|
+
return list({id(p): p for _, p in self.named_parameters()}.values()) # tied weights must not step twice
|
|
31
|
+
|
|
32
|
+
def named_parameters(self, prefix=""):
|
|
33
|
+
"""(dotted name, tensor) for every trainable tensor, named as in state_dict()."""
|
|
34
|
+
for name, val in self._attrs():
|
|
35
|
+
if isinstance(val, Tensor) and val.requires_grad:
|
|
36
|
+
yield prefix + name, val
|
|
37
|
+
elif isinstance(val, Module):
|
|
38
|
+
yield from val.named_parameters(f"{prefix}{name}.")
|
|
39
|
+
|
|
40
|
+
def train(self, mode=True):
|
|
41
|
+
self._training = mode
|
|
42
|
+
for _, val in self._attrs():
|
|
43
|
+
if isinstance(val, Module):
|
|
44
|
+
val.train(mode)
|
|
45
|
+
return self
|
|
46
|
+
|
|
47
|
+
def eval(self):
|
|
48
|
+
return self.train(False)
|
|
49
|
+
|
|
50
|
+
def to(self, device=None, dtype=None):
|
|
51
|
+
"""Move every tensor to `device` and/or cast it to `dtype` in place, so optimizers keep their references."""
|
|
52
|
+
moved = []
|
|
53
|
+
for val in self.__dict__.values():
|
|
54
|
+
items = val if isinstance(val, (list, tuple)) else [val]
|
|
55
|
+
for item in items:
|
|
56
|
+
if isinstance(item, Tensor):
|
|
57
|
+
copy = item.to(device, dtype)
|
|
58
|
+
item.data, item.device, item.grad = copy.data, copy.device, None
|
|
59
|
+
moved.append(item)
|
|
60
|
+
elif isinstance(item, Module):
|
|
61
|
+
item.to(device, dtype)
|
|
62
|
+
# realize now: a lazy MLX cast is tied to this thread's stream, and the serving engine runs on another
|
|
63
|
+
for t in moved:
|
|
64
|
+
backend.get(t.device).realize(t.data)
|
|
65
|
+
return self
|
|
66
|
+
|
|
67
|
+
def save(self, path):
|
|
68
|
+
"""Write all parameters to a .npz file through a temp file and rename, so a crash keeps the old one."""
|
|
69
|
+
path = Path(path)
|
|
70
|
+
tmp = path.with_name(path.name + ".tmp.npz") # savez appends .npz to a name without it
|
|
71
|
+
np.savez(tmp, **self.state_dict())
|
|
72
|
+
os.replace(tmp, path)
|
|
73
|
+
|
|
74
|
+
def load(self, path):
|
|
75
|
+
"""Load parameters from a .npz file written by save(); fails if any name or shape differs."""
|
|
76
|
+
with np.load(path) as data:
|
|
77
|
+
state = {k: data[k] for k in data.files}
|
|
78
|
+
expected = {k: v.shape for k, v in self.state_dict().items()}
|
|
79
|
+
got = {k: v.shape for k, v in state.items()}
|
|
80
|
+
assert got == expected, f"{path} does not match this model: {sorted(set(got) ^ set(expected))[:4] or 'shapes differ'}"
|
|
81
|
+
self.load_state_dict(state)
|
|
82
|
+
return self
|
|
83
|
+
|
|
84
|
+
def state_dict(self):
|
|
85
|
+
state = {}
|
|
86
|
+
for name, val in self._attrs():
|
|
87
|
+
if isinstance(val, Tensor) and val.requires_grad:
|
|
88
|
+
state[name] = val.numpy()
|
|
89
|
+
elif isinstance(val, np.ndarray):
|
|
90
|
+
# buffers (BatchNorm running stats); without them a reloaded model normalizes with (0, 1)
|
|
91
|
+
state[name] = val.copy()
|
|
92
|
+
elif isinstance(val, Module):
|
|
93
|
+
state.update({f"{name}.{k}": v for k, v in val.state_dict().items()})
|
|
94
|
+
return state
|
|
95
|
+
|
|
96
|
+
def load_state_dict(self, state):
|
|
97
|
+
for name, val in self._attrs():
|
|
98
|
+
if isinstance(val, Tensor) and val.requires_grad:
|
|
99
|
+
if name in state:
|
|
100
|
+
assert tuple(state[name].shape) == tuple(val.shape), (name, state[name].shape, val.shape)
|
|
101
|
+
val.data = backend.get(val.device).asarray(state[name]).astype(val.dtype)
|
|
102
|
+
elif isinstance(val, np.ndarray):
|
|
103
|
+
if name in state:
|
|
104
|
+
setattr(self, name, state[name].copy())
|
|
105
|
+
elif isinstance(val, Module):
|
|
106
|
+
child = {k[len(name) + 1:]: v for k, v in state.items() if k.startswith(name + ".")}
|
|
107
|
+
if child:
|
|
108
|
+
val.load_state_dict(child)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class Sequential(Module):
|
|
112
|
+
"""Chain modules so the output of each feeds the next."""
|
|
113
|
+
def __init__(self, *layers):
|
|
114
|
+
super().__init__()
|
|
115
|
+
self.layers = list(layers)
|
|
116
|
+
|
|
117
|
+
def forward(self, x):
|
|
118
|
+
for layer in self.layers:
|
|
119
|
+
x = layer(x)
|
|
120
|
+
return x
|
stackgrad/optim.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import math
|
|
2
|
+
|
|
3
|
+
from . import backend
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Optimizer:
|
|
7
|
+
def __init__(self, params, lr=1e-3, weight_decay=0.0):
|
|
8
|
+
self.params = list(params)
|
|
9
|
+
self.lr = lr
|
|
10
|
+
self.weight_decay = weight_decay
|
|
11
|
+
|
|
12
|
+
def zero_grad(self):
|
|
13
|
+
for p in self.params:
|
|
14
|
+
p.grad = None
|
|
15
|
+
|
|
16
|
+
def step(self):
|
|
17
|
+
raise NotImplementedError
|
|
18
|
+
|
|
19
|
+
def _realize(self, *state):
|
|
20
|
+
# lazy backends (MLX) realize here, so the next step's graph starts from arrays
|
|
21
|
+
backend.get(self.params[0].device).realize(*[p.data for p in self.params], *state)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class SGD(Optimizer):
|
|
25
|
+
"""Stochastic gradient descent with optional momentum and weight decay."""
|
|
26
|
+
def __init__(self, params, lr=1e-3, momentum=0.0, weight_decay=0.0):
|
|
27
|
+
super().__init__(params, lr, weight_decay)
|
|
28
|
+
self.momentum = momentum
|
|
29
|
+
self.velocities = [0.0] * len(self.params) # a scalar 0 broadcasts on any backend
|
|
30
|
+
|
|
31
|
+
def step(self):
|
|
32
|
+
for i, p in enumerate(self.params):
|
|
33
|
+
if not p.requires_grad or p.grad is None:
|
|
34
|
+
continue
|
|
35
|
+
grad = p.grad
|
|
36
|
+
if self.weight_decay:
|
|
37
|
+
grad = grad + self.weight_decay * p.data
|
|
38
|
+
self.velocities[i] = self.momentum * self.velocities[i] + grad
|
|
39
|
+
p.data = p.data - self.lr * self.velocities[i]
|
|
40
|
+
self._realize(*self.velocities)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Adam(Optimizer):
|
|
44
|
+
"""Adam optimizer with bias-corrected first and second moment estimates."""
|
|
45
|
+
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.0):
|
|
46
|
+
super().__init__(params, lr, weight_decay)
|
|
47
|
+
self.beta1, self.beta2 = betas
|
|
48
|
+
self.eps = eps
|
|
49
|
+
self.t = 0
|
|
50
|
+
self.m = [0.0] * len(self.params) # a scalar 0 broadcasts on any backend
|
|
51
|
+
self.v = [0.0] * len(self.params)
|
|
52
|
+
|
|
53
|
+
def step(self):
|
|
54
|
+
self.t += 1
|
|
55
|
+
for i, p in enumerate(self.params):
|
|
56
|
+
if not p.requires_grad or p.grad is None:
|
|
57
|
+
continue
|
|
58
|
+
grad = p.grad
|
|
59
|
+
if self.weight_decay:
|
|
60
|
+
grad = grad + self.weight_decay * p.data
|
|
61
|
+
self.m[i] = self.beta1 * self.m[i] + (1 - self.beta1) * grad
|
|
62
|
+
self.v[i] = self.beta2 * self.v[i] + (1 - self.beta2) * (grad * grad)
|
|
63
|
+
m_hat = self.m[i] / (1 - self.beta1 ** self.t)
|
|
64
|
+
v_hat = self.v[i] / (1 - self.beta2 ** self.t)
|
|
65
|
+
p.data = p.data - self.lr * m_hat / (v_hat ** 0.5 + self.eps)
|
|
66
|
+
self._realize(*self.m, *self.v)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class AdamW(Adam):
|
|
70
|
+
"""Adam with decoupled weight decay: shrink the weights instead of adding wd * p to the gradient."""
|
|
71
|
+
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.01):
|
|
72
|
+
super().__init__(params, lr, betas, eps, weight_decay=0.0)
|
|
73
|
+
self.decay = weight_decay
|
|
74
|
+
|
|
75
|
+
def step(self):
|
|
76
|
+
for p in self.params:
|
|
77
|
+
if p.requires_grad and p.grad is not None:
|
|
78
|
+
p.data = p.data * (1 - self.lr * self.decay)
|
|
79
|
+
super().step()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def cosine_lr(step, max_lr, min_lr, warmup_steps, total_steps):
|
|
83
|
+
"""Linear warmup to max_lr, cosine decay to min_lr at total_steps, then flat."""
|
|
84
|
+
if step < warmup_steps:
|
|
85
|
+
return max_lr * step / warmup_steps
|
|
86
|
+
if step > total_steps:
|
|
87
|
+
return min_lr
|
|
88
|
+
progress = (step - warmup_steps) / max(total_steps - warmup_steps, 1) # warmup == total: the last step is the peak
|
|
89
|
+
return min_lr + 0.5 * (1 + math.cos(math.pi * progress)) * (max_lr - min_lr)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def clip_grad_norm(params, max_norm):
|
|
93
|
+
"""Clip gradient norm across all parameters (like torch.nn.utils.clip_grad_norm_)."""
|
|
94
|
+
# one device sum and one host sync, not one per parameter
|
|
95
|
+
total_norm = float(sum((p.grad * p.grad).sum() for p in params if p.grad is not None)) ** 0.5
|
|
96
|
+
if total_norm > max_norm:
|
|
97
|
+
scale = max_norm / (total_norm + 1e-12)
|
|
98
|
+
for p in params:
|
|
99
|
+
if p.grad is not None:
|
|
100
|
+
p.grad = p.grad * scale
|
|
101
|
+
return total_norm
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
class LossScaler:
|
|
105
|
+
"""Dynamic loss scaling for autocast("float16"): halve on overflow, double after `interval` clean steps."""
|
|
106
|
+
def __init__(self, scale=2.0 ** 15, interval=1000):
|
|
107
|
+
self.scale, self.interval, self.clean = scale, interval, 0
|
|
108
|
+
|
|
109
|
+
def unscale(self, params):
|
|
110
|
+
"""Divide every gradient by the scale. Returns False, and halves the scale, if any of them overflowed."""
|
|
111
|
+
total = float(sum((p.grad * p.grad).sum() for p in params if p.grad is not None))
|
|
112
|
+
if not math.isfinite(total):
|
|
113
|
+
self.scale, self.clean = self.scale / 2, 0
|
|
114
|
+
return False
|
|
115
|
+
for p in params:
|
|
116
|
+
if p.grad is not None:
|
|
117
|
+
p.grad = p.grad / self.scale
|
|
118
|
+
self.clean += 1
|
|
119
|
+
if self.clean % self.interval == 0:
|
|
120
|
+
self.scale *= 2
|
|
121
|
+
return True
|
stackgrad/profile.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from collections import defaultdict
|
|
3
|
+
|
|
4
|
+
from . import backend
|
|
5
|
+
from . import tensor as T
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def _wait(device, out):
|
|
9
|
+
if device == "cuda":
|
|
10
|
+
import cupy
|
|
11
|
+
cupy.cuda.Device().synchronize()
|
|
12
|
+
else:
|
|
13
|
+
backend.get(device).realize(*(out if isinstance(out, tuple) else (out,)))
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class profile:
|
|
17
|
+
"""Times each primitive's forward and backward, syncing the device after each, so the total runs slower."""
|
|
18
|
+
def __init__(self):
|
|
19
|
+
self.ms = defaultdict(float)
|
|
20
|
+
self.calls = defaultdict(int)
|
|
21
|
+
|
|
22
|
+
def __enter__(self):
|
|
23
|
+
T._profiler = self
|
|
24
|
+
self.start = time.perf_counter()
|
|
25
|
+
return self
|
|
26
|
+
|
|
27
|
+
def __exit__(self, *exc):
|
|
28
|
+
self.wall_ms = (time.perf_counter() - self.start) * 1000
|
|
29
|
+
T._profiler = None
|
|
30
|
+
|
|
31
|
+
def time(self, name, device, fn, *args, **kwargs):
|
|
32
|
+
start = time.perf_counter()
|
|
33
|
+
out = fn(*args, **kwargs)
|
|
34
|
+
_wait(device, tuple(o for o in out if o is not None) if isinstance(out, tuple) else out)
|
|
35
|
+
self.ms[name] += (time.perf_counter() - start) * 1000
|
|
36
|
+
self.calls[name] += 1
|
|
37
|
+
return out
|
|
38
|
+
|
|
39
|
+
def rows(self):
|
|
40
|
+
"""(op, calls, ms) sorted by time, then the time between ops."""
|
|
41
|
+
rows = sorted(((name, self.calls[name], self.ms[name]) for name in self.ms), key=lambda r: -r[2])
|
|
42
|
+
return rows + [("between ops", 0, self.wall_ms - sum(self.ms.values()))]
|
|
43
|
+
|
|
44
|
+
def table(self):
|
|
45
|
+
return "\n".join(f"{name:28s} {calls:6d} calls {ms:9.2f} ms {ms / self.wall_ms:6.1%}" for name, calls, ms in self.rows())
|
stackgrad/quant.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from . import backend
|
|
4
|
+
from .layers import Linear
|
|
5
|
+
from .module import Module
|
|
6
|
+
from .tensor import Tensor
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def quantize(w, bits=4, group=64):
|
|
10
|
+
"""w (out, in) -> packed uint32 (out, in * bits / 32), scales, biases (out, in / group), in MLX's layout."""
|
|
11
|
+
n_out, n_in = w.shape
|
|
12
|
+
assert 32 % bits == 0 and n_in % group == 0 and group % (32 // bits) == 0, (w.shape, bits, group)
|
|
13
|
+
g = w.astype(np.float32).reshape(n_out, n_in // group, group)
|
|
14
|
+
lo, hi = g.min(axis=-1, keepdims=True), g.max(axis=-1, keepdims=True)
|
|
15
|
+
scale = np.where(hi > lo, (hi - lo) / (2**bits - 1), 1.0) # a constant group: any scale works, bias carries it
|
|
16
|
+
q = np.clip(np.round((g - lo) / scale), 0, 2**bits - 1).astype(np.uint32).reshape(n_out, -1, 32 // bits)
|
|
17
|
+
shifts = np.arange(32 // bits, dtype=np.uint32) * bits
|
|
18
|
+
packed = np.bitwise_or.reduce(q << shifts, axis=-1)
|
|
19
|
+
return packed, scale[..., 0], lo[..., 0]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def dequantize(packed, scales, biases, bits, group, xp):
|
|
23
|
+
"""The inverse of quantize, in the backend `xp` and in the scales' dtype."""
|
|
24
|
+
shifts = xp.asarray(np.arange(32 // bits, dtype=np.uint32) * bits)
|
|
25
|
+
q = (packed[..., None] >> shifts) & (2**bits - 1)
|
|
26
|
+
q = q.reshape(packed.shape[0], -1, group).astype(scales.dtype)
|
|
27
|
+
return (q * scales[..., None] + biases[..., None]).reshape(packed.shape[0], -1)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class QuantLinear(Module):
|
|
31
|
+
"""Inference-only, bias-free Linear with group-quantized weights; build it after moving the model."""
|
|
32
|
+
def __init__(self, weight, bits, group):
|
|
33
|
+
super().__init__()
|
|
34
|
+
xp = backend.get(weight.device)
|
|
35
|
+
packed, scales, biases = quantize(weight.numpy().T, bits, group) # stackgrad keeps (in, out)
|
|
36
|
+
self.device, self.bits, self.group = weight.device, bits, group
|
|
37
|
+
self.packed = xp.asarray(packed)
|
|
38
|
+
self.scales, self.biases = xp.asarray(scales, dtype=weight.dtype), xp.asarray(biases, dtype=weight.dtype)
|
|
39
|
+
|
|
40
|
+
def forward(self, x):
|
|
41
|
+
assert not x.requires_grad, "QuantLinear is inference only: run it under no_grad()"
|
|
42
|
+
assert x.device == self.device, f"input on {x.device}, weights on {self.device}"
|
|
43
|
+
xp = backend.get(self.device)
|
|
44
|
+
flat = x.data.reshape(-1, x.shape[-1])
|
|
45
|
+
fused = getattr(xp, "quant_matmul", None)
|
|
46
|
+
if fused is not None and flat.shape[0] <= 8: # decode: read the packed weights directly
|
|
47
|
+
y = fused(flat, self.packed, self.scales, self.biases, self.bits, self.group)
|
|
48
|
+
else: # prefill: dequantize once, then one large matmul
|
|
49
|
+
y = flat @ dequantize(self.packed, self.scales, self.biases, self.bits, self.group, xp).T
|
|
50
|
+
return Tensor(y.reshape(*x.shape[:-1], y.shape[-1]), device=self.device)
|
|
51
|
+
|
|
52
|
+
def nbytes(self):
|
|
53
|
+
return sum(a.nbytes for a in (self.packed, self.scales, self.biases))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def quantize_model(model, bits=4, group=64):
|
|
57
|
+
"""Replaces every TransformerLM matmul, head included, with a QuantLinear; a tied head gets its own copy."""
|
|
58
|
+
for block in model.blocks:
|
|
59
|
+
for parent in (block.attn, block.ffn):
|
|
60
|
+
for name, val in list(parent.__dict__.items()):
|
|
61
|
+
if isinstance(val, Linear):
|
|
62
|
+
assert val.bias is None, f"{name} has a bias"
|
|
63
|
+
setattr(parent, name, QuantLinear(val.weight, bits, group))
|
|
64
|
+
head = model.head.weight if model.head is not None else model.embed.weight.T
|
|
65
|
+
model.head = QuantLinear(head, bits, group)
|
|
66
|
+
return model
|
stackgrad/qwen3.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import regex as re
|
|
6
|
+
|
|
7
|
+
from .gpt import TransformerLM
|
|
8
|
+
from .state import iter_safetensors
|
|
9
|
+
from .tokenizer import Tokenizer
|
|
10
|
+
|
|
11
|
+
# Qwen's pre-tokenizer (GPT-4 style): case-insensitive contractions, one digit per token
|
|
12
|
+
PAT = re.compile(r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def config(path):
|
|
16
|
+
"""TransformerLM arguments from a Hugging Face Qwen3 config.json."""
|
|
17
|
+
c = json.loads((Path(path) / "config.json").read_text())
|
|
18
|
+
return {"vocab_size": c["vocab_size"], "context_length": c["max_position_embeddings"], "d_model": c["hidden_size"],
|
|
19
|
+
"n_layers": c["num_hidden_layers"], "n_heads": c["num_attention_heads"], "d_ff": c["intermediate_size"],
|
|
20
|
+
"n_kv_heads": c["num_key_value_heads"], "rope_theta": c["rope_theta"], "head_dim": c["head_dim"],
|
|
21
|
+
"qk_norm": True, "norm_eps": c["rms_norm_eps"], "tie_embeddings": c["tie_word_embeddings"]}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def hf_names(n_layers):
|
|
25
|
+
"""stackgrad name -> (HF tensor name, transpose?); HF Linear weights are (out, in), stackgrad's (in, out)."""
|
|
26
|
+
names = {"embed.weight": ("model.embed_tokens.weight", False), "norm.weight": ("model.norm.weight", False)}
|
|
27
|
+
for i in range(n_layers):
|
|
28
|
+
mine, hf = f"blocks.{i}.", f"model.layers.{i}."
|
|
29
|
+
names[mine + "attn_norm.weight"] = (hf + "input_layernorm.weight", False)
|
|
30
|
+
names[mine + "ffn_norm.weight"] = (hf + "post_attention_layernorm.weight", False)
|
|
31
|
+
for proj in ["q_proj", "k_proj", "v_proj", "o_proj"]:
|
|
32
|
+
names[f"{mine}attn.{proj}.weight"] = (f"{hf}self_attn.{proj}.weight", True)
|
|
33
|
+
names[mine + "attn.q_norm.weight"] = (hf + "self_attn.q_norm.weight", False)
|
|
34
|
+
names[mine + "attn.k_norm.weight"] = (hf + "self_attn.k_norm.weight", False)
|
|
35
|
+
for mine_ffn, hf_ffn in [("w1", "gate_proj"), ("w2", "down_proj"), ("w3", "up_proj")]:
|
|
36
|
+
names[f"{mine}ffn.{mine_ffn}.weight"] = (f"{hf}mlp.{hf_ffn}.weight", True)
|
|
37
|
+
return names
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def load_qwen3(path, context_length=4096):
|
|
41
|
+
"""TransformerLM from a Hugging Face Qwen3 directory (config.json and *.safetensors)."""
|
|
42
|
+
cfg = config(path) | {"context_length": context_length}
|
|
43
|
+
# shortcut: the checkpoint overwrites the random init (14.7 of a 19.6 s load); skip it if loads start to hurt
|
|
44
|
+
model = TransformerLM(**cfg)
|
|
45
|
+
params = dict(model.named_parameters())
|
|
46
|
+
names = hf_names(cfg["n_layers"])
|
|
47
|
+
# a skipped tensor would keep its random init
|
|
48
|
+
assert params.keys() == names.keys(), (sorted(params.keys() - names.keys())[:3], sorted(names.keys() - params.keys())[:3])
|
|
49
|
+
unplaced = {hf: (name, transpose) for name, (hf, transpose) in names.items()}
|
|
50
|
+
lm_head = None
|
|
51
|
+
# one tensor at a time: Qwen3-1.7B's float32 weights beside the init would take 16 GB
|
|
52
|
+
for shard in sorted(Path(path).glob("*.safetensors")):
|
|
53
|
+
for hf, w in iter_safetensors(shard):
|
|
54
|
+
hf = hf if hf.startswith(("model.", "lm_head.")) else "model." + hf # Qwen3-Embedding saves its names without the prefix
|
|
55
|
+
if hf == "lm_head.weight" and cfg["tie_embeddings"]:
|
|
56
|
+
lm_head = w
|
|
57
|
+
continue
|
|
58
|
+
name, transpose = unplaced.pop(hf) # a KeyError: the checkpoint has a tensor the mapping doesn't know
|
|
59
|
+
w = w.T if transpose else w
|
|
60
|
+
assert w.shape == params[name].shape, (name, w.shape, params[name].shape)
|
|
61
|
+
params[name].data = np.ascontiguousarray(w, dtype=np.float32)
|
|
62
|
+
assert not unplaced, f"the checkpoint lacks {sorted(unplaced)[:3]}"
|
|
63
|
+
if lm_head is not None:
|
|
64
|
+
assert np.array_equal(lm_head, params["embed.weight"].data), "lm_head differs from the tied embedding"
|
|
65
|
+
return model
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def load_tokenizer(path):
|
|
69
|
+
"""Qwen's vocab.json and merges.txt in stackgrad's BPE, with its special tokens at their own ids."""
|
|
70
|
+
added = json.loads((Path(path) / "tokenizer.json").read_text())["added_tokens"]
|
|
71
|
+
tok = Tokenizer.from_files(Path(path) / "vocab.json", Path(path) / "merges.txt", pattern=PAT, nfc=True)
|
|
72
|
+
tok = Tokenizer({**tok.vocab, **{a["id"]: a["content"].encode() for a in added}},
|
|
73
|
+
[p for p, _ in sorted(tok.ranks.items(), key=lambda kv: kv[1])],
|
|
74
|
+
[a["content"] for a in added], pattern=PAT, nfc=True)
|
|
75
|
+
assert all(tok.ids[a["content"].encode()] == a["id"] for a in added)
|
|
76
|
+
return tok
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
_TOOLS_HEADER = ("# Tools\n\nYou may call one or more functions to assist with the user query.\n\n"
|
|
80
|
+
"You are provided with function signatures within <tools></tools> XML tags:\n<tools>")
|
|
81
|
+
_TOOLS_FOOTER = ("\n</tools>\n\nFor each function call, return a json object with function name and arguments within "
|
|
82
|
+
"<tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n"
|
|
83
|
+
"</tool_call><|im_end|>\n")
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def chat_prompt(messages, think=False, tools=None):
|
|
87
|
+
"""Qwen3's chat template: messages -> prompt text ending where the reply starts; think=False skips reasoning."""
|
|
88
|
+
# shortcut: earlier assistant turns keep their <think> text (HF strips it); strip it if agent chats run long
|
|
89
|
+
text = ""
|
|
90
|
+
if tools:
|
|
91
|
+
system = messages[0]["content"] + "\n\n" if messages[0]["role"] == "system" else ""
|
|
92
|
+
text = "<|im_start|>system\n" + system + _TOOLS_HEADER + "".join("\n" + json.dumps(t, ensure_ascii=False) for t in tools) + _TOOLS_FOOTER
|
|
93
|
+
messages = messages[1:] if system else messages
|
|
94
|
+
for i, m in enumerate(messages):
|
|
95
|
+
if m["role"] != "tool":
|
|
96
|
+
text += f"<|im_start|>{m['role']}\n{m['content']}<|im_end|>\n"
|
|
97
|
+
continue
|
|
98
|
+
# consecutive tool results share one user turn
|
|
99
|
+
if i == 0 or messages[i - 1]["role"] != "tool":
|
|
100
|
+
text += "<|im_start|>user"
|
|
101
|
+
text += f"\n<tool_response>\n{m['content']}\n</tool_response>"
|
|
102
|
+
if i == len(messages) - 1 or messages[i + 1]["role"] != "tool":
|
|
103
|
+
text += "<|im_end|>\n"
|
|
104
|
+
return text + "<|im_start|>assistant\n" + ("" if think else "<think>\n\n</think>\n\n")
|
stackgrad/sampling.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
|
|
3
|
+
from . import backend
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def sample(logits, temperature=1.0, top_k=None, top_p=None):
|
|
7
|
+
"""One token id per row of logits (B, V); temperature=0 is greedy, top_k and top_p filter first."""
|
|
8
|
+
if temperature == 0:
|
|
9
|
+
return logits.argmax(axis=-1)
|
|
10
|
+
logits = logits / temperature
|
|
11
|
+
if top_k is not None:
|
|
12
|
+
kth = np.sort(logits, axis=-1)[:, -top_k][:, None]
|
|
13
|
+
logits = np.where(logits < kth, -np.inf, logits)
|
|
14
|
+
probs = np.exp(logits - logits.max(axis=-1, keepdims=True))
|
|
15
|
+
probs /= probs.sum(axis=-1, keepdims=True)
|
|
16
|
+
if top_p is not None:
|
|
17
|
+
kept = np.zeros_like(probs)
|
|
18
|
+
for row, out in zip(probs, kept):
|
|
19
|
+
# tokens under (1 - top_p) / V sum to under 1 - top_p, so the nucleus lies above: sort only those
|
|
20
|
+
cand = np.flatnonzero(row >= (1 - top_p) / len(row))
|
|
21
|
+
cand = cand[np.argsort(-row[cand])]
|
|
22
|
+
# drop a token once the likelier tokens before it already reach top_p
|
|
23
|
+
keep = cand[np.cumsum(row[cand]) - row[cand] < top_p]
|
|
24
|
+
out[keep] = row[keep]
|
|
25
|
+
probs = kept / kept.sum(axis=-1, keepdims=True)
|
|
26
|
+
return np.array([np.random.choice(len(p), p=p) for p in probs])
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def sample_on_device(logits, temperature=1.0):
|
|
30
|
+
"""sample() without top-k or top-p, run where the Tensor of logits lives, so only B ids reach the host."""
|
|
31
|
+
xp = backend.get(logits.device)
|
|
32
|
+
z = logits.data.astype(xp.float32)
|
|
33
|
+
if temperature == 0:
|
|
34
|
+
return xp.to_numpy(z.argmax(axis=-1))
|
|
35
|
+
z = z / temperature
|
|
36
|
+
cdf = xp.cumsum(xp.exp(z - z.max(axis=-1, keepdims=True)), axis=-1)
|
|
37
|
+
u = xp.asarray(np.random.rand(len(z), 1).astype(np.float32)) * cdf[:, -1:]
|
|
38
|
+
# first crossing, not a count of cdf < u: a parallel float32 cumsum dips. u < cdf[-1], so one exists
|
|
39
|
+
return xp.to_numpy((cdf >= u).argmax(axis=-1))
|