stackgrad 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
stackgrad/module.py ADDED
@@ -0,0 +1,120 @@
1
+ import os
2
+ from pathlib import Path
3
+
4
+ import numpy as np
5
+
6
+ from . import backend
7
+ from .tensor import Tensor
8
+
9
+
10
+ class Module:
11
+ """Base class for layers and models; Tensor and Module attributes are found by parameters() and state_dict()."""
12
+ def __init__(self):
13
+ self._training = True
14
+
15
+ def forward(self, *args, **kwargs):
16
+ raise NotImplementedError
17
+
18
+ def __call__(self, *args, **kwargs):
19
+ return self.forward(*args, **kwargs)
20
+
21
+ def _attrs(self):
22
+ for name, val in self.__dict__.items():
23
+ if isinstance(val, (list, tuple)):
24
+ yield from ((f"{name}.{i}", m) for i, m in enumerate(val) if isinstance(m, Module))
25
+ else:
26
+ yield name, val
27
+
28
+ def parameters(self):
29
+ """Collect every trainable `Tensor` in this module and its children."""
30
+ return list({id(p): p for _, p in self.named_parameters()}.values()) # tied weights must not step twice
31
+
32
+ def named_parameters(self, prefix=""):
33
+ """(dotted name, tensor) for every trainable tensor, named as in state_dict()."""
34
+ for name, val in self._attrs():
35
+ if isinstance(val, Tensor) and val.requires_grad:
36
+ yield prefix + name, val
37
+ elif isinstance(val, Module):
38
+ yield from val.named_parameters(f"{prefix}{name}.")
39
+
40
+ def train(self, mode=True):
41
+ self._training = mode
42
+ for _, val in self._attrs():
43
+ if isinstance(val, Module):
44
+ val.train(mode)
45
+ return self
46
+
47
+ def eval(self):
48
+ return self.train(False)
49
+
50
+ def to(self, device=None, dtype=None):
51
+ """Move every tensor to `device` and/or cast it to `dtype` in place, so optimizers keep their references."""
52
+ moved = []
53
+ for val in self.__dict__.values():
54
+ items = val if isinstance(val, (list, tuple)) else [val]
55
+ for item in items:
56
+ if isinstance(item, Tensor):
57
+ copy = item.to(device, dtype)
58
+ item.data, item.device, item.grad = copy.data, copy.device, None
59
+ moved.append(item)
60
+ elif isinstance(item, Module):
61
+ item.to(device, dtype)
62
+ # realize now: a lazy MLX cast is tied to this thread's stream, and the serving engine runs on another
63
+ for t in moved:
64
+ backend.get(t.device).realize(t.data)
65
+ return self
66
+
67
+ def save(self, path):
68
+ """Write all parameters to a .npz file through a temp file and rename, so a crash keeps the old one."""
69
+ path = Path(path)
70
+ tmp = path.with_name(path.name + ".tmp.npz") # savez appends .npz to a name without it
71
+ np.savez(tmp, **self.state_dict())
72
+ os.replace(tmp, path)
73
+
74
+ def load(self, path):
75
+ """Load parameters from a .npz file written by save(); fails if any name or shape differs."""
76
+ with np.load(path) as data:
77
+ state = {k: data[k] for k in data.files}
78
+ expected = {k: v.shape for k, v in self.state_dict().items()}
79
+ got = {k: v.shape for k, v in state.items()}
80
+ assert got == expected, f"{path} does not match this model: {sorted(set(got) ^ set(expected))[:4] or 'shapes differ'}"
81
+ self.load_state_dict(state)
82
+ return self
83
+
84
+ def state_dict(self):
85
+ state = {}
86
+ for name, val in self._attrs():
87
+ if isinstance(val, Tensor) and val.requires_grad:
88
+ state[name] = val.numpy()
89
+ elif isinstance(val, np.ndarray):
90
+ # buffers (BatchNorm running stats); without them a reloaded model normalizes with (0, 1)
91
+ state[name] = val.copy()
92
+ elif isinstance(val, Module):
93
+ state.update({f"{name}.{k}": v for k, v in val.state_dict().items()})
94
+ return state
95
+
96
+ def load_state_dict(self, state):
97
+ for name, val in self._attrs():
98
+ if isinstance(val, Tensor) and val.requires_grad:
99
+ if name in state:
100
+ assert tuple(state[name].shape) == tuple(val.shape), (name, state[name].shape, val.shape)
101
+ val.data = backend.get(val.device).asarray(state[name]).astype(val.dtype)
102
+ elif isinstance(val, np.ndarray):
103
+ if name in state:
104
+ setattr(self, name, state[name].copy())
105
+ elif isinstance(val, Module):
106
+ child = {k[len(name) + 1:]: v for k, v in state.items() if k.startswith(name + ".")}
107
+ if child:
108
+ val.load_state_dict(child)
109
+
110
+
111
+ class Sequential(Module):
112
+ """Chain modules so the output of each feeds the next."""
113
+ def __init__(self, *layers):
114
+ super().__init__()
115
+ self.layers = list(layers)
116
+
117
+ def forward(self, x):
118
+ for layer in self.layers:
119
+ x = layer(x)
120
+ return x
stackgrad/optim.py ADDED
@@ -0,0 +1,121 @@
1
+ import math
2
+
3
+ from . import backend
4
+
5
+
6
+ class Optimizer:
7
+ def __init__(self, params, lr=1e-3, weight_decay=0.0):
8
+ self.params = list(params)
9
+ self.lr = lr
10
+ self.weight_decay = weight_decay
11
+
12
+ def zero_grad(self):
13
+ for p in self.params:
14
+ p.grad = None
15
+
16
+ def step(self):
17
+ raise NotImplementedError
18
+
19
+ def _realize(self, *state):
20
+ # lazy backends (MLX) realize here, so the next step's graph starts from arrays
21
+ backend.get(self.params[0].device).realize(*[p.data for p in self.params], *state)
22
+
23
+
24
+ class SGD(Optimizer):
25
+ """Stochastic gradient descent with optional momentum and weight decay."""
26
+ def __init__(self, params, lr=1e-3, momentum=0.0, weight_decay=0.0):
27
+ super().__init__(params, lr, weight_decay)
28
+ self.momentum = momentum
29
+ self.velocities = [0.0] * len(self.params) # a scalar 0 broadcasts on any backend
30
+
31
+ def step(self):
32
+ for i, p in enumerate(self.params):
33
+ if not p.requires_grad or p.grad is None:
34
+ continue
35
+ grad = p.grad
36
+ if self.weight_decay:
37
+ grad = grad + self.weight_decay * p.data
38
+ self.velocities[i] = self.momentum * self.velocities[i] + grad
39
+ p.data = p.data - self.lr * self.velocities[i]
40
+ self._realize(*self.velocities)
41
+
42
+
43
+ class Adam(Optimizer):
44
+ """Adam optimizer with bias-corrected first and second moment estimates."""
45
+ def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.0):
46
+ super().__init__(params, lr, weight_decay)
47
+ self.beta1, self.beta2 = betas
48
+ self.eps = eps
49
+ self.t = 0
50
+ self.m = [0.0] * len(self.params) # a scalar 0 broadcasts on any backend
51
+ self.v = [0.0] * len(self.params)
52
+
53
+ def step(self):
54
+ self.t += 1
55
+ for i, p in enumerate(self.params):
56
+ if not p.requires_grad or p.grad is None:
57
+ continue
58
+ grad = p.grad
59
+ if self.weight_decay:
60
+ grad = grad + self.weight_decay * p.data
61
+ self.m[i] = self.beta1 * self.m[i] + (1 - self.beta1) * grad
62
+ self.v[i] = self.beta2 * self.v[i] + (1 - self.beta2) * (grad * grad)
63
+ m_hat = self.m[i] / (1 - self.beta1 ** self.t)
64
+ v_hat = self.v[i] / (1 - self.beta2 ** self.t)
65
+ p.data = p.data - self.lr * m_hat / (v_hat ** 0.5 + self.eps)
66
+ self._realize(*self.m, *self.v)
67
+
68
+
69
+ class AdamW(Adam):
70
+ """Adam with decoupled weight decay: shrink the weights instead of adding wd * p to the gradient."""
71
+ def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8, weight_decay=0.01):
72
+ super().__init__(params, lr, betas, eps, weight_decay=0.0)
73
+ self.decay = weight_decay
74
+
75
+ def step(self):
76
+ for p in self.params:
77
+ if p.requires_grad and p.grad is not None:
78
+ p.data = p.data * (1 - self.lr * self.decay)
79
+ super().step()
80
+
81
+
82
+ def cosine_lr(step, max_lr, min_lr, warmup_steps, total_steps):
83
+ """Linear warmup to max_lr, cosine decay to min_lr at total_steps, then flat."""
84
+ if step < warmup_steps:
85
+ return max_lr * step / warmup_steps
86
+ if step > total_steps:
87
+ return min_lr
88
+ progress = (step - warmup_steps) / max(total_steps - warmup_steps, 1) # warmup == total: the last step is the peak
89
+ return min_lr + 0.5 * (1 + math.cos(math.pi * progress)) * (max_lr - min_lr)
90
+
91
+
92
+ def clip_grad_norm(params, max_norm):
93
+ """Clip gradient norm across all parameters (like torch.nn.utils.clip_grad_norm_)."""
94
+ # one device sum and one host sync, not one per parameter
95
+ total_norm = float(sum((p.grad * p.grad).sum() for p in params if p.grad is not None)) ** 0.5
96
+ if total_norm > max_norm:
97
+ scale = max_norm / (total_norm + 1e-12)
98
+ for p in params:
99
+ if p.grad is not None:
100
+ p.grad = p.grad * scale
101
+ return total_norm
102
+
103
+
104
+ class LossScaler:
105
+ """Dynamic loss scaling for autocast("float16"): halve on overflow, double after `interval` clean steps."""
106
+ def __init__(self, scale=2.0 ** 15, interval=1000):
107
+ self.scale, self.interval, self.clean = scale, interval, 0
108
+
109
+ def unscale(self, params):
110
+ """Divide every gradient by the scale. Returns False, and halves the scale, if any of them overflowed."""
111
+ total = float(sum((p.grad * p.grad).sum() for p in params if p.grad is not None))
112
+ if not math.isfinite(total):
113
+ self.scale, self.clean = self.scale / 2, 0
114
+ return False
115
+ for p in params:
116
+ if p.grad is not None:
117
+ p.grad = p.grad / self.scale
118
+ self.clean += 1
119
+ if self.clean % self.interval == 0:
120
+ self.scale *= 2
121
+ return True
stackgrad/profile.py ADDED
@@ -0,0 +1,45 @@
1
+ import time
2
+ from collections import defaultdict
3
+
4
+ from . import backend
5
+ from . import tensor as T
6
+
7
+
8
+ def _wait(device, out):
9
+ if device == "cuda":
10
+ import cupy
11
+ cupy.cuda.Device().synchronize()
12
+ else:
13
+ backend.get(device).realize(*(out if isinstance(out, tuple) else (out,)))
14
+
15
+
16
+ class profile:
17
+ """Times each primitive's forward and backward, syncing the device after each, so the total runs slower."""
18
+ def __init__(self):
19
+ self.ms = defaultdict(float)
20
+ self.calls = defaultdict(int)
21
+
22
+ def __enter__(self):
23
+ T._profiler = self
24
+ self.start = time.perf_counter()
25
+ return self
26
+
27
+ def __exit__(self, *exc):
28
+ self.wall_ms = (time.perf_counter() - self.start) * 1000
29
+ T._profiler = None
30
+
31
+ def time(self, name, device, fn, *args, **kwargs):
32
+ start = time.perf_counter()
33
+ out = fn(*args, **kwargs)
34
+ _wait(device, tuple(o for o in out if o is not None) if isinstance(out, tuple) else out)
35
+ self.ms[name] += (time.perf_counter() - start) * 1000
36
+ self.calls[name] += 1
37
+ return out
38
+
39
+ def rows(self):
40
+ """(op, calls, ms) sorted by time, then the time between ops."""
41
+ rows = sorted(((name, self.calls[name], self.ms[name]) for name in self.ms), key=lambda r: -r[2])
42
+ return rows + [("between ops", 0, self.wall_ms - sum(self.ms.values()))]
43
+
44
+ def table(self):
45
+ return "\n".join(f"{name:28s} {calls:6d} calls {ms:9.2f} ms {ms / self.wall_ms:6.1%}" for name, calls, ms in self.rows())
stackgrad/quant.py ADDED
@@ -0,0 +1,66 @@
1
+ import numpy as np
2
+
3
+ from . import backend
4
+ from .layers import Linear
5
+ from .module import Module
6
+ from .tensor import Tensor
7
+
8
+
9
+ def quantize(w, bits=4, group=64):
10
+ """w (out, in) -> packed uint32 (out, in * bits / 32), scales, biases (out, in / group), in MLX's layout."""
11
+ n_out, n_in = w.shape
12
+ assert 32 % bits == 0 and n_in % group == 0 and group % (32 // bits) == 0, (w.shape, bits, group)
13
+ g = w.astype(np.float32).reshape(n_out, n_in // group, group)
14
+ lo, hi = g.min(axis=-1, keepdims=True), g.max(axis=-1, keepdims=True)
15
+ scale = np.where(hi > lo, (hi - lo) / (2**bits - 1), 1.0) # a constant group: any scale works, bias carries it
16
+ q = np.clip(np.round((g - lo) / scale), 0, 2**bits - 1).astype(np.uint32).reshape(n_out, -1, 32 // bits)
17
+ shifts = np.arange(32 // bits, dtype=np.uint32) * bits
18
+ packed = np.bitwise_or.reduce(q << shifts, axis=-1)
19
+ return packed, scale[..., 0], lo[..., 0]
20
+
21
+
22
+ def dequantize(packed, scales, biases, bits, group, xp):
23
+ """The inverse of quantize, in the backend `xp` and in the scales' dtype."""
24
+ shifts = xp.asarray(np.arange(32 // bits, dtype=np.uint32) * bits)
25
+ q = (packed[..., None] >> shifts) & (2**bits - 1)
26
+ q = q.reshape(packed.shape[0], -1, group).astype(scales.dtype)
27
+ return (q * scales[..., None] + biases[..., None]).reshape(packed.shape[0], -1)
28
+
29
+
30
+ class QuantLinear(Module):
31
+ """Inference-only, bias-free Linear with group-quantized weights; build it after moving the model."""
32
+ def __init__(self, weight, bits, group):
33
+ super().__init__()
34
+ xp = backend.get(weight.device)
35
+ packed, scales, biases = quantize(weight.numpy().T, bits, group) # stackgrad keeps (in, out)
36
+ self.device, self.bits, self.group = weight.device, bits, group
37
+ self.packed = xp.asarray(packed)
38
+ self.scales, self.biases = xp.asarray(scales, dtype=weight.dtype), xp.asarray(biases, dtype=weight.dtype)
39
+
40
+ def forward(self, x):
41
+ assert not x.requires_grad, "QuantLinear is inference only: run it under no_grad()"
42
+ assert x.device == self.device, f"input on {x.device}, weights on {self.device}"
43
+ xp = backend.get(self.device)
44
+ flat = x.data.reshape(-1, x.shape[-1])
45
+ fused = getattr(xp, "quant_matmul", None)
46
+ if fused is not None and flat.shape[0] <= 8: # decode: read the packed weights directly
47
+ y = fused(flat, self.packed, self.scales, self.biases, self.bits, self.group)
48
+ else: # prefill: dequantize once, then one large matmul
49
+ y = flat @ dequantize(self.packed, self.scales, self.biases, self.bits, self.group, xp).T
50
+ return Tensor(y.reshape(*x.shape[:-1], y.shape[-1]), device=self.device)
51
+
52
+ def nbytes(self):
53
+ return sum(a.nbytes for a in (self.packed, self.scales, self.biases))
54
+
55
+
56
+ def quantize_model(model, bits=4, group=64):
57
+ """Replaces every TransformerLM matmul, head included, with a QuantLinear; a tied head gets its own copy."""
58
+ for block in model.blocks:
59
+ for parent in (block.attn, block.ffn):
60
+ for name, val in list(parent.__dict__.items()):
61
+ if isinstance(val, Linear):
62
+ assert val.bias is None, f"{name} has a bias"
63
+ setattr(parent, name, QuantLinear(val.weight, bits, group))
64
+ head = model.head.weight if model.head is not None else model.embed.weight.T
65
+ model.head = QuantLinear(head, bits, group)
66
+ return model
stackgrad/qwen3.py ADDED
@@ -0,0 +1,104 @@
1
+ import json
2
+ from pathlib import Path
3
+
4
+ import numpy as np
5
+ import regex as re
6
+
7
+ from .gpt import TransformerLM
8
+ from .state import iter_safetensors
9
+ from .tokenizer import Tokenizer
10
+
11
+ # Qwen's pre-tokenizer (GPT-4 style): case-insensitive contractions, one digit per token
12
+ PAT = re.compile(r"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+")
13
+
14
+
15
+ def config(path):
16
+ """TransformerLM arguments from a Hugging Face Qwen3 config.json."""
17
+ c = json.loads((Path(path) / "config.json").read_text())
18
+ return {"vocab_size": c["vocab_size"], "context_length": c["max_position_embeddings"], "d_model": c["hidden_size"],
19
+ "n_layers": c["num_hidden_layers"], "n_heads": c["num_attention_heads"], "d_ff": c["intermediate_size"],
20
+ "n_kv_heads": c["num_key_value_heads"], "rope_theta": c["rope_theta"], "head_dim": c["head_dim"],
21
+ "qk_norm": True, "norm_eps": c["rms_norm_eps"], "tie_embeddings": c["tie_word_embeddings"]}
22
+
23
+
24
+ def hf_names(n_layers):
25
+ """stackgrad name -> (HF tensor name, transpose?); HF Linear weights are (out, in), stackgrad's (in, out)."""
26
+ names = {"embed.weight": ("model.embed_tokens.weight", False), "norm.weight": ("model.norm.weight", False)}
27
+ for i in range(n_layers):
28
+ mine, hf = f"blocks.{i}.", f"model.layers.{i}."
29
+ names[mine + "attn_norm.weight"] = (hf + "input_layernorm.weight", False)
30
+ names[mine + "ffn_norm.weight"] = (hf + "post_attention_layernorm.weight", False)
31
+ for proj in ["q_proj", "k_proj", "v_proj", "o_proj"]:
32
+ names[f"{mine}attn.{proj}.weight"] = (f"{hf}self_attn.{proj}.weight", True)
33
+ names[mine + "attn.q_norm.weight"] = (hf + "self_attn.q_norm.weight", False)
34
+ names[mine + "attn.k_norm.weight"] = (hf + "self_attn.k_norm.weight", False)
35
+ for mine_ffn, hf_ffn in [("w1", "gate_proj"), ("w2", "down_proj"), ("w3", "up_proj")]:
36
+ names[f"{mine}ffn.{mine_ffn}.weight"] = (f"{hf}mlp.{hf_ffn}.weight", True)
37
+ return names
38
+
39
+
40
+ def load_qwen3(path, context_length=4096):
41
+ """TransformerLM from a Hugging Face Qwen3 directory (config.json and *.safetensors)."""
42
+ cfg = config(path) | {"context_length": context_length}
43
+ # shortcut: the checkpoint overwrites the random init (14.7 of a 19.6 s load); skip it if loads start to hurt
44
+ model = TransformerLM(**cfg)
45
+ params = dict(model.named_parameters())
46
+ names = hf_names(cfg["n_layers"])
47
+ # a skipped tensor would keep its random init
48
+ assert params.keys() == names.keys(), (sorted(params.keys() - names.keys())[:3], sorted(names.keys() - params.keys())[:3])
49
+ unplaced = {hf: (name, transpose) for name, (hf, transpose) in names.items()}
50
+ lm_head = None
51
+ # one tensor at a time: Qwen3-1.7B's float32 weights beside the init would take 16 GB
52
+ for shard in sorted(Path(path).glob("*.safetensors")):
53
+ for hf, w in iter_safetensors(shard):
54
+ hf = hf if hf.startswith(("model.", "lm_head.")) else "model." + hf # Qwen3-Embedding saves its names without the prefix
55
+ if hf == "lm_head.weight" and cfg["tie_embeddings"]:
56
+ lm_head = w
57
+ continue
58
+ name, transpose = unplaced.pop(hf) # a KeyError: the checkpoint has a tensor the mapping doesn't know
59
+ w = w.T if transpose else w
60
+ assert w.shape == params[name].shape, (name, w.shape, params[name].shape)
61
+ params[name].data = np.ascontiguousarray(w, dtype=np.float32)
62
+ assert not unplaced, f"the checkpoint lacks {sorted(unplaced)[:3]}"
63
+ if lm_head is not None:
64
+ assert np.array_equal(lm_head, params["embed.weight"].data), "lm_head differs from the tied embedding"
65
+ return model
66
+
67
+
68
+ def load_tokenizer(path):
69
+ """Qwen's vocab.json and merges.txt in stackgrad's BPE, with its special tokens at their own ids."""
70
+ added = json.loads((Path(path) / "tokenizer.json").read_text())["added_tokens"]
71
+ tok = Tokenizer.from_files(Path(path) / "vocab.json", Path(path) / "merges.txt", pattern=PAT, nfc=True)
72
+ tok = Tokenizer({**tok.vocab, **{a["id"]: a["content"].encode() for a in added}},
73
+ [p for p, _ in sorted(tok.ranks.items(), key=lambda kv: kv[1])],
74
+ [a["content"] for a in added], pattern=PAT, nfc=True)
75
+ assert all(tok.ids[a["content"].encode()] == a["id"] for a in added)
76
+ return tok
77
+
78
+
79
+ _TOOLS_HEADER = ("# Tools\n\nYou may call one or more functions to assist with the user query.\n\n"
80
+ "You are provided with function signatures within <tools></tools> XML tags:\n<tools>")
81
+ _TOOLS_FOOTER = ("\n</tools>\n\nFor each function call, return a json object with function name and arguments within "
82
+ "<tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n"
83
+ "</tool_call><|im_end|>\n")
84
+
85
+
86
+ def chat_prompt(messages, think=False, tools=None):
87
+ """Qwen3's chat template: messages -> prompt text ending where the reply starts; think=False skips reasoning."""
88
+ # shortcut: earlier assistant turns keep their <think> text (HF strips it); strip it if agent chats run long
89
+ text = ""
90
+ if tools:
91
+ system = messages[0]["content"] + "\n\n" if messages[0]["role"] == "system" else ""
92
+ text = "<|im_start|>system\n" + system + _TOOLS_HEADER + "".join("\n" + json.dumps(t, ensure_ascii=False) for t in tools) + _TOOLS_FOOTER
93
+ messages = messages[1:] if system else messages
94
+ for i, m in enumerate(messages):
95
+ if m["role"] != "tool":
96
+ text += f"<|im_start|>{m['role']}\n{m['content']}<|im_end|>\n"
97
+ continue
98
+ # consecutive tool results share one user turn
99
+ if i == 0 or messages[i - 1]["role"] != "tool":
100
+ text += "<|im_start|>user"
101
+ text += f"\n<tool_response>\n{m['content']}\n</tool_response>"
102
+ if i == len(messages) - 1 or messages[i + 1]["role"] != "tool":
103
+ text += "<|im_end|>\n"
104
+ return text + "<|im_start|>assistant\n" + ("" if think else "<think>\n\n</think>\n\n")
stackgrad/sampling.py ADDED
@@ -0,0 +1,39 @@
1
+ import numpy as np
2
+
3
+ from . import backend
4
+
5
+
6
+ def sample(logits, temperature=1.0, top_k=None, top_p=None):
7
+ """One token id per row of logits (B, V); temperature=0 is greedy, top_k and top_p filter first."""
8
+ if temperature == 0:
9
+ return logits.argmax(axis=-1)
10
+ logits = logits / temperature
11
+ if top_k is not None:
12
+ kth = np.sort(logits, axis=-1)[:, -top_k][:, None]
13
+ logits = np.where(logits < kth, -np.inf, logits)
14
+ probs = np.exp(logits - logits.max(axis=-1, keepdims=True))
15
+ probs /= probs.sum(axis=-1, keepdims=True)
16
+ if top_p is not None:
17
+ kept = np.zeros_like(probs)
18
+ for row, out in zip(probs, kept):
19
+ # tokens under (1 - top_p) / V sum to under 1 - top_p, so the nucleus lies above: sort only those
20
+ cand = np.flatnonzero(row >= (1 - top_p) / len(row))
21
+ cand = cand[np.argsort(-row[cand])]
22
+ # drop a token once the likelier tokens before it already reach top_p
23
+ keep = cand[np.cumsum(row[cand]) - row[cand] < top_p]
24
+ out[keep] = row[keep]
25
+ probs = kept / kept.sum(axis=-1, keepdims=True)
26
+ return np.array([np.random.choice(len(p), p=p) for p in probs])
27
+
28
+
29
+ def sample_on_device(logits, temperature=1.0):
30
+ """sample() without top-k or top-p, run where the Tensor of logits lives, so only B ids reach the host."""
31
+ xp = backend.get(logits.device)
32
+ z = logits.data.astype(xp.float32)
33
+ if temperature == 0:
34
+ return xp.to_numpy(z.argmax(axis=-1))
35
+ z = z / temperature
36
+ cdf = xp.cumsum(xp.exp(z - z.max(axis=-1, keepdims=True)), axis=-1)
37
+ u = xp.asarray(np.random.rand(len(z), 1).astype(np.float32)) * cdf[:, -1:]
38
+ # first crossing, not a count of cdf < u: a parallel float32 cumsum dips. u < cdf[-1], so one exists
39
+ return xp.to_numpy((cdf >= u).argmax(axis=-1))