bitsandbytes-cpu-fork 0.50.2.dev5__py3-none-win_amd64.whl → 0.50.2.dev8__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
bitsandbytes/__init__.py CHANGED
@@ -75,7 +75,10 @@ __pdoc__ = {
75
75
  "optim.optimizer.MockArgs": False,
76
76
  }
77
77
 
78
- __version__ = "0.50.2.dev5"
78
+ __version__ = "0.50.2.dev8"
79
79
 
80
80
  # 挂载 gdn_cpu(GDN 融合内核 / patch_transformers 入口),让 bnb.gdn_cpu 可直接访问
81
81
  from . import gdn_cpu # noqa: E402,F401
82
+
83
+ # 挂载 q3bit(实验性 2/3-bit 量化 / 冻结 3-bit base + LoRA),让 bnb.q3bit 可直接访问
84
+ from . import q3bit # noqa: E402,F401
bitsandbytes/efst.py CHANGED
@@ -516,8 +516,13 @@ def collect_expert_usage(
516
516
  return
517
517
  for e in idx.detach().cpu().flatten().tolist():
518
518
  e = int(e)
519
- if e in counters[path]:
520
- counters[path][e] += 1
519
+ # Tensor-expert groups may key their counters by strings (ParameterList /
520
+ # expert container names) while top_k_index contains ints; normalize instead
521
+ # of silently counting zero (this bug made select_top_experts always pick
522
+ # the first keys for HF transformers 5.x fused-expert Mixtral).
523
+ key = e if e in counters[path] else (str(e) if str(e) in counters[path] else None)
524
+ if key is not None:
525
+ counters[path][key] += 1
521
526
  return hook
522
527
 
523
528
  for g in groups:
@@ -20,3 +20,7 @@ from .lion import Lion, Lion8bit, Lion32bit, PagedLion, PagedLion8bit, PagedLion
20
20
  from .optimizer import GlobalOptimManager
21
21
  from .rmsprop import RMSprop, RMSprop8bit, RMSprop32bit
22
22
  from .sgd import SGD, SGD8bit, SGD32bit
23
+
24
+ # 4-bit blockwise AdamW for CPU (this fork kernel; restored in 0.50.2.dev8).
25
+ # State is ~1.03 bytes/param against 8 for fp32; scalar C kernel with torch fallback.
26
+ from .adamw4bit import AdamW4bit # noqa: F401
@@ -0,0 +1,239 @@
1
+ """AdamW4bit — CPU-only 4-bit blockwise AdamW (restored in 0.50.2.dev8).
2
+
3
+ The dev2 release shipped this optimizer, but the kernel was dropped when the repository
4
+ vendored upstream bitsandbytes 0.50.2 (dev4) while the README/CLI kept documenting it.
5
+ This module restores the Python API and the scalar C kernel (``csrc/cpu_ops.cpp``,
6
+ ``coptimizer_update_4bit_blockwise_cpu``); it falls back to a pure-torch implementation
7
+ when the loaded native library does not export that symbol (the Windows wheel).
8
+
9
+ State cost: two 4-bit codes per byte for m and v plus one fp32 absmax per 256 elements
10
+ => ~1.03 bytes/param instead of 8 for fp32 AdamW (~7.8x less). The scalar kernel is
11
+ ~4x slower than the AVX2 8-bit kernel per element; the optimizer is a small fraction of
12
+ a CPU training step, so this trades speed for memory. (The dev2 AVX2 path for 4-bit was
13
+ known to produce wrong codes and stays disabled; only the verified scalar path runs.)
14
+
15
+ Storage contract (must match csrc): state1/state2 are ceil(n/2) uint8 each, two 4-bit
16
+ codes per byte, HIGH nibble = even index; absmax1/absmax2 have ceil(n/256) fp32 each;
17
+ qmap is a 256-entry float table of which only the first 16 codes are used
18
+ (qmap[c] = c*2/15 - 1, i.e. 16 symmetric linear levels; code 8 is the closest to zero).
19
+ """
20
+ from __future__ import annotations
21
+
22
+ import ctypes
23
+ import os
24
+
25
+ import torch
26
+
27
+ __all__ = ["AdamW4bit", "native_available", "BLOCKSIZE"]
28
+
29
+ BLOCKSIZE = 256 # must equal kOptBlockSize in csrc/cpu_ops.cpp
30
+ OPT_ADAM = 0
31
+
32
+ _NATIVE = None
33
+ _NATIVE_TRIED = False
34
+ _FORCED_OFF = False
35
+
36
+
37
+ def _native():
38
+ """Return (lib) if it exports coptimizer_update_4bit_blockwise_cpu, else None."""
39
+ global _NATIVE, _NATIVE_TRIED
40
+ if _FORCED_OFF:
41
+ return None
42
+ if _NATIVE_TRIED:
43
+ return _NATIVE
44
+ _NATIVE_TRIED = True
45
+ if os.environ.get("BNB_ADAMW4BIT_FORCE_TORCH", "").strip() in ("1", "true", "yes"):
46
+ return None
47
+ try:
48
+ from ..gdn_cpu import load_native
49
+
50
+ lib = load_native()
51
+ lib.coptimizer_update_4bit_blockwise_cpu
52
+ except Exception: # noqa: BLE001 - no native library is a supported configuration
53
+ return None
54
+ _F = ctypes.c_float
55
+ _U8P = ctypes.POINTER(ctypes.c_ubyte)
56
+ _FP = ctypes.POINTER(_F)
57
+ fn = lib.coptimizer_update_4bit_blockwise_cpu
58
+ fn.restype = None
59
+ fn.argtypes = [
60
+ ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, _U8P, _U8P,
61
+ _F, _F, _F, _F, _F, ctypes.c_int, _F, _FP, _FP, _FP, _FP,
62
+ _F, _F, ctypes.c_int, ctypes.c_longlong, ctypes.c_int,
63
+ ]
64
+ _NATIVE = lib
65
+ return _NATIVE
66
+
67
+
68
+ def native_available() -> bool:
69
+ return _native() is not None
70
+
71
+
72
+ def _set_force_torch(flag: bool) -> None:
73
+ """Testing hook: force the pure-torch implementation."""
74
+ global _FORCED_OFF, _NATIVE_TRIED, _NATIVE
75
+ _FORCED_OFF = bool(flag)
76
+ _NATIVE_TRIED = False
77
+ _NATIVE = None
78
+
79
+
80
+ def _qmap() -> torch.Tensor:
81
+ q = torch.empty(256, dtype=torch.float32)
82
+ for i in range(256):
83
+ q[i] = (i & 0x0F) * (2.0 / 15.0) - 1.0
84
+ return q
85
+
86
+
87
+ _QMAP = _qmap()
88
+
89
+
90
+ def _unpack4(state: torch.Tensor, n: int) -> torch.Tensor:
91
+ """uint8 packed nibbles -> int codes (length n)."""
92
+ b = state[: (n + 1) // 2]
93
+ codes = torch.empty(n, dtype=torch.uint8)
94
+ codes[0::2] = b[: (n + 1) // 2] >> 4
95
+ codes[1::2] = b[: n // 2] & 0x0F
96
+ return codes.to(torch.long) # uint8 indices are treated as deprecated bool masks
97
+
98
+
99
+ def _pack4(codes: torch.Tensor) -> torch.Tensor:
100
+ """int codes (length n) -> uint8 packed nibbles (ceil(n/2)); high nibble = even."""
101
+ n = codes.numel()
102
+ even = codes[0::2].to(torch.uint8)
103
+ odd = codes[1::2].to(torch.uint8)
104
+ out = torch.empty((n + 1) // 2, dtype=torch.uint8)
105
+ out[: even.numel()] = (even << 4) | odd
106
+ return out
107
+
108
+
109
+ def _fallback_step(opt: "AdamW4bit", group: dict, p: torch.Tensor) -> None:
110
+ """Pure-torch equivalent of the scalar C kernel (same arithmetic, same packing)."""
111
+ g = p.grad
112
+ n = p.numel()
113
+ blocks = (n + BLOCKSIZE - 1) // BLOCKSIZE
114
+ st = opt.state[p]
115
+ if "state1" not in st:
116
+ st["state1"] = torch.zeros((n + 1) // 2, dtype=torch.uint8)
117
+ st["state2"] = torch.zeros((n + 1) // 2, dtype=torch.uint8)
118
+ st["absmax1"] = torch.zeros(blocks, dtype=torch.float32)
119
+ st["absmax2"] = torch.zeros(blocks, dtype=torch.float32)
120
+ st["step"] = 0
121
+ st["step"] += 1
122
+ step = st["step"]
123
+ beta1, beta2 = group["betas"]
124
+ eps, lr, wd = group["eps"], group["lr"], group["weight_decay"]
125
+ pad = blocks * BLOCKSIZE - n
126
+ gf = g.detach().float().reshape(-1)
127
+ pf = p.detach().float().reshape(-1)
128
+ if pad:
129
+ gf = torch.cat([gf, torch.zeros(pad)])
130
+ pf = torch.cat([pf, torch.zeros(pad)])
131
+ s1 = _QMAP[_unpack4(st["state1"], n)].to(torch.float32)
132
+ s2 = _QMAP[_unpack4(st["state2"], n)].to(torch.float32)
133
+ am1 = st["absmax1"].repeat_interleave(BLOCKSIZE)[:n]
134
+ am2 = st["absmax2"].repeat_interleave(BLOCKSIZE)[:n]
135
+ if pad:
136
+ s1 = torch.cat([s1, torch.zeros(pad)])
137
+ s2 = torch.cat([s2, torch.zeros(pad)])
138
+ am1 = torch.cat([am1, torch.zeros(pad)])
139
+ am2 = torch.cat([am2, torch.zeros(pad)])
140
+ s1 = (s1 * am1).view(blocks, BLOCKSIZE)
141
+ s2 = (s2 * am2).view(blocks, BLOCKSIZE)
142
+ gv = gf.view(blocks, BLOCKSIZE)
143
+ pv = pf.view(blocks, BLOCKSIZE)
144
+ s2 = s2 * beta2 + (1.0 - beta2) * gv * gv
145
+ s1 = s1 * beta1 + (1.0 - beta1) * gv
146
+ c1 = 1.0 - beta1 ** step
147
+ c2 = (1.0 - beta2 ** step) ** 0.5
148
+ step_size = -lr * c2 / c1
149
+ pn = pv + step_size * s1 / (torch.sqrt(s2) + c2 * eps)
150
+ if wd > 0:
151
+ pn = pn * (1.0 - lr * wd)
152
+ # new block absmax + requantise (zero block -> code 8, exactly like the kernel)
153
+ n1 = s1.abs().amax(dim=1)
154
+ n2 = s2.abs().amax(dim=1)
155
+ inv1 = torch.where(n1 > 0, 1.0 / n1, torch.zeros_like(n1))
156
+ inv2 = torch.where(n2 > 0, 1.0 / n2, torch.zeros_like(n2))
157
+ code1 = torch.round((s1 * inv1[:, None] + 1.0) * 7.5).clamp(0, 15)
158
+ code2 = torch.round((s2 * inv2[:, None] + 1.0) * 7.5).clamp(0, 15)
159
+ zero1 = (n1 == 0)[:, None]
160
+ zero2 = (n2 == 0)[:, None]
161
+ code1 = torch.where(zero1, torch.full_like(code1, 8), code1)
162
+ code2 = torch.where(zero2, torch.full_like(code2, 8), code2)
163
+ st["absmax1"] = n1.to(torch.float32)
164
+ st["absmax2"] = n2.to(torch.float32)
165
+ st["state1"] = _pack4(code1.reshape(-1).to(torch.uint8))
166
+ st["state2"] = _pack4(code2.reshape(-1).to(torch.uint8))
167
+ with torch.no_grad():
168
+ p.copy_(pn.reshape(-1)[:n].reshape(p.shape).to(p.dtype))
169
+
170
+
171
+ class AdamW4bit(torch.optim.Optimizer):
172
+ """CPU 4-bit blockwise AdamW; drop-in for ``torch.optim.AdamW`` on fp32 params.
173
+
174
+ >>> opt = AdamW4bit(model.parameters(), lr=1e-4, weight_decay=0.01)
175
+ """
176
+
177
+ def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
178
+ weight_decay=0.0, blocksize=BLOCKSIZE):
179
+ if blocksize != BLOCKSIZE:
180
+ raise ValueError("blocksize must be %d (compiled into cpu_ops.cpp)" % BLOCKSIZE)
181
+ super().__init__(params, dict(lr=lr, betas=betas, eps=eps,
182
+ weight_decay=weight_decay))
183
+ self._qmap = _QMAP.clone()
184
+
185
+ @torch.no_grad()
186
+ def step(self, closure=None):
187
+ loss = None
188
+ if closure is not None:
189
+ with torch.enable_grad():
190
+ loss = closure()
191
+ lib = _native()
192
+ for group in self.param_groups:
193
+ for p in group["params"]:
194
+ if p.grad is None:
195
+ continue
196
+ if p.dtype != torch.float32:
197
+ raise TypeError("AdamW4bit supports fp32 params only; got %s" % p.dtype)
198
+ if not p.is_contiguous():
199
+ p.data = p.data.contiguous()
200
+ g = p.grad
201
+ if not g.is_contiguous():
202
+ g = g.contiguous()
203
+ if lib is None:
204
+ _fallback_step(self, group, p)
205
+ continue
206
+ n = p.numel()
207
+ blocks = (n + BLOCKSIZE - 1) // BLOCKSIZE
208
+ st = self.state[p]
209
+ if "state1" not in st:
210
+ st["state1"] = torch.zeros((n + 1) // 2, dtype=torch.uint8)
211
+ st["state2"] = torch.zeros((n + 1) // 2, dtype=torch.uint8)
212
+ st["absmax1"] = torch.zeros(blocks, dtype=torch.float32)
213
+ st["absmax2"] = torch.zeros(blocks, dtype=torch.float32)
214
+ st["step"] = 0
215
+ st["step"] += 1
216
+ b1, b2 = group["betas"]
217
+ lib.coptimizer_update_4bit_blockwise_cpu(
218
+ OPT_ADAM,
219
+ g.data_ptr(), p.data_ptr(),
220
+ ctypes.cast(st["state1"].data_ptr(), ctypes.POINTER(ctypes.c_ubyte)),
221
+ ctypes.cast(st["state2"].data_ptr(), ctypes.POINTER(ctypes.c_ubyte)),
222
+ b1, b2, 0.0, 0.0, group["eps"],
223
+ st["step"], group["lr"],
224
+ ctypes.cast(self._qmap.data_ptr(), ctypes.POINTER(ctypes.c_float)),
225
+ ctypes.cast(self._qmap.data_ptr(), ctypes.POINTER(ctypes.c_float)),
226
+ ctypes.cast(st["absmax1"].data_ptr(), ctypes.POINTER(ctypes.c_float)),
227
+ ctypes.cast(st["absmax2"].data_ptr(), ctypes.POINTER(ctypes.c_float)),
228
+ group["weight_decay"], 1.0, 0, n, 0,
229
+ )
230
+ return loss
231
+
232
+ def state_bytes(self) -> int:
233
+ """Actual optimizer-state bytes held (the metric this optimizer exists for)."""
234
+ tot = 0
235
+ for st in self.state.values():
236
+ for k in ("state1", "state2", "absmax1", "absmax2"):
237
+ if k in st:
238
+ tot += st[k].numel() * st[k].element_size()
239
+ return tot