bitsandbytes-cpu-fork 0.50.2.dev6__py3-none-win_amd64.whl → 0.50.2.dev9__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bitsandbytes/__init__.py +4 -1
- bitsandbytes/efst.py +7 -2
- bitsandbytes/libbitsandbytes_cpu.dll +0 -0
- bitsandbytes/optim/__init__.py +4 -0
- bitsandbytes/optim/adamw4bit.py +239 -0
- bitsandbytes/q3bit.py +481 -0
- bitsandbytes_cpu_cli.py +67 -17
- bitsandbytes_cpu_fork-0.50.2.dev9.dist-info/METADATA +346 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6.dist-info → bitsandbytes_cpu_fork-0.50.2.dev9.dist-info}/RECORD +13 -11
- bitsandbytes_cpu_fork-0.50.2.dev6.dist-info/METADATA +0 -452
- {bitsandbytes_cpu_fork-0.50.2.dev6.dist-info → bitsandbytes_cpu_fork-0.50.2.dev9.dist-info}/WHEEL +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6.dist-info → bitsandbytes_cpu_fork-0.50.2.dev9.dist-info}/entry_points.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6.dist-info → bitsandbytes_cpu_fork-0.50.2.dev9.dist-info}/licenses/LICENSE +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6.dist-info → bitsandbytes_cpu_fork-0.50.2.dev9.dist-info}/top_level.txt +0 -0
bitsandbytes/__init__.py
CHANGED
|
@@ -75,7 +75,10 @@ __pdoc__ = {
|
|
|
75
75
|
"optim.optimizer.MockArgs": False,
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
-
__version__ = "0.50.2.
|
|
78
|
+
__version__ = "0.50.2.dev9"
|
|
79
79
|
|
|
80
80
|
# 挂载 gdn_cpu(GDN 融合内核 / patch_transformers 入口),让 bnb.gdn_cpu 可直接访问
|
|
81
81
|
from . import gdn_cpu # noqa: E402,F401
|
|
82
|
+
|
|
83
|
+
# 挂载 q3bit(实验性 2/3-bit 量化 / 冻结 3-bit base + LoRA),让 bnb.q3bit 可直接访问
|
|
84
|
+
from . import q3bit # noqa: E402,F401
|
bitsandbytes/efst.py
CHANGED
|
@@ -516,8 +516,13 @@ def collect_expert_usage(
|
|
|
516
516
|
return
|
|
517
517
|
for e in idx.detach().cpu().flatten().tolist():
|
|
518
518
|
e = int(e)
|
|
519
|
-
|
|
520
|
-
|
|
519
|
+
# Tensor-expert groups may key their counters by strings (ParameterList /
|
|
520
|
+
# expert container names) while top_k_index contains ints; normalize instead
|
|
521
|
+
# of silently counting zero (this bug made select_top_experts always pick
|
|
522
|
+
# the first keys for HF transformers 5.x fused-expert Mixtral).
|
|
523
|
+
key = e if e in counters[path] else (str(e) if str(e) in counters[path] else None)
|
|
524
|
+
if key is not None:
|
|
525
|
+
counters[path][key] += 1
|
|
521
526
|
return hook
|
|
522
527
|
|
|
523
528
|
for g in groups:
|
|
Binary file
|
bitsandbytes/optim/__init__.py
CHANGED
|
@@ -20,3 +20,7 @@ from .lion import Lion, Lion8bit, Lion32bit, PagedLion, PagedLion8bit, PagedLion
|
|
|
20
20
|
from .optimizer import GlobalOptimManager
|
|
21
21
|
from .rmsprop import RMSprop, RMSprop8bit, RMSprop32bit
|
|
22
22
|
from .sgd import SGD, SGD8bit, SGD32bit
|
|
23
|
+
|
|
24
|
+
# 4-bit blockwise AdamW for CPU (this fork kernel; restored in 0.50.2.dev8).
|
|
25
|
+
# State is ~1.03 bytes/param against 8 for fp32; scalar C kernel with torch fallback.
|
|
26
|
+
from .adamw4bit import AdamW4bit # noqa: F401
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""AdamW4bit — CPU-only 4-bit blockwise AdamW (restored in 0.50.2.dev8).
|
|
2
|
+
|
|
3
|
+
The dev2 release shipped this optimizer, but the kernel was dropped when the repository
|
|
4
|
+
vendored upstream bitsandbytes 0.50.2 (dev4) while the README/CLI kept documenting it.
|
|
5
|
+
This module restores the Python API and the scalar C kernel (``csrc/cpu_ops.cpp``,
|
|
6
|
+
``coptimizer_update_4bit_blockwise_cpu``); it falls back to a pure-torch implementation
|
|
7
|
+
when the loaded native library does not export that symbol (the Windows wheel).
|
|
8
|
+
|
|
9
|
+
State cost: two 4-bit codes per byte for m and v plus one fp32 absmax per 256 elements
|
|
10
|
+
=> ~1.03 bytes/param instead of 8 for fp32 AdamW (~7.8x less). The scalar kernel is
|
|
11
|
+
~4x slower than the AVX2 8-bit kernel per element; the optimizer is a small fraction of
|
|
12
|
+
a CPU training step, so this trades speed for memory. (The dev2 AVX2 path for 4-bit was
|
|
13
|
+
known to produce wrong codes and stays disabled; only the verified scalar path runs.)
|
|
14
|
+
|
|
15
|
+
Storage contract (must match csrc): state1/state2 are ceil(n/2) uint8 each, two 4-bit
|
|
16
|
+
codes per byte, HIGH nibble = even index; absmax1/absmax2 have ceil(n/256) fp32 each;
|
|
17
|
+
qmap is a 256-entry float table of which only the first 16 codes are used
|
|
18
|
+
(qmap[c] = c*2/15 - 1, i.e. 16 symmetric linear levels; code 8 is the closest to zero).
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import ctypes
|
|
23
|
+
import os
|
|
24
|
+
|
|
25
|
+
import torch
|
|
26
|
+
|
|
27
|
+
__all__ = ["AdamW4bit", "native_available", "BLOCKSIZE"]
|
|
28
|
+
|
|
29
|
+
BLOCKSIZE = 256 # must equal kOptBlockSize in csrc/cpu_ops.cpp
|
|
30
|
+
OPT_ADAM = 0
|
|
31
|
+
|
|
32
|
+
_NATIVE = None
|
|
33
|
+
_NATIVE_TRIED = False
|
|
34
|
+
_FORCED_OFF = False
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _native():
|
|
38
|
+
"""Return (lib) if it exports coptimizer_update_4bit_blockwise_cpu, else None."""
|
|
39
|
+
global _NATIVE, _NATIVE_TRIED
|
|
40
|
+
if _FORCED_OFF:
|
|
41
|
+
return None
|
|
42
|
+
if _NATIVE_TRIED:
|
|
43
|
+
return _NATIVE
|
|
44
|
+
_NATIVE_TRIED = True
|
|
45
|
+
if os.environ.get("BNB_ADAMW4BIT_FORCE_TORCH", "").strip() in ("1", "true", "yes"):
|
|
46
|
+
return None
|
|
47
|
+
try:
|
|
48
|
+
from ..gdn_cpu import load_native
|
|
49
|
+
|
|
50
|
+
lib = load_native()
|
|
51
|
+
lib.coptimizer_update_4bit_blockwise_cpu
|
|
52
|
+
except Exception: # noqa: BLE001 - no native library is a supported configuration
|
|
53
|
+
return None
|
|
54
|
+
_F = ctypes.c_float
|
|
55
|
+
_U8P = ctypes.POINTER(ctypes.c_ubyte)
|
|
56
|
+
_FP = ctypes.POINTER(_F)
|
|
57
|
+
fn = lib.coptimizer_update_4bit_blockwise_cpu
|
|
58
|
+
fn.restype = None
|
|
59
|
+
fn.argtypes = [
|
|
60
|
+
ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p, _U8P, _U8P,
|
|
61
|
+
_F, _F, _F, _F, _F, ctypes.c_int, _F, _FP, _FP, _FP, _FP,
|
|
62
|
+
_F, _F, ctypes.c_int, ctypes.c_longlong, ctypes.c_int,
|
|
63
|
+
]
|
|
64
|
+
_NATIVE = lib
|
|
65
|
+
return _NATIVE
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def native_available() -> bool:
|
|
69
|
+
return _native() is not None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _set_force_torch(flag: bool) -> None:
|
|
73
|
+
"""Testing hook: force the pure-torch implementation."""
|
|
74
|
+
global _FORCED_OFF, _NATIVE_TRIED, _NATIVE
|
|
75
|
+
_FORCED_OFF = bool(flag)
|
|
76
|
+
_NATIVE_TRIED = False
|
|
77
|
+
_NATIVE = None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _qmap() -> torch.Tensor:
|
|
81
|
+
q = torch.empty(256, dtype=torch.float32)
|
|
82
|
+
for i in range(256):
|
|
83
|
+
q[i] = (i & 0x0F) * (2.0 / 15.0) - 1.0
|
|
84
|
+
return q
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
_QMAP = _qmap()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _unpack4(state: torch.Tensor, n: int) -> torch.Tensor:
|
|
91
|
+
"""uint8 packed nibbles -> int codes (length n)."""
|
|
92
|
+
b = state[: (n + 1) // 2]
|
|
93
|
+
codes = torch.empty(n, dtype=torch.uint8)
|
|
94
|
+
codes[0::2] = b[: (n + 1) // 2] >> 4
|
|
95
|
+
codes[1::2] = b[: n // 2] & 0x0F
|
|
96
|
+
return codes.to(torch.long) # uint8 indices are treated as deprecated bool masks
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _pack4(codes: torch.Tensor) -> torch.Tensor:
|
|
100
|
+
"""int codes (length n) -> uint8 packed nibbles (ceil(n/2)); high nibble = even."""
|
|
101
|
+
n = codes.numel()
|
|
102
|
+
even = codes[0::2].to(torch.uint8)
|
|
103
|
+
odd = codes[1::2].to(torch.uint8)
|
|
104
|
+
out = torch.empty((n + 1) // 2, dtype=torch.uint8)
|
|
105
|
+
out[: even.numel()] = (even << 4) | odd
|
|
106
|
+
return out
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _fallback_step(opt: "AdamW4bit", group: dict, p: torch.Tensor) -> None:
|
|
110
|
+
"""Pure-torch equivalent of the scalar C kernel (same arithmetic, same packing)."""
|
|
111
|
+
g = p.grad
|
|
112
|
+
n = p.numel()
|
|
113
|
+
blocks = (n + BLOCKSIZE - 1) // BLOCKSIZE
|
|
114
|
+
st = opt.state[p]
|
|
115
|
+
if "state1" not in st:
|
|
116
|
+
st["state1"] = torch.zeros((n + 1) // 2, dtype=torch.uint8)
|
|
117
|
+
st["state2"] = torch.zeros((n + 1) // 2, dtype=torch.uint8)
|
|
118
|
+
st["absmax1"] = torch.zeros(blocks, dtype=torch.float32)
|
|
119
|
+
st["absmax2"] = torch.zeros(blocks, dtype=torch.float32)
|
|
120
|
+
st["step"] = 0
|
|
121
|
+
st["step"] += 1
|
|
122
|
+
step = st["step"]
|
|
123
|
+
beta1, beta2 = group["betas"]
|
|
124
|
+
eps, lr, wd = group["eps"], group["lr"], group["weight_decay"]
|
|
125
|
+
pad = blocks * BLOCKSIZE - n
|
|
126
|
+
gf = g.detach().float().reshape(-1)
|
|
127
|
+
pf = p.detach().float().reshape(-1)
|
|
128
|
+
if pad:
|
|
129
|
+
gf = torch.cat([gf, torch.zeros(pad)])
|
|
130
|
+
pf = torch.cat([pf, torch.zeros(pad)])
|
|
131
|
+
s1 = _QMAP[_unpack4(st["state1"], n)].to(torch.float32)
|
|
132
|
+
s2 = _QMAP[_unpack4(st["state2"], n)].to(torch.float32)
|
|
133
|
+
am1 = st["absmax1"].repeat_interleave(BLOCKSIZE)[:n]
|
|
134
|
+
am2 = st["absmax2"].repeat_interleave(BLOCKSIZE)[:n]
|
|
135
|
+
if pad:
|
|
136
|
+
s1 = torch.cat([s1, torch.zeros(pad)])
|
|
137
|
+
s2 = torch.cat([s2, torch.zeros(pad)])
|
|
138
|
+
am1 = torch.cat([am1, torch.zeros(pad)])
|
|
139
|
+
am2 = torch.cat([am2, torch.zeros(pad)])
|
|
140
|
+
s1 = (s1 * am1).view(blocks, BLOCKSIZE)
|
|
141
|
+
s2 = (s2 * am2).view(blocks, BLOCKSIZE)
|
|
142
|
+
gv = gf.view(blocks, BLOCKSIZE)
|
|
143
|
+
pv = pf.view(blocks, BLOCKSIZE)
|
|
144
|
+
s2 = s2 * beta2 + (1.0 - beta2) * gv * gv
|
|
145
|
+
s1 = s1 * beta1 + (1.0 - beta1) * gv
|
|
146
|
+
c1 = 1.0 - beta1 ** step
|
|
147
|
+
c2 = (1.0 - beta2 ** step) ** 0.5
|
|
148
|
+
step_size = -lr * c2 / c1
|
|
149
|
+
pn = pv + step_size * s1 / (torch.sqrt(s2) + c2 * eps)
|
|
150
|
+
if wd > 0:
|
|
151
|
+
pn = pn * (1.0 - lr * wd)
|
|
152
|
+
# new block absmax + requantise (zero block -> code 8, exactly like the kernel)
|
|
153
|
+
n1 = s1.abs().amax(dim=1)
|
|
154
|
+
n2 = s2.abs().amax(dim=1)
|
|
155
|
+
inv1 = torch.where(n1 > 0, 1.0 / n1, torch.zeros_like(n1))
|
|
156
|
+
inv2 = torch.where(n2 > 0, 1.0 / n2, torch.zeros_like(n2))
|
|
157
|
+
code1 = torch.round((s1 * inv1[:, None] + 1.0) * 7.5).clamp(0, 15)
|
|
158
|
+
code2 = torch.round((s2 * inv2[:, None] + 1.0) * 7.5).clamp(0, 15)
|
|
159
|
+
zero1 = (n1 == 0)[:, None]
|
|
160
|
+
zero2 = (n2 == 0)[:, None]
|
|
161
|
+
code1 = torch.where(zero1, torch.full_like(code1, 8), code1)
|
|
162
|
+
code2 = torch.where(zero2, torch.full_like(code2, 8), code2)
|
|
163
|
+
st["absmax1"] = n1.to(torch.float32)
|
|
164
|
+
st["absmax2"] = n2.to(torch.float32)
|
|
165
|
+
st["state1"] = _pack4(code1.reshape(-1).to(torch.uint8))
|
|
166
|
+
st["state2"] = _pack4(code2.reshape(-1).to(torch.uint8))
|
|
167
|
+
with torch.no_grad():
|
|
168
|
+
p.copy_(pn.reshape(-1)[:n].reshape(p.shape).to(p.dtype))
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class AdamW4bit(torch.optim.Optimizer):
|
|
172
|
+
"""CPU 4-bit blockwise AdamW; drop-in for ``torch.optim.AdamW`` on fp32 params.
|
|
173
|
+
|
|
174
|
+
>>> opt = AdamW4bit(model.parameters(), lr=1e-4, weight_decay=0.01)
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
|
|
178
|
+
weight_decay=0.0, blocksize=BLOCKSIZE):
|
|
179
|
+
if blocksize != BLOCKSIZE:
|
|
180
|
+
raise ValueError("blocksize must be %d (compiled into cpu_ops.cpp)" % BLOCKSIZE)
|
|
181
|
+
super().__init__(params, dict(lr=lr, betas=betas, eps=eps,
|
|
182
|
+
weight_decay=weight_decay))
|
|
183
|
+
self._qmap = _QMAP.clone()
|
|
184
|
+
|
|
185
|
+
@torch.no_grad()
|
|
186
|
+
def step(self, closure=None):
|
|
187
|
+
loss = None
|
|
188
|
+
if closure is not None:
|
|
189
|
+
with torch.enable_grad():
|
|
190
|
+
loss = closure()
|
|
191
|
+
lib = _native()
|
|
192
|
+
for group in self.param_groups:
|
|
193
|
+
for p in group["params"]:
|
|
194
|
+
if p.grad is None:
|
|
195
|
+
continue
|
|
196
|
+
if p.dtype != torch.float32:
|
|
197
|
+
raise TypeError("AdamW4bit supports fp32 params only; got %s" % p.dtype)
|
|
198
|
+
if not p.is_contiguous():
|
|
199
|
+
p.data = p.data.contiguous()
|
|
200
|
+
g = p.grad
|
|
201
|
+
if not g.is_contiguous():
|
|
202
|
+
g = g.contiguous()
|
|
203
|
+
if lib is None:
|
|
204
|
+
_fallback_step(self, group, p)
|
|
205
|
+
continue
|
|
206
|
+
n = p.numel()
|
|
207
|
+
blocks = (n + BLOCKSIZE - 1) // BLOCKSIZE
|
|
208
|
+
st = self.state[p]
|
|
209
|
+
if "state1" not in st:
|
|
210
|
+
st["state1"] = torch.zeros((n + 1) // 2, dtype=torch.uint8)
|
|
211
|
+
st["state2"] = torch.zeros((n + 1) // 2, dtype=torch.uint8)
|
|
212
|
+
st["absmax1"] = torch.zeros(blocks, dtype=torch.float32)
|
|
213
|
+
st["absmax2"] = torch.zeros(blocks, dtype=torch.float32)
|
|
214
|
+
st["step"] = 0
|
|
215
|
+
st["step"] += 1
|
|
216
|
+
b1, b2 = group["betas"]
|
|
217
|
+
lib.coptimizer_update_4bit_blockwise_cpu(
|
|
218
|
+
OPT_ADAM,
|
|
219
|
+
g.data_ptr(), p.data_ptr(),
|
|
220
|
+
ctypes.cast(st["state1"].data_ptr(), ctypes.POINTER(ctypes.c_ubyte)),
|
|
221
|
+
ctypes.cast(st["state2"].data_ptr(), ctypes.POINTER(ctypes.c_ubyte)),
|
|
222
|
+
b1, b2, 0.0, 0.0, group["eps"],
|
|
223
|
+
st["step"], group["lr"],
|
|
224
|
+
ctypes.cast(self._qmap.data_ptr(), ctypes.POINTER(ctypes.c_float)),
|
|
225
|
+
ctypes.cast(self._qmap.data_ptr(), ctypes.POINTER(ctypes.c_float)),
|
|
226
|
+
ctypes.cast(st["absmax1"].data_ptr(), ctypes.POINTER(ctypes.c_float)),
|
|
227
|
+
ctypes.cast(st["absmax2"].data_ptr(), ctypes.POINTER(ctypes.c_float)),
|
|
228
|
+
group["weight_decay"], 1.0, 0, n, 0,
|
|
229
|
+
)
|
|
230
|
+
return loss
|
|
231
|
+
|
|
232
|
+
def state_bytes(self) -> int:
|
|
233
|
+
"""Actual optimizer-state bytes held (the metric this optimizer exists for)."""
|
|
234
|
+
tot = 0
|
|
235
|
+
for st in self.state.values():
|
|
236
|
+
for k in ("state1", "state2", "absmax1", "absmax2"):
|
|
237
|
+
if k in st:
|
|
238
|
+
tot += st[k].numel() * st[k].element_size()
|
|
239
|
+
return tot
|