bitsandbytes-cpu-fork 0.50.2.dev6__py3-none-win_amd64.whl → 0.50.2.dev9__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bitsandbytes/__init__.py +4 -1
- bitsandbytes/efst.py +7 -2
- bitsandbytes/libbitsandbytes_cpu.dll +0 -0
- bitsandbytes/optim/__init__.py +4 -0
- bitsandbytes/optim/adamw4bit.py +239 -0
- bitsandbytes/q3bit.py +481 -0
- bitsandbytes_cpu_cli.py +67 -17
- bitsandbytes_cpu_fork-0.50.2.dev9.dist-info/METADATA +346 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6.dist-info → bitsandbytes_cpu_fork-0.50.2.dev9.dist-info}/RECORD +13 -11
- bitsandbytes_cpu_fork-0.50.2.dev6.dist-info/METADATA +0 -452
- {bitsandbytes_cpu_fork-0.50.2.dev6.dist-info → bitsandbytes_cpu_fork-0.50.2.dev9.dist-info}/WHEEL +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6.dist-info → bitsandbytes_cpu_fork-0.50.2.dev9.dist-info}/entry_points.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6.dist-info → bitsandbytes_cpu_fork-0.50.2.dev9.dist-info}/licenses/LICENSE +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6.dist-info → bitsandbytes_cpu_fork-0.50.2.dev9.dist-info}/top_level.txt +0 -0
bitsandbytes/q3bit.py
ADDED
|
@@ -0,0 +1,481 @@
|
|
|
1
|
+
"""3-bit (and experimental 2-bit) blockwise quantisation for CPU training.
|
|
2
|
+
|
|
3
|
+
This is the productised version of the CPU-forge 2/3-bit research kernels (report
|
|
4
|
+
§7.139–§7.150): a frozen base weight is stored as blockwise 3-bit codes (≈11% of
|
|
5
|
+
fp32 memory) and can be used for LoRA training or inference. The heavy lifting is done
|
|
6
|
+
by the C kernels added to ``csrc/cpu_ops.cpp`` (AVX2 / NEON / scalar fallback):
|
|
7
|
+
|
|
8
|
+
* ``cbdequantize_3bit_cpu`` – unpack packed codes into fp32
|
|
9
|
+
* ``cbq3bit_gemm_cpu`` – C[M,N] = A[M,K] @ dequant(B)[N,K]^T without materialising B
|
|
10
|
+
* ``cbq3bit_gemm_t_cpu`` – transposed variant for a custom backward (currently unused
|
|
11
|
+
by default: torch's oneDNN matmul wins for small layers)
|
|
12
|
+
|
|
13
|
+
On machines whose native library does not export those symbols (the current Windows
|
|
14
|
+
wheel predates them), every function transparently falls back to a pure-torch
|
|
15
|
+
implementation, so the API is portable; only the speed differs.
|
|
16
|
+
|
|
17
|
+
Layout (identical to the research scripts and the C kernels): blocks of ``block``
|
|
18
|
+
elements, one fp32 absmax per block, symmetric codes, block % 8 == 0 (3-bit) or
|
|
19
|
+
block % 4 == 0 (2-bit), value(code) = code * absmax/center - absmax with
|
|
20
|
+
center = (2^bits - 1) / 2.
|
|
21
|
+
|
|
22
|
+
Examples
|
|
23
|
+
--------
|
|
24
|
+
>>> import torch, bitsandbytes as bnb
|
|
25
|
+
>>> from bitsandbytes import q3bit
|
|
26
|
+
>>> layer = torch.nn.Linear(1024, 1024)
|
|
27
|
+
>>> ql = q3bit.QuantLinear3bit.from_linear(layer, lora_r=8)
|
|
28
|
+
>>> y = ql(torch.randn(2, 1024)) # frozen 3-bit base + trainable LoRA
|
|
29
|
+
>>> packed, absmax = q3bit.quantize_3bit(layer.weight.detach())
|
|
30
|
+
>>> w = q3bit.dequantize_3bit(packed, absmax, layer.weight.shape)
|
|
31
|
+
>>> logits = q3bit.gemm_3bit(torch.randn(2, 1024), packed, absmax) # no fp32 materialisation (native)
|
|
32
|
+
"""
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import ctypes
|
|
36
|
+
import os
|
|
37
|
+
from typing import Optional, Tuple
|
|
38
|
+
|
|
39
|
+
import torch
|
|
40
|
+
import torch.nn as nn
|
|
41
|
+
import torch.nn.functional as F
|
|
42
|
+
|
|
43
|
+
__all__ = [
|
|
44
|
+
"pack_3bit",
|
|
45
|
+
"unpack_3bit",
|
|
46
|
+
"quantize_3bit",
|
|
47
|
+
"dequantize_3bit",
|
|
48
|
+
"quantize_2bit",
|
|
49
|
+
"dequantize_2bit",
|
|
50
|
+
"gemm_3bit",
|
|
51
|
+
"gemm_3bit_transposed",
|
|
52
|
+
"QuantLinear3bit",
|
|
53
|
+
"convert_linears_3bit",
|
|
54
|
+
"native_available",
|
|
55
|
+
]
|
|
56
|
+
|
|
57
|
+
_NATIVE = None
|
|
58
|
+
_NATIVE_TRIED = False
|
|
59
|
+
_NATIVE_FORCED_OFF = False
|
|
60
|
+
_CENTER = {2: 1.5, 3: 3.5}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# ---------------------------------------------------------------------------
|
|
64
|
+
# native library access
|
|
65
|
+
# ---------------------------------------------------------------------------
|
|
66
|
+
def _native():
|
|
67
|
+
"""Return the native CDLL if it exports the q3bit symbols, else None."""
|
|
68
|
+
global _NATIVE, _NATIVE_TRIED
|
|
69
|
+
if _NATIVE_FORCED_OFF:
|
|
70
|
+
return None
|
|
71
|
+
if _NATIVE_TRIED:
|
|
72
|
+
return _NATIVE
|
|
73
|
+
_NATIVE_TRIED = True
|
|
74
|
+
if os.environ.get("BNB_Q3BIT_FORCE_TORCH", "").strip() in ("1", "true", "yes"):
|
|
75
|
+
return None
|
|
76
|
+
try:
|
|
77
|
+
from .gdn_cpu import load_native
|
|
78
|
+
|
|
79
|
+
lib = load_native()
|
|
80
|
+
except Exception: # noqa: BLE001 - no native library is a supported configuration
|
|
81
|
+
return None
|
|
82
|
+
try:
|
|
83
|
+
lib.cbdequantize_3bit_cpu.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
|
|
84
|
+
ctypes.c_void_p, ctypes.c_longlong,
|
|
85
|
+
ctypes.c_longlong]
|
|
86
|
+
lib.cbdequantize_3bit_cpu.restype = ctypes.c_longlong
|
|
87
|
+
lib.cbdequantize_2bit_cpu.argtypes = lib.cbdequantize_3bit_cpu.argtypes
|
|
88
|
+
lib.cbdequantize_2bit_cpu.restype = ctypes.c_longlong
|
|
89
|
+
lib.cbq3bit_gemm_cpu.argtypes = [ctypes.c_void_p, ctypes.c_longlong,
|
|
90
|
+
ctypes.c_void_p, ctypes.c_void_p,
|
|
91
|
+
ctypes.c_void_p, ctypes.c_longlong,
|
|
92
|
+
ctypes.c_longlong, ctypes.c_longlong,
|
|
93
|
+
ctypes.c_longlong, ctypes.c_longlong]
|
|
94
|
+
lib.cbq3bit_gemm_cpu.restype = ctypes.c_longlong
|
|
95
|
+
lib.cbq3bit_gemm_t_cpu.argtypes = lib.cbq3bit_gemm_cpu.argtypes
|
|
96
|
+
lib.cbq3bit_gemm_t_cpu.restype = ctypes.c_longlong
|
|
97
|
+
except AttributeError:
|
|
98
|
+
return None
|
|
99
|
+
_NATIVE = lib
|
|
100
|
+
return _NATIVE
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def native_available() -> bool:
|
|
104
|
+
"""True when the loaded native library exports the 2/3-bit kernels."""
|
|
105
|
+
return _native() is not None
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _set_force_torch(flag: bool) -> None:
|
|
109
|
+
"""Testing hook: force the pure-torch fallback path."""
|
|
110
|
+
global _NATIVE_FORCED_OFF, _NATIVE_TRIED, _NATIVE
|
|
111
|
+
_NATIVE_FORCED_OFF = bool(flag)
|
|
112
|
+
_NATIVE_TRIED = False
|
|
113
|
+
_NATIVE = None
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
# ---------------------------------------------------------------------------
|
|
117
|
+
# packing (torch; also used to create the packed weights)
|
|
118
|
+
# ---------------------------------------------------------------------------
|
|
119
|
+
def _levels(bits: int) -> int:
|
|
120
|
+
return 1 << bits
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _bytes_per_block(bits: int, block: int) -> int:
|
|
124
|
+
if bits == 2:
|
|
125
|
+
return block // 4
|
|
126
|
+
if bits == 3:
|
|
127
|
+
return block // 8 * 3
|
|
128
|
+
raise ValueError("only 2/3-bit supported")
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def pack_3bit(codes: torch.Tensor) -> torch.Tensor:
|
|
132
|
+
"""Pack uint8 codes in [0,7] (shape (nblocks, block), block % 8 == 0) into bytes."""
|
|
133
|
+
nb, block = codes.shape
|
|
134
|
+
c = codes.view(nb, block // 8, 8).to(torch.int32)
|
|
135
|
+
v = (c[..., 0] | (c[..., 1] << 3) | (c[..., 2] << 6) | (c[..., 3] << 9)
|
|
136
|
+
| (c[..., 4] << 12) | (c[..., 5] << 15) | (c[..., 6] << 18) | (c[..., 7] << 21))
|
|
137
|
+
b0 = (v & 0xFF).to(torch.uint8)
|
|
138
|
+
b1 = ((v >> 8) & 0xFF).to(torch.uint8)
|
|
139
|
+
b2 = ((v >> 16) & 0xFF).to(torch.uint8)
|
|
140
|
+
return torch.stack([b0, b1, b2], dim=-1).reshape(nb, -1)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def unpack_3bit(packed: torch.Tensor, block: int) -> torch.Tensor:
|
|
144
|
+
"""Inverse of :func:`pack_3bit` (returns (nblocks, block) uint8 codes).
|
|
145
|
+
|
|
146
|
+
``packed`` may be the per-row 2-D layout (N, blocks_per_row*bytes) as well as the
|
|
147
|
+
flat (nblocks, bytes) layout; both are views of the same block sequence.
|
|
148
|
+
"""
|
|
149
|
+
groups = block // 8
|
|
150
|
+
nb = int(packed.numel()) // 3 // groups
|
|
151
|
+
p = packed.reshape(nb, groups, 3).to(torch.int32)
|
|
152
|
+
v = p[..., 0] | (p[..., 1] << 8) | (p[..., 2] << 16)
|
|
153
|
+
return torch.stack([(v >> (3 * i)) & 7 for i in range(8)], dim=-1).reshape(nb, block).to(torch.uint8)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _pack_2bit(codes: torch.Tensor) -> torch.Tensor:
|
|
157
|
+
nb, block = codes.shape
|
|
158
|
+
c = codes.view(nb, block // 4, 4).to(torch.int32)
|
|
159
|
+
v = c[..., 0] | (c[..., 1] << 2) | (c[..., 2] << 4) | (c[..., 3] << 6)
|
|
160
|
+
return v.to(torch.uint8).reshape(nb, -1)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _unpack_2bit(packed: torch.Tensor, block: int) -> torch.Tensor:
|
|
164
|
+
groups = block // 4
|
|
165
|
+
nb = int(packed.numel()) // groups
|
|
166
|
+
v = packed.reshape(nb, groups).to(torch.int32)
|
|
167
|
+
return torch.stack([(v >> (2 * i)) & 3 for i in range(4)], dim=-1).reshape(nb, block).to(torch.uint8)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
# ---------------------------------------------------------------------------
|
|
171
|
+
# quantise / dequantise
|
|
172
|
+
# ---------------------------------------------------------------------------
|
|
173
|
+
def _quantize(w: torch.Tensor, bits: int, block: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Size, int]:
|
|
174
|
+
if block % (8 if bits == 3 else 4) != 0 or block <= 0:
|
|
175
|
+
raise ValueError("block must be a positive multiple of 8 for 3-bit / 4 for 2-bit")
|
|
176
|
+
x = w.detach().float().contiguous()
|
|
177
|
+
shape = x.shape
|
|
178
|
+
flat = x.reshape(-1)
|
|
179
|
+
n = flat.numel()
|
|
180
|
+
pad = (block - n % block) % block
|
|
181
|
+
if pad:
|
|
182
|
+
flat = torch.cat([flat, torch.zeros(pad, dtype=flat.dtype)])
|
|
183
|
+
nb = flat.numel() // block
|
|
184
|
+
blk = flat.view(nb, block)
|
|
185
|
+
absmax = blk.abs().amax(dim=1)
|
|
186
|
+
center = _CENTER[bits]
|
|
187
|
+
scale = (absmax / center).clamp_min(1e-12)
|
|
188
|
+
q = torch.round(blk / scale.unsqueeze(1) + center).clamp(0, _levels(bits) - 1).to(torch.uint8)
|
|
189
|
+
packed = pack_3bit(q) if bits == 3 else _pack_2bit(q)
|
|
190
|
+
absmax = absmax.float()
|
|
191
|
+
# For a 2-D weight whose last dimension is block-aligned, keep the per-row block
|
|
192
|
+
# layout (N, nblocks_per_row * bytes) so the fused GEMM can index rows directly.
|
|
193
|
+
if len(shape) == 2 and shape[1] % block == 0:
|
|
194
|
+
nb_per_row = shape[1] // block
|
|
195
|
+
bytes_per_block = packed.shape[1]
|
|
196
|
+
packed = packed.reshape(shape[0], nb_per_row * bytes_per_block)
|
|
197
|
+
absmax = absmax.reshape(shape[0], nb_per_row)
|
|
198
|
+
return packed, absmax, shape, n
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def quantize_3bit(w: torch.Tensor, block: int = 64) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
202
|
+
"""Blockwise symmetric 3-bit quantisation.
|
|
203
|
+
|
|
204
|
+
Returns ``(packed_uint8, absmax_float32)``; the byte layout matches the C kernels
|
|
205
|
+
(3 bytes per 8 weights, one fp32 absmax per ``block`` weights).
|
|
206
|
+
"""
|
|
207
|
+
packed, absmax, _, _ = _quantize(w, 3, block)
|
|
208
|
+
return packed, absmax
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def quantize_2bit(w: torch.Tensor, block: int = 64) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
212
|
+
"""Experimental 2-bit variant (higher error, smaller memory; same layout rules)."""
|
|
213
|
+
packed, absmax, _, _ = _quantize(w, 2, block)
|
|
214
|
+
return packed, absmax
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _dequantize_torch(packed: torch.Tensor, absmax: torch.Tensor, shape, n: int,
|
|
218
|
+
bits: int, block: int) -> torch.Tensor:
|
|
219
|
+
# flatten: the 2-D per-row layout is just a view of the same block sequence
|
|
220
|
+
absmax_f = absmax.reshape(-1).to(torch.float32)
|
|
221
|
+
packed_f = packed.reshape(-1, packed.shape[-1])
|
|
222
|
+
codes = unpack_3bit(packed_f, block) if bits == 3 else _unpack_2bit(packed_f, block)
|
|
223
|
+
center = _CENTER[bits]
|
|
224
|
+
scale = absmax_f / center
|
|
225
|
+
vals = codes.float() * scale.unsqueeze(1) - absmax_f.unsqueeze(1)
|
|
226
|
+
return vals.reshape(-1)[:n].reshape(shape)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def dequantize_3bit(packed: torch.Tensor, absmax: torch.Tensor, shape,
|
|
230
|
+
block: int = 64, dtype: Optional[torch.dtype] = None) -> torch.Tensor:
|
|
231
|
+
"""Unpack 3-bit codes back to a tensor of ``shape``.
|
|
232
|
+
|
|
233
|
+
The native kernel is used when available (exact same arithmetic); otherwise the
|
|
234
|
+
pure-torch unpack is used. ``packed``/``absmax`` are never modified.
|
|
235
|
+
"""
|
|
236
|
+
n = 1
|
|
237
|
+
for s in shape:
|
|
238
|
+
n *= int(s)
|
|
239
|
+
packed = packed.contiguous()
|
|
240
|
+
absmax = absmax.contiguous()
|
|
241
|
+
nblocks = int(absmax.numel())
|
|
242
|
+
if native_available() and packed.is_contiguous() and absmax.dtype in (torch.float32,):
|
|
243
|
+
lib = _native()
|
|
244
|
+
# the kernel writes nblocks*block floats (the last block is padded)
|
|
245
|
+
out = torch.empty(nblocks * int(block), dtype=torch.float32)
|
|
246
|
+
rc = lib.cbdequantize_3bit_cpu(packed.data_ptr(), absmax.data_ptr(),
|
|
247
|
+
out.data_ptr(), nblocks, int(block))
|
|
248
|
+
if rc == 0:
|
|
249
|
+
return out[:n].reshape(shape).to(dtype or torch.float32)
|
|
250
|
+
out = _dequantize_torch(packed, absmax, shape, n, 3, block)
|
|
251
|
+
return out.to(dtype or torch.float32)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def dequantize_2bit(packed: torch.Tensor, absmax: torch.Tensor, shape,
|
|
255
|
+
block: int = 64, dtype: Optional[torch.dtype] = None) -> torch.Tensor:
|
|
256
|
+
n = 1
|
|
257
|
+
for s in shape:
|
|
258
|
+
n *= int(s)
|
|
259
|
+
packed = packed.contiguous()
|
|
260
|
+
absmax = absmax.contiguous()
|
|
261
|
+
nblocks = int(absmax.numel())
|
|
262
|
+
if native_available():
|
|
263
|
+
lib = _native()
|
|
264
|
+
out = torch.empty(nblocks * int(block), dtype=torch.float32)
|
|
265
|
+
rc = lib.cbdequantize_2bit_cpu(packed.data_ptr(), absmax.data_ptr(),
|
|
266
|
+
out.data_ptr(), nblocks, int(block))
|
|
267
|
+
if rc == 0:
|
|
268
|
+
return out[:n].reshape(shape).to(dtype or torch.float32)
|
|
269
|
+
out = _dequantize_torch(packed, absmax, shape, n, 2, block)
|
|
270
|
+
return out.to(dtype or torch.float32)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
# ---------------------------------------------------------------------------
|
|
274
|
+
# GEMM
|
|
275
|
+
# ---------------------------------------------------------------------------
|
|
276
|
+
def _check_gemm_args(x: torch.Tensor, packed: torch.Tensor, absmax: torch.Tensor,
|
|
277
|
+
block: int) -> Tuple[int, int, torch.Tensor]:
|
|
278
|
+
"""Validate args and normalise ``packed`` to the per-row (N, blocks*bytes) layout."""
|
|
279
|
+
if x.dim() != 2:
|
|
280
|
+
raise ValueError("x must be 2-D (M, K)")
|
|
281
|
+
if packed.dim() != 2:
|
|
282
|
+
raise ValueError("packed weight must be 2-D")
|
|
283
|
+
K = x.shape[1]
|
|
284
|
+
if K % block != 0:
|
|
285
|
+
raise ValueError("K (%d) must be divisible by block (%d)" % (K, block))
|
|
286
|
+
nb_per_row = K // block
|
|
287
|
+
bpb = _bytes_per_block(3, block)
|
|
288
|
+
bytes_per_row = nb_per_row * bpb
|
|
289
|
+
if packed.shape[1] == bytes_per_row:
|
|
290
|
+
pass # already per-row layout
|
|
291
|
+
elif packed.shape[1] == bpb:
|
|
292
|
+
# flat (nblocks, bytes) layout from an older/simple quantizer call
|
|
293
|
+
if packed.shape[0] % nb_per_row != 0:
|
|
294
|
+
raise ValueError("packed block count is not a multiple of blocks-per-row")
|
|
295
|
+
packed = packed.reshape(-1, bytes_per_row)
|
|
296
|
+
else:
|
|
297
|
+
raise ValueError("packed row width %d does not match K/block (=%d)"
|
|
298
|
+
% (packed.shape[1], bytes_per_row))
|
|
299
|
+
if int(absmax.numel()) != packed.shape[0] * nb_per_row:
|
|
300
|
+
raise ValueError("absmax count %%d does not match packed blocks")
|
|
301
|
+
if packed.shape[0] == 0:
|
|
302
|
+
raise ValueError("empty packed weight")
|
|
303
|
+
return packed.shape[0], K, packed
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def gemm_3bit(x: torch.Tensor, packed: torch.Tensor, absmax: torch.Tensor,
|
|
307
|
+
block: int = 64, threads: int = 0, ntile: int = 64) -> torch.Tensor:
|
|
308
|
+
"""``x @ dequant(packed).T`` without materialising the fp32 weight (native path).
|
|
309
|
+
|
|
310
|
+
``x`` is (M, K), the packed weight is (N, K) row-blockwise 3-bit. The native kernel
|
|
311
|
+
is used when available (it never writes the fp32 weight); otherwise this falls back
|
|
312
|
+
to ``x @ dequantize_3bit(...).T`` in torch.
|
|
313
|
+
"""
|
|
314
|
+
N, K, packed = _check_gemm_args(x, packed, absmax, block)
|
|
315
|
+
if native_available():
|
|
316
|
+
x = x.contiguous().float()
|
|
317
|
+
packed = packed.contiguous()
|
|
318
|
+
absmax = absmax.contiguous()
|
|
319
|
+
out = torch.empty(x.shape[0], N, dtype=torch.float32)
|
|
320
|
+
lib = _native()
|
|
321
|
+
rc = lib.cbq3bit_gemm_cpu(x.data_ptr(), int(x.shape[0]), packed.data_ptr(),
|
|
322
|
+
absmax.data_ptr(), out.data_ptr(), N, K, int(block),
|
|
323
|
+
int(threads), int(ntile))
|
|
324
|
+
if rc == 0:
|
|
325
|
+
return out
|
|
326
|
+
w = dequantize_3bit(packed, absmax, (N, K), block=block)
|
|
327
|
+
return x.float() @ w.t()
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _check_gemm_t_args(dy: torch.Tensor, packed: torch.Tensor, absmax: torch.Tensor,
|
|
331
|
+
block: int) -> Tuple[int, int, int, torch.Tensor]:
|
|
332
|
+
"""Validate the transposed GEMM: here dy is (M, N) and K comes from the packed rows."""
|
|
333
|
+
if dy.dim() != 2:
|
|
334
|
+
raise ValueError("dy must be 2-D (M, N)")
|
|
335
|
+
if packed.dim() != 2:
|
|
336
|
+
raise ValueError("packed weight must be 2-D")
|
|
337
|
+
bpb = _bytes_per_block(3, block)
|
|
338
|
+
if packed.shape[1] % bpb != 0:
|
|
339
|
+
raise ValueError("packed row width is not a multiple of bytes-per-block")
|
|
340
|
+
nb_per_row = packed.shape[1] // bpb
|
|
341
|
+
K = nb_per_row * block
|
|
342
|
+
N = packed.shape[0]
|
|
343
|
+
if dy.shape[1] != N:
|
|
344
|
+
raise ValueError("dy.shape[1] (%d) != packed rows (%d)" % (dy.shape[1], N))
|
|
345
|
+
if int(absmax.numel()) != N * nb_per_row:
|
|
346
|
+
raise ValueError("absmax count does not match packed blocks")
|
|
347
|
+
return dy.shape[0], N, K, packed
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def gemm_3bit_transposed(dy: torch.Tensor, packed: torch.Tensor, absmax: torch.Tensor,
|
|
351
|
+
block: int = 64, threads: int = 0, ntile: int = 64) -> torch.Tensor:
|
|
352
|
+
"""``dy @ dequant(packed)`` (i.e. the transposed GEMM used by the backward pass)."""
|
|
353
|
+
M, N, K, packed = _check_gemm_t_args(dy, packed, absmax, block)
|
|
354
|
+
if native_available():
|
|
355
|
+
dy = dy.contiguous().float()
|
|
356
|
+
packed = packed.contiguous()
|
|
357
|
+
absmax = absmax.contiguous()
|
|
358
|
+
out = torch.empty(M, K, dtype=torch.float32)
|
|
359
|
+
lib = _native()
|
|
360
|
+
rc = lib.cbq3bit_gemm_t_cpu(dy.data_ptr(), int(M), packed.data_ptr(),
|
|
361
|
+
absmax.data_ptr(), out.data_ptr(), N, K, int(block),
|
|
362
|
+
int(threads), int(ntile))
|
|
363
|
+
if rc == 0:
|
|
364
|
+
return out
|
|
365
|
+
w = dequantize_3bit(packed, absmax, (N, K), block=block)
|
|
366
|
+
return dy.float() @ w
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
# ---------------------------------------------------------------------------
|
|
370
|
+
# module: frozen 3-bit base + trainable LoRA
|
|
371
|
+
# ---------------------------------------------------------------------------
|
|
372
|
+
class QuantLinear3bit(nn.Module):
|
|
373
|
+
"""A frozen 3-bit ``nn.Linear`` replacement with trainable LoRA adapters.
|
|
374
|
+
|
|
375
|
+
The base matrix is stored as blockwise 3-bit codes (≈11% of fp32 memory) and is
|
|
376
|
+
frozen; only ``lora_A``/``lora_B`` are trainable. In grad mode the forward
|
|
377
|
+
dequantises the weight (native kernel when available) and runs a regular
|
|
378
|
+
matmul/``F.linear`` so autograd handles the input gradient and the LoRA
|
|
379
|
+
gradients; in ``no_grad`` mode on a native build it calls the fused 3-bit GEMM
|
|
380
|
+
and never materialises the fp32 weight.
|
|
381
|
+
"""
|
|
382
|
+
|
|
383
|
+
def __init__(self, in_features: int, out_features: int, bias: bool = True,
|
|
384
|
+
block: int = 64, lora_r: int = 8, lora_alpha: int = 16,
|
|
385
|
+
dtype: torch.dtype = torch.float32, device=None):
|
|
386
|
+
super().__init__()
|
|
387
|
+
if block % 8 != 0:
|
|
388
|
+
raise ValueError("block must be a multiple of 8")
|
|
389
|
+
self.in_features = int(in_features)
|
|
390
|
+
self.out_features = int(out_features)
|
|
391
|
+
self.block = int(block)
|
|
392
|
+
self.bits = 3
|
|
393
|
+
# the buffers are filled by from_linear() / load_from_linear()
|
|
394
|
+
self.register_buffer("packed", None)
|
|
395
|
+
self.register_buffer("absmax", None)
|
|
396
|
+
self.bias = nn.Parameter(torch.zeros(self.out_features, dtype=dtype,
|
|
397
|
+
device=device), requires_grad=False) if bias else None
|
|
398
|
+
self.lora_A = nn.Parameter(torch.empty(lora_r, self.in_features, dtype=dtype,
|
|
399
|
+
device=device))
|
|
400
|
+
self.lora_B = nn.Parameter(torch.zeros(self.out_features, lora_r, dtype=dtype,
|
|
401
|
+
device=device))
|
|
402
|
+
nn.init.kaiming_uniform_(self.lora_A, a=5 ** 0.5)
|
|
403
|
+
self.scale = lora_alpha / lora_r
|
|
404
|
+
|
|
405
|
+
@classmethod
|
|
406
|
+
def from_linear(cls, linear: nn.Linear, block: int = 64, lora_r: int = 8,
|
|
407
|
+
lora_alpha: int = 16) -> "QuantLinear3bit":
|
|
408
|
+
"""Build a :class:`QuantLinear3bit` from an existing ``nn.Linear``."""
|
|
409
|
+
m = cls(linear.in_features, linear.out_features, bias=linear.bias is not None,
|
|
410
|
+
block=block, lora_r=lora_r, lora_alpha=lora_alpha,
|
|
411
|
+
dtype=linear.weight.dtype, device=linear.weight.device)
|
|
412
|
+
m.load_from_linear(linear)
|
|
413
|
+
return m
|
|
414
|
+
|
|
415
|
+
def load_from_linear(self, linear: nn.Linear) -> None:
|
|
416
|
+
"""Quantise ``linear.weight`` into this module and copy/freeze the bias."""
|
|
417
|
+
w = linear.weight.detach()
|
|
418
|
+
packed, absmax = quantize_3bit(w, self.block)
|
|
419
|
+
self.packed = packed
|
|
420
|
+
self.absmax = absmax
|
|
421
|
+
if linear.bias is not None:
|
|
422
|
+
with torch.no_grad():
|
|
423
|
+
self.bias.copy_(linear.bias.detach())
|
|
424
|
+
|
|
425
|
+
def weight(self) -> torch.Tensor:
|
|
426
|
+
"""Dequantise the base weight (fp32) — allocates N*K floats."""
|
|
427
|
+
if self.packed is None:
|
|
428
|
+
raise RuntimeError("QuantLinear3bit has no packed weight; use from_linear()")
|
|
429
|
+
return dequantize_3bit(self.packed, self.absmax,
|
|
430
|
+
(self.out_features, self.in_features), block=self.block)
|
|
431
|
+
|
|
432
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
433
|
+
if self.packed is None:
|
|
434
|
+
raise RuntimeError("QuantLinear3bit has no packed weight; use from_linear()")
|
|
435
|
+
lead = x.shape[:-1]
|
|
436
|
+
x2 = x.reshape(-1, self.in_features)
|
|
437
|
+
base: torch.Tensor
|
|
438
|
+
if torch.is_grad_enabled():
|
|
439
|
+
# dequantise once (native kernel when available) then let autograd compute
|
|
440
|
+
# the input gradient through the frozen weight.
|
|
441
|
+
w = self.weight().to(x.dtype)
|
|
442
|
+
base = F.linear(x2, w, self.bias)
|
|
443
|
+
elif self.in_features % self.block == 0:
|
|
444
|
+
base = gemm_3bit(x2, self.packed, self.absmax, block=self.block)
|
|
445
|
+
if self.bias is not None:
|
|
446
|
+
base = base + self.bias
|
|
447
|
+
else:
|
|
448
|
+
w = self.weight().to(x.dtype)
|
|
449
|
+
base = F.linear(x2, w, self.bias)
|
|
450
|
+
lora = (x2 @ self.lora_A.t()) @ self.lora_B.t() * self.scale
|
|
451
|
+
return (base + lora).reshape(*lead, self.out_features)
|
|
452
|
+
|
|
453
|
+
def extra_repr(self) -> str:
|
|
454
|
+
return ("in_features=%d, out_features=%d, bits=3, block=%d, lora_r=%d"
|
|
455
|
+
% (self.in_features, self.out_features, self.block, self.lora_A.shape[0]))
|
|
456
|
+
|
|
457
|
+
|
|
458
|
+
def convert_linears_3bit(model: nn.Module, block: int = 64, lora_r: int = 8,
|
|
459
|
+
lora_alpha: int = 16, skip_names: Tuple[str, ...] = ("lm_head",),
|
|
460
|
+
name_prefix: str = "") -> int:
|
|
461
|
+
"""Replace every ``nn.Linear`` child with :class:`QuantLinear3bit` (in place).
|
|
462
|
+
|
|
463
|
+
Layers whose *leaf* name is in ``skip_names`` are kept in fp32 (``lm_head`` by
|
|
464
|
+
default, because logits need the full unquantised matrix on CPU). Returns the
|
|
465
|
+
number of replaced layers; the base weights become frozen and only the LoRA
|
|
466
|
+
adapters are trainable.
|
|
467
|
+
"""
|
|
468
|
+
n = 0
|
|
469
|
+
for name, child in list(model.named_children()):
|
|
470
|
+
full = f"{name_prefix}.{name}" if name_prefix else name
|
|
471
|
+
if isinstance(child, nn.Linear):
|
|
472
|
+
if name in skip_names:
|
|
473
|
+
continue
|
|
474
|
+
setattr(model, name, QuantLinear3bit.from_linear(
|
|
475
|
+
child, block=block, lora_r=lora_r, lora_alpha=lora_alpha))
|
|
476
|
+
n += 1
|
|
477
|
+
else:
|
|
478
|
+
n += convert_linears_3bit(child, block=block, lora_r=lora_r,
|
|
479
|
+
lora_alpha=lora_alpha, skip_names=skip_names,
|
|
480
|
+
name_prefix=full)
|
|
481
|
+
return n
|
bitsandbytes_cpu_cli.py
CHANGED
|
@@ -86,6 +86,9 @@ SECTIONS = [
|
|
|
86
86
|
"fused Gated DeltaNet for Qwen3-Next / Qwen3.5 linear attention", [], PY),
|
|
87
87
|
("disk_balancer",
|
|
88
88
|
"spills cold parameters to another disk when RAM runs out", [], PY),
|
|
89
|
+
("bitsandbytes.q3bit",
|
|
90
|
+
"frozen 3-bit weights + LoRA (~11% of fp32 memory); native kernels with a "
|
|
91
|
+
"pure-torch fallback", [], PY),
|
|
89
92
|
("bitsandbytes-cpu",
|
|
90
93
|
"this command; pip also installs it as bnb-cpu", [], PY),
|
|
91
94
|
]),
|
|
@@ -273,6 +276,8 @@ SECTIONS = [
|
|
|
273
276
|
("what you get",
|
|
274
277
|
"1.031 bytes of state per parameter, against 2.062 for the 8-bit optimizer "
|
|
275
278
|
"and 8.0 for torch AdamW, all measured at 1M parameters", [
|
|
279
|
+
"import torch, torch.nn as nn",
|
|
280
|
+
"from bitsandbytes.optim import AdamW4bit",
|
|
276
281
|
"p = nn.Parameter(torch.randn(4096))",
|
|
277
282
|
"opt = AdamW4bit([p], lr=1e-3)",
|
|
278
283
|
"p.grad = torch.randn_like(p)",
|
|
@@ -294,29 +299,30 @@ SECTIONS = [
|
|
|
294
299
|
("does it train as well as fp32",
|
|
295
300
|
"it tracks fp32 rather than matching it: quantised state cannot reproduce an "
|
|
296
301
|
"fp32 trajectory, and that is not the criterion", [
|
|
297
|
-
"# measured over 120 steps on the real
|
|
298
|
-
"#
|
|
299
|
-
"#
|
|
300
|
-
"#
|
|
301
|
-
"#
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
"
|
|
306
|
-
|
|
307
|
-
"#
|
|
308
|
-
"#
|
|
309
|
-
"# the
|
|
302
|
+
"# dev2 measured two loss curves within 0.27% over 120 steps on the real",
|
|
303
|
+
"# 30.49M image-AR model",
|
|
304
|
+
"# our 20-step 1024-param check: parameter relL2 1.2e-3 vs torch AdamW",
|
|
305
|
+
"# the restored scalar kernel matches the pure-torch fallback to 1.2e-7",
|
|
306
|
+
"# over 10 steps (test_adamw4bit.py), so the maths agrees exactly",
|
|
307
|
+
], PY),
|
|
308
|
+
("the step is slower, so use it for memory not speed",
|
|
309
|
+
"scalar 4-bit kernel: 28.6 ms vs 3.3 ms (8-bit AVX2) and 8.2 ms (fp32 torch) "
|
|
310
|
+
"per step at 1M params, measured on the R5-4500U with 6 threads", [
|
|
311
|
+
"# the dev2 AVX2 4-bit path produced wrong codes and stayed disabled;",
|
|
312
|
+
"# what runs today is the verified scalar path",
|
|
313
|
+
"# the optimizer is a few percent of a real training step at 30M params,",
|
|
314
|
+
"# so the wall-clock cost is small while the state saving is 7.8x",
|
|
315
|
+
"# older Windows wheels run the pure-torch fallback (dev9+ is native)",
|
|
310
316
|
], PY),
|
|
311
317
|
("traps",
|
|
312
318
|
"the failure modes seen while building this, each one silent at first", [
|
|
313
319
|
"# 1. torch.optim.AdamW is NOT replaced: this is a separate class, and",
|
|
314
320
|
"# AdamW4bit requires fp32 params (the kernel's dtype 0).",
|
|
315
321
|
"# 2. blocksize is compiled in at 256; passing another value raises.",
|
|
316
|
-
"# 3.
|
|
317
|
-
"# coptimizer_update_4bit_blockwise_cpu
|
|
318
|
-
"#
|
|
319
|
-
"#
|
|
322
|
+
"# 3. older Windows wheels (< dev9) do not export",
|
|
323
|
+
"# coptimizer_update_4bit_blockwise_cpu (pre-dev9 Windows wheels),",
|
|
324
|
+
"# this class transparently uses a pure-torch implementation with the",
|
|
325
|
+
"# same numbers and the same 1.031 B/param state, just slower.",
|
|
320
326
|
"# 4. quantised state is not a drop-in for exact reproducibility runs:",
|
|
321
327
|
"# same seed, same data, different trajectory from fp32.",
|
|
322
328
|
], PY),
|
|
@@ -518,6 +524,50 @@ SECTIONS = [
|
|
|
518
524
|
" ...",
|
|
519
525
|
], PY),
|
|
520
526
|
]),
|
|
527
|
+
("q3bit", "3-bit frozen base + LoRA (experimental)", [
|
|
528
|
+
("quantize_3bit(w, block=64)",
|
|
529
|
+
"returns (packed uint8, absmax): blockwise symmetric 3-bit codes, 3 bytes per 8 "
|
|
530
|
+
"weights; the layout is the one the C kernels use", [
|
|
531
|
+
"import torch",
|
|
532
|
+
"from bitsandbytes import q3bit",
|
|
533
|
+
"w = torch.randn(256, 256)",
|
|
534
|
+
"packed, absmax = q3bit.quantize_3bit(w, block=64)",
|
|
535
|
+
"back = q3bit.dequantize_3bit(packed, absmax, w.shape)",
|
|
536
|
+
"rel = ((w - back).norm() / w.norm()).item()",
|
|
537
|
+
"assert rel < 0.30, rel",
|
|
538
|
+
], PY),
|
|
539
|
+
("dequantize_3bit(packed, absmax, shape, block=64)",
|
|
540
|
+
"uses the native AVX2 kernel when the loaded library exports it, else the torch "
|
|
541
|
+
"unpack; the two agree to float rounding", [], PY),
|
|
542
|
+
("gemm_3bit(A, packed, absmax, block=64)",
|
|
543
|
+
"A @ dequant(B).T without writing the fp32 weight on the native path; K % block "
|
|
544
|
+
"must be 0", [
|
|
545
|
+
"import torch",
|
|
546
|
+
"from bitsandbytes import q3bit",
|
|
547
|
+
"w = torch.randn(128, 128)",
|
|
548
|
+
"packed, absmax = q3bit.quantize_3bit(w)",
|
|
549
|
+
"A = torch.randn(4, 128)",
|
|
550
|
+
"out = q3bit.gemm_3bit(A, packed, absmax)",
|
|
551
|
+
"ref = A @ q3bit.dequantize_3bit(packed, absmax, w.shape).t()",
|
|
552
|
+
"assert (out - ref).abs().max().item() < 1e-5",
|
|
553
|
+
], PY),
|
|
554
|
+
("QuantLinear3bit.from_linear(linear, lora_r=8)",
|
|
555
|
+
"frozen 3-bit base + trainable LoRA; only lora_A/lora_B require grad", [
|
|
556
|
+
"import torch, torch.nn as nn",
|
|
557
|
+
"from bitsandbytes import q3bit",
|
|
558
|
+
"lin = nn.Linear(128, 96)",
|
|
559
|
+
"ql = q3bit.QuantLinear3bit.from_linear(lin, lora_r=4)",
|
|
560
|
+
"y = ql(torch.randn(2, 128))",
|
|
561
|
+
"assert y.shape == (2, 96)",
|
|
562
|
+
"assert [n for n, p in ql.named_parameters() if p.requires_grad] == "
|
|
563
|
+
"['lora_A', 'lora_B']",
|
|
564
|
+
], PY),
|
|
565
|
+
("convert_linears_3bit(model, lora_r=8)",
|
|
566
|
+
"replaces every nn.Linear in place and returns the count; lm_head stays fp32", []),
|
|
567
|
+
("experimental / portability",
|
|
568
|
+
"since dev9 the Windows wheel ships a CI-built MSVC DLL; older wheels use the pure-torch "
|
|
569
|
+
"fallback, while the dev9 Windows and Linux wheels use the native kernels", []),
|
|
570
|
+
]),
|
|
521
571
|
("layers", "Reference: bitsandbytes.nn", [
|
|
522
572
|
("Linear4bit(input_features, output_features, bias=True, compute_dtype=None,\n"
|
|
523
573
|
" compress_statistics=True, quant_type='fp4',\n"
|