bitsandbytes-cpu-fork 0.50.2.dev6__py3-none-win_amd64.whl → 0.50.2.dev9__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
bitsandbytes/q3bit.py ADDED
@@ -0,0 +1,481 @@
1
+ """3-bit (and experimental 2-bit) blockwise quantisation for CPU training.
2
+
3
+ This is the productised version of the CPU-forge 2/3-bit research kernels (report
4
+ §7.139–§7.150): a frozen base weight is stored as blockwise 3-bit codes (≈11% of
5
+ fp32 memory) and can be used for LoRA training or inference. The heavy lifting is done
6
+ by the C kernels added to ``csrc/cpu_ops.cpp`` (AVX2 / NEON / scalar fallback):
7
+
8
+ * ``cbdequantize_3bit_cpu`` – unpack packed codes into fp32
9
+ * ``cbq3bit_gemm_cpu`` – C[M,N] = A[M,K] @ dequant(B)[N,K]^T without materialising B
10
+ * ``cbq3bit_gemm_t_cpu`` – transposed variant for a custom backward (currently unused
11
+ by default: torch's oneDNN matmul wins for small layers)
12
+
13
+ On machines whose native library does not export those symbols (the current Windows
14
+ wheel predates them), every function transparently falls back to a pure-torch
15
+ implementation, so the API is portable; only the speed differs.
16
+
17
+ Layout (identical to the research scripts and the C kernels): blocks of ``block``
18
+ elements, one fp32 absmax per block, symmetric codes, block % 8 == 0 (3-bit) or
19
+ block % 4 == 0 (2-bit), value(code) = code * absmax/center - absmax with
20
+ center = (2^bits - 1) / 2.
21
+
22
+ Examples
23
+ --------
24
+ >>> import torch, bitsandbytes as bnb
25
+ >>> from bitsandbytes import q3bit
26
+ >>> layer = torch.nn.Linear(1024, 1024)
27
+ >>> ql = q3bit.QuantLinear3bit.from_linear(layer, lora_r=8)
28
+ >>> y = ql(torch.randn(2, 1024)) # frozen 3-bit base + trainable LoRA
29
+ >>> packed, absmax = q3bit.quantize_3bit(layer.weight.detach())
30
+ >>> w = q3bit.dequantize_3bit(packed, absmax, layer.weight.shape)
31
+ >>> logits = q3bit.gemm_3bit(torch.randn(2, 1024), packed, absmax) # no fp32 materialisation (native)
32
+ """
33
+ from __future__ import annotations
34
+
35
+ import ctypes
36
+ import os
37
+ from typing import Optional, Tuple
38
+
39
+ import torch
40
+ import torch.nn as nn
41
+ import torch.nn.functional as F
42
+
43
+ __all__ = [
44
+ "pack_3bit",
45
+ "unpack_3bit",
46
+ "quantize_3bit",
47
+ "dequantize_3bit",
48
+ "quantize_2bit",
49
+ "dequantize_2bit",
50
+ "gemm_3bit",
51
+ "gemm_3bit_transposed",
52
+ "QuantLinear3bit",
53
+ "convert_linears_3bit",
54
+ "native_available",
55
+ ]
56
+
57
+ _NATIVE = None
58
+ _NATIVE_TRIED = False
59
+ _NATIVE_FORCED_OFF = False
60
+ _CENTER = {2: 1.5, 3: 3.5}
61
+
62
+
63
+ # ---------------------------------------------------------------------------
64
+ # native library access
65
+ # ---------------------------------------------------------------------------
66
+ def _native():
67
+ """Return the native CDLL if it exports the q3bit symbols, else None."""
68
+ global _NATIVE, _NATIVE_TRIED
69
+ if _NATIVE_FORCED_OFF:
70
+ return None
71
+ if _NATIVE_TRIED:
72
+ return _NATIVE
73
+ _NATIVE_TRIED = True
74
+ if os.environ.get("BNB_Q3BIT_FORCE_TORCH", "").strip() in ("1", "true", "yes"):
75
+ return None
76
+ try:
77
+ from .gdn_cpu import load_native
78
+
79
+ lib = load_native()
80
+ except Exception: # noqa: BLE001 - no native library is a supported configuration
81
+ return None
82
+ try:
83
+ lib.cbdequantize_3bit_cpu.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
84
+ ctypes.c_void_p, ctypes.c_longlong,
85
+ ctypes.c_longlong]
86
+ lib.cbdequantize_3bit_cpu.restype = ctypes.c_longlong
87
+ lib.cbdequantize_2bit_cpu.argtypes = lib.cbdequantize_3bit_cpu.argtypes
88
+ lib.cbdequantize_2bit_cpu.restype = ctypes.c_longlong
89
+ lib.cbq3bit_gemm_cpu.argtypes = [ctypes.c_void_p, ctypes.c_longlong,
90
+ ctypes.c_void_p, ctypes.c_void_p,
91
+ ctypes.c_void_p, ctypes.c_longlong,
92
+ ctypes.c_longlong, ctypes.c_longlong,
93
+ ctypes.c_longlong, ctypes.c_longlong]
94
+ lib.cbq3bit_gemm_cpu.restype = ctypes.c_longlong
95
+ lib.cbq3bit_gemm_t_cpu.argtypes = lib.cbq3bit_gemm_cpu.argtypes
96
+ lib.cbq3bit_gemm_t_cpu.restype = ctypes.c_longlong
97
+ except AttributeError:
98
+ return None
99
+ _NATIVE = lib
100
+ return _NATIVE
101
+
102
+
103
+ def native_available() -> bool:
104
+ """True when the loaded native library exports the 2/3-bit kernels."""
105
+ return _native() is not None
106
+
107
+
108
+ def _set_force_torch(flag: bool) -> None:
109
+ """Testing hook: force the pure-torch fallback path."""
110
+ global _NATIVE_FORCED_OFF, _NATIVE_TRIED, _NATIVE
111
+ _NATIVE_FORCED_OFF = bool(flag)
112
+ _NATIVE_TRIED = False
113
+ _NATIVE = None
114
+
115
+
116
+ # ---------------------------------------------------------------------------
117
+ # packing (torch; also used to create the packed weights)
118
+ # ---------------------------------------------------------------------------
119
+ def _levels(bits: int) -> int:
120
+ return 1 << bits
121
+
122
+
123
+ def _bytes_per_block(bits: int, block: int) -> int:
124
+ if bits == 2:
125
+ return block // 4
126
+ if bits == 3:
127
+ return block // 8 * 3
128
+ raise ValueError("only 2/3-bit supported")
129
+
130
+
131
+ def pack_3bit(codes: torch.Tensor) -> torch.Tensor:
132
+ """Pack uint8 codes in [0,7] (shape (nblocks, block), block % 8 == 0) into bytes."""
133
+ nb, block = codes.shape
134
+ c = codes.view(nb, block // 8, 8).to(torch.int32)
135
+ v = (c[..., 0] | (c[..., 1] << 3) | (c[..., 2] << 6) | (c[..., 3] << 9)
136
+ | (c[..., 4] << 12) | (c[..., 5] << 15) | (c[..., 6] << 18) | (c[..., 7] << 21))
137
+ b0 = (v & 0xFF).to(torch.uint8)
138
+ b1 = ((v >> 8) & 0xFF).to(torch.uint8)
139
+ b2 = ((v >> 16) & 0xFF).to(torch.uint8)
140
+ return torch.stack([b0, b1, b2], dim=-1).reshape(nb, -1)
141
+
142
+
143
+ def unpack_3bit(packed: torch.Tensor, block: int) -> torch.Tensor:
144
+ """Inverse of :func:`pack_3bit` (returns (nblocks, block) uint8 codes).
145
+
146
+ ``packed`` may be the per-row 2-D layout (N, blocks_per_row*bytes) as well as the
147
+ flat (nblocks, bytes) layout; both are views of the same block sequence.
148
+ """
149
+ groups = block // 8
150
+ nb = int(packed.numel()) // 3 // groups
151
+ p = packed.reshape(nb, groups, 3).to(torch.int32)
152
+ v = p[..., 0] | (p[..., 1] << 8) | (p[..., 2] << 16)
153
+ return torch.stack([(v >> (3 * i)) & 7 for i in range(8)], dim=-1).reshape(nb, block).to(torch.uint8)
154
+
155
+
156
+ def _pack_2bit(codes: torch.Tensor) -> torch.Tensor:
157
+ nb, block = codes.shape
158
+ c = codes.view(nb, block // 4, 4).to(torch.int32)
159
+ v = c[..., 0] | (c[..., 1] << 2) | (c[..., 2] << 4) | (c[..., 3] << 6)
160
+ return v.to(torch.uint8).reshape(nb, -1)
161
+
162
+
163
+ def _unpack_2bit(packed: torch.Tensor, block: int) -> torch.Tensor:
164
+ groups = block // 4
165
+ nb = int(packed.numel()) // groups
166
+ v = packed.reshape(nb, groups).to(torch.int32)
167
+ return torch.stack([(v >> (2 * i)) & 3 for i in range(4)], dim=-1).reshape(nb, block).to(torch.uint8)
168
+
169
+
170
+ # ---------------------------------------------------------------------------
171
+ # quantise / dequantise
172
+ # ---------------------------------------------------------------------------
173
+ def _quantize(w: torch.Tensor, bits: int, block: int) -> Tuple[torch.Tensor, torch.Tensor, torch.Size, int]:
174
+ if block % (8 if bits == 3 else 4) != 0 or block <= 0:
175
+ raise ValueError("block must be a positive multiple of 8 for 3-bit / 4 for 2-bit")
176
+ x = w.detach().float().contiguous()
177
+ shape = x.shape
178
+ flat = x.reshape(-1)
179
+ n = flat.numel()
180
+ pad = (block - n % block) % block
181
+ if pad:
182
+ flat = torch.cat([flat, torch.zeros(pad, dtype=flat.dtype)])
183
+ nb = flat.numel() // block
184
+ blk = flat.view(nb, block)
185
+ absmax = blk.abs().amax(dim=1)
186
+ center = _CENTER[bits]
187
+ scale = (absmax / center).clamp_min(1e-12)
188
+ q = torch.round(blk / scale.unsqueeze(1) + center).clamp(0, _levels(bits) - 1).to(torch.uint8)
189
+ packed = pack_3bit(q) if bits == 3 else _pack_2bit(q)
190
+ absmax = absmax.float()
191
+ # For a 2-D weight whose last dimension is block-aligned, keep the per-row block
192
+ # layout (N, nblocks_per_row * bytes) so the fused GEMM can index rows directly.
193
+ if len(shape) == 2 and shape[1] % block == 0:
194
+ nb_per_row = shape[1] // block
195
+ bytes_per_block = packed.shape[1]
196
+ packed = packed.reshape(shape[0], nb_per_row * bytes_per_block)
197
+ absmax = absmax.reshape(shape[0], nb_per_row)
198
+ return packed, absmax, shape, n
199
+
200
+
201
+ def quantize_3bit(w: torch.Tensor, block: int = 64) -> Tuple[torch.Tensor, torch.Tensor]:
202
+ """Blockwise symmetric 3-bit quantisation.
203
+
204
+ Returns ``(packed_uint8, absmax_float32)``; the byte layout matches the C kernels
205
+ (3 bytes per 8 weights, one fp32 absmax per ``block`` weights).
206
+ """
207
+ packed, absmax, _, _ = _quantize(w, 3, block)
208
+ return packed, absmax
209
+
210
+
211
+ def quantize_2bit(w: torch.Tensor, block: int = 64) -> Tuple[torch.Tensor, torch.Tensor]:
212
+ """Experimental 2-bit variant (higher error, smaller memory; same layout rules)."""
213
+ packed, absmax, _, _ = _quantize(w, 2, block)
214
+ return packed, absmax
215
+
216
+
217
+ def _dequantize_torch(packed: torch.Tensor, absmax: torch.Tensor, shape, n: int,
218
+ bits: int, block: int) -> torch.Tensor:
219
+ # flatten: the 2-D per-row layout is just a view of the same block sequence
220
+ absmax_f = absmax.reshape(-1).to(torch.float32)
221
+ packed_f = packed.reshape(-1, packed.shape[-1])
222
+ codes = unpack_3bit(packed_f, block) if bits == 3 else _unpack_2bit(packed_f, block)
223
+ center = _CENTER[bits]
224
+ scale = absmax_f / center
225
+ vals = codes.float() * scale.unsqueeze(1) - absmax_f.unsqueeze(1)
226
+ return vals.reshape(-1)[:n].reshape(shape)
227
+
228
+
229
+ def dequantize_3bit(packed: torch.Tensor, absmax: torch.Tensor, shape,
230
+ block: int = 64, dtype: Optional[torch.dtype] = None) -> torch.Tensor:
231
+ """Unpack 3-bit codes back to a tensor of ``shape``.
232
+
233
+ The native kernel is used when available (exact same arithmetic); otherwise the
234
+ pure-torch unpack is used. ``packed``/``absmax`` are never modified.
235
+ """
236
+ n = 1
237
+ for s in shape:
238
+ n *= int(s)
239
+ packed = packed.contiguous()
240
+ absmax = absmax.contiguous()
241
+ nblocks = int(absmax.numel())
242
+ if native_available() and packed.is_contiguous() and absmax.dtype in (torch.float32,):
243
+ lib = _native()
244
+ # the kernel writes nblocks*block floats (the last block is padded)
245
+ out = torch.empty(nblocks * int(block), dtype=torch.float32)
246
+ rc = lib.cbdequantize_3bit_cpu(packed.data_ptr(), absmax.data_ptr(),
247
+ out.data_ptr(), nblocks, int(block))
248
+ if rc == 0:
249
+ return out[:n].reshape(shape).to(dtype or torch.float32)
250
+ out = _dequantize_torch(packed, absmax, shape, n, 3, block)
251
+ return out.to(dtype or torch.float32)
252
+
253
+
254
+ def dequantize_2bit(packed: torch.Tensor, absmax: torch.Tensor, shape,
255
+ block: int = 64, dtype: Optional[torch.dtype] = None) -> torch.Tensor:
256
+ n = 1
257
+ for s in shape:
258
+ n *= int(s)
259
+ packed = packed.contiguous()
260
+ absmax = absmax.contiguous()
261
+ nblocks = int(absmax.numel())
262
+ if native_available():
263
+ lib = _native()
264
+ out = torch.empty(nblocks * int(block), dtype=torch.float32)
265
+ rc = lib.cbdequantize_2bit_cpu(packed.data_ptr(), absmax.data_ptr(),
266
+ out.data_ptr(), nblocks, int(block))
267
+ if rc == 0:
268
+ return out[:n].reshape(shape).to(dtype or torch.float32)
269
+ out = _dequantize_torch(packed, absmax, shape, n, 2, block)
270
+ return out.to(dtype or torch.float32)
271
+
272
+
273
+ # ---------------------------------------------------------------------------
274
+ # GEMM
275
+ # ---------------------------------------------------------------------------
276
+ def _check_gemm_args(x: torch.Tensor, packed: torch.Tensor, absmax: torch.Tensor,
277
+ block: int) -> Tuple[int, int, torch.Tensor]:
278
+ """Validate args and normalise ``packed`` to the per-row (N, blocks*bytes) layout."""
279
+ if x.dim() != 2:
280
+ raise ValueError("x must be 2-D (M, K)")
281
+ if packed.dim() != 2:
282
+ raise ValueError("packed weight must be 2-D")
283
+ K = x.shape[1]
284
+ if K % block != 0:
285
+ raise ValueError("K (%d) must be divisible by block (%d)" % (K, block))
286
+ nb_per_row = K // block
287
+ bpb = _bytes_per_block(3, block)
288
+ bytes_per_row = nb_per_row * bpb
289
+ if packed.shape[1] == bytes_per_row:
290
+ pass # already per-row layout
291
+ elif packed.shape[1] == bpb:
292
+ # flat (nblocks, bytes) layout from an older/simple quantizer call
293
+ if packed.shape[0] % nb_per_row != 0:
294
+ raise ValueError("packed block count is not a multiple of blocks-per-row")
295
+ packed = packed.reshape(-1, bytes_per_row)
296
+ else:
297
+ raise ValueError("packed row width %d does not match K/block (=%d)"
298
+ % (packed.shape[1], bytes_per_row))
299
+ if int(absmax.numel()) != packed.shape[0] * nb_per_row:
300
+ raise ValueError("absmax count %%d does not match packed blocks")
301
+ if packed.shape[0] == 0:
302
+ raise ValueError("empty packed weight")
303
+ return packed.shape[0], K, packed
304
+
305
+
306
+ def gemm_3bit(x: torch.Tensor, packed: torch.Tensor, absmax: torch.Tensor,
307
+ block: int = 64, threads: int = 0, ntile: int = 64) -> torch.Tensor:
308
+ """``x @ dequant(packed).T`` without materialising the fp32 weight (native path).
309
+
310
+ ``x`` is (M, K), the packed weight is (N, K) row-blockwise 3-bit. The native kernel
311
+ is used when available (it never writes the fp32 weight); otherwise this falls back
312
+ to ``x @ dequantize_3bit(...).T`` in torch.
313
+ """
314
+ N, K, packed = _check_gemm_args(x, packed, absmax, block)
315
+ if native_available():
316
+ x = x.contiguous().float()
317
+ packed = packed.contiguous()
318
+ absmax = absmax.contiguous()
319
+ out = torch.empty(x.shape[0], N, dtype=torch.float32)
320
+ lib = _native()
321
+ rc = lib.cbq3bit_gemm_cpu(x.data_ptr(), int(x.shape[0]), packed.data_ptr(),
322
+ absmax.data_ptr(), out.data_ptr(), N, K, int(block),
323
+ int(threads), int(ntile))
324
+ if rc == 0:
325
+ return out
326
+ w = dequantize_3bit(packed, absmax, (N, K), block=block)
327
+ return x.float() @ w.t()
328
+
329
+
330
+ def _check_gemm_t_args(dy: torch.Tensor, packed: torch.Tensor, absmax: torch.Tensor,
331
+ block: int) -> Tuple[int, int, int, torch.Tensor]:
332
+ """Validate the transposed GEMM: here dy is (M, N) and K comes from the packed rows."""
333
+ if dy.dim() != 2:
334
+ raise ValueError("dy must be 2-D (M, N)")
335
+ if packed.dim() != 2:
336
+ raise ValueError("packed weight must be 2-D")
337
+ bpb = _bytes_per_block(3, block)
338
+ if packed.shape[1] % bpb != 0:
339
+ raise ValueError("packed row width is not a multiple of bytes-per-block")
340
+ nb_per_row = packed.shape[1] // bpb
341
+ K = nb_per_row * block
342
+ N = packed.shape[0]
343
+ if dy.shape[1] != N:
344
+ raise ValueError("dy.shape[1] (%d) != packed rows (%d)" % (dy.shape[1], N))
345
+ if int(absmax.numel()) != N * nb_per_row:
346
+ raise ValueError("absmax count does not match packed blocks")
347
+ return dy.shape[0], N, K, packed
348
+
349
+
350
+ def gemm_3bit_transposed(dy: torch.Tensor, packed: torch.Tensor, absmax: torch.Tensor,
351
+ block: int = 64, threads: int = 0, ntile: int = 64) -> torch.Tensor:
352
+ """``dy @ dequant(packed)`` (i.e. the transposed GEMM used by the backward pass)."""
353
+ M, N, K, packed = _check_gemm_t_args(dy, packed, absmax, block)
354
+ if native_available():
355
+ dy = dy.contiguous().float()
356
+ packed = packed.contiguous()
357
+ absmax = absmax.contiguous()
358
+ out = torch.empty(M, K, dtype=torch.float32)
359
+ lib = _native()
360
+ rc = lib.cbq3bit_gemm_t_cpu(dy.data_ptr(), int(M), packed.data_ptr(),
361
+ absmax.data_ptr(), out.data_ptr(), N, K, int(block),
362
+ int(threads), int(ntile))
363
+ if rc == 0:
364
+ return out
365
+ w = dequantize_3bit(packed, absmax, (N, K), block=block)
366
+ return dy.float() @ w
367
+
368
+
369
+ # ---------------------------------------------------------------------------
370
+ # module: frozen 3-bit base + trainable LoRA
371
+ # ---------------------------------------------------------------------------
372
+ class QuantLinear3bit(nn.Module):
373
+ """A frozen 3-bit ``nn.Linear`` replacement with trainable LoRA adapters.
374
+
375
+ The base matrix is stored as blockwise 3-bit codes (≈11% of fp32 memory) and is
376
+ frozen; only ``lora_A``/``lora_B`` are trainable. In grad mode the forward
377
+ dequantises the weight (native kernel when available) and runs a regular
378
+ matmul/``F.linear`` so autograd handles the input gradient and the LoRA
379
+ gradients; in ``no_grad`` mode on a native build it calls the fused 3-bit GEMM
380
+ and never materialises the fp32 weight.
381
+ """
382
+
383
+ def __init__(self, in_features: int, out_features: int, bias: bool = True,
384
+ block: int = 64, lora_r: int = 8, lora_alpha: int = 16,
385
+ dtype: torch.dtype = torch.float32, device=None):
386
+ super().__init__()
387
+ if block % 8 != 0:
388
+ raise ValueError("block must be a multiple of 8")
389
+ self.in_features = int(in_features)
390
+ self.out_features = int(out_features)
391
+ self.block = int(block)
392
+ self.bits = 3
393
+ # the buffers are filled by from_linear() / load_from_linear()
394
+ self.register_buffer("packed", None)
395
+ self.register_buffer("absmax", None)
396
+ self.bias = nn.Parameter(torch.zeros(self.out_features, dtype=dtype,
397
+ device=device), requires_grad=False) if bias else None
398
+ self.lora_A = nn.Parameter(torch.empty(lora_r, self.in_features, dtype=dtype,
399
+ device=device))
400
+ self.lora_B = nn.Parameter(torch.zeros(self.out_features, lora_r, dtype=dtype,
401
+ device=device))
402
+ nn.init.kaiming_uniform_(self.lora_A, a=5 ** 0.5)
403
+ self.scale = lora_alpha / lora_r
404
+
405
+ @classmethod
406
+ def from_linear(cls, linear: nn.Linear, block: int = 64, lora_r: int = 8,
407
+ lora_alpha: int = 16) -> "QuantLinear3bit":
408
+ """Build a :class:`QuantLinear3bit` from an existing ``nn.Linear``."""
409
+ m = cls(linear.in_features, linear.out_features, bias=linear.bias is not None,
410
+ block=block, lora_r=lora_r, lora_alpha=lora_alpha,
411
+ dtype=linear.weight.dtype, device=linear.weight.device)
412
+ m.load_from_linear(linear)
413
+ return m
414
+
415
+ def load_from_linear(self, linear: nn.Linear) -> None:
416
+ """Quantise ``linear.weight`` into this module and copy/freeze the bias."""
417
+ w = linear.weight.detach()
418
+ packed, absmax = quantize_3bit(w, self.block)
419
+ self.packed = packed
420
+ self.absmax = absmax
421
+ if linear.bias is not None:
422
+ with torch.no_grad():
423
+ self.bias.copy_(linear.bias.detach())
424
+
425
+ def weight(self) -> torch.Tensor:
426
+ """Dequantise the base weight (fp32) — allocates N*K floats."""
427
+ if self.packed is None:
428
+ raise RuntimeError("QuantLinear3bit has no packed weight; use from_linear()")
429
+ return dequantize_3bit(self.packed, self.absmax,
430
+ (self.out_features, self.in_features), block=self.block)
431
+
432
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
433
+ if self.packed is None:
434
+ raise RuntimeError("QuantLinear3bit has no packed weight; use from_linear()")
435
+ lead = x.shape[:-1]
436
+ x2 = x.reshape(-1, self.in_features)
437
+ base: torch.Tensor
438
+ if torch.is_grad_enabled():
439
+ # dequantise once (native kernel when available) then let autograd compute
440
+ # the input gradient through the frozen weight.
441
+ w = self.weight().to(x.dtype)
442
+ base = F.linear(x2, w, self.bias)
443
+ elif self.in_features % self.block == 0:
444
+ base = gemm_3bit(x2, self.packed, self.absmax, block=self.block)
445
+ if self.bias is not None:
446
+ base = base + self.bias
447
+ else:
448
+ w = self.weight().to(x.dtype)
449
+ base = F.linear(x2, w, self.bias)
450
+ lora = (x2 @ self.lora_A.t()) @ self.lora_B.t() * self.scale
451
+ return (base + lora).reshape(*lead, self.out_features)
452
+
453
+ def extra_repr(self) -> str:
454
+ return ("in_features=%d, out_features=%d, bits=3, block=%d, lora_r=%d"
455
+ % (self.in_features, self.out_features, self.block, self.lora_A.shape[0]))
456
+
457
+
458
+ def convert_linears_3bit(model: nn.Module, block: int = 64, lora_r: int = 8,
459
+ lora_alpha: int = 16, skip_names: Tuple[str, ...] = ("lm_head",),
460
+ name_prefix: str = "") -> int:
461
+ """Replace every ``nn.Linear`` child with :class:`QuantLinear3bit` (in place).
462
+
463
+ Layers whose *leaf* name is in ``skip_names`` are kept in fp32 (``lm_head`` by
464
+ default, because logits need the full unquantised matrix on CPU). Returns the
465
+ number of replaced layers; the base weights become frozen and only the LoRA
466
+ adapters are trainable.
467
+ """
468
+ n = 0
469
+ for name, child in list(model.named_children()):
470
+ full = f"{name_prefix}.{name}" if name_prefix else name
471
+ if isinstance(child, nn.Linear):
472
+ if name in skip_names:
473
+ continue
474
+ setattr(model, name, QuantLinear3bit.from_linear(
475
+ child, block=block, lora_r=lora_r, lora_alpha=lora_alpha))
476
+ n += 1
477
+ else:
478
+ n += convert_linears_3bit(child, block=block, lora_r=lora_r,
479
+ lora_alpha=lora_alpha, skip_names=skip_names,
480
+ name_prefix=full)
481
+ return n
bitsandbytes_cpu_cli.py CHANGED
@@ -86,6 +86,9 @@ SECTIONS = [
86
86
  "fused Gated DeltaNet for Qwen3-Next / Qwen3.5 linear attention", [], PY),
87
87
  ("disk_balancer",
88
88
  "spills cold parameters to another disk when RAM runs out", [], PY),
89
+ ("bitsandbytes.q3bit",
90
+ "frozen 3-bit weights + LoRA (~11% of fp32 memory); native kernels with a "
91
+ "pure-torch fallback", [], PY),
89
92
  ("bitsandbytes-cpu",
90
93
  "this command; pip also installs it as bnb-cpu", [], PY),
91
94
  ]),
@@ -273,6 +276,8 @@ SECTIONS = [
273
276
  ("what you get",
274
277
  "1.031 bytes of state per parameter, against 2.062 for the 8-bit optimizer "
275
278
  "and 8.0 for torch AdamW, all measured at 1M parameters", [
279
+ "import torch, torch.nn as nn",
280
+ "from bitsandbytes.optim import AdamW4bit",
276
281
  "p = nn.Parameter(torch.randn(4096))",
277
282
  "opt = AdamW4bit([p], lr=1e-3)",
278
283
  "p.grad = torch.randn_like(p)",
@@ -294,29 +299,30 @@ SECTIONS = [
294
299
  ("does it train as well as fp32",
295
300
  "it tracks fp32 rather than matching it: quantised state cannot reproduce an "
296
301
  "fp32 trajectory, and that is not the criterion", [
297
- "# measured over 120 steps on the real 30.49M image-AR model:",
298
- "# the two loss curves stay within 0.27% of each other",
299
- "# on a 200-step MLP run the 4-bit run ended at 0.0198 against fp32's",
300
- "# 0.0240 -- neither worse nor identical, simply a different path",
301
- "# the kernel's state codes are identical to an independent numpy",
302
- "# implementation over 50 steps, so the maths agrees exactly",
303
- ], PY),
304
- ("the step is faster, unlike 8-bit",
305
- "0.658 ms against 1.682 ms for the 8-bit kernel at 1M parameters (2.56x), "
306
- "because it moves half the bytes", [
307
- "# both are AVX2 kernels in the same library, so the comparison isolates",
308
- "# state width rather than implementation language",
309
- "# the scalar fallback is 7.99 ms -- the AVX2 path is 12x faster than it",
302
+ "# dev2 measured two loss curves within 0.27% over 120 steps on the real",
303
+ "# 30.49M image-AR model",
304
+ "# our 20-step 1024-param check: parameter relL2 1.2e-3 vs torch AdamW",
305
+ "# the restored scalar kernel matches the pure-torch fallback to 1.2e-7",
306
+ "# over 10 steps (test_adamw4bit.py), so the maths agrees exactly",
307
+ ], PY),
308
+ ("the step is slower, so use it for memory not speed",
309
+ "scalar 4-bit kernel: 28.6 ms vs 3.3 ms (8-bit AVX2) and 8.2 ms (fp32 torch) "
310
+ "per step at 1M params, measured on the R5-4500U with 6 threads", [
311
+ "# the dev2 AVX2 4-bit path produced wrong codes and stayed disabled;",
312
+ "# what runs today is the verified scalar path",
313
+ "# the optimizer is a few percent of a real training step at 30M params,",
314
+ "# so the wall-clock cost is small while the state saving is 7.8x",
315
+ "# older Windows wheels run the pure-torch fallback (dev9+ is native)",
310
316
  ], PY),
311
317
  ("traps",
312
318
  "the failure modes seen while building this, each one silent at first", [
313
319
  "# 1. torch.optim.AdamW is NOT replaced: this is a separate class, and",
314
320
  "# AdamW4bit requires fp32 params (the kernel's dtype 0).",
315
321
  "# 2. blocksize is compiled in at 256; passing another value raises.",
316
- "# 3. if the shared library does not export",
317
- "# coptimizer_update_4bit_blockwise_cpu, import fails loudly rather",
318
- "# than falling back -- three earlier versions of the search silently",
319
- "# picked a DLL without the kernel and only failed at the first step.",
322
+ "# 3. older Windows wheels (< dev9) do not export",
323
+ "# coptimizer_update_4bit_blockwise_cpu (pre-dev9 Windows wheels),",
324
+ "# this class transparently uses a pure-torch implementation with the",
325
+ "# same numbers and the same 1.031 B/param state, just slower.",
320
326
  "# 4. quantised state is not a drop-in for exact reproducibility runs:",
321
327
  "# same seed, same data, different trajectory from fp32.",
322
328
  ], PY),
@@ -518,6 +524,50 @@ SECTIONS = [
518
524
  " ...",
519
525
  ], PY),
520
526
  ]),
527
+ ("q3bit", "3-bit frozen base + LoRA (experimental)", [
528
+ ("quantize_3bit(w, block=64)",
529
+ "returns (packed uint8, absmax): blockwise symmetric 3-bit codes, 3 bytes per 8 "
530
+ "weights; the layout is the one the C kernels use", [
531
+ "import torch",
532
+ "from bitsandbytes import q3bit",
533
+ "w = torch.randn(256, 256)",
534
+ "packed, absmax = q3bit.quantize_3bit(w, block=64)",
535
+ "back = q3bit.dequantize_3bit(packed, absmax, w.shape)",
536
+ "rel = ((w - back).norm() / w.norm()).item()",
537
+ "assert rel < 0.30, rel",
538
+ ], PY),
539
+ ("dequantize_3bit(packed, absmax, shape, block=64)",
540
+ "uses the native AVX2 kernel when the loaded library exports it, else the torch "
541
+ "unpack; the two agree to float rounding", [], PY),
542
+ ("gemm_3bit(A, packed, absmax, block=64)",
543
+ "A @ dequant(B).T without writing the fp32 weight on the native path; K % block "
544
+ "must be 0", [
545
+ "import torch",
546
+ "from bitsandbytes import q3bit",
547
+ "w = torch.randn(128, 128)",
548
+ "packed, absmax = q3bit.quantize_3bit(w)",
549
+ "A = torch.randn(4, 128)",
550
+ "out = q3bit.gemm_3bit(A, packed, absmax)",
551
+ "ref = A @ q3bit.dequantize_3bit(packed, absmax, w.shape).t()",
552
+ "assert (out - ref).abs().max().item() < 1e-5",
553
+ ], PY),
554
+ ("QuantLinear3bit.from_linear(linear, lora_r=8)",
555
+ "frozen 3-bit base + trainable LoRA; only lora_A/lora_B require grad", [
556
+ "import torch, torch.nn as nn",
557
+ "from bitsandbytes import q3bit",
558
+ "lin = nn.Linear(128, 96)",
559
+ "ql = q3bit.QuantLinear3bit.from_linear(lin, lora_r=4)",
560
+ "y = ql(torch.randn(2, 128))",
561
+ "assert y.shape == (2, 96)",
562
+ "assert [n for n, p in ql.named_parameters() if p.requires_grad] == "
563
+ "['lora_A', 'lora_B']",
564
+ ], PY),
565
+ ("convert_linears_3bit(model, lora_r=8)",
566
+ "replaces every nn.Linear in place and returns the count; lm_head stays fp32", []),
567
+ ("experimental / portability",
568
+ "since dev9 the Windows wheel ships a CI-built MSVC DLL; older wheels use the pure-torch "
569
+ "fallback, while the dev9 Windows and Linux wheels use the native kernels", []),
570
+ ]),
521
571
  ("layers", "Reference: bitsandbytes.nn", [
522
572
  ("Linear4bit(input_features, output_features, bias=True, compute_dtype=None,\n"
523
573
  " compress_statistics=True, quant_type='fp4',\n"