bitsandbytes-cpu-fork 0.50.2.dev9__py3-none-win_amd64.whl → 0.50.2.dev11__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. bitsandbytes/__init__.py +84 -84
  2. bitsandbytes/__main__.py +4 -4
  3. bitsandbytes/_cpu_caps.py +110 -110
  4. bitsandbytes/_ops.py +526 -526
  5. bitsandbytes/autograd/_functions.py +491 -491
  6. bitsandbytes/backends/cpu/ops.py +819 -819
  7. bitsandbytes/backends/cuda/ops.py +1212 -1212
  8. bitsandbytes/backends/default/ops.py +632 -632
  9. bitsandbytes/backends/hpu/ops.py +53 -53
  10. bitsandbytes/backends/mps/ops.py +277 -277
  11. bitsandbytes/backends/triton/kernels_4bit.py +583 -583
  12. bitsandbytes/backends/triton/kernels_8bit_quant.py +195 -195
  13. bitsandbytes/backends/triton/kernels_optim.py +1177 -1177
  14. bitsandbytes/backends/triton/ops.py +304 -304
  15. bitsandbytes/backends/utils.py +94 -94
  16. bitsandbytes/backends/xpu/ops.py +305 -305
  17. bitsandbytes/cextension.py +447 -447
  18. bitsandbytes/cli.py +50 -50
  19. bitsandbytes/consts.py +12 -12
  20. bitsandbytes/cuda_specs.py +111 -111
  21. bitsandbytes/detect_cpu.py +189 -189
  22. bitsandbytes/diagnostics/cuda.py +193 -193
  23. bitsandbytes/diagnostics/main.py +134 -134
  24. bitsandbytes/diagnostics/utils.py +12 -12
  25. bitsandbytes/disk_balancer.py +1687 -1687
  26. bitsandbytes/efst.py +782 -782
  27. bitsandbytes/functional.py +1836 -1836
  28. bitsandbytes/fused/__init__.py +21 -0
  29. bitsandbytes/fused/_native.py +66 -0
  30. bitsandbytes/fused/_ref.py +62 -0
  31. bitsandbytes/gdn_cpu.py +616 -616
  32. bitsandbytes/libbitsandbytes_cpu.dll +0 -0
  33. bitsandbytes/nn/__init__.py +19 -19
  34. bitsandbytes/nn/modules.py +1220 -1220
  35. bitsandbytes/nn/parametrize.py +206 -206
  36. bitsandbytes/optim/__init__.py +26 -26
  37. bitsandbytes/optim/adagrad.py +187 -187
  38. bitsandbytes/optim/adam.py +346 -346
  39. bitsandbytes/optim/adamw.py +337 -337
  40. bitsandbytes/optim/adamw4bit.py +239 -239
  41. bitsandbytes/optim/ademamix.py +410 -410
  42. bitsandbytes/optim/lamb.py +190 -190
  43. bitsandbytes/optim/lars.py +259 -259
  44. bitsandbytes/optim/lion.py +266 -266
  45. bitsandbytes/optim/optimizer.py +756 -756
  46. bitsandbytes/optim/rmsprop.py +170 -170
  47. bitsandbytes/optim/sgd.py +152 -152
  48. bitsandbytes/q3bit.py +487 -481
  49. bitsandbytes/torch_cpu_kit.py +794 -794
  50. bitsandbytes/utils.py +208 -208
  51. bitsandbytes_cpu_cli.py +1062 -1062
  52. {bitsandbytes_cpu_fork-0.50.2.dev9.dist-info → bitsandbytes_cpu_fork-0.50.2.dev11.dist-info}/METADATA +73 -11
  53. bitsandbytes_cpu_fork-0.50.2.dev11.dist-info/RECORD +69 -0
  54. {bitsandbytes_cpu_fork-0.50.2.dev9.dist-info → bitsandbytes_cpu_fork-0.50.2.dev11.dist-info}/licenses/LICENSE +21 -21
  55. bitsandbytes_cpu_fork-0.50.2.dev9.dist-info/RECORD +0 -66
  56. {bitsandbytes_cpu_fork-0.50.2.dev9.dist-info → bitsandbytes_cpu_fork-0.50.2.dev11.dist-info}/WHEEL +0 -0
  57. {bitsandbytes_cpu_fork-0.50.2.dev9.dist-info → bitsandbytes_cpu_fork-0.50.2.dev11.dist-info}/entry_points.txt +0 -0
  58. {bitsandbytes_cpu_fork-0.50.2.dev9.dist-info → bitsandbytes_cpu_fork-0.50.2.dev11.dist-info}/top_level.txt +0 -0
bitsandbytes/_ops.py CHANGED
@@ -1,526 +1,526 @@
1
- from collections.abc import Sequence
2
- from typing import Optional
3
-
4
- import torch
5
-
6
- register_fake = torch.library.register_fake
7
- register_kernel = torch.library.register_kernel
8
-
9
- # Int8 mixed precision matmul + dequant + bias
10
- torch.library.define(
11
- "bitsandbytes::int8_mixed_scaled_mm",
12
- "(Tensor A, Tensor CA, Tensor CB, Tensor SCA, Tensor SCB, Tensor? outlier_cols=None, Tensor? bias=None) -> (Tensor, Tensor?)",
13
- )
14
-
15
-
16
- @register_fake("bitsandbytes::int8_mixed_scaled_mm")
17
- def _(
18
- A: torch.Tensor,
19
- CA: torch.Tensor,
20
- CB: torch.Tensor,
21
- SCA: torch.Tensor,
22
- SCB: torch.Tensor,
23
- outlier_cols: Optional[torch.Tensor] = None,
24
- bias: Optional[torch.Tensor] = None,
25
- ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
26
- shapeC = (*CA.shape[:-1], CB.shape[0])
27
-
28
- out = torch.empty(shapeC, device=A.device, dtype=A.dtype)
29
-
30
- outlier_cols = torch.library.get_ctx().new_dynamic_size()
31
- subA = A.new_empty(outlier_cols, dtype=torch.int64)
32
-
33
- return out, subA
34
-
35
-
36
- # Higher level op: int8 matmul + dequant + bias
37
- torch.library.define(
38
- "bitsandbytes::int8_scaled_mm",
39
- "(Tensor A, Tensor B, Tensor row_stats, Tensor col_stats, Tensor? bias=None, ScalarType? dtype=None) -> Tensor",
40
- )
41
-
42
-
43
- @register_fake("bitsandbytes::int8_scaled_mm")
44
- def _(
45
- A: torch.Tensor,
46
- B: torch.Tensor,
47
- row_stats: torch.Tensor,
48
- col_stats: torch.Tensor,
49
- bias: Optional[torch.Tensor] = None,
50
- dtype: Optional[torch.dtype] = None,
51
- ) -> torch.Tensor:
52
- shapeC = (*A.shape[:-1], B.shape[0])
53
- return torch.empty(shapeC, device=A.device, dtype=dtype or torch.float16)
54
-
55
-
56
- torch.library.define(
57
- "bitsandbytes::int8_linear_matmul",
58
- "(Tensor A, Tensor B) -> Tensor",
59
- )
60
-
61
-
62
- @register_fake("bitsandbytes::int8_linear_matmul")
63
- def _(A: torch.Tensor, B: torch.Tensor):
64
- torch._check(A.dtype == torch.int8, lambda: "A must be int8")
65
- torch._check(B.dtype == torch.int8, lambda: "B must be int8")
66
- shapeC = (*A.shape[:-1], B.shape[0])
67
- return torch.empty(shapeC, device=A.device, dtype=torch.int32)
68
-
69
-
70
- # More info on `out` overloads:
71
- # https://github.com/pytorch/pytorch/issues/125044
72
- torch.library.define(
73
- "bitsandbytes::int8_linear_matmul.out",
74
- "(Tensor A, Tensor B, Tensor! out) -> ()",
75
- )
76
-
77
-
78
- @register_fake("bitsandbytes::int8_linear_matmul.out")
79
- def _(A: torch.Tensor, B: torch.Tensor, out: torch.Tensor):
80
- shapeC = (*A.shape[:-1], B.shape[0])
81
-
82
- torch._check(A.dtype == torch.int8, lambda: "A must be int8")
83
- torch._check(B.dtype == torch.int8, lambda: "B must be int8")
84
- torch._check(out.shape == shapeC, lambda: f"Expected out.shape == {shapeC}, got {out.shape}")
85
- torch._check(out.device == A.device, lambda: f"Expected out.device == {A.device}, got {out.device}")
86
- torch._check(out.dtype == torch.int32, lambda: f"Expected out.dtype == int32, got {out.dtype}")
87
-
88
-
89
- torch.library.define(
90
- "bitsandbytes::int8_vectorwise_quant",
91
- "(Tensor A, float threshold=0.0) -> (Tensor, Tensor, Tensor?)",
92
- )
93
-
94
-
95
- @register_fake("bitsandbytes::int8_vectorwise_quant")
96
- def _(A: torch.Tensor, threshold=0.0):
97
- out_row = torch.empty(A.shape, device=A.device, dtype=torch.int8)
98
- row_stats = torch.empty(A.numel() // A.shape[-1], device=A.device, dtype=torch.float32)
99
-
100
- if threshold == 0.0:
101
- return out_row, row_stats, None
102
-
103
- outlier_cols = torch.library.get_ctx().new_dynamic_size()
104
-
105
- return out_row, row_stats, A.new_empty(outlier_cols, dtype=torch.int64)
106
-
107
-
108
- torch.library.define("bitsandbytes::int8_vectorwise_dequant", "(Tensor A, Tensor stats) -> Tensor")
109
-
110
-
111
- @register_fake("bitsandbytes::int8_vectorwise_dequant")
112
- def _(A: torch.Tensor, stats: torch.Tensor) -> torch.Tensor:
113
- torch._check(A.dtype == torch.int8, lambda: "A must be int8")
114
- return torch.empty_like(A, dtype=torch.float32)
115
-
116
-
117
- # Default PyTorch-native implementation
118
- @register_kernel("bitsandbytes::int8_vectorwise_dequant", "default")
119
- def _(A: torch.Tensor, stats: torch.Tensor):
120
- # To dequantize we divide by 127, or multiply by the reciprocal.
121
- return A * stats.view(-1, 1) * 7.874015718698502e-3
122
-
123
-
124
- torch.library.define(
125
- "bitsandbytes::int8_mm_dequant",
126
- "(Tensor A, Tensor row_stats, Tensor col_stats, ScalarType? dtype=None, Tensor? bias=None) -> Tensor",
127
- )
128
-
129
-
130
- @register_fake("bitsandbytes::int8_mm_dequant")
131
- def _(
132
- A: torch.Tensor,
133
- row_stats: torch.Tensor,
134
- col_stats: torch.Tensor,
135
- dtype: Optional[torch.dtype] = None,
136
- bias: Optional[torch.Tensor] = None,
137
- ) -> torch.Tensor:
138
- torch._check(A.dtype == torch.int32, lambda: "A must be int32")
139
- return torch.empty_like(A, dtype=dtype or torch.float16)
140
-
141
-
142
- torch.library.define(
143
- "bitsandbytes::int8_double_quant",
144
- "(Tensor A, float threshold=0.0) -> (Tensor, Tensor, Tensor, Tensor, Tensor?)",
145
- )
146
-
147
-
148
- @register_fake("bitsandbytes::int8_double_quant")
149
- def _(
150
- A: torch.Tensor,
151
- threshold=0.0,
152
- ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
153
- out_row = torch.empty_like(A, dtype=torch.int8)
154
- out_col = torch.empty_like(A, dtype=torch.int8)
155
- row_stats = torch.empty(A.numel() // A.shape[-1], device=A.device, dtype=torch.float32)
156
- col_stats = torch.empty(A.shape[-1], device=A.device, dtype=torch.float32)
157
- outlier_n = torch.library.get_ctx().new_dynamic_size()
158
- outlier_cols = A.new_empty(outlier_n, dtype=torch.int64)
159
- return out_row, out_col, row_stats, col_stats, outlier_cols
160
-
161
-
162
- torch.library.define(
163
- "bitsandbytes::dequantize_4bit",
164
- "(Tensor A, Tensor absmax, int blocksize, str quant_type, int[] shape, ScalarType dtype) -> Tensor",
165
- )
166
-
167
-
168
- @register_fake("bitsandbytes::dequantize_4bit")
169
- def _(
170
- A: torch.Tensor,
171
- absmax: torch.Tensor,
172
- blocksize: int,
173
- quant_type: str,
174
- shape: Sequence[int],
175
- dtype: torch.dtype,
176
- ) -> torch.Tensor:
177
- torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
178
- torch._check(quant_type in ("nf4", "fp4"), lambda: f"quant_type must be 'nf4' or 'fp4', got {quant_type!r}")
179
- torch._check(absmax.dtype == torch.float32, lambda: f"absmax must be float32, got {absmax.dtype}")
180
- torch._check(
181
- dtype in (torch.float16, torch.bfloat16, torch.float32),
182
- lambda: f"Blockwise 4bit dequantization only supports 16/32-bit floats, but got {dtype}",
183
- )
184
- return torch.empty(shape, dtype=dtype, device=A.device)
185
-
186
-
187
- torch.library.define(
188
- "bitsandbytes::dequantize_4bit.out",
189
- "(Tensor A, Tensor absmax, int blocksize, str quant_type, int[] shape, ScalarType dtype, Tensor! out) -> ()",
190
- )
191
-
192
-
193
- @register_fake("bitsandbytes::dequantize_4bit.out")
194
- def _(
195
- A: torch.Tensor,
196
- absmax: torch.Tensor,
197
- blocksize: int,
198
- quant_type: str,
199
- shape: Sequence[int],
200
- dtype: torch.dtype,
201
- out: torch.Tensor,
202
- ) -> None:
203
- torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
204
- torch._check(quant_type in ("nf4", "fp4"), lambda: f"quant_type must be 'nf4' or 'fp4', got {quant_type!r}")
205
- torch._check(absmax.dtype == torch.float32, lambda: f"absmax must be float32, got {absmax.dtype}")
206
- torch._check(
207
- dtype in (torch.float16, torch.bfloat16, torch.float32),
208
- lambda: f"Blockwise 4bit dequantization only supports 16/32-bit floats, but got {dtype}",
209
- )
210
- torch._check(out.shape == shape, lambda: f"Expected out.shape == {shape}, got {out.shape}")
211
- torch._check(out.device == A.device, lambda: f"Expected out.device == {A.device}, got {out.device}")
212
- torch._check(out.dtype == dtype, lambda: f"Expected out.dtype == {dtype}, got {out.dtype}")
213
-
214
-
215
- torch.library.define(
216
- "bitsandbytes::quantize_4bit",
217
- "(Tensor A, int blocksize, str quant_type, ScalarType quant_storage) -> (Tensor, Tensor)",
218
- )
219
-
220
-
221
- @register_fake("bitsandbytes::quantize_4bit")
222
- def _(
223
- A: torch.Tensor, blocksize: int, quant_type: str, quant_storage: torch.dtype
224
- ) -> tuple[torch.Tensor, torch.Tensor]:
225
- torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
226
- torch._check(quant_type in ("nf4", "fp4"), lambda: f"quant_type must be 'nf4' or 'fp4', got {quant_type!r}")
227
- torch._check(
228
- A.dtype in (torch.float16, torch.bfloat16, torch.float32),
229
- lambda: f"Blockwise 4bit quantization only supports 16/32-bit floats, but got {A.dtype}",
230
- )
231
-
232
- n = A.numel()
233
- blocks = -(n // -blocksize)
234
- absmax = torch.empty((blocks,), device=A.device, dtype=torch.float32)
235
- out = torch.empty(((n + 1) // (quant_storage.itemsize * 2), 1), device=A.device, dtype=quant_storage)
236
- return out, absmax
237
-
238
-
239
- torch.library.define(
240
- "bitsandbytes::gemm_4bit",
241
- "(Tensor A, Tensor B, int[] shapeB, Tensor absmax, int blocksize, str quant_type, "
242
- "Tensor? bias=None, Tensor? absmax_8bit=None, Tensor? absmax_code=None, Tensor? absmax_offset=None) -> Tensor",
243
- )
244
-
245
-
246
- @register_fake("bitsandbytes::gemm_4bit")
247
- def _(
248
- A: torch.Tensor,
249
- B: torch.Tensor,
250
- shapeB: Sequence[int],
251
- absmax: torch.Tensor,
252
- blocksize: int,
253
- quant_type: str,
254
- bias: Optional[torch.Tensor] = None,
255
- absmax_8bit: Optional[torch.Tensor] = None,
256
- absmax_code: Optional[torch.Tensor] = None,
257
- absmax_offset: Optional[torch.Tensor] = None,
258
- ) -> torch.Tensor:
259
- torch._check(len(shapeB) == 2, lambda: f"shapeB must be 2D [N, K], got {list(shapeB)}")
260
- torch._check(A.shape[-1] == shapeB[1], lambda: f"A inner dim ({A.shape[-1]}) must match shapeB ({shapeB[1]})")
261
- torch._check(
262
- A.dtype in (torch.float16, torch.bfloat16, torch.float32),
263
- lambda: f"A must be float16, bfloat16, or float32, got {A.dtype}",
264
- )
265
- torch._check(
266
- B.dtype in (torch.uint8, torch.bfloat16, torch.float16, torch.float32),
267
- lambda: f"B must be backed by storage of type uint8, bfloat16, float16, or float32, got {B.dtype}",
268
- )
269
- torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
270
- torch._check(quant_type in ("nf4", "fp4"), lambda: f"quant_type must be 'nf4' or 'fp4', got {quant_type!r}")
271
- torch._check(absmax.dtype == torch.float32, lambda: f"absmax must be float32, got {absmax.dtype}")
272
- if absmax_8bit is not None:
273
- torch._check(absmax_8bit.ndim == 1, lambda: f"absmax_8bit must be 1D, got {absmax_8bit.ndim}D")
274
- torch._check(absmax_8bit.dtype == torch.uint8, lambda: f"absmax_8bit must be uint8, got {absmax_8bit.dtype}")
275
- torch._check(absmax_code is not None, lambda: "absmax_code required when absmax_8bit is provided")
276
- torch._check(absmax_code.ndim == 1, lambda: f"absmax_code must be 1D, got {absmax_code.ndim}D")
277
- torch._check(
278
- absmax_code.shape[0] == 256, lambda: f"absmax_code must have 256 entries, got {absmax_code.shape[0]}"
279
- )
280
- torch._check(
281
- absmax_code.dtype == torch.float32, lambda: f"absmax_code must be float32, got {absmax_code.dtype}"
282
- )
283
- torch._check(absmax_offset is not None, lambda: "absmax_offset required when absmax_8bit is provided")
284
- torch._check(
285
- absmax_offset.ndim == 0, lambda: f"absmax_offset must be a scalar (0-dim), got {absmax_offset.ndim}D"
286
- )
287
- torch._check(
288
- absmax_offset.dtype == torch.float32, lambda: f"absmax_offset must be float32, got {absmax_offset.dtype}"
289
- )
290
- if bias is not None:
291
- torch._check(bias.ndim == 1, lambda: f"bias must be 1D, got {bias.ndim}D")
292
- torch._check(bias.shape[0] == shapeB[0], lambda: f"bias length ({bias.shape[0]}) must match N ({shapeB[0]})")
293
- torch._check(bias.dtype == A.dtype, lambda: f"bias dtype ({bias.dtype}) must match A dtype ({A.dtype})")
294
- N = shapeB[0]
295
- return torch.empty((*A.shape[:-1], N), dtype=A.dtype, device=A.device)
296
-
297
-
298
- torch.library.define(
299
- "bitsandbytes::dequantize_blockwise",
300
- "(Tensor A, Tensor absmax, Tensor code, int blocksize, ScalarType dtype) -> Tensor",
301
- )
302
-
303
-
304
- @register_fake("bitsandbytes::dequantize_blockwise")
305
- def _(A: torch.Tensor, absmax: torch.Tensor, code: torch.Tensor, blocksize: int, dtype: torch.dtype) -> torch.Tensor:
306
- torch._check(blocksize > 0, lambda: f"blocksize must be positive, got {blocksize}")
307
- torch._check(A.dtype == torch.uint8, lambda: f"A must be uint8, got {A.dtype}")
308
- torch._check(
309
- dtype in (torch.float16, torch.bfloat16, torch.float32),
310
- lambda: f"Blockwise dequantization only supports 16/32-bit floats, but got {dtype}",
311
- )
312
- return torch.empty_like(A, dtype=dtype)
313
-
314
-
315
- torch.library.define(
316
- "bitsandbytes::dequantize_blockwise.out",
317
- "(Tensor A, Tensor absmax, Tensor code, int blocksize, ScalarType dtype, Tensor! out) -> ()",
318
- )
319
-
320
-
321
- @register_fake("bitsandbytes::dequantize_blockwise.out")
322
- def _(
323
- A: torch.Tensor, absmax: torch.Tensor, code: torch.Tensor, blocksize: int, dtype: torch.dtype, out: torch.Tensor
324
- ):
325
- torch._check(blocksize > 0, lambda: f"blocksize must be positive, got {blocksize}")
326
- torch._check(A.dtype == torch.uint8, lambda: f"A must be uint8, got {A.dtype}")
327
- torch._check(
328
- dtype in (torch.float16, torch.bfloat16, torch.float32),
329
- lambda: f"Blockwise dequantization only supports 16/32-bit floats, but got {dtype}",
330
- )
331
- torch._check(out.shape == A.shape, lambda: f"Expected out.shape == {A.shape}, got {out.shape}")
332
- torch._check(out.device == A.device, lambda: f"Expected out.device == {A.device}, got {out.device}")
333
- torch._check(out.dtype == dtype, lambda: f"Expected out.dtype == {dtype}, got {out.dtype}")
334
-
335
-
336
- torch.library.define("bitsandbytes::quantize_blockwise", "(Tensor A, Tensor code, int blocksize) -> (Tensor, Tensor)")
337
-
338
-
339
- @register_fake("bitsandbytes::quantize_blockwise")
340
- def _(A: torch.Tensor, code: torch.Tensor, blocksize: int) -> tuple[torch.Tensor, torch.Tensor]:
341
- torch._check(blocksize > 0, lambda: f"blocksize must be positive, got {blocksize}")
342
- torch._check(
343
- A.dtype in (torch.float16, torch.bfloat16, torch.float32),
344
- lambda: f"Blockwise quantization only supports 16/32-bit floats, but got {A.dtype}",
345
- )
346
- n = A.numel()
347
- blocks = -(n // -blocksize)
348
- absmax = torch.empty((blocks,), device=A.device, dtype=torch.float32)
349
- out = torch.empty_like(A, dtype=torch.uint8)
350
- return out, absmax
351
-
352
-
353
- torch.library.define(
354
- "bitsandbytes::gemm_8bit",
355
- "(Tensor A, Tensor B, Tensor absmax, int blocksize) -> Tensor",
356
- )
357
-
358
-
359
- @register_fake("bitsandbytes::gemm_8bit")
360
- def _(A: torch.Tensor, B: torch.Tensor, absmax: torch.Tensor, blocksize: int) -> torch.Tensor:
361
- torch._check(blocksize > 0, lambda: f"blocksize must be positive, got {blocksize}")
362
- torch._check(A.dtype == torch.float32, lambda: f"A must be float32, got {A.dtype}")
363
- torch._check(B.dtype == torch.uint8, lambda: f"B must be uint8, got {B.dtype}")
364
- torch._check(absmax.dtype == torch.float32, lambda: f"absmax must be float32, got {absmax.dtype}")
365
- N = B.shape[0]
366
- return torch.empty((*A.shape[:-1], N), device=A.device, dtype=A.dtype)
367
-
368
-
369
- torch.library.define(
370
- "bitsandbytes::gemv_4bit",
371
- "(Tensor A, Tensor B, int[] shapeB, Tensor absmax, Tensor code, int blocksize) -> Tensor",
372
- )
373
-
374
-
375
- @register_fake("bitsandbytes::gemv_4bit")
376
- def _(
377
- A: torch.Tensor, B: torch.Tensor, shapeB: Sequence[int], absmax: torch.Tensor, code: torch.Tensor, blocksize: int
378
- ) -> torch.Tensor:
379
- torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
380
- torch._check(
381
- A.dtype in (torch.float16, torch.bfloat16, torch.float32),
382
- lambda: f"A must be float16, bfloat16, or float32, got {A.dtype}",
383
- )
384
- torch._check(
385
- B.dtype in (torch.uint8, torch.bfloat16, torch.float16, torch.float32),
386
- lambda: f"B must be backed by storage of type uint8, bfloat16, float16, or float32, got {B.dtype}",
387
- )
388
- shape = (*A.shape[:-1], shapeB[0])
389
- return torch.empty(shape, device=A.device, dtype=A.dtype)
390
-
391
-
392
- torch.library.define(
393
- "bitsandbytes::gemv_4bit.out",
394
- "(Tensor A, Tensor B, int[] shapeB, Tensor absmax, Tensor code, int blocksize, Tensor! out) -> ()",
395
- )
396
-
397
-
398
- @register_fake("bitsandbytes::gemv_4bit.out")
399
- def _(
400
- A: torch.Tensor,
401
- B: torch.Tensor,
402
- shapeB: Sequence[int],
403
- absmax: torch.Tensor,
404
- code: torch.Tensor,
405
- blocksize: int,
406
- out: torch.Tensor,
407
- ) -> None:
408
- torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
409
- torch._check(
410
- A.dtype in (torch.float16, torch.bfloat16, torch.float32),
411
- lambda: f"A must be float16, bfloat16, or float32, got {A.dtype}",
412
- )
413
- torch._check(
414
- B.dtype in (torch.uint8, torch.bfloat16, torch.float16, torch.float32),
415
- lambda: f"B must be backed by storage of type uint8, bfloat16, float16, or float32, got {B.dtype}",
416
- )
417
- torch._check(
418
- out.shape == (*A.shape[:-1], shapeB[0]),
419
- lambda: f"Expected out.shape == {(*A.shape[:-1], shapeB[0])}, got {out.shape}",
420
- )
421
- torch._check(out.device == A.device, lambda: f"Expected out.device == {A.device}, got {out.device}")
422
- torch._check(out.dtype == A.dtype, lambda: f"Expected out.dtype == {A.dtype}, got {out.dtype}")
423
-
424
-
425
- torch.library.define(
426
- "bitsandbytes::optimizer_update_32bit",
427
- "(str optimizer_name, Tensor(a0!) g, Tensor(a1!) p, Tensor(a2!) state1, Tensor(a3!)? state2, Tensor(a4!)? unorm_vec, float max_unorm, float param_norm, float beta1, float beta2, float beta3, float alpha, float eps, float weight_decay, int step, float lr, float gnorm_scale, bool skip_zeros=False) -> ()",
428
- )
429
-
430
-
431
- @register_fake("bitsandbytes::optimizer_update_32bit")
432
- def _(
433
- optimizer_name: str,
434
- g: torch.Tensor,
435
- p: torch.Tensor,
436
- state1: torch.Tensor,
437
- state2: Optional[torch.Tensor],
438
- unorm_vec: Optional[torch.Tensor],
439
- max_unorm: float,
440
- param_norm: float,
441
- beta1: float,
442
- beta2: float,
443
- beta3: float,
444
- alpha: float,
445
- eps: float,
446
- weight_decay: float,
447
- step: int,
448
- lr: float,
449
- gnorm_scale: float,
450
- skip_zeros=False,
451
- ) -> None:
452
- torch._check(
453
- g.numel() == p.numel(),
454
- lambda: f"g and p must have the same number of elements, got {g.numel()} and {p.numel()}",
455
- )
456
- compute_dtypes = [torch.float16, torch.bfloat16, torch.float32]
457
-
458
- torch._check(
459
- g.dtype in compute_dtypes,
460
- lambda: f"g must be bfloat16, float16, or float32, got {g.dtype}",
461
- )
462
- torch._check(
463
- g.dtype == p.dtype,
464
- lambda: f"Expected all tensors to have the same dtype, got g.dtype={g.dtype}, p.dtype={p.dtype}",
465
- )
466
-
467
-
468
- torch.library.define(
469
- "bitsandbytes::optimizer_update_8bit_blockwise",
470
- "(str optimizer_name, Tensor(a0!) g, Tensor(a1!) p, Tensor(a2!) state1, Tensor(a3!)? state2, float beta1, float beta2, float beta3, float alpha, float eps, int step, float lr, Tensor(a4!) qmap1, Tensor(a5!)? qmap2, Tensor(a6!) absmax1, Tensor(a7!)? absmax2, float weight_decay, float gnorm_scale, bool skip_zeros=False) -> ()",
471
- )
472
-
473
-
474
- @register_fake("bitsandbytes::optimizer_update_8bit_blockwise")
475
- def _(
476
- optimizer_name: str,
477
- g: torch.Tensor,
478
- p: torch.Tensor,
479
- state1: torch.Tensor,
480
- state2: Optional[torch.Tensor],
481
- beta1: float,
482
- beta2: float,
483
- beta3: float,
484
- alpha: float,
485
- eps: float,
486
- step: int,
487
- lr: float,
488
- qmap1: torch.Tensor,
489
- qmap2: Optional[torch.Tensor],
490
- absmax1: torch.Tensor,
491
- absmax2: Optional[torch.Tensor],
492
- weight_decay: float,
493
- gnorm_scale: float,
494
- skip_zeros=False,
495
- ) -> None:
496
- torch._check(
497
- g.numel() == p.numel(),
498
- lambda: f"g and p must have the same number of elements, got {g.numel()} and {p.numel()}",
499
- )
500
- compute_dtypes = [torch.float16, torch.bfloat16, torch.float32]
501
-
502
- torch._check(
503
- g.dtype in compute_dtypes,
504
- lambda: f"g must be bfloat16, float16, or float32, got {g.dtype}",
505
- )
506
- torch._check(
507
- g.dtype == p.dtype,
508
- lambda: f"Expected all tensors to have the same dtype, got g.dtype={g.dtype}, p.dtype={p.dtype}",
509
- )
510
- torch._check(
511
- state1.dtype == torch.uint8,
512
- lambda: f"state1 must be uint8, got {state1.dtype}",
513
- )
514
- torch._check(
515
- qmap1.dtype == absmax1.dtype == torch.float32,
516
- lambda: f"Expected qmap1 and absmax1 to be float32, got qmap1.dtype={qmap1.dtype}, absmax1.dtype={absmax1.dtype}",
517
- )
518
- if state2 is not None:
519
- torch._check(
520
- state2.dtype == torch.uint8,
521
- lambda: f"state2 must be uint8, got {state2.dtype}",
522
- )
523
- torch._check(
524
- qmap2.dtype == absmax2.dtype == torch.float32,
525
- lambda: f"Expected qmap2 and absmax2 to be float32, got qmap2.dtype={qmap2.dtype}, absmax2.dtype={absmax2.dtype}",
526
- )
1
+ from collections.abc import Sequence
2
+ from typing import Optional
3
+
4
+ import torch
5
+
6
+ register_fake = torch.library.register_fake
7
+ register_kernel = torch.library.register_kernel
8
+
9
+ # Int8 mixed precision matmul + dequant + bias
10
+ torch.library.define(
11
+ "bitsandbytes::int8_mixed_scaled_mm",
12
+ "(Tensor A, Tensor CA, Tensor CB, Tensor SCA, Tensor SCB, Tensor? outlier_cols=None, Tensor? bias=None) -> (Tensor, Tensor?)",
13
+ )
14
+
15
+
16
+ @register_fake("bitsandbytes::int8_mixed_scaled_mm")
17
+ def _(
18
+ A: torch.Tensor,
19
+ CA: torch.Tensor,
20
+ CB: torch.Tensor,
21
+ SCA: torch.Tensor,
22
+ SCB: torch.Tensor,
23
+ outlier_cols: Optional[torch.Tensor] = None,
24
+ bias: Optional[torch.Tensor] = None,
25
+ ) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
26
+ shapeC = (*CA.shape[:-1], CB.shape[0])
27
+
28
+ out = torch.empty(shapeC, device=A.device, dtype=A.dtype)
29
+
30
+ outlier_cols = torch.library.get_ctx().new_dynamic_size()
31
+ subA = A.new_empty(outlier_cols, dtype=torch.int64)
32
+
33
+ return out, subA
34
+
35
+
36
+ # Higher level op: int8 matmul + dequant + bias
37
+ torch.library.define(
38
+ "bitsandbytes::int8_scaled_mm",
39
+ "(Tensor A, Tensor B, Tensor row_stats, Tensor col_stats, Tensor? bias=None, ScalarType? dtype=None) -> Tensor",
40
+ )
41
+
42
+
43
+ @register_fake("bitsandbytes::int8_scaled_mm")
44
+ def _(
45
+ A: torch.Tensor,
46
+ B: torch.Tensor,
47
+ row_stats: torch.Tensor,
48
+ col_stats: torch.Tensor,
49
+ bias: Optional[torch.Tensor] = None,
50
+ dtype: Optional[torch.dtype] = None,
51
+ ) -> torch.Tensor:
52
+ shapeC = (*A.shape[:-1], B.shape[0])
53
+ return torch.empty(shapeC, device=A.device, dtype=dtype or torch.float16)
54
+
55
+
56
+ torch.library.define(
57
+ "bitsandbytes::int8_linear_matmul",
58
+ "(Tensor A, Tensor B) -> Tensor",
59
+ )
60
+
61
+
62
+ @register_fake("bitsandbytes::int8_linear_matmul")
63
+ def _(A: torch.Tensor, B: torch.Tensor):
64
+ torch._check(A.dtype == torch.int8, lambda: "A must be int8")
65
+ torch._check(B.dtype == torch.int8, lambda: "B must be int8")
66
+ shapeC = (*A.shape[:-1], B.shape[0])
67
+ return torch.empty(shapeC, device=A.device, dtype=torch.int32)
68
+
69
+
70
+ # More info on `out` overloads:
71
+ # https://github.com/pytorch/pytorch/issues/125044
72
+ torch.library.define(
73
+ "bitsandbytes::int8_linear_matmul.out",
74
+ "(Tensor A, Tensor B, Tensor! out) -> ()",
75
+ )
76
+
77
+
78
+ @register_fake("bitsandbytes::int8_linear_matmul.out")
79
+ def _(A: torch.Tensor, B: torch.Tensor, out: torch.Tensor):
80
+ shapeC = (*A.shape[:-1], B.shape[0])
81
+
82
+ torch._check(A.dtype == torch.int8, lambda: "A must be int8")
83
+ torch._check(B.dtype == torch.int8, lambda: "B must be int8")
84
+ torch._check(out.shape == shapeC, lambda: f"Expected out.shape == {shapeC}, got {out.shape}")
85
+ torch._check(out.device == A.device, lambda: f"Expected out.device == {A.device}, got {out.device}")
86
+ torch._check(out.dtype == torch.int32, lambda: f"Expected out.dtype == int32, got {out.dtype}")
87
+
88
+
89
+ torch.library.define(
90
+ "bitsandbytes::int8_vectorwise_quant",
91
+ "(Tensor A, float threshold=0.0) -> (Tensor, Tensor, Tensor?)",
92
+ )
93
+
94
+
95
+ @register_fake("bitsandbytes::int8_vectorwise_quant")
96
+ def _(A: torch.Tensor, threshold=0.0):
97
+ out_row = torch.empty(A.shape, device=A.device, dtype=torch.int8)
98
+ row_stats = torch.empty(A.numel() // A.shape[-1], device=A.device, dtype=torch.float32)
99
+
100
+ if threshold == 0.0:
101
+ return out_row, row_stats, None
102
+
103
+ outlier_cols = torch.library.get_ctx().new_dynamic_size()
104
+
105
+ return out_row, row_stats, A.new_empty(outlier_cols, dtype=torch.int64)
106
+
107
+
108
+ torch.library.define("bitsandbytes::int8_vectorwise_dequant", "(Tensor A, Tensor stats) -> Tensor")
109
+
110
+
111
+ @register_fake("bitsandbytes::int8_vectorwise_dequant")
112
+ def _(A: torch.Tensor, stats: torch.Tensor) -> torch.Tensor:
113
+ torch._check(A.dtype == torch.int8, lambda: "A must be int8")
114
+ return torch.empty_like(A, dtype=torch.float32)
115
+
116
+
117
+ # Default PyTorch-native implementation
118
+ @register_kernel("bitsandbytes::int8_vectorwise_dequant", "default")
119
+ def _(A: torch.Tensor, stats: torch.Tensor):
120
+ # To dequantize we divide by 127, or multiply by the reciprocal.
121
+ return A * stats.view(-1, 1) * 7.874015718698502e-3
122
+
123
+
124
+ torch.library.define(
125
+ "bitsandbytes::int8_mm_dequant",
126
+ "(Tensor A, Tensor row_stats, Tensor col_stats, ScalarType? dtype=None, Tensor? bias=None) -> Tensor",
127
+ )
128
+
129
+
130
+ @register_fake("bitsandbytes::int8_mm_dequant")
131
+ def _(
132
+ A: torch.Tensor,
133
+ row_stats: torch.Tensor,
134
+ col_stats: torch.Tensor,
135
+ dtype: Optional[torch.dtype] = None,
136
+ bias: Optional[torch.Tensor] = None,
137
+ ) -> torch.Tensor:
138
+ torch._check(A.dtype == torch.int32, lambda: "A must be int32")
139
+ return torch.empty_like(A, dtype=dtype or torch.float16)
140
+
141
+
142
+ torch.library.define(
143
+ "bitsandbytes::int8_double_quant",
144
+ "(Tensor A, float threshold=0.0) -> (Tensor, Tensor, Tensor, Tensor, Tensor?)",
145
+ )
146
+
147
+
148
+ @register_fake("bitsandbytes::int8_double_quant")
149
+ def _(
150
+ A: torch.Tensor,
151
+ threshold=0.0,
152
+ ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
153
+ out_row = torch.empty_like(A, dtype=torch.int8)
154
+ out_col = torch.empty_like(A, dtype=torch.int8)
155
+ row_stats = torch.empty(A.numel() // A.shape[-1], device=A.device, dtype=torch.float32)
156
+ col_stats = torch.empty(A.shape[-1], device=A.device, dtype=torch.float32)
157
+ outlier_n = torch.library.get_ctx().new_dynamic_size()
158
+ outlier_cols = A.new_empty(outlier_n, dtype=torch.int64)
159
+ return out_row, out_col, row_stats, col_stats, outlier_cols
160
+
161
+
162
+ torch.library.define(
163
+ "bitsandbytes::dequantize_4bit",
164
+ "(Tensor A, Tensor absmax, int blocksize, str quant_type, int[] shape, ScalarType dtype) -> Tensor",
165
+ )
166
+
167
+
168
+ @register_fake("bitsandbytes::dequantize_4bit")
169
+ def _(
170
+ A: torch.Tensor,
171
+ absmax: torch.Tensor,
172
+ blocksize: int,
173
+ quant_type: str,
174
+ shape: Sequence[int],
175
+ dtype: torch.dtype,
176
+ ) -> torch.Tensor:
177
+ torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
178
+ torch._check(quant_type in ("nf4", "fp4"), lambda: f"quant_type must be 'nf4' or 'fp4', got {quant_type!r}")
179
+ torch._check(absmax.dtype == torch.float32, lambda: f"absmax must be float32, got {absmax.dtype}")
180
+ torch._check(
181
+ dtype in (torch.float16, torch.bfloat16, torch.float32),
182
+ lambda: f"Blockwise 4bit dequantization only supports 16/32-bit floats, but got {dtype}",
183
+ )
184
+ return torch.empty(shape, dtype=dtype, device=A.device)
185
+
186
+
187
+ torch.library.define(
188
+ "bitsandbytes::dequantize_4bit.out",
189
+ "(Tensor A, Tensor absmax, int blocksize, str quant_type, int[] shape, ScalarType dtype, Tensor! out) -> ()",
190
+ )
191
+
192
+
193
+ @register_fake("bitsandbytes::dequantize_4bit.out")
194
+ def _(
195
+ A: torch.Tensor,
196
+ absmax: torch.Tensor,
197
+ blocksize: int,
198
+ quant_type: str,
199
+ shape: Sequence[int],
200
+ dtype: torch.dtype,
201
+ out: torch.Tensor,
202
+ ) -> None:
203
+ torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
204
+ torch._check(quant_type in ("nf4", "fp4"), lambda: f"quant_type must be 'nf4' or 'fp4', got {quant_type!r}")
205
+ torch._check(absmax.dtype == torch.float32, lambda: f"absmax must be float32, got {absmax.dtype}")
206
+ torch._check(
207
+ dtype in (torch.float16, torch.bfloat16, torch.float32),
208
+ lambda: f"Blockwise 4bit dequantization only supports 16/32-bit floats, but got {dtype}",
209
+ )
210
+ torch._check(out.shape == shape, lambda: f"Expected out.shape == {shape}, got {out.shape}")
211
+ torch._check(out.device == A.device, lambda: f"Expected out.device == {A.device}, got {out.device}")
212
+ torch._check(out.dtype == dtype, lambda: f"Expected out.dtype == {dtype}, got {out.dtype}")
213
+
214
+
215
+ torch.library.define(
216
+ "bitsandbytes::quantize_4bit",
217
+ "(Tensor A, int blocksize, str quant_type, ScalarType quant_storage) -> (Tensor, Tensor)",
218
+ )
219
+
220
+
221
+ @register_fake("bitsandbytes::quantize_4bit")
222
+ def _(
223
+ A: torch.Tensor, blocksize: int, quant_type: str, quant_storage: torch.dtype
224
+ ) -> tuple[torch.Tensor, torch.Tensor]:
225
+ torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
226
+ torch._check(quant_type in ("nf4", "fp4"), lambda: f"quant_type must be 'nf4' or 'fp4', got {quant_type!r}")
227
+ torch._check(
228
+ A.dtype in (torch.float16, torch.bfloat16, torch.float32),
229
+ lambda: f"Blockwise 4bit quantization only supports 16/32-bit floats, but got {A.dtype}",
230
+ )
231
+
232
+ n = A.numel()
233
+ blocks = -(n // -blocksize)
234
+ absmax = torch.empty((blocks,), device=A.device, dtype=torch.float32)
235
+ out = torch.empty(((n + 1) // (quant_storage.itemsize * 2), 1), device=A.device, dtype=quant_storage)
236
+ return out, absmax
237
+
238
+
239
+ torch.library.define(
240
+ "bitsandbytes::gemm_4bit",
241
+ "(Tensor A, Tensor B, int[] shapeB, Tensor absmax, int blocksize, str quant_type, "
242
+ "Tensor? bias=None, Tensor? absmax_8bit=None, Tensor? absmax_code=None, Tensor? absmax_offset=None) -> Tensor",
243
+ )
244
+
245
+
246
+ @register_fake("bitsandbytes::gemm_4bit")
247
+ def _(
248
+ A: torch.Tensor,
249
+ B: torch.Tensor,
250
+ shapeB: Sequence[int],
251
+ absmax: torch.Tensor,
252
+ blocksize: int,
253
+ quant_type: str,
254
+ bias: Optional[torch.Tensor] = None,
255
+ absmax_8bit: Optional[torch.Tensor] = None,
256
+ absmax_code: Optional[torch.Tensor] = None,
257
+ absmax_offset: Optional[torch.Tensor] = None,
258
+ ) -> torch.Tensor:
259
+ torch._check(len(shapeB) == 2, lambda: f"shapeB must be 2D [N, K], got {list(shapeB)}")
260
+ torch._check(A.shape[-1] == shapeB[1], lambda: f"A inner dim ({A.shape[-1]}) must match shapeB ({shapeB[1]})")
261
+ torch._check(
262
+ A.dtype in (torch.float16, torch.bfloat16, torch.float32),
263
+ lambda: f"A must be float16, bfloat16, or float32, got {A.dtype}",
264
+ )
265
+ torch._check(
266
+ B.dtype in (torch.uint8, torch.bfloat16, torch.float16, torch.float32),
267
+ lambda: f"B must be backed by storage of type uint8, bfloat16, float16, or float32, got {B.dtype}",
268
+ )
269
+ torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
270
+ torch._check(quant_type in ("nf4", "fp4"), lambda: f"quant_type must be 'nf4' or 'fp4', got {quant_type!r}")
271
+ torch._check(absmax.dtype == torch.float32, lambda: f"absmax must be float32, got {absmax.dtype}")
272
+ if absmax_8bit is not None:
273
+ torch._check(absmax_8bit.ndim == 1, lambda: f"absmax_8bit must be 1D, got {absmax_8bit.ndim}D")
274
+ torch._check(absmax_8bit.dtype == torch.uint8, lambda: f"absmax_8bit must be uint8, got {absmax_8bit.dtype}")
275
+ torch._check(absmax_code is not None, lambda: "absmax_code required when absmax_8bit is provided")
276
+ torch._check(absmax_code.ndim == 1, lambda: f"absmax_code must be 1D, got {absmax_code.ndim}D")
277
+ torch._check(
278
+ absmax_code.shape[0] == 256, lambda: f"absmax_code must have 256 entries, got {absmax_code.shape[0]}"
279
+ )
280
+ torch._check(
281
+ absmax_code.dtype == torch.float32, lambda: f"absmax_code must be float32, got {absmax_code.dtype}"
282
+ )
283
+ torch._check(absmax_offset is not None, lambda: "absmax_offset required when absmax_8bit is provided")
284
+ torch._check(
285
+ absmax_offset.ndim == 0, lambda: f"absmax_offset must be a scalar (0-dim), got {absmax_offset.ndim}D"
286
+ )
287
+ torch._check(
288
+ absmax_offset.dtype == torch.float32, lambda: f"absmax_offset must be float32, got {absmax_offset.dtype}"
289
+ )
290
+ if bias is not None:
291
+ torch._check(bias.ndim == 1, lambda: f"bias must be 1D, got {bias.ndim}D")
292
+ torch._check(bias.shape[0] == shapeB[0], lambda: f"bias length ({bias.shape[0]}) must match N ({shapeB[0]})")
293
+ torch._check(bias.dtype == A.dtype, lambda: f"bias dtype ({bias.dtype}) must match A dtype ({A.dtype})")
294
+ N = shapeB[0]
295
+ return torch.empty((*A.shape[:-1], N), dtype=A.dtype, device=A.device)
296
+
297
+
298
+ torch.library.define(
299
+ "bitsandbytes::dequantize_blockwise",
300
+ "(Tensor A, Tensor absmax, Tensor code, int blocksize, ScalarType dtype) -> Tensor",
301
+ )
302
+
303
+
304
+ @register_fake("bitsandbytes::dequantize_blockwise")
305
+ def _(A: torch.Tensor, absmax: torch.Tensor, code: torch.Tensor, blocksize: int, dtype: torch.dtype) -> torch.Tensor:
306
+ torch._check(blocksize > 0, lambda: f"blocksize must be positive, got {blocksize}")
307
+ torch._check(A.dtype == torch.uint8, lambda: f"A must be uint8, got {A.dtype}")
308
+ torch._check(
309
+ dtype in (torch.float16, torch.bfloat16, torch.float32),
310
+ lambda: f"Blockwise dequantization only supports 16/32-bit floats, but got {dtype}",
311
+ )
312
+ return torch.empty_like(A, dtype=dtype)
313
+
314
+
315
+ torch.library.define(
316
+ "bitsandbytes::dequantize_blockwise.out",
317
+ "(Tensor A, Tensor absmax, Tensor code, int blocksize, ScalarType dtype, Tensor! out) -> ()",
318
+ )
319
+
320
+
321
+ @register_fake("bitsandbytes::dequantize_blockwise.out")
322
+ def _(
323
+ A: torch.Tensor, absmax: torch.Tensor, code: torch.Tensor, blocksize: int, dtype: torch.dtype, out: torch.Tensor
324
+ ):
325
+ torch._check(blocksize > 0, lambda: f"blocksize must be positive, got {blocksize}")
326
+ torch._check(A.dtype == torch.uint8, lambda: f"A must be uint8, got {A.dtype}")
327
+ torch._check(
328
+ dtype in (torch.float16, torch.bfloat16, torch.float32),
329
+ lambda: f"Blockwise dequantization only supports 16/32-bit floats, but got {dtype}",
330
+ )
331
+ torch._check(out.shape == A.shape, lambda: f"Expected out.shape == {A.shape}, got {out.shape}")
332
+ torch._check(out.device == A.device, lambda: f"Expected out.device == {A.device}, got {out.device}")
333
+ torch._check(out.dtype == dtype, lambda: f"Expected out.dtype == {dtype}, got {out.dtype}")
334
+
335
+
336
+ torch.library.define("bitsandbytes::quantize_blockwise", "(Tensor A, Tensor code, int blocksize) -> (Tensor, Tensor)")
337
+
338
+
339
+ @register_fake("bitsandbytes::quantize_blockwise")
340
+ def _(A: torch.Tensor, code: torch.Tensor, blocksize: int) -> tuple[torch.Tensor, torch.Tensor]:
341
+ torch._check(blocksize > 0, lambda: f"blocksize must be positive, got {blocksize}")
342
+ torch._check(
343
+ A.dtype in (torch.float16, torch.bfloat16, torch.float32),
344
+ lambda: f"Blockwise quantization only supports 16/32-bit floats, but got {A.dtype}",
345
+ )
346
+ n = A.numel()
347
+ blocks = -(n // -blocksize)
348
+ absmax = torch.empty((blocks,), device=A.device, dtype=torch.float32)
349
+ out = torch.empty_like(A, dtype=torch.uint8)
350
+ return out, absmax
351
+
352
+
353
+ torch.library.define(
354
+ "bitsandbytes::gemm_8bit",
355
+ "(Tensor A, Tensor B, Tensor absmax, int blocksize) -> Tensor",
356
+ )
357
+
358
+
359
+ @register_fake("bitsandbytes::gemm_8bit")
360
+ def _(A: torch.Tensor, B: torch.Tensor, absmax: torch.Tensor, blocksize: int) -> torch.Tensor:
361
+ torch._check(blocksize > 0, lambda: f"blocksize must be positive, got {blocksize}")
362
+ torch._check(A.dtype == torch.float32, lambda: f"A must be float32, got {A.dtype}")
363
+ torch._check(B.dtype == torch.uint8, lambda: f"B must be uint8, got {B.dtype}")
364
+ torch._check(absmax.dtype == torch.float32, lambda: f"absmax must be float32, got {absmax.dtype}")
365
+ N = B.shape[0]
366
+ return torch.empty((*A.shape[:-1], N), device=A.device, dtype=A.dtype)
367
+
368
+
369
+ torch.library.define(
370
+ "bitsandbytes::gemv_4bit",
371
+ "(Tensor A, Tensor B, int[] shapeB, Tensor absmax, Tensor code, int blocksize) -> Tensor",
372
+ )
373
+
374
+
375
+ @register_fake("bitsandbytes::gemv_4bit")
376
+ def _(
377
+ A: torch.Tensor, B: torch.Tensor, shapeB: Sequence[int], absmax: torch.Tensor, code: torch.Tensor, blocksize: int
378
+ ) -> torch.Tensor:
379
+ torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
380
+ torch._check(
381
+ A.dtype in (torch.float16, torch.bfloat16, torch.float32),
382
+ lambda: f"A must be float16, bfloat16, or float32, got {A.dtype}",
383
+ )
384
+ torch._check(
385
+ B.dtype in (torch.uint8, torch.bfloat16, torch.float16, torch.float32),
386
+ lambda: f"B must be backed by storage of type uint8, bfloat16, float16, or float32, got {B.dtype}",
387
+ )
388
+ shape = (*A.shape[:-1], shapeB[0])
389
+ return torch.empty(shape, device=A.device, dtype=A.dtype)
390
+
391
+
392
+ torch.library.define(
393
+ "bitsandbytes::gemv_4bit.out",
394
+ "(Tensor A, Tensor B, int[] shapeB, Tensor absmax, Tensor code, int blocksize, Tensor! out) -> ()",
395
+ )
396
+
397
+
398
+ @register_fake("bitsandbytes::gemv_4bit.out")
399
+ def _(
400
+ A: torch.Tensor,
401
+ B: torch.Tensor,
402
+ shapeB: Sequence[int],
403
+ absmax: torch.Tensor,
404
+ code: torch.Tensor,
405
+ blocksize: int,
406
+ out: torch.Tensor,
407
+ ) -> None:
408
+ torch._check(blocksize in (32, 64, 128, 256, 512, 1024, 2048, 4096), lambda: f"invalid blocksize {blocksize}")
409
+ torch._check(
410
+ A.dtype in (torch.float16, torch.bfloat16, torch.float32),
411
+ lambda: f"A must be float16, bfloat16, or float32, got {A.dtype}",
412
+ )
413
+ torch._check(
414
+ B.dtype in (torch.uint8, torch.bfloat16, torch.float16, torch.float32),
415
+ lambda: f"B must be backed by storage of type uint8, bfloat16, float16, or float32, got {B.dtype}",
416
+ )
417
+ torch._check(
418
+ out.shape == (*A.shape[:-1], shapeB[0]),
419
+ lambda: f"Expected out.shape == {(*A.shape[:-1], shapeB[0])}, got {out.shape}",
420
+ )
421
+ torch._check(out.device == A.device, lambda: f"Expected out.device == {A.device}, got {out.device}")
422
+ torch._check(out.dtype == A.dtype, lambda: f"Expected out.dtype == {A.dtype}, got {out.dtype}")
423
+
424
+
425
+ torch.library.define(
426
+ "bitsandbytes::optimizer_update_32bit",
427
+ "(str optimizer_name, Tensor(a0!) g, Tensor(a1!) p, Tensor(a2!) state1, Tensor(a3!)? state2, Tensor(a4!)? unorm_vec, float max_unorm, float param_norm, float beta1, float beta2, float beta3, float alpha, float eps, float weight_decay, int step, float lr, float gnorm_scale, bool skip_zeros=False) -> ()",
428
+ )
429
+
430
+
431
+ @register_fake("bitsandbytes::optimizer_update_32bit")
432
+ def _(
433
+ optimizer_name: str,
434
+ g: torch.Tensor,
435
+ p: torch.Tensor,
436
+ state1: torch.Tensor,
437
+ state2: Optional[torch.Tensor],
438
+ unorm_vec: Optional[torch.Tensor],
439
+ max_unorm: float,
440
+ param_norm: float,
441
+ beta1: float,
442
+ beta2: float,
443
+ beta3: float,
444
+ alpha: float,
445
+ eps: float,
446
+ weight_decay: float,
447
+ step: int,
448
+ lr: float,
449
+ gnorm_scale: float,
450
+ skip_zeros=False,
451
+ ) -> None:
452
+ torch._check(
453
+ g.numel() == p.numel(),
454
+ lambda: f"g and p must have the same number of elements, got {g.numel()} and {p.numel()}",
455
+ )
456
+ compute_dtypes = [torch.float16, torch.bfloat16, torch.float32]
457
+
458
+ torch._check(
459
+ g.dtype in compute_dtypes,
460
+ lambda: f"g must be bfloat16, float16, or float32, got {g.dtype}",
461
+ )
462
+ torch._check(
463
+ g.dtype == p.dtype,
464
+ lambda: f"Expected all tensors to have the same dtype, got g.dtype={g.dtype}, p.dtype={p.dtype}",
465
+ )
466
+
467
+
468
+ torch.library.define(
469
+ "bitsandbytes::optimizer_update_8bit_blockwise",
470
+ "(str optimizer_name, Tensor(a0!) g, Tensor(a1!) p, Tensor(a2!) state1, Tensor(a3!)? state2, float beta1, float beta2, float beta3, float alpha, float eps, int step, float lr, Tensor(a4!) qmap1, Tensor(a5!)? qmap2, Tensor(a6!) absmax1, Tensor(a7!)? absmax2, float weight_decay, float gnorm_scale, bool skip_zeros=False) -> ()",
471
+ )
472
+
473
+
474
+ @register_fake("bitsandbytes::optimizer_update_8bit_blockwise")
475
+ def _(
476
+ optimizer_name: str,
477
+ g: torch.Tensor,
478
+ p: torch.Tensor,
479
+ state1: torch.Tensor,
480
+ state2: Optional[torch.Tensor],
481
+ beta1: float,
482
+ beta2: float,
483
+ beta3: float,
484
+ alpha: float,
485
+ eps: float,
486
+ step: int,
487
+ lr: float,
488
+ qmap1: torch.Tensor,
489
+ qmap2: Optional[torch.Tensor],
490
+ absmax1: torch.Tensor,
491
+ absmax2: Optional[torch.Tensor],
492
+ weight_decay: float,
493
+ gnorm_scale: float,
494
+ skip_zeros=False,
495
+ ) -> None:
496
+ torch._check(
497
+ g.numel() == p.numel(),
498
+ lambda: f"g and p must have the same number of elements, got {g.numel()} and {p.numel()}",
499
+ )
500
+ compute_dtypes = [torch.float16, torch.bfloat16, torch.float32]
501
+
502
+ torch._check(
503
+ g.dtype in compute_dtypes,
504
+ lambda: f"g must be bfloat16, float16, or float32, got {g.dtype}",
505
+ )
506
+ torch._check(
507
+ g.dtype == p.dtype,
508
+ lambda: f"Expected all tensors to have the same dtype, got g.dtype={g.dtype}, p.dtype={p.dtype}",
509
+ )
510
+ torch._check(
511
+ state1.dtype == torch.uint8,
512
+ lambda: f"state1 must be uint8, got {state1.dtype}",
513
+ )
514
+ torch._check(
515
+ qmap1.dtype == absmax1.dtype == torch.float32,
516
+ lambda: f"Expected qmap1 and absmax1 to be float32, got qmap1.dtype={qmap1.dtype}, absmax1.dtype={absmax1.dtype}",
517
+ )
518
+ if state2 is not None:
519
+ torch._check(
520
+ state2.dtype == torch.uint8,
521
+ lambda: f"state2 must be uint8, got {state2.dtype}",
522
+ )
523
+ torch._check(
524
+ qmap2.dtype == absmax2.dtype == torch.float32,
525
+ lambda: f"Expected qmap2 and absmax2 to be float32, got qmap2.dtype={qmap2.dtype}, absmax2.dtype={absmax2.dtype}",
526
+ )