bitsandbytes-cpu-fork 0.50.2.dev10__tar.gz → 0.50.2.dev12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/MANIFEST.in +19 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10/bitsandbytes_cpu_fork.egg-info → bitsandbytes_cpu_fork-0.50.2.dev12}/PKG-INFO +149 -9
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/README.md +148 -8
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/__init__.py +1 -1
- bitsandbytes_cpu_fork-0.50.2.dev12/bitsandbytes/libbitsandbytes_cpu.dll +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/nn/modules.py +89 -10
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12/bitsandbytes_cpu_fork.egg-info}/PKG-INFO +149 -9
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/cpu_cache.h +160 -15
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/cpu_ops.cpp +69 -13
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/test_cpu_cache.cpp +1 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/API_REFERENCE.md +1 -1
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/API_REFERENCE_EN.md +1 -1
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/MOTION_BOUNDARY.md +3 -3
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/MOTION_BOUNDARY_EN.md +3 -3
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/QUICKSTART.md +51 -20
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/QUICKSTART_EN.md +56 -20
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/TECHNICAL_GUIDE.md +17 -15
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/TECHNICAL_GUIDE_EN.md +17 -15
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/TECH_REPORT.md +28 -8
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/TECH_REPORT_EN.md +33 -8
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/check_docs.py +29 -6
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_linear4bit.py +56 -0
- bitsandbytes_cpu_fork-0.50.2.dev10/bitsandbytes/libbitsandbytes_cpu.dll +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/LICENSE +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/__main__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/_cpu_caps.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/_ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/autograd/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/autograd/_functions.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/cpu/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/cpu/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/cuda/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/cuda/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/default/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/default/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/hpu/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/hpu/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/mps/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/mps/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/triton/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/triton/kernels_4bit.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/triton/kernels_8bit_quant.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/triton/kernels_optim.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/triton/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/utils.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/xpu/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/backends/xpu/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/cextension.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/cli.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/consts.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/cuda_specs.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/detect_cpu.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/diagnostics/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/diagnostics/cuda.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/diagnostics/main.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/diagnostics/utils.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/disk_balancer.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/efst.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/functional.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/fused/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/fused/_native.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/fused/_ref.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/gdn_cpu.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/nn/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/nn/parametrize.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/adagrad.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/adam.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/adamw.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/adamw4bit.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/ademamix.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/lamb.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/lars.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/lion.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/optimizer.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/rmsprop.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/optim/sgd.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/py.typed +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/q3bit.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/torch_cpu_kit.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/utils.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/vcomp140.dll +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes_cpu_cli.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes_cpu_fork.egg-info/SOURCES.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes_cpu_fork.egg-info/dependency_links.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes_cpu_fork.egg-info/entry_points.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes_cpu_fork.egg-info/requires.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes_cpu_fork.egg-info/top_level.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/build_linux.sh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/build_manual/build_manual.bat +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/build_manual/export.def +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/build_manual/export_avx512.def +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/build_manual/export_generic.def +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/build_manual/selftest_win.bat +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/build_termux.sh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/avx2_gemv_4bit.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/common.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/common.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/compat.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/compat_device.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/cpu_features.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/cpu_gdn.cpp +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/cpu_ops.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/gemm_4bit.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/gemm_4bit_common.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/gemm_4bit_mma.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/gemm_4bit_simt.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/gemm_4bit_simt.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/gemm_4bit_sm75.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/gemm_4bit_sm75.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/gemm_4bit_sm80.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/gemm_4bit_sm80.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/kernels.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/kernels.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/ops.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/ops.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/pythonInterface.cpp +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/test_avx2_gemv_4bit.cpp +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/xpu_kernels.cpp +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/xpu_kernels.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/xpu_ops.cpp +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/csrc/xpu_ops.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/EPYC_VALIDATION.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/ISA_MATRIX.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/docs_cpu/WHEEL_PLATFORM_NOTES.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/examples/compile_inference.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/examples/cpu/cpu_training.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/examples/int8_inference_huggingface.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/examples/xpu/benchmark_paged_memory.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/examples/xpu/paged_xpu_training.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/pyproject.toml +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/selftest_cpu.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/setup.cfg +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/setup.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_autograd.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_cpu_e2e.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_cuda_setup_evaluator.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_functional.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_generation.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_linear8bitlt.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_modules.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_optim.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_parametrize.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tests/test_rmsnorm_fwd_cpu.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/carve_selftest.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/create_github_release.ps1 +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/dump_signatures.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/gen_api_reference.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/mft_recover.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/mft_recover_gui.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/mft_recover_gui.manifest +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/mft_selftest.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/mftcore.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/patch_dev9.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/publish.ps1 +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/recovery_pipeline_selftest.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/sector_carve.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/sector_carve_gui.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/sector_mirror.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/sector_mirror_gui.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/train_llm_sft.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/verify_help_examples.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/verify_linux_cli.sh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/tools/verify_train_release.py +0 -0
|
@@ -28,6 +28,25 @@ graft tools
|
|
|
28
28
|
# three. Their .c sources stay in the sdist and in the repository.
|
|
29
29
|
global-exclude *.exe
|
|
30
30
|
global-exclude *.exe.manifest
|
|
31
|
+
# Not shipping byte-code: a stale __pycache__ inside csrc/ or tools/ can otherwise end up in the
|
|
32
|
+
# sdist, and a .pyc built by a different interpreter is worse than useless to the recipient
|
|
33
|
+
# (it is silently ignored, or it shadows nothing). The source is always there.
|
|
34
|
+
global-exclude *.pyc
|
|
35
|
+
global-exclude *.pyo
|
|
36
|
+
prune tools/__pycache__
|
|
37
|
+
# Build outputs are not sources. The tools/ directory holds compiled .obj files next to the .c
|
|
38
|
+
# they came from, and a local Linux build leaves libbitsandbytes_*.so in bitsandbytes/. A sdist
|
|
39
|
+
# built from such a tree carried 5.5 MB of them (a 4.1 MB AVX-512 .so among them) on every
|
|
40
|
+
# platform, because an egg-info SOURCES.txt written by that earlier build kept them listed.
|
|
41
|
+
# The Windows DLL stays: it is the prebuilt kernel this fork ships.
|
|
42
|
+
global-exclude *.obj
|
|
43
|
+
global-exclude *.pdb
|
|
44
|
+
global-exclude *.o
|
|
45
|
+
global-exclude *.a
|
|
46
|
+
global-exclude *.lib
|
|
47
|
+
global-exclude *.exp
|
|
48
|
+
global-exclude *.so
|
|
49
|
+
global-exclude *.dylib
|
|
31
50
|
include build_linux.sh
|
|
32
51
|
include build_termux.sh
|
|
33
52
|
include selftest_cpu.c
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: bitsandbytes-cpu-fork
|
|
3
|
-
Version: 0.50.2.
|
|
3
|
+
Version: 0.50.2.dev12
|
|
4
4
|
Summary: bitsandbytes with a pure-CPU training backend (AVX2 / NEON): 8-bit optimizers, fused GDN, gemm_8bit, disk balancer. No GPU required.
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: repository, https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU
|
|
@@ -36,12 +36,12 @@ Dynamic: license-file
|
|
|
36
36
|
> toolkit and no driver** — just an AVX2 x86-64 CPU (or ARM64 NEON), the CPU build of PyTorch,
|
|
37
37
|
> and 12–16 GB of RAM.
|
|
38
38
|
|
|
39
|
-
**Version: `0.50.2.
|
|
39
|
+
**Version: `0.50.2.dev12`** · import name is unchanged (`import bitsandbytes as bnb`) · MIT.
|
|
40
40
|
|
|
41
41
|
What the CPU backend provides: 8-bit blockwise optimizers (≈2.3 bytes of state per parameter
|
|
42
42
|
instead of 8; the 4-bit optimizer ≈1.03), 4/8-bit layers that run on CPU tensors, the fused
|
|
43
43
|
Gated DeltaNet kernel for Qwen3-Next / Qwen3.5 linear attention, fused `gemm_8bit`, fused
|
|
44
|
-
RMSNorm/SwiGLU, an SSD cold-memory balancer, and a CPU training/concurrency toolbox.
|
|
44
|
+
RMSNorm/SwiGLU (with NT-store fast paths for large outputs), an SSD cold-memory balancer, and a CPU training/concurrency toolbox.
|
|
45
45
|
|
|
46
46
|
---
|
|
47
47
|
|
|
@@ -57,6 +57,15 @@ pip install bitsandbytes-cpu-fork
|
|
|
57
57
|
python -m bitsandbytes # prints version / platform / library info
|
|
58
58
|
```
|
|
59
59
|
|
|
60
|
+
If upstream `bitsandbytes` is already installed, **uninstall bitsandbytes first**: this fork
|
|
61
|
+
deliberately keeps the `bitsandbytes` import name (so transformers / peft / diffusers integrations
|
|
62
|
+
need no change), and pip will let the two distributions overwrite each other's files.
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip uninstall -y bitsandbytes
|
|
66
|
+
pip install bitsandbytes-cpu-fork
|
|
67
|
+
```
|
|
68
|
+
|
|
60
69
|
Requirements: Python ≥ 3.10 (64-bit), PyTorch ≥ 2.0 CPU build, x86-64 with AVX2 (a generic
|
|
61
70
|
SSE2 fallback is shipped) or ARM64 with NEON, Windows 10/11 or Linux glibc ≥ 2.17.
|
|
62
71
|
|
|
@@ -130,6 +139,31 @@ layer = layer.to("cpu") # .to() is what quantises the weights
|
|
|
130
139
|
y = layer(torch.randn(2, 1024))
|
|
131
140
|
```
|
|
132
141
|
|
|
142
|
+
Quantising an **existing** fp32/bf16 layer (the usual case) means moving the weights in first:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
orig = torch.nn.Linear(1024, 1024)
|
|
146
|
+
layer = bnb.nn.Linear4bit(orig.in_features, orig.out_features,
|
|
147
|
+
quant_type="nf4", compute_dtype=torch.float32)
|
|
148
|
+
layer.weight = bnb.nn.Params4bit(orig.weight.detach().clone(), quant_type="nf4", blocksize=64)
|
|
149
|
+
layer = layer.to("cpu") # quantisation happens here; calling forward() before this raises
|
|
150
|
+
out = layer(torch.randn(2, 1024))
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Checkpoints round-trip through the ordinary API -- the 4-bit payload and the quantization state are
|
|
154
|
+
saved together and restored by `load_state_dict()`:
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
sd = layer.state_dict()
|
|
158
|
+
fresh = bnb.nn.Linear4bit(1024, 1024, quant_type="nf4", compute_dtype=torch.float32)
|
|
159
|
+
fresh.load_state_dict(sd)
|
|
160
|
+
fresh = fresh.to("cpu") # already quantized: this keeps the loaded state
|
|
161
|
+
assert torch.equal(layer(x), fresh(x))
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Only if you split the state dict by hand do you need the low-level form:
|
|
165
|
+
`bnb.nn.Params4bit.from_prequantized(quantized_stats=sd, data=sd.pop("weight"), device="cpu")`.
|
|
166
|
+
|
|
133
167
|
### Fused Gated DeltaNet (Qwen3-Next / Qwen3.5 linear attention)
|
|
134
168
|
|
|
135
169
|
```python
|
|
@@ -197,6 +231,36 @@ Also: `estimate_training_memory_gb()`, `spawn_training_jobs(cmds, plan)` (logs p
|
|
|
197
231
|
|
|
198
232
|
---
|
|
199
233
|
|
|
234
|
+
## Threading: which knob actually controls the CPU kernels
|
|
235
|
+
|
|
236
|
+
The CPU kernels parallelise with **OpenMP**. `torch.set_num_threads()` only affects PyTorch's own
|
|
237
|
+
operators; it does **not** change how many threads a bnb kernel uses. Use the environment, set
|
|
238
|
+
before the process starts (it is read at import time):
|
|
239
|
+
|
|
240
|
+
```bash
|
|
241
|
+
OMP_NUM_THREADS=4 python train.py
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
Measured on the 6-core reference laptop (4-bit dequantize, 2048x4096, one process at a time):
|
|
245
|
+
|
|
246
|
+
| setting | time |
|
|
247
|
+
|---|---|
|
|
248
|
+
| `torch.set_num_threads(1)` | 4.5 ms |
|
|
249
|
+
| `torch.set_num_threads(6)` | 5.2 ms -- not the knob |
|
|
250
|
+
| `OMP_NUM_THREADS=1` | 6.7 ms |
|
|
251
|
+
| `OMP_NUM_THREADS=6` | 4.4 ms |
|
|
252
|
+
|
|
253
|
+
Two consequences for training loops:
|
|
254
|
+
|
|
255
|
+
- **Match the two settings.** `torch.set_num_threads(6)` on a 6-thread CPU plus the default
|
|
256
|
+
OpenMP pool oversubscribes the machine; set `OMP_NUM_THREADS` to the same value.
|
|
257
|
+
- **DataLoader workers inherit the environment.** Set `OMP_NUM_THREADS` before workers are forked,
|
|
258
|
+
otherwise every worker starts a full OpenMP pool.
|
|
259
|
+
|
|
260
|
+
`torch_cpu_kit.tune_threads()` (above) calibrates a real step instead of guessing.
|
|
261
|
+
|
|
262
|
+
---
|
|
263
|
+
|
|
200
264
|
## What is inside the CPU backend
|
|
201
265
|
|
|
202
266
|
| Area | What it does |
|
|
@@ -308,6 +372,47 @@ the full trade-off tables, including negative results.
|
|
|
308
372
|
|
|
309
373
|
---
|
|
310
374
|
|
|
375
|
+
### What the quantized paths cost
|
|
376
|
+
|
|
377
|
+
Numbers from one 6-core reference laptop (R5-4500U, AVX2, no GPU), one process at a time. Read them
|
|
378
|
+
as the shape of the trade-off, not as universal constants.
|
|
379
|
+
|
|
380
|
+
**4-bit layer inference on CPU is a memory win, not a speed win.** Weight memory drops ~20x
|
|
381
|
+
(a 2048x2048 layer: 4.0 MB fp32 -> 0.2 MB) but every forward pass dequantizes, so it is slower than
|
|
382
|
+
the fp32 layer:
|
|
383
|
+
|
|
384
|
+
| batch | fp32 | 4-bit | 4-bit / fp32 |
|
|
385
|
+
|---|---|---|---|
|
|
386
|
+
| 1 | 1.33 ms | 5.32 ms | 4.00x slower |
|
|
387
|
+
| 8 | 4.34 ms | 7.66 ms | 1.76x slower |
|
|
388
|
+
| 64 | 5.67 ms | 9.57 ms | 1.69x slower |
|
|
389
|
+
|
|
390
|
+
**8-bit optimizer: the win depends on the tensor size, not only on the parameter count.** The same
|
|
391
|
+
32M parameters, arranged differently (`optimizer.step()` only):
|
|
392
|
+
|
|
393
|
+
| layout | fp32 | 8-bit | ratio |
|
|
394
|
+
|---|---|---|---|
|
|
395
|
+
| 1 tensor x 32M | 177.1 ms | 93.6 ms | 0.53x (faster) |
|
|
396
|
+
| 32 tensors x 1M | 155.8 ms | 127.0 ms | 0.82x |
|
|
397
|
+
| 128 tensors x 250k | 126.9 ms | 154.7 ms | 1.22x (slower) |
|
|
398
|
+
|
|
399
|
+
There is a fixed **per-tensor** cost, so 8-bit pays off once a tensor holds roughly **1M elements**
|
|
400
|
+
or more; on many tiny tensors it can be slower than fp32. Real transformer layouts are dominated by
|
|
401
|
+
large matrices, which is why full-model numbers still favour it (GPT2-124M, 148 tensors: 445 ms vs
|
|
402
|
+
688 ms fp32).
|
|
403
|
+
|
|
404
|
+
**8-bit vs 4-bit vs fp32, three-way** (`optimizer.step()`, real GPT-2 architectures):
|
|
405
|
+
|
|
406
|
+
| model | fp32 | 8-bit | 4-bit | state per parameter |
|
|
407
|
+
|---|---|---|---|---|
|
|
408
|
+
| GPT2-124M (148 tensors) | 688 ms | **445 ms** | 1151 ms | 8.00 / 2.04 / 1.03 B |
|
|
409
|
+
| GPT2-345M (292 tensors) | 1962 ms | **1304 ms** | 3995 ms | 8.00 / 2.04 / 1.03 B |
|
|
410
|
+
|
|
411
|
+
Default to 8-bit. Reach for 4-bit only when the optimizer state has to be smaller than the 4x
|
|
412
|
+
reduction 8-bit already gives, and accept the ~2x slower step.
|
|
413
|
+
|
|
414
|
+
---
|
|
415
|
+
|
|
311
416
|
## Platform limitations
|
|
312
417
|
|
|
313
418
|
- **CPU only.** CUDA/ROCm/XPU kernels are not built; a GPU is never required or used.
|
|
@@ -365,12 +470,12 @@ the zig manylinux_2_17 portable build and the devtoolset-8 manylinux_2_17 **Open
|
|
|
365
470
|
```bash
|
|
366
471
|
bash build_linux.sh --selftest # C kernels: 9 checks (quantize/gemm_8bit/gemv_4bit/
|
|
367
472
|
# AdamW8bit/vector-vs-scalar/2-3bit dequant/fused GEMM)
|
|
368
|
-
python test_efst_regressions.py # EFST: nested blocks, 3-D tensor experts, routing usage
|
|
369
|
-
python test_q3bit_kernels.py # 2/3-bit research prototype (old DLL-based tests)
|
|
370
|
-
python test_q3bit_api.py # public q3bit API: pack/dequant/gemm, QuantLinear3bit, fallback
|
|
371
|
-
python test_adamw4bit.py # restored 4-bit optimizer: state bytes, native vs torch, fp32 tracking
|
|
372
|
-
python test_concurrency_plan.py # planner / memory estimate / tune_threads / launcher
|
|
373
|
-
python test_disk_balancer_safety.py # data-safety rules for the disk balancer
|
|
473
|
+
python scripts/test/test_efst_regressions.py # EFST: nested blocks, 3-D tensor experts, routing usage
|
|
474
|
+
python scripts/test/test_q3bit_kernels.py # 2/3-bit research prototype (old DLL-based tests)
|
|
475
|
+
python scripts/test/test_q3bit_api.py # public q3bit API: pack/dequant/gemm, QuantLinear3bit, fallback
|
|
476
|
+
python scripts/test/test_adamw4bit.py # restored 4-bit optimizer: state bytes, native vs torch, fp32 tracking
|
|
477
|
+
python scripts/test/test_concurrency_plan.py # planner / memory estimate / tune_threads / launcher
|
|
478
|
+
python scripts/test/test_disk_balancer_safety.py # data-safety rules for the disk balancer
|
|
374
479
|
```
|
|
375
480
|
|
|
376
481
|
## Documentation
|
|
@@ -384,6 +489,38 @@ The long-form docs live in the repository (`bitsandbytes/docs_cpu/`, Chinese and
|
|
|
384
489
|
- [`ISA_MATRIX.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/ISA_MATRIX.md) — x86 AVX2/generic/AVX-512 + aarch64 test matrix
|
|
385
490
|
- [`WHEEL_PLATFORM_NOTES.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/WHEEL_PLATFORM_NOTES.md) — which wheel runs where, and why
|
|
386
491
|
|
|
492
|
+
## Video on CPU (verified)
|
|
493
|
+
|
|
494
|
+
Inference is verified: LCM-LoRA 3.7–4.4× (512² / 192²-6f), ONNX Runtime fp32 UNet + async VAE
|
|
495
|
+
**43 frames 257.0 s vs 280.9 s (1.09×)** with per-frame PSNR **73.2 dB** vs torch fp32 (18-frame
|
|
496
|
+
control run: 71.95 dB); int8 UNet is unusable (11.04 dB). Training (AnimateDiff 5D LoRA) runs
|
|
497
|
+
end-to-end on CPU (~11.2 s/step, 5.9 GB, loss/checkpoint/sample produced) but **learned motion
|
|
498
|
+
is not demonstrated** — all seven demo runs report `has_global_motion=False`; temporal-only LoRA
|
|
499
|
+
measured harmful, spatial rank4/151 steps is the usable recipe so far. Details and negative
|
|
500
|
+
results: `VIDEO_GUIDE.md`.
|
|
501
|
+
|
|
502
|
+
## Project extras (outside the library API)
|
|
503
|
+
|
|
504
|
+
The repository also ships the tooling used to build and verify the releases; none of it is
|
|
505
|
+
imported by the library:
|
|
506
|
+
|
|
507
|
+
- **Audio capability benches** — `scripts/bench/audio/`: ASR `faster-whisper-small` int8
|
|
508
|
+
**5.8x realtime / WER 0**, Piper TTS **15-18x**, Kokoro-82M **1.9-2.1x**, training-cost tiers
|
|
509
|
+
(tiny CNN / 19M VITS-proxy / 185M Whisper-proxy) and a real KWS training demo. Weights are
|
|
510
|
+
never committed; bad input exits 2.
|
|
511
|
+
- **Video E2E pipeline** — `scripts/video/` + `VIDEO_GUIDE.md`: stage-wise generation/training,
|
|
512
|
+
motion diagnostics, and the six dev11 perf fixes (all six cores, no grad checkpointing,
|
|
513
|
+
flush denormals, 5-thread training default).
|
|
514
|
+
- **NTFS recovery toolkit** — `bitsandbytes/tools/`: `sector_mirror` (bad-sector mirror with
|
|
515
|
+
resume), `mft_recover` (deleted files straight from `$MFT`), `sector_carve` (PNG/JPEG/GIF/
|
|
516
|
+
ZIP incl. docx-xlsx-pptx/PDF/MP4); each has a CLI + GUI and a byte-for-byte selftest.
|
|
517
|
+
- **Release engineering** — `tools/`: `normalize_sdist.py` + `check_sdist_identity.py` audit the
|
|
518
|
+
**uploaded bytes** (2-DLL policy, sha256-only; 16 selftest + 20 adversarial cases), plus the
|
|
519
|
+
normalize-limit and docs-gate suites. dev11's sdist was rebuilt twice from `git archive` and
|
|
520
|
+
matched on the normalized tar sha.
|
|
521
|
+
- **AR experiments** — `gt1030/ar_*`, `scripts/lib/small_image_model_v2.py` (documented in the
|
|
522
|
+
devlog; not part of the library API).
|
|
523
|
+
|
|
387
524
|
## Changelog
|
|
388
525
|
|
|
389
526
|
| Version | Changes |
|
|
@@ -397,6 +534,9 @@ The long-form docs live in the repository (`bitsandbytes/docs_cpu/`, Chinese and
|
|
|
397
534
|
| `0.50.2.dev7` | documentation release: README rewritten for the CPU fork (install/wheels/platform matrix, feature table, measured performance, limitations, tests, full dev1→dev7 changelog, EFST/disk-balancer/toolbox examples) |
|
|
398
535
|
| `0.50.2.dev8` | **restored `AdamW4bit`** (the kernel was dropped when dev4 vendored upstream; now scalar C kernel + torch fallback, 1.031 B/param); **new `bitsandbytes.q3bit` API** — `quantize_3bit`/`dequantize_3bit`/`gemm_3bit`/`gemm_3bit_transposed`, `QuantLinear3bit` (frozen 3-bit base + LoRA), `convert_linears_3bit`, native kernels on Linux with identical pure-torch fallback; CLI reference now includes 4bitopt/q3bit and all 43 examples execute in CI-style verification |
|
|
399
536
|
| `0.50.2.dev9` | **Windows native kernels**: CI now compiles the MSVC DLL from the current csrc and the export list includes the 4-bit optimizer and 2/3-bit symbols (they were silently missing before); **q3bit real-model validation** on Qwen3-1.7B (frozen 3-bit base + LoRA, eval 4.34→4.03 over 80 steps, RSS −67%, native step 7.7 s vs fp32 5.5 s); fixed the causal-LM **double-shift** label bug in the iGPU training kit (Windows package v0.1.2) |
|
|
537
|
+
| `0.50.2.dev10` | **NTFS recovery toolkit** (`sector_carve`, `mft_recover`, CLI + GUI, selftests) and the **video E2E pipeline** (generation/training, LCM and motion diagnostics); AVX2 4-bit GEMV fixes (wrong FP4 codebook, lazy-init race, non-divisible blocksize); gt1030 probe scripts and docs |
|
|
538
|
+
| `0.50.2.dev12` | description sync with the verified video results (43 frames 257.0 vs 280.9 s, per-frame PSNR 73.2 dB; training is loop-verified only), 4-bit CPU usability audit fixes (9 issues), `exp/ue8m0/` negative-result archive |
|
|
539
|
+
| `0.50.2.dev11` | **NT-store fast paths**: 4-bit dequant writes bypass the cache above 16 MiB (1.7–2.0× at 32–64 MB) and RMSNorm chooses NT by total footprint (x+res+out vs per-CCX L3); CPUID L3-sharing mask fix and the NT threshold wired to one macro; **byte-reproducible sdist** with an identity gate that audits the uploaded bytes (`--sdist`, 2-DLL policy, 20 adversarial cases) plus a docs gate requiring all six ZH/EN docs; repo regrouping into `scripts/*`; UTF-8 BOM strip; 5 reproducible CPU audio benches; 6 video perf fixes and 3 AR fixes |
|
|
400
540
|
|
|
401
541
|
## License and attribution
|
|
402
542
|
|
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
> toolkit and no driver** — just an AVX2 x86-64 CPU (or ARM64 NEON), the CPU build of PyTorch,
|
|
6
6
|
> and 12–16 GB of RAM.
|
|
7
7
|
|
|
8
|
-
**Version: `0.50.2.
|
|
8
|
+
**Version: `0.50.2.dev12`** · import name is unchanged (`import bitsandbytes as bnb`) · MIT.
|
|
9
9
|
|
|
10
10
|
What the CPU backend provides: 8-bit blockwise optimizers (≈2.3 bytes of state per parameter
|
|
11
11
|
instead of 8; the 4-bit optimizer ≈1.03), 4/8-bit layers that run on CPU tensors, the fused
|
|
12
12
|
Gated DeltaNet kernel for Qwen3-Next / Qwen3.5 linear attention, fused `gemm_8bit`, fused
|
|
13
|
-
RMSNorm/SwiGLU, an SSD cold-memory balancer, and a CPU training/concurrency toolbox.
|
|
13
|
+
RMSNorm/SwiGLU (with NT-store fast paths for large outputs), an SSD cold-memory balancer, and a CPU training/concurrency toolbox.
|
|
14
14
|
|
|
15
15
|
---
|
|
16
16
|
|
|
@@ -26,6 +26,15 @@ pip install bitsandbytes-cpu-fork
|
|
|
26
26
|
python -m bitsandbytes # prints version / platform / library info
|
|
27
27
|
```
|
|
28
28
|
|
|
29
|
+
If upstream `bitsandbytes` is already installed, **uninstall bitsandbytes first**: this fork
|
|
30
|
+
deliberately keeps the `bitsandbytes` import name (so transformers / peft / diffusers integrations
|
|
31
|
+
need no change), and pip will let the two distributions overwrite each other's files.
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip uninstall -y bitsandbytes
|
|
35
|
+
pip install bitsandbytes-cpu-fork
|
|
36
|
+
```
|
|
37
|
+
|
|
29
38
|
Requirements: Python ≥ 3.10 (64-bit), PyTorch ≥ 2.0 CPU build, x86-64 with AVX2 (a generic
|
|
30
39
|
SSE2 fallback is shipped) or ARM64 with NEON, Windows 10/11 or Linux glibc ≥ 2.17.
|
|
31
40
|
|
|
@@ -99,6 +108,31 @@ layer = layer.to("cpu") # .to() is what quantises the weights
|
|
|
99
108
|
y = layer(torch.randn(2, 1024))
|
|
100
109
|
```
|
|
101
110
|
|
|
111
|
+
Quantising an **existing** fp32/bf16 layer (the usual case) means moving the weights in first:
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
orig = torch.nn.Linear(1024, 1024)
|
|
115
|
+
layer = bnb.nn.Linear4bit(orig.in_features, orig.out_features,
|
|
116
|
+
quant_type="nf4", compute_dtype=torch.float32)
|
|
117
|
+
layer.weight = bnb.nn.Params4bit(orig.weight.detach().clone(), quant_type="nf4", blocksize=64)
|
|
118
|
+
layer = layer.to("cpu") # quantisation happens here; calling forward() before this raises
|
|
119
|
+
out = layer(torch.randn(2, 1024))
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
Checkpoints round-trip through the ordinary API -- the 4-bit payload and the quantization state are
|
|
123
|
+
saved together and restored by `load_state_dict()`:
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
sd = layer.state_dict()
|
|
127
|
+
fresh = bnb.nn.Linear4bit(1024, 1024, quant_type="nf4", compute_dtype=torch.float32)
|
|
128
|
+
fresh.load_state_dict(sd)
|
|
129
|
+
fresh = fresh.to("cpu") # already quantized: this keeps the loaded state
|
|
130
|
+
assert torch.equal(layer(x), fresh(x))
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Only if you split the state dict by hand do you need the low-level form:
|
|
134
|
+
`bnb.nn.Params4bit.from_prequantized(quantized_stats=sd, data=sd.pop("weight"), device="cpu")`.
|
|
135
|
+
|
|
102
136
|
### Fused Gated DeltaNet (Qwen3-Next / Qwen3.5 linear attention)
|
|
103
137
|
|
|
104
138
|
```python
|
|
@@ -166,6 +200,36 @@ Also: `estimate_training_memory_gb()`, `spawn_training_jobs(cmds, plan)` (logs p
|
|
|
166
200
|
|
|
167
201
|
---
|
|
168
202
|
|
|
203
|
+
## Threading: which knob actually controls the CPU kernels
|
|
204
|
+
|
|
205
|
+
The CPU kernels parallelise with **OpenMP**. `torch.set_num_threads()` only affects PyTorch's own
|
|
206
|
+
operators; it does **not** change how many threads a bnb kernel uses. Use the environment, set
|
|
207
|
+
before the process starts (it is read at import time):
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
OMP_NUM_THREADS=4 python train.py
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Measured on the 6-core reference laptop (4-bit dequantize, 2048x4096, one process at a time):
|
|
214
|
+
|
|
215
|
+
| setting | time |
|
|
216
|
+
|---|---|
|
|
217
|
+
| `torch.set_num_threads(1)` | 4.5 ms |
|
|
218
|
+
| `torch.set_num_threads(6)` | 5.2 ms -- not the knob |
|
|
219
|
+
| `OMP_NUM_THREADS=1` | 6.7 ms |
|
|
220
|
+
| `OMP_NUM_THREADS=6` | 4.4 ms |
|
|
221
|
+
|
|
222
|
+
Two consequences for training loops:
|
|
223
|
+
|
|
224
|
+
- **Match the two settings.** `torch.set_num_threads(6)` on a 6-thread CPU plus the default
|
|
225
|
+
OpenMP pool oversubscribes the machine; set `OMP_NUM_THREADS` to the same value.
|
|
226
|
+
- **DataLoader workers inherit the environment.** Set `OMP_NUM_THREADS` before workers are forked,
|
|
227
|
+
otherwise every worker starts a full OpenMP pool.
|
|
228
|
+
|
|
229
|
+
`torch_cpu_kit.tune_threads()` (above) calibrates a real step instead of guessing.
|
|
230
|
+
|
|
231
|
+
---
|
|
232
|
+
|
|
169
233
|
## What is inside the CPU backend
|
|
170
234
|
|
|
171
235
|
| Area | What it does |
|
|
@@ -277,6 +341,47 @@ the full trade-off tables, including negative results.
|
|
|
277
341
|
|
|
278
342
|
---
|
|
279
343
|
|
|
344
|
+
### What the quantized paths cost
|
|
345
|
+
|
|
346
|
+
Numbers from one 6-core reference laptop (R5-4500U, AVX2, no GPU), one process at a time. Read them
|
|
347
|
+
as the shape of the trade-off, not as universal constants.
|
|
348
|
+
|
|
349
|
+
**4-bit layer inference on CPU is a memory win, not a speed win.** Weight memory drops ~20x
|
|
350
|
+
(a 2048x2048 layer: 4.0 MB fp32 -> 0.2 MB) but every forward pass dequantizes, so it is slower than
|
|
351
|
+
the fp32 layer:
|
|
352
|
+
|
|
353
|
+
| batch | fp32 | 4-bit | 4-bit / fp32 |
|
|
354
|
+
|---|---|---|---|
|
|
355
|
+
| 1 | 1.33 ms | 5.32 ms | 4.00x slower |
|
|
356
|
+
| 8 | 4.34 ms | 7.66 ms | 1.76x slower |
|
|
357
|
+
| 64 | 5.67 ms | 9.57 ms | 1.69x slower |
|
|
358
|
+
|
|
359
|
+
**8-bit optimizer: the win depends on the tensor size, not only on the parameter count.** The same
|
|
360
|
+
32M parameters, arranged differently (`optimizer.step()` only):
|
|
361
|
+
|
|
362
|
+
| layout | fp32 | 8-bit | ratio |
|
|
363
|
+
|---|---|---|---|
|
|
364
|
+
| 1 tensor x 32M | 177.1 ms | 93.6 ms | 0.53x (faster) |
|
|
365
|
+
| 32 tensors x 1M | 155.8 ms | 127.0 ms | 0.82x |
|
|
366
|
+
| 128 tensors x 250k | 126.9 ms | 154.7 ms | 1.22x (slower) |
|
|
367
|
+
|
|
368
|
+
There is a fixed **per-tensor** cost, so 8-bit pays off once a tensor holds roughly **1M elements**
|
|
369
|
+
or more; on many tiny tensors it can be slower than fp32. Real transformer layouts are dominated by
|
|
370
|
+
large matrices, which is why full-model numbers still favour it (GPT2-124M, 148 tensors: 445 ms vs
|
|
371
|
+
688 ms fp32).
|
|
372
|
+
|
|
373
|
+
**8-bit vs 4-bit vs fp32, three-way** (`optimizer.step()`, real GPT-2 architectures):
|
|
374
|
+
|
|
375
|
+
| model | fp32 | 8-bit | 4-bit | state per parameter |
|
|
376
|
+
|---|---|---|---|---|
|
|
377
|
+
| GPT2-124M (148 tensors) | 688 ms | **445 ms** | 1151 ms | 8.00 / 2.04 / 1.03 B |
|
|
378
|
+
| GPT2-345M (292 tensors) | 1962 ms | **1304 ms** | 3995 ms | 8.00 / 2.04 / 1.03 B |
|
|
379
|
+
|
|
380
|
+
Default to 8-bit. Reach for 4-bit only when the optimizer state has to be smaller than the 4x
|
|
381
|
+
reduction 8-bit already gives, and accept the ~2x slower step.
|
|
382
|
+
|
|
383
|
+
---
|
|
384
|
+
|
|
280
385
|
## Platform limitations
|
|
281
386
|
|
|
282
387
|
- **CPU only.** CUDA/ROCm/XPU kernels are not built; a GPU is never required or used.
|
|
@@ -334,12 +439,12 @@ the zig manylinux_2_17 portable build and the devtoolset-8 manylinux_2_17 **Open
|
|
|
334
439
|
```bash
|
|
335
440
|
bash build_linux.sh --selftest # C kernels: 9 checks (quantize/gemm_8bit/gemv_4bit/
|
|
336
441
|
# AdamW8bit/vector-vs-scalar/2-3bit dequant/fused GEMM)
|
|
337
|
-
python test_efst_regressions.py # EFST: nested blocks, 3-D tensor experts, routing usage
|
|
338
|
-
python test_q3bit_kernels.py # 2/3-bit research prototype (old DLL-based tests)
|
|
339
|
-
python test_q3bit_api.py # public q3bit API: pack/dequant/gemm, QuantLinear3bit, fallback
|
|
340
|
-
python test_adamw4bit.py # restored 4-bit optimizer: state bytes, native vs torch, fp32 tracking
|
|
341
|
-
python test_concurrency_plan.py # planner / memory estimate / tune_threads / launcher
|
|
342
|
-
python test_disk_balancer_safety.py # data-safety rules for the disk balancer
|
|
442
|
+
python scripts/test/test_efst_regressions.py # EFST: nested blocks, 3-D tensor experts, routing usage
|
|
443
|
+
python scripts/test/test_q3bit_kernels.py # 2/3-bit research prototype (old DLL-based tests)
|
|
444
|
+
python scripts/test/test_q3bit_api.py # public q3bit API: pack/dequant/gemm, QuantLinear3bit, fallback
|
|
445
|
+
python scripts/test/test_adamw4bit.py # restored 4-bit optimizer: state bytes, native vs torch, fp32 tracking
|
|
446
|
+
python scripts/test/test_concurrency_plan.py # planner / memory estimate / tune_threads / launcher
|
|
447
|
+
python scripts/test/test_disk_balancer_safety.py # data-safety rules for the disk balancer
|
|
343
448
|
```
|
|
344
449
|
|
|
345
450
|
## Documentation
|
|
@@ -353,6 +458,38 @@ The long-form docs live in the repository (`bitsandbytes/docs_cpu/`, Chinese and
|
|
|
353
458
|
- [`ISA_MATRIX.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/ISA_MATRIX.md) — x86 AVX2/generic/AVX-512 + aarch64 test matrix
|
|
354
459
|
- [`WHEEL_PLATFORM_NOTES.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/WHEEL_PLATFORM_NOTES.md) — which wheel runs where, and why
|
|
355
460
|
|
|
461
|
+
## Video on CPU (verified)
|
|
462
|
+
|
|
463
|
+
Inference is verified: LCM-LoRA 3.7–4.4× (512² / 192²-6f), ONNX Runtime fp32 UNet + async VAE
|
|
464
|
+
**43 frames 257.0 s vs 280.9 s (1.09×)** with per-frame PSNR **73.2 dB** vs torch fp32 (18-frame
|
|
465
|
+
control run: 71.95 dB); int8 UNet is unusable (11.04 dB). Training (AnimateDiff 5D LoRA) runs
|
|
466
|
+
end-to-end on CPU (~11.2 s/step, 5.9 GB, loss/checkpoint/sample produced) but **learned motion
|
|
467
|
+
is not demonstrated** — all seven demo runs report `has_global_motion=False`; temporal-only LoRA
|
|
468
|
+
measured harmful, spatial rank4/151 steps is the usable recipe so far. Details and negative
|
|
469
|
+
results: `VIDEO_GUIDE.md`.
|
|
470
|
+
|
|
471
|
+
## Project extras (outside the library API)
|
|
472
|
+
|
|
473
|
+
The repository also ships the tooling used to build and verify the releases; none of it is
|
|
474
|
+
imported by the library:
|
|
475
|
+
|
|
476
|
+
- **Audio capability benches** — `scripts/bench/audio/`: ASR `faster-whisper-small` int8
|
|
477
|
+
**5.8x realtime / WER 0**, Piper TTS **15-18x**, Kokoro-82M **1.9-2.1x**, training-cost tiers
|
|
478
|
+
(tiny CNN / 19M VITS-proxy / 185M Whisper-proxy) and a real KWS training demo. Weights are
|
|
479
|
+
never committed; bad input exits 2.
|
|
480
|
+
- **Video E2E pipeline** — `scripts/video/` + `VIDEO_GUIDE.md`: stage-wise generation/training,
|
|
481
|
+
motion diagnostics, and the six dev11 perf fixes (all six cores, no grad checkpointing,
|
|
482
|
+
flush denormals, 5-thread training default).
|
|
483
|
+
- **NTFS recovery toolkit** — `bitsandbytes/tools/`: `sector_mirror` (bad-sector mirror with
|
|
484
|
+
resume), `mft_recover` (deleted files straight from `$MFT`), `sector_carve` (PNG/JPEG/GIF/
|
|
485
|
+
ZIP incl. docx-xlsx-pptx/PDF/MP4); each has a CLI + GUI and a byte-for-byte selftest.
|
|
486
|
+
- **Release engineering** — `tools/`: `normalize_sdist.py` + `check_sdist_identity.py` audit the
|
|
487
|
+
**uploaded bytes** (2-DLL policy, sha256-only; 16 selftest + 20 adversarial cases), plus the
|
|
488
|
+
normalize-limit and docs-gate suites. dev11's sdist was rebuilt twice from `git archive` and
|
|
489
|
+
matched on the normalized tar sha.
|
|
490
|
+
- **AR experiments** — `gt1030/ar_*`, `scripts/lib/small_image_model_v2.py` (documented in the
|
|
491
|
+
devlog; not part of the library API).
|
|
492
|
+
|
|
356
493
|
## Changelog
|
|
357
494
|
|
|
358
495
|
| Version | Changes |
|
|
@@ -366,6 +503,9 @@ The long-form docs live in the repository (`bitsandbytes/docs_cpu/`, Chinese and
|
|
|
366
503
|
| `0.50.2.dev7` | documentation release: README rewritten for the CPU fork (install/wheels/platform matrix, feature table, measured performance, limitations, tests, full dev1→dev7 changelog, EFST/disk-balancer/toolbox examples) |
|
|
367
504
|
| `0.50.2.dev8` | **restored `AdamW4bit`** (the kernel was dropped when dev4 vendored upstream; now scalar C kernel + torch fallback, 1.031 B/param); **new `bitsandbytes.q3bit` API** — `quantize_3bit`/`dequantize_3bit`/`gemm_3bit`/`gemm_3bit_transposed`, `QuantLinear3bit` (frozen 3-bit base + LoRA), `convert_linears_3bit`, native kernels on Linux with identical pure-torch fallback; CLI reference now includes 4bitopt/q3bit and all 43 examples execute in CI-style verification |
|
|
368
505
|
| `0.50.2.dev9` | **Windows native kernels**: CI now compiles the MSVC DLL from the current csrc and the export list includes the 4-bit optimizer and 2/3-bit symbols (they were silently missing before); **q3bit real-model validation** on Qwen3-1.7B (frozen 3-bit base + LoRA, eval 4.34→4.03 over 80 steps, RSS −67%, native step 7.7 s vs fp32 5.5 s); fixed the causal-LM **double-shift** label bug in the iGPU training kit (Windows package v0.1.2) |
|
|
506
|
+
| `0.50.2.dev10` | **NTFS recovery toolkit** (`sector_carve`, `mft_recover`, CLI + GUI, selftests) and the **video E2E pipeline** (generation/training, LCM and motion diagnostics); AVX2 4-bit GEMV fixes (wrong FP4 codebook, lazy-init race, non-divisible blocksize); gt1030 probe scripts and docs |
|
|
507
|
+
| `0.50.2.dev12` | description sync with the verified video results (43 frames 257.0 vs 280.9 s, per-frame PSNR 73.2 dB; training is loop-verified only), 4-bit CPU usability audit fixes (9 issues), `exp/ue8m0/` negative-result archive |
|
|
508
|
+
| `0.50.2.dev11` | **NT-store fast paths**: 4-bit dequant writes bypass the cache above 16 MiB (1.7–2.0× at 32–64 MB) and RMSNorm chooses NT by total footprint (x+res+out vs per-CCX L3); CPUID L3-sharing mask fix and the NT threshold wired to one macro; **byte-reproducible sdist** with an identity gate that audits the uploaded bytes (`--sdist`, 2-DLL policy, 20 adversarial cases) plus a docs gate requiring all six ZH/EN docs; repo regrouping into `scripts/*`; UTF-8 BOM strip; 5 reproducible CPU audio benches; 6 video perf fixes and 3 AR fixes |
|
|
369
509
|
|
|
370
510
|
## License and attribution
|
|
371
511
|
|
|
Binary file
|
{bitsandbytes_cpu_fork-0.50.2.dev10 → bitsandbytes_cpu_fork-0.50.2.dev12}/bitsandbytes/nn/modules.py
RENAMED
|
@@ -485,20 +485,39 @@ class Params4bit(torch.nn.Parameter):
|
|
|
485
485
|
|
|
486
486
|
|
|
487
487
|
def fix_4bit_weight_quant_state_from_module(module: Union["Embedding4bit", "Linear4bit"]):
|
|
488
|
+
"""Make sure a 4-bit module that is about to run has its quantization state attached.
|
|
489
|
+
|
|
490
|
+
Weights are quantized by ``.to(device)``, not by construction, so a freshly built layer that
|
|
491
|
+
never saw ``.to(...)`` still holds unquantized weights. That is by far the most common way to
|
|
492
|
+
hit this function, and it used to surface as a bare ``AssertionError`` (or, on CUDA builds, as
|
|
493
|
+
advice to move the layer to a CUDA device -- useless on this CPU-only fork). It now raises with
|
|
494
|
+
the fix in it.
|
|
495
|
+
"""
|
|
488
496
|
if getattr(module.weight, "quant_state", None) is not None:
|
|
489
497
|
return
|
|
490
498
|
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
499
|
+
quant_state = getattr(module, "quant_state", None)
|
|
500
|
+
if quant_state is None:
|
|
501
|
+
raise RuntimeError(
|
|
502
|
+
f"{type(module).__name__} has no quantization state: its weights are still unquantized. "
|
|
503
|
+
"Weights are quantized by calling `.to(device)` on the layer, for example "
|
|
504
|
+
"`layer = layer.to('cpu')` on this CPU-only build. If you are restoring a checkpoint, "
|
|
505
|
+
"load a state dict that contains the 4-bit entries "
|
|
506
|
+
"(weight.absmax / weight.quant_map / weight.quant_state.bitsandbytes__*) "
|
|
507
|
+
"with `load_state_dict()`."
|
|
494
508
|
)
|
|
495
509
|
|
|
496
|
-
#
|
|
497
|
-
#
|
|
498
|
-
|
|
510
|
+
# The state was dropped from the parameter when it got converted (e.g. FSDP); the module still
|
|
511
|
+
# holds it, so recover it here.
|
|
512
|
+
if module.weight.shape[1] != 1:
|
|
513
|
+
raise RuntimeError(
|
|
514
|
+
f"{type(module).__name__} weight is not 4-bit packed (shape {tuple(module.weight.shape)}), "
|
|
515
|
+
"so the lost quantization state cannot be restored. Re-quantize the layer with `.to(device)`."
|
|
516
|
+
)
|
|
517
|
+
logger.debug("recovering lost 4-bit quantization state from the module")
|
|
499
518
|
if not isinstance(module.weight, Params4bit):
|
|
500
519
|
module.weight = Params4bit(module.weight, quant_storage=module.quant_storage, bnb_quantized=True)
|
|
501
|
-
module.weight.quant_state =
|
|
520
|
+
module.weight.quant_state = quant_state
|
|
502
521
|
|
|
503
522
|
|
|
504
523
|
class Linear4bit(nn.Linear):
|
|
@@ -606,6 +625,62 @@ class Linear4bit(nn.Linear):
|
|
|
606
625
|
for k, v in self.weight.quant_state.as_dict(packed=True).items():
|
|
607
626
|
destination[prefix + "weight." + k] = v if keep_vars else v.detach()
|
|
608
627
|
|
|
628
|
+
def _load_from_state_dict(
|
|
629
|
+
self,
|
|
630
|
+
state_dict,
|
|
631
|
+
prefix,
|
|
632
|
+
local_metadata,
|
|
633
|
+
strict,
|
|
634
|
+
missing_keys,
|
|
635
|
+
unexpected_keys,
|
|
636
|
+
error_msgs,
|
|
637
|
+
):
|
|
638
|
+
"""Load a state dict saved by :meth:`_save_to_state_dict`, quantization state included.
|
|
639
|
+
|
|
640
|
+
The 4-bit payload lives in ``weight`` while the extra entries written by
|
|
641
|
+
``QuantState.as_dict(packed=True)`` (``weight.absmax``, ``weight.quant_map``,
|
|
642
|
+
``weight.quant_state.bitsandbytes__nf4`` ...) are not parameters or buffers, so PyTorch's
|
|
643
|
+
default loader reports them as unexpected and a plain ``load_state_dict()`` of a quantized
|
|
644
|
+
layer used to fail. Rebuild the ``QuantState`` here instead.
|
|
645
|
+
"""
|
|
646
|
+
weight_key = prefix + "weight"
|
|
647
|
+
stats_prefix = weight_key + "."
|
|
648
|
+
quantized_stats = {}
|
|
649
|
+
for key in list(state_dict.keys()):
|
|
650
|
+
if not key.startswith(stats_prefix):
|
|
651
|
+
continue
|
|
652
|
+
suffix = key[len(stats_prefix):]
|
|
653
|
+
if suffix in QuantState.valid_qs_keys or suffix.split(".")[-1] in QuantState.valid_qs_type_keys:
|
|
654
|
+
quantized_stats[suffix] = state_dict.pop(key)
|
|
655
|
+
|
|
656
|
+
if not quantized_stats or weight_key not in state_dict:
|
|
657
|
+
# Unquantized checkpoint (or a nested state dict): keep the default behaviour.
|
|
658
|
+
super()._load_from_state_dict(
|
|
659
|
+
state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs,
|
|
660
|
+
)
|
|
661
|
+
return
|
|
662
|
+
|
|
663
|
+
data = state_dict.pop(weight_key)
|
|
664
|
+
super()._load_from_state_dict(
|
|
665
|
+
state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs,
|
|
666
|
+
)
|
|
667
|
+
if weight_key in missing_keys: # `weight` was consumed above, not missing
|
|
668
|
+
missing_keys.remove(weight_key)
|
|
669
|
+
try:
|
|
670
|
+
self.weight = Params4bit.from_prequantized(
|
|
671
|
+
data=data, quantized_stats=quantized_stats, device=self.weight.device, module=self,
|
|
672
|
+
)
|
|
673
|
+
except Exception as exc: # noqa: BLE001 - reported through torch's error channel
|
|
674
|
+
error_msgs.append(
|
|
675
|
+
f"{type(self).__name__} could not rebuild its 4-bit quantization state from the "
|
|
676
|
+
f"checkpoint ({type(exc).__name__}: {exc}). Save the layer with `state_dict()` and "
|
|
677
|
+
"load it with `load_state_dict()`, or restore it manually with "
|
|
678
|
+
"`Params4bit.from_prequantized(...)`.",
|
|
679
|
+
)
|
|
680
|
+
return
|
|
681
|
+
self.quant_state = self.weight.quant_state
|
|
682
|
+
self.compute_type_is_set = self.compute_dtype is not None
|
|
683
|
+
|
|
609
684
|
def forward(self, x: torch.Tensor):
|
|
610
685
|
fix_4bit_weight_quant_state_from_module(self)
|
|
611
686
|
quant_state = self.weight.quant_state
|
|
@@ -861,7 +936,10 @@ class Embedding8bit(nn.Embedding):
|
|
|
861
936
|
|
|
862
937
|
def forward(self, input: Tensor) -> Tensor:
|
|
863
938
|
if not hasattr(self.weight, "SCB"):
|
|
864
|
-
raise RuntimeError(
|
|
939
|
+
raise RuntimeError(
|
|
940
|
+
"Embedding layer is not quantized: weights are quantized by calling `.to(device)` "
|
|
941
|
+
"on the layer, for example `.to('cpu')` on this CPU-only build.",
|
|
942
|
+
)
|
|
865
943
|
|
|
866
944
|
rows = self.weight.data
|
|
867
945
|
row_stats = self.weight.SCB
|
|
@@ -1144,8 +1222,9 @@ class Linear8bitLt(nn.Linear):
|
|
|
1144
1222
|
if weight_scb is None:
|
|
1145
1223
|
# buffers not yet initialized, can't access them directly without quantizing first
|
|
1146
1224
|
raise RuntimeError(
|
|
1147
|
-
"Loading a quantized checkpoint into non-quantized Linear8bitLt is "
|
|
1148
|
-
"
|
|
1225
|
+
"Loading a quantized checkpoint into non-quantized Linear8bitLt is not "
|
|
1226
|
+
"supported. Call `.to(device)` on the module before `load_state_dict()` "
|
|
1227
|
+
"(on this CPU-only build: `.to('cpu')`).",
|
|
1149
1228
|
)
|
|
1150
1229
|
|
|
1151
1230
|
input_param = state_dict[key]
|