bitsandbytes-cpu-fork 0.50.2.dev6__tar.gz → 0.50.2.dev9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bitsandbytes_cpu_fork-0.50.2.dev9/PKG-INFO +346 -0
- bitsandbytes_cpu_fork-0.50.2.dev9/README.md +315 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/__init__.py +4 -1
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/efst.py +7 -2
- bitsandbytes_cpu_fork-0.50.2.dev9/bitsandbytes/libbitsandbytes_cpu.dll +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/__init__.py +4 -0
- bitsandbytes_cpu_fork-0.50.2.dev9/bitsandbytes/optim/adamw4bit.py +239 -0
- bitsandbytes_cpu_fork-0.50.2.dev9/bitsandbytes/q3bit.py +481 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes_cpu_cli.py +67 -17
- bitsandbytes_cpu_fork-0.50.2.dev9/bitsandbytes_cpu_fork.egg-info/PKG-INFO +346 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes_cpu_fork.egg-info/SOURCES.txt +3 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/build_manual/export.def +44 -39
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/cpu_ops.cpp +210 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/cpu_ops.h +7 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/pythonInterface.cpp +13 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/API_REFERENCE.md +67 -204
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/API_REFERENCE_EN.md +67 -196
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/create_github_release.ps1 +22 -22
- bitsandbytes_cpu_fork-0.50.2.dev9/tools/patch_dev9.py +75 -0
- bitsandbytes_cpu_fork-0.50.2.dev6/PKG-INFO +0 -452
- bitsandbytes_cpu_fork-0.50.2.dev6/README.md +0 -421
- bitsandbytes_cpu_fork-0.50.2.dev6/bitsandbytes/libbitsandbytes_cpu.dll +0 -0
- bitsandbytes_cpu_fork-0.50.2.dev6/bitsandbytes_cpu_fork.egg-info/PKG-INFO +0 -452
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/LICENSE +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/MANIFEST.in +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/__main__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/_cpu_caps.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/_ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/autograd/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/autograd/_functions.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/cpu/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/cpu/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/cuda/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/cuda/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/default/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/default/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/hpu/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/hpu/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/mps/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/mps/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/triton/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/triton/kernels_4bit.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/triton/kernels_8bit_quant.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/triton/kernels_optim.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/triton/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/utils.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/xpu/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/backends/xpu/ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/cextension.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/cli.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/consts.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/cuda_specs.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/detect_cpu.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/diagnostics/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/diagnostics/cuda.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/diagnostics/main.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/diagnostics/utils.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/disk_balancer.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/functional.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/gdn_cpu.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/nn/__init__.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/nn/modules.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/nn/parametrize.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/adagrad.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/adam.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/adamw.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/ademamix.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/lamb.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/lars.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/lion.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/optimizer.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/rmsprop.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/optim/sgd.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/py.typed +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/torch_cpu_kit.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/utils.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes/vcomp140.dll +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes_cpu_fork.egg-info/dependency_links.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes_cpu_fork.egg-info/entry_points.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes_cpu_fork.egg-info/requires.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/bitsandbytes_cpu_fork.egg-info/top_level.txt +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/build_linux.sh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/build_manual/build_manual.bat +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/build_manual/export_avx512.def +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/build_manual/export_generic.def +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/build_manual/selftest_win.bat +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/build_termux.sh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/avx2_gemv_4bit.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/common.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/common.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/compat.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/compat_device.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/cpu_cache.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/cpu_features.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/cpu_gdn.cpp +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/gemm_4bit.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/gemm_4bit_common.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/gemm_4bit_mma.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/gemm_4bit_simt.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/gemm_4bit_simt.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/gemm_4bit_sm75.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/gemm_4bit_sm75.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/gemm_4bit_sm80.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/gemm_4bit_sm80.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/kernels.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/kernels.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/ops.cu +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/ops.cuh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/test_avx2_gemv_4bit.cpp +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/test_cpu_cache.cpp +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/xpu_kernels.cpp +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/xpu_kernels.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/xpu_ops.cpp +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/csrc/xpu_ops.h +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/EPYC_VALIDATION.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/ISA_MATRIX.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/MOTION_BOUNDARY.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/MOTION_BOUNDARY_EN.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/QUICKSTART.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/QUICKSTART_EN.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/TECHNICAL_GUIDE.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/TECHNICAL_GUIDE_EN.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/TECH_REPORT.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/TECH_REPORT_EN.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/WHEEL_PLATFORM_NOTES.md +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/docs_cpu/check_docs.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/examples/compile_inference.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/examples/cpu/cpu_training.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/examples/int8_inference_huggingface.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/examples/xpu/benchmark_paged_memory.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/examples/xpu/paged_xpu_training.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/pyproject.toml +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/selftest_cpu.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/setup.cfg +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/setup.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_autograd.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_cpu_e2e.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_cuda_setup_evaluator.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_functional.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_generation.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_linear4bit.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_linear8bitlt.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_modules.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_ops.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_optim.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_parametrize.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tests/test_rmsnorm_fwd_cpu.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/dump_signatures.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/gen_api_reference.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/publish.ps1 +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/sector_carve.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/sector_carve_gui.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/sector_mirror.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/sector_mirror_gui.c +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/train_llm_sft.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/verify_help_examples.py +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/verify_linux_cli.sh +0 -0
- {bitsandbytes_cpu_fork-0.50.2.dev6 → bitsandbytes_cpu_fork-0.50.2.dev9}/tools/verify_train_release.py +0 -0
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bitsandbytes-cpu-fork
|
|
3
|
+
Version: 0.50.2.dev9
|
|
4
|
+
Summary: bitsandbytes with a pure-CPU training backend (AVX2 / NEON): 8-bit optimizers, fused GDN, gemm_8bit, disk balancer. No GPU required.
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: repository, https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU
|
|
7
|
+
Project-URL: Upstream bitsandbytes, https://github.com/bitsandbytes-foundation/bitsandbytes
|
|
8
|
+
Keywords: cpu,optimizers,optimization,8-bit,quantization,compression,avx2,neon
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Operating System :: Microsoft :: Windows
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Programming Language :: C++
|
|
15
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE
|
|
25
|
+
Requires-Dist: numpy>=1.17
|
|
26
|
+
Requires-Dist: packaging>=20.9
|
|
27
|
+
Requires-Dist: psutil>=5.9
|
|
28
|
+
Provides-Extra: test
|
|
29
|
+
Requires-Dist: pytest~=8.3; extra == "test"
|
|
30
|
+
Dynamic: license-file
|
|
31
|
+
|
|
32
|
+
# bitsandbytes CPU fork — `bitsandbytes-cpu-fork`
|
|
33
|
+
|
|
34
|
+
> **Not upstream `bitsandbytes`.** This is the CPU fork: the CUDA kernels are replaced by a
|
|
35
|
+
> pure-CPU backend, so it runs on ordinary laptops and Mini-PCs with **no NVIDIA GPU, no CUDA
|
|
36
|
+
> toolkit and no driver** — just an AVX2 x86-64 CPU (or ARM64 NEON), the CPU build of PyTorch,
|
|
37
|
+
> and 12–16 GB of RAM.
|
|
38
|
+
|
|
39
|
+
**Version: `0.50.2.dev9`** · import name is unchanged (`import bitsandbytes as bnb`) · MIT.
|
|
40
|
+
|
|
41
|
+
What the CPU backend provides: 8-bit blockwise optimizers (≈2.3 bytes of state per parameter
|
|
42
|
+
instead of 8; the 4-bit optimizer ≈1.03), 4/8-bit layers that run on CPU tensors, the fused
|
|
43
|
+
Gated DeltaNet kernel for Qwen3-Next / Qwen3.5 linear attention, fused `gemm_8bit`, fused
|
|
44
|
+
RMSNorm/SwiGLU, an SSD cold-memory balancer, and a CPU training/concurrency toolbox.
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## Install
|
|
49
|
+
|
|
50
|
+
`torch` is **deliberately not a dependency**: on PyPI the name `torch` resolves to the CUDA
|
|
51
|
+
build, which pulls ~2.5 GB of `nvidia-*` wheels onto machines with no NVIDIA device. Install
|
|
52
|
+
the CPU build first, then this fork:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
pip install torch --index-url https://download.pytorch.org/whl/cpu
|
|
56
|
+
pip install bitsandbytes-cpu-fork
|
|
57
|
+
python -m bitsandbytes # prints version / platform / library info
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Requirements: Python ≥ 3.10 (64-bit), PyTorch ≥ 2.0 CPU build, x86-64 with AVX2 (a generic
|
|
61
|
+
SSE2 fallback is shipped) or ARM64 with NEON, Windows 10/11 or Linux glibc ≥ 2.17.
|
|
62
|
+
|
|
63
|
+
### Wheels and platform support (dev7)
|
|
64
|
+
|
|
65
|
+
| Artifact | Platform | Parallel | Notes |
|
|
66
|
+
|---|---|---|---|
|
|
67
|
+
| `win_amd64.whl` | Windows 10/11 x64 | MSVC OpenMP (bundled) | prebuilt DLL, no compiler needed |
|
|
68
|
+
| `manylinux_2_17_x86_64.whl` | Linux, glibc ≥ 2.17 | **OpenMP (bundled libgomp)** | portable wheel: AVX2 + generic `.so`; the bundled AVX-512 `.so` is single-threaded — set `BNB_CPU_DISABLE_AVX512=1` to use the AVX2 OpenMP path |
|
|
69
|
+
| `manylinux_2_39_x86_64.whl` | Linux, glibc ≥ 2.39 | OpenMP (native build) | native wheel incl. AVX-512/BF16 kernels |
|
|
70
|
+
| `sdist` | any platform with a C++ compiler + OpenMP | OpenMP | builds `csrc` locally (aarch64 Linux / Android Termux supported this way) |
|
|
71
|
+
|
|
72
|
+
pip picks the best matching wheel automatically. If your glibc is older than 2.17 (or the
|
|
73
|
+
platform is not x86-64), install the sdist: `pip install --no-binary bitsandbytes-cpu-fork
|
|
74
|
+
bitsandbytes-cpu-fork`.
|
|
75
|
+
|
|
76
|
+
### Verify the install
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
python -m bitsandbytes # version, platform, which native library loads
|
|
80
|
+
bitsandbytes-cpu detect # CPU model, cores, SIMD, RAM, recommended thread count
|
|
81
|
+
bitsandbytes-cpu doctor # which .dll/.so will load and which kernel symbols it has
|
|
82
|
+
bitsandbytes-cpu selftest # 4-bit layer + 8-bit optimizer step + GDN kernel
|
|
83
|
+
bitsandbytes-cpu help # full feature reference with runnable examples
|
|
84
|
+
bitsandbytes-cpu help 8bitopt # one section: 8-bit optimizers
|
|
85
|
+
bitsandbytes-cpu help 4bit # one section: 4-bit layers on a CPU
|
|
86
|
+
bitsandbytes-cpu help kernels # one section: fused kernels, called directly
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
`bitsandbytes-cpu` is also installed as `bnb-cpu`.
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## Quick start
|
|
94
|
+
|
|
95
|
+
### 8-bit optimizer (≈2.3 B of state per parameter)
|
|
96
|
+
|
|
97
|
+
```python
|
|
98
|
+
import torch, bitsandbytes as bnb
|
|
99
|
+
|
|
100
|
+
p = torch.nn.Parameter(torch.randn(1024, 1024))
|
|
101
|
+
opt = bnb.optim.AdamW8bit([p], lr=1e-4)
|
|
102
|
+
p.grad = torch.randn_like(p)
|
|
103
|
+
opt.step()
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
### 4-bit optimizer (half the state of 8-bit, ~1.03 B/param)
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
import torch, bitsandbytes as bnb
|
|
110
|
+
|
|
111
|
+
p = torch.nn.Parameter(torch.randn(1024, 1024))
|
|
112
|
+
opt = bnb.optim.AdamW4bit([p], lr=1e-4) # fp32 parameters only
|
|
113
|
+
p.grad = torch.randn_like(p)
|
|
114
|
+
opt.step()
|
|
115
|
+
print(opt.state_bytes()) # ~1.031 bytes per parameter
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
The 4-bit kernel is scalar (the AVX2 packing was never correct), so it is ~8x slower
|
|
119
|
+
than the AVX2 8-bit kernel per element — use it to save state memory, not for speed.
|
|
120
|
+
On wheels without the kernel symbol (Windows) it falls back to torch with the same
|
|
121
|
+
numbers.
|
|
122
|
+
|
|
123
|
+
### 4-bit layer on CPU
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
import torch, bitsandbytes as bnb
|
|
127
|
+
|
|
128
|
+
layer = bnb.nn.Linear4bit(1024, 1024, quant_type="nf4", compute_dtype=torch.float32)
|
|
129
|
+
layer = layer.to("cpu") # .to() is what quantises the weights
|
|
130
|
+
y = layer(torch.randn(2, 1024))
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
### Fused Gated DeltaNet (Qwen3-Next / Qwen3.5 linear attention)
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from bitsandbytes.gdn_cpu import patch_transformers
|
|
137
|
+
|
|
138
|
+
patched = patch_transformers() # routes the transformers torch fallback to the fused CPU kernel
|
|
139
|
+
# patched == True means the hybrid-attention models now use the fused fwd/bwd kernel
|
|
140
|
+
# (transformers 5.x and older models both supported; see docs_cpu/EPYC_VALIDATION.md for HW notes)
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
### EFST — frozen experts + expert LoRA for MoE training
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
from bitsandbytes.efst import EFSTConfig, apply_efst
|
|
147
|
+
|
|
148
|
+
cfg = EFSTConfig(
|
|
149
|
+
top_k=2, # keep the 2 hottest experts per layer
|
|
150
|
+
calibration_dataloader=calib_batches,
|
|
151
|
+
calibration_forward_fn=lambda batch: model(**batch),
|
|
152
|
+
lora=True, lora_r=8, lora_alpha=16, # train LoRA adapters inside the selected experts
|
|
153
|
+
)
|
|
154
|
+
info = apply_efst(model, cfg)
|
|
155
|
+
print(info.summary()) # expert groups, trainable before/after
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Works with classic `ModuleList` experts and with the fused 3-D tensor experts used by HF
|
|
159
|
+
transformers 5.x Mixtral / Qwen3-Next (routing-usage calibration included).
|
|
160
|
+
|
|
161
|
+
### Disk load balancer — spill cold tensors to SSD when RAM is tight
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
import torch
|
|
165
|
+
from bitsandbytes.disk_balancer import DiskLoadBalancer, DiskBalancerConfig
|
|
166
|
+
|
|
167
|
+
bal = DiskLoadBalancer(DiskBalancerConfig(mode="auto"))
|
|
168
|
+
bal.start() # picks a safe local SSD cache dir
|
|
169
|
+
t = torch.arange(2 * 1024 * 1024, dtype=torch.float32)
|
|
170
|
+
bal.put_cold("cold_embeddings", t, tier="ssd")
|
|
171
|
+
bal.wait_writes()
|
|
172
|
+
back = bal.get_cold_shape("cold_embeddings", t.shape)
|
|
173
|
+
bal.cleanup() # deletes only its own cache dir
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
Safety rules: on Linux/WSL only native ext4/xfs/btrfs are used — 9P host drives (`/mnt/[drive]`),
|
|
177
|
+
network filesystems, read-only mounts, tmpfs and `/` are rejected; explicit unsafe
|
|
178
|
+
`--flash_paths` raise. In-flight write bytes are capped (`max_inflight_bytes`) so offload cannot
|
|
179
|
+
silently pin source tensors; quota and free-space limits are enforced.
|
|
180
|
+
|
|
181
|
+
### CPU concurrency planner / thread calibration (`torch_cpu_kit`)
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
from bitsandbytes import torch_cpu_kit as tck
|
|
185
|
+
|
|
186
|
+
plan = tck.plan_concurrency_for_training(
|
|
187
|
+
800_000_000, mode="lora", trainable_fraction=0.005, activation_gb=0.5)
|
|
188
|
+
print(plan["advice"]) # e.g. "3 job(s) x 2 thread(s) each ..."
|
|
189
|
+
print(plan["env"]) # OMP/MKL env for each job
|
|
190
|
+
|
|
191
|
+
# Measured on a real step, not guessed (hybrid-attention models often prefer fewer threads):
|
|
192
|
+
best = tck.tune_threads(lambda: train_step(model, batch))
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Also: `estimate_training_memory_gb()`, `spawn_training_jobs(cmds, plan)` (logs prefixed
|
|
196
|
+
`[jobN]`), `apply_env()`, `init()`, `mem_report()`.
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
## What is inside the CPU backend
|
|
201
|
+
|
|
202
|
+
| Area | What it does |
|
|
203
|
+
|---|---|
|
|
204
|
+
| 8-bit optimizers | AdamW/Adam/AdEMAMix/LAMB/LARS/Lion + paged variants, blockwise 8-bit state with single-pass dequant→update→requant kernels (AVX2/NEON) |
|
|
205
|
+
| 4-bit optimizer | `AdamW4bit`: ≈1.03 bytes of state per parameter (scalar kernel; native on Linux and, since dev9, Windows) |
|
|
206
|
+
| Quantisation | blockwise 8-bit, NF4/FP4 4-bit quantize/dequantize; 4-bit GEMV inference kernel |
|
|
207
|
+
| `gemm_8bit` | `out = A @ dequant8(B)` with weights kept uint8 (¼ DRAM traffic), no fp32 temporary (`bnb.functional.fused_dequant_linear_8bit`) |
|
|
208
|
+
| GDN | fused Gated DeltaNet forward + chunk-checkpointed backward (`gdn_fwd_cpu` / `gdn_bwd_cpu`), plus `patch_transformers()` / `patch_fla()` |
|
|
209
|
+
| Fused elementwise | RMSNorm fwd/bwd, SwiGLU, add-scale, NT-store variants |
|
|
210
|
+
| CPU detection | CPUID/XCR0 probe with OS-state checks; the AVX-512 library is only loaded when AVX-512 **and** BF16 are present, otherwise AVX2 (or the generic zero-AVX artifact) |
|
|
211
|
+
| Tooling | `torch_cpu_kit`, `disk_balancer`, `efst`, `bitsandbytes-cpu` CLI |
|
|
212
|
+
|
|
213
|
+
The library never loads a CUDA/ROCm/XPU library: `bnb.cuda`/`bnb.optim` CPU paths work, and
|
|
214
|
+
`import bitsandbytes` succeeds on a CPU-only host with no CUDA anywhere.
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
## 3-bit frozen base + LoRA (`bitsandbytes.q3bit`, experimental)
|
|
219
|
+
|
|
220
|
+
A frozen weight can be stored as blockwise **3-bit** codes (≈11% of fp32 memory) and
|
|
221
|
+
trained through LoRA; the native Linux wheels contain the C dequant / fused-GEMM
|
|
222
|
+
kernels, other platforms fall back to a pure-torch implementation with identical
|
|
223
|
+
numbers (slower):
|
|
224
|
+
|
|
225
|
+
```python
|
|
226
|
+
import torch, torch.nn as nn
|
|
227
|
+
from bitsandbytes import q3bit
|
|
228
|
+
|
|
229
|
+
w = torch.randn(1024, 1024)
|
|
230
|
+
packed, absmax = q3bit.quantize_3bit(w, block=64) # 3 bytes per 8 weights
|
|
231
|
+
back = q3bit.dequantize_3bit(packed, absmax, w.shape) # native kernel when available
|
|
232
|
+
|
|
233
|
+
layer = nn.Linear(1024, 1024)
|
|
234
|
+
ql = q3bit.QuantLinear3bit.from_linear(layer, lora_r=8) # frozen 3-bit base + LoRA
|
|
235
|
+
y = ql(torch.randn(2, 1024)) # only lora_A/lora_B train
|
|
236
|
+
print([n for n, p in ql.named_parameters() if p.requires_grad]) # ['lora_A', 'lora_B']
|
|
237
|
+
|
|
238
|
+
# freeze every Linear in a model at 3 bits (lm_head stays fp32):
|
|
239
|
+
q3bit.convert_linears_3bit(model, lora_r=8)
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
Functions: `quantize_3bit` / `dequantize_3bit` / `gemm_3bit` (A @ dequant(B).T without
|
|
243
|
+
materialising B) / `gemm_3bit_transposed`; 2-bit variants are available as
|
|
244
|
+
`quantize_2bit` / `dequantize_2bit` (dequant only). `Bits` are blockwise symmetric
|
|
245
|
+
(block=64, one absmax per block) and match the C kernel layout exactly.
|
|
246
|
+
|
|
247
|
+
## Performance notes (measured on this fork's target laptops)
|
|
248
|
+
|
|
249
|
+
All numbers are from the repodata in `bench_results/` and the technical report; hardware:
|
|
250
|
+
R5-4500U (6C/6T, AVX2, 15.4 GB) / i5-10400 (6C/12T, AVX2, 12 GB), no NVIDIA GPU.
|
|
251
|
+
|
|
252
|
+
| Scenario | Result |
|
|
253
|
+
|---|---|
|
|
254
|
+
| 17M-parameter AdamW8bit step | ≈34.7 ms (6 threads) / ≈38.6 ms (1 thread); 7–10 GB/s effective |
|
|
255
|
+
| 8-bit vs fp32 optimizer traffic | ≈16 B/element vs ≈28 B/element (1.7× less bandwidth) |
|
|
256
|
+
| Concurrency packing (synthetic jobs) | 6 jobs × 1 thread **5619 tok/s** vs 1 job × 6 threads **3843 tok/s** (+46%) |
|
|
257
|
+
| Real model (Qwen3.5-0.8B LoRA, seq 64) | 4 threads beat 6 by **7–10%** — measure with `tune_threads()` |
|
|
258
|
+
| Fused lm_head 3-bit GEMM (research) | 14–22% faster than fp32 matmul, 89% less weight memory |
|
|
259
|
+
| 3-bit frozen base + LoRA (Qwen3-1.7B, CPU) | 80 steps: eval loss 4.34 → **4.03**; RSS **2.9 GB vs 8.7 GB** fp32; step 7.7 s vs 5.5 s (native Windows) |
|
|
260
|
+
| AdamW4bit step (1M params) | 28.6 ms vs 3.3 ms for the AVX2 8-bit kernel; state 1.031 vs 2.062 B/param |
|
|
261
|
+
|
|
262
|
+
Not every workload is faster than fp32: 4-bit/8-bit kernels trade compute for memory traffic,
|
|
263
|
+
and small per-layer matmuls are better left to oneDNN. See `bench_results/` and the report for
|
|
264
|
+
the full trade-off tables, including negative results.
|
|
265
|
+
|
|
266
|
+
---
|
|
267
|
+
|
|
268
|
+
## Platform limitations
|
|
269
|
+
|
|
270
|
+
- **CPU only.** CUDA/ROCm/XPU kernels are not built; a GPU is never required or used.
|
|
271
|
+
- **Windows wheel**: from dev9 on it ships the MSVC DLL built by CI from the current
|
|
272
|
+
`csrc`, which exports the 2/3-bit dequant/GEMM and 4-bit optimizer kernels; they run
|
|
273
|
+
natively on Windows (verified: dequant maxdiff 1.2e-6, 4-bit native vs torch 1.2e-7).
|
|
274
|
+
Wheels older than dev9 fall back to pure torch for those symbols.
|
|
275
|
+
- **manylinux_2_17 wheel**: the AVX-512 library inside it is single-threaded (the parallel
|
|
276
|
+
build is only for the native `manylinux_2_39` wheel). On AVX-512 machines set
|
|
277
|
+
`BNB_CPU_DISABLE_AVX512=1` to run the AVX2 OpenMP path.
|
|
278
|
+
- **aarch64**: there is no PyPI wheel yet; build from the sdist (`build_linux.sh`) or use the
|
|
279
|
+
Termux instructions in `docs_cpu/TERMUX_GUIDE.md`.
|
|
280
|
+
- **macOS**: no wheel is published and the CPU backend is not tested there; the sdist build may
|
|
281
|
+
work with clang + OpenMP, but treat it as unsupported.
|
|
282
|
+
- 3-bit (and 2-bit dequant-only) is **experimental**: the API and kernels are covered
|
|
283
|
+
by tests, but there is no long-run quality study yet; treat it as a research feature.
|
|
284
|
+
- `torch` must be installed separately as the CPU build (see above).
|
|
285
|
+
|
|
286
|
+
## Build from source
|
|
287
|
+
|
|
288
|
+
```bash
|
|
289
|
+
# Linux (x86-64 AVX2 or aarch64 NEON; g++/clang + OpenMP)
|
|
290
|
+
sudo apt install g++ libomp-dev # Debian/Ubuntu
|
|
291
|
+
bash build_linux.sh # -> bitsandbytes/libbitsandbytes_cpu.so
|
|
292
|
+
bash build_linux.sh --selftest # also build + run the torch-free C self-test
|
|
293
|
+
|
|
294
|
+
# Windows (MSVC, manual build, no CMake)
|
|
295
|
+
build_manual\build_manual.bat amd # run inside "x64 Native Tools Command Prompt"
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
Two special wheel builds used by this project are documented in
|
|
299
|
+
[`docs_cpu/WHEEL_PLATFORM_NOTES.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/WHEEL_PLATFORM_NOTES.md):
|
|
300
|
+
the zig manylinux_2_17 portable build and the devtoolset-8 manylinux_2_17 **OpenMP** build
|
|
301
|
+
(`build_oldgomp_linux.sh`, bundles libgomp).
|
|
302
|
+
|
|
303
|
+
## Tests
|
|
304
|
+
|
|
305
|
+
```bash
|
|
306
|
+
bash build_linux.sh --selftest # C kernels: 9 checks (quantize/gemm_8bit/gemv_4bit/
|
|
307
|
+
# AdamW8bit/vector-vs-scalar/2-3bit dequant/fused GEMM)
|
|
308
|
+
python test_efst_regressions.py # EFST: nested blocks, 3-D tensor experts, routing usage
|
|
309
|
+
python test_q3bit_kernels.py # 2/3-bit research prototype (old DLL-based tests)
|
|
310
|
+
python test_q3bit_api.py # public q3bit API: pack/dequant/gemm, QuantLinear3bit, fallback
|
|
311
|
+
python test_adamw4bit.py # restored 4-bit optimizer: state bytes, native vs torch, fp32 tracking
|
|
312
|
+
python test_concurrency_plan.py # planner / memory estimate / tune_threads / launcher
|
|
313
|
+
python test_disk_balancer_safety.py # data-safety rules for the disk balancer
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
## Documentation
|
|
317
|
+
|
|
318
|
+
The long-form docs live in the repository (`bitsandbytes/docs_cpu/`, Chinese and English):
|
|
319
|
+
|
|
320
|
+
- [`QUICKSTART.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/QUICKSTART.md) / `QUICKSTART_EN.md` — commands, features, disaster recovery
|
|
321
|
+
- [`API_REFERENCE.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/API_REFERENCE.md) / `API_REFERENCE_EN.md` — every public API
|
|
322
|
+
- [`TECHNICAL_GUIDE.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/TECHNICAL_GUIDE.md) — kernel architecture (AVX2/NEON paths, layouts, sync rules)
|
|
323
|
+
- [`EPYC_VALIDATION.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/EPYC_VALIDATION.md) — real AVX-512/BF16 machine validation
|
|
324
|
+
- [`ISA_MATRIX.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/ISA_MATRIX.md) — x86 AVX2/generic/AVX-512 + aarch64 test matrix
|
|
325
|
+
- [`WHEEL_PLATFORM_NOTES.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/WHEEL_PLATFORM_NOTES.md) — which wheel runs where, and why
|
|
326
|
+
|
|
327
|
+
## Changelog
|
|
328
|
+
|
|
329
|
+
| Version | Changes |
|
|
330
|
+
|---|---|
|
|
331
|
+
| `0.50.2.dev1` | First CPU-fork release: AVX2/NEON CPU backend (8-bit optimizers, fused GDN, `gemm_8bit`, 4/8-bit quantisation) |
|
|
332
|
+
| `0.50.2.dev2` | AdamW4bit packaged; PyPI release |
|
|
333
|
+
| `0.50.2.dev3` | README / API reference / summary document the 4-bit optimizer |
|
|
334
|
+
| `0.50.2.dev4` | Linux wheels (manylinux_2_17 portable + manylinux_2_39 OpenMP); libgcc-free CPUID detection; AVX-512 BF16 loader safety; portability and data-safety fixes |
|
|
335
|
+
| `0.50.2.dev5` | WSL/Linux disk balancer (ext4-only, 9P rejected, in-flight byte cap); `torch_cpu_kit` concurrency planner / memory estimator / `tune_threads`; optional `kernels` warning fix; EPYC validation docs |
|
|
336
|
+
| `0.50.2.dev6` | manylinux_2_17 wheel is now **OpenMP** (devtoolset-8, bundled libgomp → 3–4× optimizer-kernel speedup on old glibc); csrc 2/3-bit dequant + fused GEMM kernels with 9-item C self-test; EFST fused tensor-expert routing fix; GCC 8 `<cpuid.h>` portability fix |
|
|
337
|
+
| `0.50.2.dev7` | documentation release: README rewritten for the CPU fork (install/wheels/platform matrix, feature table, measured performance, limitations, tests, full dev1→dev7 changelog, EFST/disk-balancer/toolbox examples) |
|
|
338
|
+
| `0.50.2.dev8` | **restored `AdamW4bit`** (the kernel was dropped when dev4 vendored upstream; now scalar C kernel + torch fallback, 1.031 B/param); **new `bitsandbytes.q3bit` API** — `quantize_3bit`/`dequantize_3bit`/`gemm_3bit`/`gemm_3bit_transposed`, `QuantLinear3bit` (frozen 3-bit base + LoRA), `convert_linears_3bit`, native kernels on Linux with identical pure-torch fallback; CLI reference now includes 4bitopt/q3bit and all 43 examples execute in CI-style verification |
|
|
339
|
+
| `0.50.2.dev9` | **Windows native kernels**: CI now compiles the MSVC DLL from the current csrc and the export list includes the 4-bit optimizer and 2/3-bit symbols (they were silently missing before); **q3bit real-model validation** on Qwen3-1.7B (frozen 3-bit base + LoRA, eval 4.34→4.03 over 80 steps, RSS −67%, native step 7.7 s vs fp32 5.5 s); fixed the causal-LM **double-shift** label bug in the iGPU training kit (Windows package v0.1.2) |
|
|
340
|
+
|
|
341
|
+
## License and attribution
|
|
342
|
+
|
|
343
|
+
`bitsandbytes-cpu-fork` is MIT licensed, like upstream
|
|
344
|
+
[bitsandbytes](https://github.com/bitsandbytes-foundation/bitsandbytes) (this fork is based on
|
|
345
|
+
upstream `0.50.2`); the CUDA kernels were removed and replaced by the CPU backend described
|
|
346
|
+
above. See `NOTICE.md` for details and upstream attribution.
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
# bitsandbytes CPU fork — `bitsandbytes-cpu-fork`
|
|
2
|
+
|
|
3
|
+
> **Not upstream `bitsandbytes`.** This is the CPU fork: the CUDA kernels are replaced by a
|
|
4
|
+
> pure-CPU backend, so it runs on ordinary laptops and Mini-PCs with **no NVIDIA GPU, no CUDA
|
|
5
|
+
> toolkit and no driver** — just an AVX2 x86-64 CPU (or ARM64 NEON), the CPU build of PyTorch,
|
|
6
|
+
> and 12–16 GB of RAM.
|
|
7
|
+
|
|
8
|
+
**Version: `0.50.2.dev9`** · import name is unchanged (`import bitsandbytes as bnb`) · MIT.
|
|
9
|
+
|
|
10
|
+
What the CPU backend provides: 8-bit blockwise optimizers (≈2.3 bytes of state per parameter
|
|
11
|
+
instead of 8; the 4-bit optimizer ≈1.03), 4/8-bit layers that run on CPU tensors, the fused
|
|
12
|
+
Gated DeltaNet kernel for Qwen3-Next / Qwen3.5 linear attention, fused `gemm_8bit`, fused
|
|
13
|
+
RMSNorm/SwiGLU, an SSD cold-memory balancer, and a CPU training/concurrency toolbox.
|
|
14
|
+
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
## Install
|
|
18
|
+
|
|
19
|
+
`torch` is **deliberately not a dependency**: on PyPI the name `torch` resolves to the CUDA
|
|
20
|
+
build, which pulls ~2.5 GB of `nvidia-*` wheels onto machines with no NVIDIA device. Install
|
|
21
|
+
the CPU build first, then this fork:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install torch --index-url https://download.pytorch.org/whl/cpu
|
|
25
|
+
pip install bitsandbytes-cpu-fork
|
|
26
|
+
python -m bitsandbytes # prints version / platform / library info
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Requirements: Python ≥ 3.10 (64-bit), PyTorch ≥ 2.0 CPU build, x86-64 with AVX2 (a generic
|
|
30
|
+
SSE2 fallback is shipped) or ARM64 with NEON, Windows 10/11 or Linux glibc ≥ 2.17.
|
|
31
|
+
|
|
32
|
+
### Wheels and platform support (dev7)
|
|
33
|
+
|
|
34
|
+
| Artifact | Platform | Parallel | Notes |
|
|
35
|
+
|---|---|---|---|
|
|
36
|
+
| `win_amd64.whl` | Windows 10/11 x64 | MSVC OpenMP (bundled) | prebuilt DLL, no compiler needed |
|
|
37
|
+
| `manylinux_2_17_x86_64.whl` | Linux, glibc ≥ 2.17 | **OpenMP (bundled libgomp)** | portable wheel: AVX2 + generic `.so`; the bundled AVX-512 `.so` is single-threaded — set `BNB_CPU_DISABLE_AVX512=1` to use the AVX2 OpenMP path |
|
|
38
|
+
| `manylinux_2_39_x86_64.whl` | Linux, glibc ≥ 2.39 | OpenMP (native build) | native wheel incl. AVX-512/BF16 kernels |
|
|
39
|
+
| `sdist` | any platform with a C++ compiler + OpenMP | OpenMP | builds `csrc` locally (aarch64 Linux / Android Termux supported this way) |
|
|
40
|
+
|
|
41
|
+
pip picks the best matching wheel automatically. If your glibc is older than 2.17 (or the
|
|
42
|
+
platform is not x86-64), install the sdist: `pip install --no-binary bitsandbytes-cpu-fork
|
|
43
|
+
bitsandbytes-cpu-fork`.
|
|
44
|
+
|
|
45
|
+
### Verify the install
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
python -m bitsandbytes # version, platform, which native library loads
|
|
49
|
+
bitsandbytes-cpu detect # CPU model, cores, SIMD, RAM, recommended thread count
|
|
50
|
+
bitsandbytes-cpu doctor # which .dll/.so will load and which kernel symbols it has
|
|
51
|
+
bitsandbytes-cpu selftest # 4-bit layer + 8-bit optimizer step + GDN kernel
|
|
52
|
+
bitsandbytes-cpu help # full feature reference with runnable examples
|
|
53
|
+
bitsandbytes-cpu help 8bitopt # one section: 8-bit optimizers
|
|
54
|
+
bitsandbytes-cpu help 4bit # one section: 4-bit layers on a CPU
|
|
55
|
+
bitsandbytes-cpu help kernels # one section: fused kernels, called directly
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
`bitsandbytes-cpu` is also installed as `bnb-cpu`.
|
|
59
|
+
|
|
60
|
+
---
|
|
61
|
+
|
|
62
|
+
## Quick start
|
|
63
|
+
|
|
64
|
+
### 8-bit optimizer (≈2.3 B of state per parameter)
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
import torch, bitsandbytes as bnb
|
|
68
|
+
|
|
69
|
+
p = torch.nn.Parameter(torch.randn(1024, 1024))
|
|
70
|
+
opt = bnb.optim.AdamW8bit([p], lr=1e-4)
|
|
71
|
+
p.grad = torch.randn_like(p)
|
|
72
|
+
opt.step()
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### 4-bit optimizer (half the state of 8-bit, ~1.03 B/param)
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import torch, bitsandbytes as bnb
|
|
79
|
+
|
|
80
|
+
p = torch.nn.Parameter(torch.randn(1024, 1024))
|
|
81
|
+
opt = bnb.optim.AdamW4bit([p], lr=1e-4) # fp32 parameters only
|
|
82
|
+
p.grad = torch.randn_like(p)
|
|
83
|
+
opt.step()
|
|
84
|
+
print(opt.state_bytes()) # ~1.031 bytes per parameter
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
The 4-bit kernel is scalar (the AVX2 packing was never correct), so it is ~8x slower
|
|
88
|
+
than the AVX2 8-bit kernel per element — use it to save state memory, not for speed.
|
|
89
|
+
On wheels without the kernel symbol (Windows) it falls back to torch with the same
|
|
90
|
+
numbers.
|
|
91
|
+
|
|
92
|
+
### 4-bit layer on CPU
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
import torch, bitsandbytes as bnb
|
|
96
|
+
|
|
97
|
+
layer = bnb.nn.Linear4bit(1024, 1024, quant_type="nf4", compute_dtype=torch.float32)
|
|
98
|
+
layer = layer.to("cpu") # .to() is what quantises the weights
|
|
99
|
+
y = layer(torch.randn(2, 1024))
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Fused Gated DeltaNet (Qwen3-Next / Qwen3.5 linear attention)
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
from bitsandbytes.gdn_cpu import patch_transformers
|
|
106
|
+
|
|
107
|
+
patched = patch_transformers() # routes the transformers torch fallback to the fused CPU kernel
|
|
108
|
+
# patched == True means the hybrid-attention models now use the fused fwd/bwd kernel
|
|
109
|
+
# (transformers 5.x and older models both supported; see docs_cpu/EPYC_VALIDATION.md for HW notes)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
### EFST — frozen experts + expert LoRA for MoE training
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
from bitsandbytes.efst import EFSTConfig, apply_efst
|
|
116
|
+
|
|
117
|
+
cfg = EFSTConfig(
|
|
118
|
+
top_k=2, # keep the 2 hottest experts per layer
|
|
119
|
+
calibration_dataloader=calib_batches,
|
|
120
|
+
calibration_forward_fn=lambda batch: model(**batch),
|
|
121
|
+
lora=True, lora_r=8, lora_alpha=16, # train LoRA adapters inside the selected experts
|
|
122
|
+
)
|
|
123
|
+
info = apply_efst(model, cfg)
|
|
124
|
+
print(info.summary()) # expert groups, trainable before/after
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Works with classic `ModuleList` experts and with the fused 3-D tensor experts used by HF
|
|
128
|
+
transformers 5.x Mixtral / Qwen3-Next (routing-usage calibration included).
|
|
129
|
+
|
|
130
|
+
### Disk load balancer — spill cold tensors to SSD when RAM is tight
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
import torch
|
|
134
|
+
from bitsandbytes.disk_balancer import DiskLoadBalancer, DiskBalancerConfig
|
|
135
|
+
|
|
136
|
+
bal = DiskLoadBalancer(DiskBalancerConfig(mode="auto"))
|
|
137
|
+
bal.start() # picks a safe local SSD cache dir
|
|
138
|
+
t = torch.arange(2 * 1024 * 1024, dtype=torch.float32)
|
|
139
|
+
bal.put_cold("cold_embeddings", t, tier="ssd")
|
|
140
|
+
bal.wait_writes()
|
|
141
|
+
back = bal.get_cold_shape("cold_embeddings", t.shape)
|
|
142
|
+
bal.cleanup() # deletes only its own cache dir
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
Safety rules: on Linux/WSL only native ext4/xfs/btrfs are used — 9P host drives (`/mnt/[drive]`),
|
|
146
|
+
network filesystems, read-only mounts, tmpfs and `/` are rejected; explicit unsafe
|
|
147
|
+
`--flash_paths` raise. In-flight write bytes are capped (`max_inflight_bytes`) so offload cannot
|
|
148
|
+
silently pin source tensors; quota and free-space limits are enforced.
|
|
149
|
+
|
|
150
|
+
### CPU concurrency planner / thread calibration (`torch_cpu_kit`)
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
from bitsandbytes import torch_cpu_kit as tck
|
|
154
|
+
|
|
155
|
+
plan = tck.plan_concurrency_for_training(
|
|
156
|
+
800_000_000, mode="lora", trainable_fraction=0.005, activation_gb=0.5)
|
|
157
|
+
print(plan["advice"]) # e.g. "3 job(s) x 2 thread(s) each ..."
|
|
158
|
+
print(plan["env"]) # OMP/MKL env for each job
|
|
159
|
+
|
|
160
|
+
# Measured on a real step, not guessed (hybrid-attention models often prefer fewer threads):
|
|
161
|
+
best = tck.tune_threads(lambda: train_step(model, batch))
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
Also: `estimate_training_memory_gb()`, `spawn_training_jobs(cmds, plan)` (logs prefixed
|
|
165
|
+
`[jobN]`), `apply_env()`, `init()`, `mem_report()`.
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## What is inside the CPU backend
|
|
170
|
+
|
|
171
|
+
| Area | What it does |
|
|
172
|
+
|---|---|
|
|
173
|
+
| 8-bit optimizers | AdamW/Adam/AdEMAMix/LAMB/LARS/Lion + paged variants, blockwise 8-bit state with single-pass dequant→update→requant kernels (AVX2/NEON) |
|
|
174
|
+
| 4-bit optimizer | `AdamW4bit`: ≈1.03 bytes of state per parameter (scalar kernel; native on Linux and, since dev9, Windows) |
|
|
175
|
+
| Quantisation | blockwise 8-bit, NF4/FP4 4-bit quantize/dequantize; 4-bit GEMV inference kernel |
|
|
176
|
+
| `gemm_8bit` | `out = A @ dequant8(B)` with weights kept uint8 (¼ DRAM traffic), no fp32 temporary (`bnb.functional.fused_dequant_linear_8bit`) |
|
|
177
|
+
| GDN | fused Gated DeltaNet forward + chunk-checkpointed backward (`gdn_fwd_cpu` / `gdn_bwd_cpu`), plus `patch_transformers()` / `patch_fla()` |
|
|
178
|
+
| Fused elementwise | RMSNorm fwd/bwd, SwiGLU, add-scale, NT-store variants |
|
|
179
|
+
| CPU detection | CPUID/XCR0 probe with OS-state checks; the AVX-512 library is only loaded when AVX-512 **and** BF16 are present, otherwise AVX2 (or the generic zero-AVX artifact) |
|
|
180
|
+
| Tooling | `torch_cpu_kit`, `disk_balancer`, `efst`, `bitsandbytes-cpu` CLI |
|
|
181
|
+
|
|
182
|
+
The library never loads a CUDA/ROCm/XPU library: `bnb.cuda`/`bnb.optim` CPU paths work, and
|
|
183
|
+
`import bitsandbytes` succeeds on a CPU-only host with no CUDA anywhere.
|
|
184
|
+
|
|
185
|
+
---
|
|
186
|
+
|
|
187
|
+
## 3-bit frozen base + LoRA (`bitsandbytes.q3bit`, experimental)
|
|
188
|
+
|
|
189
|
+
A frozen weight can be stored as blockwise **3-bit** codes (≈11% of fp32 memory) and
|
|
190
|
+
trained through LoRA; the native Linux wheels contain the C dequant / fused-GEMM
|
|
191
|
+
kernels, other platforms fall back to a pure-torch implementation with identical
|
|
192
|
+
numbers (slower):
|
|
193
|
+
|
|
194
|
+
```python
|
|
195
|
+
import torch, torch.nn as nn
|
|
196
|
+
from bitsandbytes import q3bit
|
|
197
|
+
|
|
198
|
+
w = torch.randn(1024, 1024)
|
|
199
|
+
packed, absmax = q3bit.quantize_3bit(w, block=64) # 3 bytes per 8 weights
|
|
200
|
+
back = q3bit.dequantize_3bit(packed, absmax, w.shape) # native kernel when available
|
|
201
|
+
|
|
202
|
+
layer = nn.Linear(1024, 1024)
|
|
203
|
+
ql = q3bit.QuantLinear3bit.from_linear(layer, lora_r=8) # frozen 3-bit base + LoRA
|
|
204
|
+
y = ql(torch.randn(2, 1024)) # only lora_A/lora_B train
|
|
205
|
+
print([n for n, p in ql.named_parameters() if p.requires_grad]) # ['lora_A', 'lora_B']
|
|
206
|
+
|
|
207
|
+
# freeze every Linear in a model at 3 bits (lm_head stays fp32):
|
|
208
|
+
q3bit.convert_linears_3bit(model, lora_r=8)
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
Functions: `quantize_3bit` / `dequantize_3bit` / `gemm_3bit` (A @ dequant(B).T without
|
|
212
|
+
materialising B) / `gemm_3bit_transposed`; 2-bit variants are available as
|
|
213
|
+
`quantize_2bit` / `dequantize_2bit` (dequant only). `Bits` are blockwise symmetric
|
|
214
|
+
(block=64, one absmax per block) and match the C kernel layout exactly.
|
|
215
|
+
|
|
216
|
+
## Performance notes (measured on this fork's target laptops)
|
|
217
|
+
|
|
218
|
+
All numbers are from the repodata in `bench_results/` and the technical report; hardware:
|
|
219
|
+
R5-4500U (6C/6T, AVX2, 15.4 GB) / i5-10400 (6C/12T, AVX2, 12 GB), no NVIDIA GPU.
|
|
220
|
+
|
|
221
|
+
| Scenario | Result |
|
|
222
|
+
|---|---|
|
|
223
|
+
| 17M-parameter AdamW8bit step | ≈34.7 ms (6 threads) / ≈38.6 ms (1 thread); 7–10 GB/s effective |
|
|
224
|
+
| 8-bit vs fp32 optimizer traffic | ≈16 B/element vs ≈28 B/element (1.7× less bandwidth) |
|
|
225
|
+
| Concurrency packing (synthetic jobs) | 6 jobs × 1 thread **5619 tok/s** vs 1 job × 6 threads **3843 tok/s** (+46%) |
|
|
226
|
+
| Real model (Qwen3.5-0.8B LoRA, seq 64) | 4 threads beat 6 by **7–10%** — measure with `tune_threads()` |
|
|
227
|
+
| Fused lm_head 3-bit GEMM (research) | 14–22% faster than fp32 matmul, 89% less weight memory |
|
|
228
|
+
| 3-bit frozen base + LoRA (Qwen3-1.7B, CPU) | 80 steps: eval loss 4.34 → **4.03**; RSS **2.9 GB vs 8.7 GB** fp32; step 7.7 s vs 5.5 s (native Windows) |
|
|
229
|
+
| AdamW4bit step (1M params) | 28.6 ms vs 3.3 ms for the AVX2 8-bit kernel; state 1.031 vs 2.062 B/param |
|
|
230
|
+
|
|
231
|
+
Not every workload is faster than fp32: 4-bit/8-bit kernels trade compute for memory traffic,
|
|
232
|
+
and small per-layer matmuls are better left to oneDNN. See `bench_results/` and the report for
|
|
233
|
+
the full trade-off tables, including negative results.
|
|
234
|
+
|
|
235
|
+
---
|
|
236
|
+
|
|
237
|
+
## Platform limitations
|
|
238
|
+
|
|
239
|
+
- **CPU only.** CUDA/ROCm/XPU kernels are not built; a GPU is never required or used.
|
|
240
|
+
- **Windows wheel**: from dev9 on it ships the MSVC DLL built by CI from the current
|
|
241
|
+
`csrc`, which exports the 2/3-bit dequant/GEMM and 4-bit optimizer kernels; they run
|
|
242
|
+
natively on Windows (verified: dequant maxdiff 1.2e-6, 4-bit native vs torch 1.2e-7).
|
|
243
|
+
Wheels older than dev9 fall back to pure torch for those symbols.
|
|
244
|
+
- **manylinux_2_17 wheel**: the AVX-512 library inside it is single-threaded (the parallel
|
|
245
|
+
build is only for the native `manylinux_2_39` wheel). On AVX-512 machines set
|
|
246
|
+
`BNB_CPU_DISABLE_AVX512=1` to run the AVX2 OpenMP path.
|
|
247
|
+
- **aarch64**: there is no PyPI wheel yet; build from the sdist (`build_linux.sh`) or use the
|
|
248
|
+
Termux instructions in `docs_cpu/TERMUX_GUIDE.md`.
|
|
249
|
+
- **macOS**: no wheel is published and the CPU backend is not tested there; the sdist build may
|
|
250
|
+
work with clang + OpenMP, but treat it as unsupported.
|
|
251
|
+
- 3-bit (and 2-bit dequant-only) is **experimental**: the API and kernels are covered
|
|
252
|
+
by tests, but there is no long-run quality study yet; treat it as a research feature.
|
|
253
|
+
- `torch` must be installed separately as the CPU build (see above).
|
|
254
|
+
|
|
255
|
+
## Build from source
|
|
256
|
+
|
|
257
|
+
```bash
|
|
258
|
+
# Linux (x86-64 AVX2 or aarch64 NEON; g++/clang + OpenMP)
|
|
259
|
+
sudo apt install g++ libomp-dev # Debian/Ubuntu
|
|
260
|
+
bash build_linux.sh # -> bitsandbytes/libbitsandbytes_cpu.so
|
|
261
|
+
bash build_linux.sh --selftest # also build + run the torch-free C self-test
|
|
262
|
+
|
|
263
|
+
# Windows (MSVC, manual build, no CMake)
|
|
264
|
+
build_manual\build_manual.bat amd # run inside "x64 Native Tools Command Prompt"
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
Two special wheel builds used by this project are documented in
|
|
268
|
+
[`docs_cpu/WHEEL_PLATFORM_NOTES.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/WHEEL_PLATFORM_NOTES.md):
|
|
269
|
+
the zig manylinux_2_17 portable build and the devtoolset-8 manylinux_2_17 **OpenMP** build
|
|
270
|
+
(`build_oldgomp_linux.sh`, bundles libgomp).
|
|
271
|
+
|
|
272
|
+
## Tests
|
|
273
|
+
|
|
274
|
+
```bash
|
|
275
|
+
bash build_linux.sh --selftest # C kernels: 9 checks (quantize/gemm_8bit/gemv_4bit/
|
|
276
|
+
# AdamW8bit/vector-vs-scalar/2-3bit dequant/fused GEMM)
|
|
277
|
+
python test_efst_regressions.py # EFST: nested blocks, 3-D tensor experts, routing usage
|
|
278
|
+
python test_q3bit_kernels.py # 2/3-bit research prototype (old DLL-based tests)
|
|
279
|
+
python test_q3bit_api.py # public q3bit API: pack/dequant/gemm, QuantLinear3bit, fallback
|
|
280
|
+
python test_adamw4bit.py # restored 4-bit optimizer: state bytes, native vs torch, fp32 tracking
|
|
281
|
+
python test_concurrency_plan.py # planner / memory estimate / tune_threads / launcher
|
|
282
|
+
python test_disk_balancer_safety.py # data-safety rules for the disk balancer
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
## Documentation
|
|
286
|
+
|
|
287
|
+
The long-form docs live in the repository (`bitsandbytes/docs_cpu/`, Chinese and English):
|
|
288
|
+
|
|
289
|
+
- [`QUICKSTART.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/QUICKSTART.md) / `QUICKSTART_EN.md` — commands, features, disaster recovery
|
|
290
|
+
- [`API_REFERENCE.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/API_REFERENCE.md) / `API_REFERENCE_EN.md` — every public API
|
|
291
|
+
- [`TECHNICAL_GUIDE.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/TECHNICAL_GUIDE.md) — kernel architecture (AVX2/NEON paths, layouts, sync rules)
|
|
292
|
+
- [`EPYC_VALIDATION.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/EPYC_VALIDATION.md) — real AVX-512/BF16 machine validation
|
|
293
|
+
- [`ISA_MATRIX.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/ISA_MATRIX.md) — x86 AVX2/generic/AVX-512 + aarch64 test matrix
|
|
294
|
+
- [`WHEEL_PLATFORM_NOTES.md`](https://github.com/Ganyvnanoaa09-0906/bitsandbytes-CPU/blob/HEAD/bitsandbytes/docs_cpu/WHEEL_PLATFORM_NOTES.md) — which wheel runs where, and why
|
|
295
|
+
|
|
296
|
+
## Changelog
|
|
297
|
+
|
|
298
|
+
| Version | Changes |
|
|
299
|
+
|---|---|
|
|
300
|
+
| `0.50.2.dev1` | First CPU-fork release: AVX2/NEON CPU backend (8-bit optimizers, fused GDN, `gemm_8bit`, 4/8-bit quantisation) |
|
|
301
|
+
| `0.50.2.dev2` | AdamW4bit packaged; PyPI release |
|
|
302
|
+
| `0.50.2.dev3` | README / API reference / summary document the 4-bit optimizer |
|
|
303
|
+
| `0.50.2.dev4` | Linux wheels (manylinux_2_17 portable + manylinux_2_39 OpenMP); libgcc-free CPUID detection; AVX-512 BF16 loader safety; portability and data-safety fixes |
|
|
304
|
+
| `0.50.2.dev5` | WSL/Linux disk balancer (ext4-only, 9P rejected, in-flight byte cap); `torch_cpu_kit` concurrency planner / memory estimator / `tune_threads`; optional `kernels` warning fix; EPYC validation docs |
|
|
305
|
+
| `0.50.2.dev6` | manylinux_2_17 wheel is now **OpenMP** (devtoolset-8, bundled libgomp → 3–4× optimizer-kernel speedup on old glibc); csrc 2/3-bit dequant + fused GEMM kernels with 9-item C self-test; EFST fused tensor-expert routing fix; GCC 8 `<cpuid.h>` portability fix |
|
|
306
|
+
| `0.50.2.dev7` | documentation release: README rewritten for the CPU fork (install/wheels/platform matrix, feature table, measured performance, limitations, tests, full dev1→dev7 changelog, EFST/disk-balancer/toolbox examples) |
|
|
307
|
+
| `0.50.2.dev8` | **restored `AdamW4bit`** (the kernel was dropped when dev4 vendored upstream; now scalar C kernel + torch fallback, 1.031 B/param); **new `bitsandbytes.q3bit` API** — `quantize_3bit`/`dequantize_3bit`/`gemm_3bit`/`gemm_3bit_transposed`, `QuantLinear3bit` (frozen 3-bit base + LoRA), `convert_linears_3bit`, native kernels on Linux with identical pure-torch fallback; CLI reference now includes 4bitopt/q3bit and all 43 examples execute in CI-style verification |
|
|
308
|
+
| `0.50.2.dev9` | **Windows native kernels**: CI now compiles the MSVC DLL from the current csrc and the export list includes the 4-bit optimizer and 2/3-bit symbols (they were silently missing before); **q3bit real-model validation** on Qwen3-1.7B (frozen 3-bit base + LoRA, eval 4.34→4.03 over 80 steps, RSS −67%, native step 7.7 s vs fp32 5.5 s); fixed the causal-LM **double-shift** label bug in the iGPU training kit (Windows package v0.1.2) |
|
|
309
|
+
|
|
310
|
+
## License and attribution
|
|
311
|
+
|
|
312
|
+
`bitsandbytes-cpu-fork` is MIT licensed, like upstream
|
|
313
|
+
[bitsandbytes](https://github.com/bitsandbytes-foundation/bitsandbytes) (this fork is based on
|
|
314
|
+
upstream `0.50.2`); the CUDA kernels were removed and replaced by the CPU backend described
|
|
315
|
+
above. See `NOTICE.md` for details and upstream attribution.
|