bitsandbytes 0.50.0__py3-none-win_arm64.whl

bitsandbytes/cextension.py

@@ -0,0 +1,405 @@
+import ctypes as ct
+import functools
+import logging
+import os
+from pathlib import Path
+import re
+from typing import Optional
+
+import torch
+
+from bitsandbytes.consts import DYNAMIC_LIBRARY_SUFFIX, PACKAGE_DIR
+from bitsandbytes.cuda_specs import (
+    CUDASpecs,
+    get_cuda_specs,
+    get_cuda_version_tuple,
+    get_rocm_gpu_arch,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def get_cuda_bnb_library_path(cuda_specs: CUDASpecs) -> Path:
+    """
+    Get the path to the best matching CUDA/ROCm BNB native library for the given specs.
+
+    When no override is set, selects from packaged libraries using the following priority:
+    1. Exact version match.
+    2. Highest packaged version <= runtime version, same major (e.g. runtime 12.9, ship 12.8).
+    3. Lowest packaged version > runtime version, same major (e.g. runtime 12.0, ship 12.1).
+    No cross-major fallback: if no same-major library exists, returns the exact non-existent
+    path so the caller raises a clear "not found" error.
+    A warning is logged when falling back. Override env vars bypass selection entirely
+    and load the named version with no fallback. The returned path is not guaranteed to
+    exist when no packaged libs are found, or when an override names an absent version.
+    """
+    is_hip = bool(torch.version.hip)
+    prefix = "rocm" if is_hip else "cuda"
+    override_var = "BNB_ROCM_VERSION" if is_hip else "BNB_CUDA_VERSION"
+
+    override_value = os.environ.get(override_var)
+
+    if override_value is not None:
+        if not override_value.isdigit():
+            raise RuntimeError(f"{override_var}={override_value!r}: value must be digits only (e.g. '124' for 12.4).")
+        library_name = f"libbitsandbytes_{prefix}{override_value}{DYNAMIC_LIBRARY_SUFFIX}"
+        logger.warning(
+            f"WARNING: {override_var}={override_value} environment variable detected; loading {library_name}.\n"
+            f"This overrides automatic {'ROCm' if is_hip else 'CUDA'} version selection.\n"
+            f"If this was unintended clear the variable and retry: unset {override_var}\n",
+        )
+        return PACKAGE_DIR / library_name
+
+    available = _find_cuda_libs(prefix, is_hip)
+    runtime_version = cuda_specs.cuda_version_tuple
+
+    if not available:
+        return PACKAGE_DIR / f"libbitsandbytes_{prefix}{cuda_specs.cuda_version_string}{DYNAMIC_LIBRARY_SUFFIX}"
+
+    if runtime_version in available:
+        return available[runtime_version]
+
+    lower = [v for v in available if v[0] == runtime_version[0] and v < runtime_version]
+    if lower:
+        selected = max(lower)
+    else:
+        higher_same = [v for v in available if v[0] == runtime_version[0] and v > runtime_version]
+        if higher_same:
+            selected = min(higher_same)
+        else:
+            # No same-major library available. Return the non-existent exact path so
+            # get_native_library() raises a clear "not found" error.
+            return PACKAGE_DIR / f"libbitsandbytes_{prefix}{cuda_specs.cuda_version_string}{DYNAMIC_LIBRARY_SUFFIX}"
+
+    logger.warning(
+        f"No prebuilt binary for {'ROCm' if is_hip else 'CUDA'} "
+        f"{runtime_version[0]}.{runtime_version[1]}, loading "
+        f"{'ROCm' if is_hip else 'CUDA'} {selected[0]}.{selected[1]} instead. "
+        f"Set {override_var} to override."
+    )
+    return available[selected]
+
+
+class BNBNativeLibrary:
+    _lib: ct.CDLL
+    compiled_with_cuda = False
+
+    def __init__(self, lib: ct.CDLL):
+        self._lib = lib
+
+    @functools.cache  # noqa: B019
+    def __getattr__(self, name):
+        fn = getattr(self._lib, name, None)
+
+        if fn is not None:
+            return fn
+
+        def throw_on_call(*args, **kwargs):
+            raise RuntimeError(
+                f"Method '{name}' not available in CPU-only version of bitsandbytes.\n"
+                "Reinstall with GPU support or use CUDA-enabled hardware."
+            )
+
+        return throw_on_call
+
+    def __getitem__(self, item):
+        return self.__getattr__(item)
+
+
+class CudaBNBNativeLibrary(BNBNativeLibrary):
+    compiled_with_cuda = True
+
+    def __init__(self, lib: ct.CDLL):
+        super().__init__(lib)
+        lib.get_context.restype = ct.c_void_p
+        lib.cget_managed_ptr.restype = ct.c_void_p
+
+
+class XpuBNBNativeLibrary(BNBNativeLibrary):
+    """XPU native library with SYCL USM paged memory support."""
+
+    def __init__(self, lib: ct.CDLL):
+        super().__init__(lib)
+        if hasattr(lib, "cget_managed_ptr"):
+            lib.cget_managed_ptr.restype = ct.c_void_p
+
+
+def _split_cuda_version(compact: str, is_hip: bool) -> tuple[int, int]:
+    """Split a compact CUDA/ROCm version string from a library filename into (major, minor).
+
+    CUDA: major is always 2 digits (11, 12, 13...), e.g. '118' -> (11, 8), '132' -> (13, 2).
+    ROCm: major is always 1 digit for now (6, 7...), e.g. '72' -> (7, 2), '713' -> (7, 13).
+    Note: revisit if ROCm major reaches 10.
+    """
+    if is_hip:
+        return int(compact[:1]), int(compact[1:])
+    return int(compact[:2]), int(compact[2:])
+
+
+def _find_cuda_libs(prefix: str, is_hip: bool) -> dict[tuple[int, int], Path]:
+    """Return a {(major, minor): Path} mapping for all packaged CUDA/ROCm library files."""
+    result = {}
+    for lib in PACKAGE_DIR.glob(f"libbitsandbytes_{prefix}*{DYNAMIC_LIBRARY_SUFFIX}"):
+        match = re.search(rf"{prefix}(\d+)", lib.name)
+        if match:
+            try:
+                result[_split_cuda_version(match.group(1), is_hip)] = lib
+            except (ValueError, IndexError):
+                continue
+    return result
+
+
+def get_available_cuda_binary_versions() -> list[str]:
+    """Get formatted CUDA/ROCm versions from existing library files."""
+    is_hip = bool(torch.version.hip)
+    prefix = "rocm" if is_hip else "cuda"
+    return sorted(f"{major}.{minor}" for major, minor in _find_cuda_libs(prefix, is_hip))
+
+
+def parse_cuda_version(version_str: str) -> str:
+    """Convert a raw version code string (e.g. '118', '713') to a dotted version (e.g. '11.8', '7.13')."""
+    if version_str.isdigit():
+        is_hip = bool(torch.version.hip)
+        try:
+            major, minor = _split_cuda_version(version_str, is_hip)
+            return f"{major}.{minor}"
+        except (ValueError, IndexError):
+            pass
+    return version_str
+
+
+class ErrorHandlerMockBNBNativeLibrary(BNBNativeLibrary):
+    """
+    Mock library handler that defers errors until native methods are called.
+
+    This class serves as a fallback when the native bitsandbytes library fails to load.
+    It captures the original error and generates detailed troubleshooting guidance.
+
+    Key behaviors:
+    - Allows attribute access and method assignment without immediate errors
+    - Throws a RuntimeError with diagnostic information only when a native method is called, as otherwise it would error out on import, breaking backward compatibility
+    - Handles both missing CUDA dependencies and version mismatch scenarios
+
+    Error scenarios covered:
+    1. Missing shared library dependencies (e.g., libcudart.so not in LD_LIBRARY_PATH or through PyTorch CUDA installation)
+    2. CUDA version mismatch between PyTorch and available pre-compiled binaries
+    3. Completely missing pre-compiled binaries when CUDA is detected
+    4. Custom BNB_CUDA_VERSION or BNB_ROCM_VERSION override but mismatch
+    5. CPU-only installation attempts when GPU functionality is requested
+
+    """
+
+    def __init__(self, error_msg: str):
+        self.error_msg = error_msg
+        self.available_versions = get_available_cuda_binary_versions()
+        override_value = os.environ.get("BNB_ROCM_VERSION") if HIP_ENVIRONMENT else os.environ.get("BNB_CUDA_VERSION")
+        user_version = get_cuda_version_tuple()
+        user_version_str = f"{user_version[0]}.{user_version[1]}" if user_version else "unknown"
+        self.requested_version = parse_cuda_version(override_value) if override_value else user_version_str
+
+        # Pre-generate the error message based on error type
+        if "cannot open shared object file" in error_msg:
+            self.formatted_error = self._format_dependency_error()
+        else:  # lib loading errors
+            self.formatted_error = self._format_lib_error_message(
+                available_versions=self.available_versions,
+                user_cuda_version=user_version_str,
+                original_error=f"Original error: {self.error_msg}\n" if self.error_msg else "",
+                requested_version=self.requested_version,
+            )
+
+    def _format_lib_error_message(
+        self,
+        available_versions: list[str],
+        user_cuda_version: str,
+        original_error: str = "",
+        requested_version: Optional[str] = None,
+    ) -> str:
+        """Format detailed error message for library loading failures"""
+        analysis = ""
+        no_cpu_lib_found = "libbitsandbytes_cpu.so: cannot open" in original_error
+        no_cuda_lib_found = f"{BNB_BACKEND} binary not found" in original_error
+
+        if no_cpu_lib_found:
+            analysis = "\n🚨 Failed to load CPU-only bitsandbytes library 🚨\n\n"
+
+        elif no_cuda_lib_found:
+            version_list_str = "\n  - " + "\n  - ".join(available_versions) if available_versions else "NONE"
+            analysis = (
+                (
+                    f"\n🚨 {BNB_BACKEND} VERSION MISMATCH 🚨\n"
+                    f"Requested {BNB_BACKEND} version:          {requested_version}\n"
+                    f"Detected PyTorch {BNB_BACKEND} version:   {user_cuda_version}\n"
+                    f"Available pre-compiled versions: {version_list_str}\n\n"
+                    "This means:\n"
+                    "The version you're trying to use is NOT distributed with this package\n\n"
+                )
+                if available_versions
+                else "\n🚨 Forgot to compile the bitsandbytes library? 🚨\n"
+                "1. You're not using the package but checked-out the source code\n"
+                "2. You MUST compile from source\n\n"
+            )
+
+        base_msg = "Attempted to use bitsandbytes native library functionality but it's not available.\n\n"
+
+        troubleshooting = (
+            (
+                f"This typically happens when:\n"
+                f"1. bitsandbytes doesn't ship with a pre-compiled binary for your {BNB_BACKEND} version\n"
+                f"2. The library wasn't compiled properly during installation from source\n\n"
+            )
+            if no_cuda_lib_found
+            else f"This typically happens when you checked the code out from source and your torch installation doesn't detect {BNB_BACKEND} on your machine.\n\n"
+        )
+
+        note = (
+            (
+                f"bitsandbytes tried to find a compatible {BNB_BACKEND} binary but none could be loaded.\n"
+                f"If your {BNB_BACKEND} version isn't among the available pre-compiled versions above, you must compile from source.\n\n"
+            )
+            if no_cuda_lib_found
+            else ""
+        )
+
+        compile_instructions = (
+            ("COMPILE FROM SOURCE for CPU-only:\n  `cmake -DCOMPUTE_BACKEND=cpu -S . && make`\n\n")
+            if not no_cuda_lib_found
+            else (
+                "You have two options:\n"
+                "1. COMPILE FROM SOURCE (required if no binary exists):\n"
+                "   https://huggingface.co/docs/bitsandbytes/main/en/installation#cuda-compile\n"
+                "2. Use BNB_CUDA_VERSION to specify a DIFFERENT CUDA version from the detected one, which is installed on your machine and matching an available pre-compiled version listed above\n\n"
+            )
+            if not HIP_ENVIRONMENT
+            else (
+                "You have two options:\n"
+                "1. COMPILE FROM SOURCE as mentioned here:\n"
+                "   https://huggingface.co/docs/bitsandbytes/main/en/installation?backend=AMD+ROCm#amd-gpu\n"
+                "2. Use BNB_ROCM_VERSION to specify a DIFFERENT ROCm version from the detected one, matching the version the library was built with.\n\n"
+            )
+        )
+
+        diagnostics = (
+            f"🔍 Run this command for detailed diagnostics:\n"
+            f"python -m bitsandbytes\n\n"
+            f"If you've tried everything and still have issues:\n"
+            f"1. Include ALL version info (operating system, bitsandbytes, pytorch, {BNB_BACKEND.lower()}, python)\n"
+            f"2. Describe what you've tried in detail\n"
+            f"3. Open an issue with this information:\n"
+            f"   https://github.com/bitsandbytes-foundation/bitsandbytes/issues\n\n"
+        )
+
+        return f"{analysis}{base_msg}{troubleshooting}{note}{compile_instructions}{original_error}\n{diagnostics}"
+
+    def _format_dependency_error(self) -> str:
+        """Format error message for missing shared libraries"""
+        # Extract missing library name from error
+        error_parts = self.error_msg.split(":")
+        missing_lib = error_parts[0].strip() if len(error_parts) > 0 else "unknown library"
+        cuda_major_version = (
+            self.requested_version.split(".")[0] if "." in self.requested_version else self.requested_version
+        )
+
+        return (
+            f"\n🚨 {BNB_BACKEND} SETUP ERROR: Missing dependency: {missing_lib} 🚨\n\n"
+            f"{BNB_BACKEND} {cuda_major_version}.x runtime libraries were not found in the LD_LIBRARY_PATH.\n\n"
+            f"To fix this, make sure that:\n"
+            f"1. You have installed {BNB_BACKEND} {cuda_major_version}.x toolkit on your system\n"
+            f"2. The {BNB_BACKEND} runtime libraries are in your LD_LIBRARY_PATH\n\n"
+            f"You can add them with (and persist the change by adding the line to your .bashrc):\n"
+            f"   export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/{BNB_BACKEND.lower()}-{cuda_major_version}.x/"
+            f"{'lib64' if not HIP_ENVIRONMENT else 'lib'}\n\n"
+            f"Original error: {self.error_msg}\n\n"
+            f"🔍 Run this command for detailed diagnostics:\n"
+            f"python -m bitsandbytes\n\n"
+            f"If you've tried everything and still have issues:\n"
+            f"1. Include ALL version info (operating system, bitsandbytes, pytorch, {BNB_BACKEND.lower()}, python)\n"
+            f"2. Describe what you've tried in detail\n"
+            f"3. Open an issue with this information:\n"
+            f"   https://github.com/bitsandbytes-foundation/bitsandbytes/issues\n\n"
+        )
+
+    def __getattr__(self, name):
+        """Return a dummy function that throws when called, rather than on attribute access"""
+
+        def throw_on_call(*args, **kwargs):
+            raise RuntimeError(f"{self.formatted_error}Native code method attempted to call: lib.{name}()")
+
+        return throw_on_call
+
+    def __getitem__(self, name):
+        return self.__getattr__(name)
+
+
+def get_xpu_bnb_library_path() -> Path:
+    """Get the path to the XPU native library matching the oneAPI toolchain.
+
+    Prefers the versioned library (e.g. libbitsandbytes_xpu2026) matching the first
+    4 digits of torch.version.xpu, falling back to an unversioned libbitsandbytes_xpu.
+    """
+    xpu_version = getattr(torch.version, "xpu", None)
+    if xpu_version:
+        versioned = PACKAGE_DIR / f"libbitsandbytes_xpu{xpu_version[:4]}{DYNAMIC_LIBRARY_SUFFIX}"
+        if versioned.exists():
+            return versioned
+    return PACKAGE_DIR / f"libbitsandbytes_xpu{DYNAMIC_LIBRARY_SUFFIX}"
+
+
+def get_native_library() -> BNBNativeLibrary:
+    """
+    Load CUDA library XOR CPU, as the latter contains a subset of symbols of the former.
+    """
+    cuda_specs = get_cuda_specs()
+    binary_path = PACKAGE_DIR / f"libbitsandbytes_cpu{DYNAMIC_LIBRARY_SUFFIX}"
+
+    if cuda_specs:
+        cuda_binary_path = get_cuda_bnb_library_path(cuda_specs)
+
+        if not cuda_binary_path.exists():
+            raise RuntimeError(f"No compatible {BNB_BACKEND} binary found at {cuda_binary_path}")
+
+        binary_path = cuda_binary_path
+
+    if torch._C._has_xpu:
+        binary_path = get_xpu_bnb_library_path()
+
+    logger.debug(f"Loading bitsandbytes native library from: {binary_path}")
+
+    # Try to load the library - any errors will propagate up
+    dll = ct.cdll.LoadLibrary(str(binary_path))
+
+    if hasattr(dll, "get_context"):  # only a CUDA-built library exposes this
+        return CudaBNBNativeLibrary(dll)
+
+    if torch._C._has_xpu:
+        return XpuBNBNativeLibrary(dll)
+
+    return BNBNativeLibrary(dll)
+
+
+ROCM_GPU_ARCH = get_rocm_gpu_arch()
+
+HIP_ENVIRONMENT = False
+BNB_BACKEND = "CPU"
+if torch.version.hip:
+    HIP_ENVIRONMENT = True
+    BNB_BACKEND = "ROCm"
+elif torch.cuda.is_available():
+    BNB_BACKEND = "CUDA"
+elif torch._C._has_xpu:
+    BNB_BACKEND = "XPU"
+
+try:
+    lib = get_native_library()
+except Exception as e:
+    if BNB_BACKEND in ("CPU", "XPU"):
+        lib = ErrorHandlerMockBNBNativeLibrary("XPU/CPU can run without native library.")
+    else:
+        error_msg = str(e)
+        logger.error(
+            f"bitsandbytes library load error: {error_msg}",
+            exc_info=True,
+        )
+
+        # create a mock with error messaging as fallback
+        lib = ErrorHandlerMockBNBNativeLibrary(error_msg)

bitsandbytes/consts.py

@@ -0,0 +1,12 @@
+from pathlib import Path
+import platform
+
+DYNAMIC_LIBRARY_SUFFIX = {
+    "Darwin": ".dylib",
+    "Linux": ".so",
+    "Windows": ".dll",
+}.get(platform.system(), ".so")
+
+PACKAGE_DIR = Path(__file__).parent
+PACKAGE_GITHUB_URL = "https://github.com/TimDettmers/bitsandbytes"
+NONPYTORCH_DOC_URL = "https://github.com/TimDettmers/bitsandbytes/blob/main/docs/source/nonpytorchcuda.mdx"

bitsandbytes/cuda_specs.py

@@ -0,0 +1,111 @@
+import dataclasses
+from functools import lru_cache
+import logging
+import platform
+import re
+import subprocess
+from typing import Optional
+
+import torch
+
+
+@dataclasses.dataclass(frozen=True)
+class CUDASpecs:
+    highest_compute_capability: tuple[int, int]
+    cuda_version_string: str
+    cuda_version_tuple: tuple[int, int]
+
+    @property
+    def has_imma(self) -> bool:
+        return torch.version.hip or self.highest_compute_capability >= (7, 5)
+
+
+def get_compute_capabilities() -> list[tuple[int, int]]:
+    return sorted(torch.cuda.get_device_capability(torch.cuda.device(i)) for i in range(torch.cuda.device_count()))
+
+
+@lru_cache(None)
+def get_cuda_version_tuple() -> Optional[tuple[int, int]]:
+    """Get CUDA/HIP version as a tuple of (major, minor)."""
+    try:
+        if torch.version.cuda:
+            version_str = torch.version.cuda
+        elif torch.version.hip:
+            version_str = torch.version.hip
+        else:
+            return None
+
+        parts = version_str.split(".")
+        if len(parts) >= 2:
+            return tuple(map(int, parts[:2]))
+        return None
+    except (AttributeError, ValueError, IndexError):
+        return None
+
+
+def get_cuda_version_string() -> Optional[str]:
+    """Get CUDA/HIP version as a string."""
+    version_tuple = get_cuda_version_tuple()
+    if version_tuple is None:
+        return None
+    major, minor = version_tuple
+    return f"{major}{minor}"
+
+
+def get_cuda_specs() -> Optional[CUDASpecs]:
+    """Get CUDA/HIP specifications."""
+    if not torch.cuda.is_available():
+        return None
+
+    try:
+        compute_capabilities = get_compute_capabilities()
+        if not compute_capabilities:
+            return None
+
+        version_tuple = get_cuda_version_tuple()
+        if version_tuple is None:
+            return None
+
+        version_string = get_cuda_version_string()
+        if version_string is None:
+            return None
+
+        return CUDASpecs(
+            highest_compute_capability=compute_capabilities[-1],
+            cuda_version_string=version_string,
+            cuda_version_tuple=version_tuple,
+        )
+    except Exception:
+        return None
+
+
+def get_rocm_gpu_arch() -> str:
+    """Get ROCm GPU architecture."""
+    logger = logging.getLogger(__name__)
+    try:
+        if torch.version.hip:
+            # On Windows, use hipinfo.exe; on Linux, use rocminfo
+            if platform.system() == "Windows":
+                cmd = ["hipinfo.exe"]
+                arch_pattern = r"gcnArchName:\s+gfx([a-zA-Z\d]+)"
+            else:
+                cmd = ["rocminfo"]
+                arch_pattern = r"Name:\s+gfx([a-zA-Z\d]+)"
+
+            result = subprocess.run(cmd, capture_output=True, text=True)
+            match = re.search(arch_pattern, result.stdout)
+            if match:
+                return "gfx" + match.group(1)
+            else:
+                return "unknown"
+        else:
+            return "unknown"
+    except Exception as e:
+        logger.error(f"Could not detect ROCm GPU architecture: {e}")
+        if torch.cuda.is_available():
+            logger.warning(
+                """
+ROCm GPU architecture detection failed despite ROCm being available.
+                """,
+            )
+        return "unknown"