transformers 5.0.0rc0__py3-none-any.whl → 5.0.0rc1__py3-none-any.whl

transformers/quantizers/quantizer_bnb_8bit.py

@@ -25,6 +25,8 @@ from ..utils import (
     is_accelerate_available,
     is_bitsandbytes_available,
     is_torch_available,
+    is_torch_hpu_available,
+    is_torch_npu_available,
     is_torch_xpu_available,
     logging,
 )
@@ -35,34 +37,20 @@ if is_torch_available():
     import torch
 
     from ..core_model_loading import WeightConverter
-    from ..pytorch_utils import Conv1D
 
 logger = logging.get_logger(__name__)
 
 
 class Bnb8BitHfQuantizer(HfQuantizer):
     """
-    8-bit quantization from bitsandbytes quantization method:
-        before loading: converts transformer layers into Linear8bitLt during loading: load 16bit weight and pass to the
-        layer object after: quantizes individual weights in Linear8bitLt into 8bit at fitst .cuda() call
-    saving:
-        from state dict, as usual; saves weights and 'SCB' component
-    loading:
-        need to locate SCB component and pass to the Linear8bitLt object
+    8-bit quantization from bitsandbytes quantization method
     """
 
-    use_keep_in_fp32_modules = True
-    requires_parameters_quantization = True
     requires_calibration = False
 
-    required_packages = ["bitsandbytes", "accelerate"]
-
     def __init__(self, quantization_config, **kwargs):
         super().__init__(quantization_config, **kwargs)
 
-        if self.quantization_config.llm_int8_skip_modules is not None:
-            self.modules_to_not_convert = self.quantization_config.llm_int8_skip_modules
-
     def validate_environment(self, *args, **kwargs):
         if not is_accelerate_available():
             raise ImportError(
@@ -78,17 +66,9 @@ class Bnb8BitHfQuantizer(HfQuantizer):
         validate_bnb_backend_availability(raise_exception=True)
 
         device_map = kwargs.get("device_map")
-        if (
-            device_map is not None
-            and isinstance(device_map, dict)
-            and not self.quantization_config.llm_int8_enable_fp32_cpu_offload
-        ):
-            device_map_without_lm_head = {
-                key: device_map[key] for key in device_map if key not in self.modules_to_not_convert
-            }
-            if set(device_map.values()) == {"cpu"}:
-                pass
-            elif "cpu" in device_map_without_lm_head.values() or "disk" in device_map_without_lm_head.values():
+        if not self.quantization_config.llm_int8_enable_fp32_cpu_offload and isinstance(device_map, dict):
+            values = set(device_map.values())
+            if values != {"cpu"} and ("cpu" in values or "disk" in values):
                 raise ValueError(
                     "Some modules are dispatched on the CPU or the disk. Make sure you have enough GPU RAM to fit the "
                     "quantized model. If you want to dispatch the model on the CPU or the disk while keeping these modules "
@@ -120,6 +100,10 @@ class Bnb8BitHfQuantizer(HfQuantizer):
         if device_map is None:
             if torch.cuda.is_available():
                 device_map = {"": torch.cuda.current_device()}
+            elif is_torch_npu_available():
+                device_map = {"": f"npu:{torch.npu.current_device()}"}
+            elif is_torch_hpu_available():
+                device_map = {"": f"hpu:{torch.hpu.current_device()}"}
             elif is_torch_xpu_available():
                 device_map = {"": torch.xpu.current_device()}
             else:
@@ -132,61 +116,14 @@ class Bnb8BitHfQuantizer(HfQuantizer):
         return device_map
 
     def adjust_target_dtype(self, target_dtype: "torch.dtype") -> "torch.dtype":
-        if target_dtype != torch.int8:
-            logger.info("target_dtype {target_dtype} is replaced by `torch.int8` for 8-bit BnB quantization")
         return torch.int8
 
-    def update_unexpected_keys(self, model, unexpected_keys: list[str]) -> list[str]:
-        bnb_keys = ["SCB", "weight_format"]
-        return [k for k in unexpected_keys if not any(k.endswith(x) for x in bnb_keys)]
-
     def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool:
         import bitsandbytes as bnb
 
         module, name = get_module_from_name(model, param_name)
         return isinstance(module, bnb.nn.Linear8bitLt) and name != "bias"
 
-    def create_quantized_param(
-        self,
-        model: "PreTrainedModel",
-        param_value: "torch.Tensor",
-        param_name: str,
-        target_device: "torch.device",
-        **kwargs,
-    ):
-        import bitsandbytes as bnb
-
-        module, tensor_name = get_module_from_name(model, param_name)
-
-        if self.pre_quantized and not self.is_serializable():
-            raise ValueError(
-                "Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. "
-                "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."
-            )
-        # Those 2 can only happen when self.pre_quantized == True
-        if tensor_name == "SCB":
-            setattr(module.weight, "SCB", param_value.to(target_device))
-            return
-        # It's not used, but it's getting serialized for BC reason...
-        elif tensor_name == "weight_format":
-            return
-
-        # Support models using `Conv1D` in place of `nn.Linear` (e.g. openai-community/gpt2) by transposing the weight matrix prior to quantization.
-        # Since weights are saved in the correct "orientation", we skip transposing when loading.
-        if issubclass(module.source_cls, Conv1D) and not self.pre_quantized:
-            param_value = param_value.T
-
-        old_value = getattr(module, tensor_name)
-        kwargs = old_value.__dict__
-        kwargs.pop("_is_hf_initialized", None)
-        # Need to pop SCB and reset it because of bnb internals that modifies its value when switching devices ...
-        SCB = kwargs.pop("SCB", None)
-        new_value = bnb.nn.Int8Params(param_value.to("cpu"), requires_grad=False, **kwargs).to(target_device)
-        if SCB is not None:
-            setattr(new_value, "SCB", SCB)
-        # Set it to the module
-        module._parameters[tensor_name] = new_value
-
     def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs):
         model.is_loaded_in_8bit = True
         model.is_8bit_serializable = self.is_serializable()
@@ -201,23 +138,14 @@ class Bnb8BitHfQuantizer(HfQuantizer):
     ):
         from ..integrations import replace_with_bnb_linear
 
-        llm_int8_enable_fp32_cpu_offload = self.quantization_config.llm_int8_enable_fp32_cpu_offload
-
         self.modules_to_not_convert = self.get_modules_to_not_convert(
             model, self.quantization_config.llm_int8_skip_modules, keep_in_fp32_modules
         )
 
-        # Extend `self.modules_to_not_convert` to keys that are supposed to be offloaded to `cpu` or `disk`
-        if isinstance(device_map, dict) and len(device_map.keys()) > 1:
-            keys_on_cpu = [key for key, value in device_map.items() if value in ["disk", "cpu"]]
-
-            if len(keys_on_cpu) > 0 and not llm_int8_enable_fp32_cpu_offload:
-                raise ValueError(
-                    "If you want to offload some keys to `cpu` or `disk`, you need to set "
-                    "`llm_int8_enable_fp32_cpu_offload=True`. Note that these modules will not be "
-                    " converted to 8-bit but kept in 32-bit."
-                )
-            self.modules_to_not_convert.extend(keys_on_cpu)
+        if self.quantization_config.llm_int8_enable_fp32_cpu_offload:
+            if isinstance(device_map, dict):
+                keys_on_cpu = [key for key, value in device_map.items() if value in ["disk", "cpu"]]
+                self.modules_to_not_convert.extend(keys_on_cpu)
 
         model = replace_with_bnb_linear(
             model,
@@ -226,9 +154,7 @@ class Bnb8BitHfQuantizer(HfQuantizer):
             pre_quantized=self.pre_quantized,
         )
 
-        model.config.quantization_config = self.quantization_config
-
-    def is_serializable(self, safe_serialization=None):
+    def is_serializable(self):
         return True
 
     @property
@@ -238,9 +164,7 @@ class Bnb8BitHfQuantizer(HfQuantizer):
     def _dequantize(self, model):
         from ..integrations import dequantize_and_replace
 
-        model = dequantize_and_replace(
-            model, self.modules_to_not_convert, quantization_config=self.quantization_config
-        )
+        model = dequantize_and_replace(model, quantization_config=self.quantization_config)
         return model
 
     def get_quantize_ops(self):

transformers/quantizers/quantizer_compressed_tensors.py

@@ -31,7 +31,6 @@ class CompressedTensorsHfQuantizer(HfQuantizer):
     """
 
     requires_calibration = True
-    required_packages = ["compressed_tensors"]
 
     def __init__(self, quantization_config: CompressedTensorsConfig, **kwargs):
         super().__init__(quantization_config, **kwargs)
@@ -58,9 +57,6 @@ class CompressedTensorsHfQuantizer(HfQuantizer):
                 "Using `compressed_tensors` quantized models requires the compressed-tensors library: "
                 "`pip install compressed-tensors`"
             )
-        if not is_torch_available():
-            # torch already should be installed as part of compressed tensors
-            raise ImportError("torch is required for using compressed-tensors quantization")
 
     def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype":
         if dtype is None:
@@ -113,6 +109,6 @@ class CompressedTensorsHfQuantizer(HfQuantizer):
         # models need to be decompressed carry out qat
         return not self.run_compressed or not self.quantization_config.is_quantization_compressed
 
-    def is_serializable(self, safe_serialization=None) -> bool:
+    def is_serializable(self) -> bool:
         """Models quantized using compressed tensors can be saved to disk"""
         return True

transformers/quantizers/quantizer_eetq.py

@@ -19,7 +19,7 @@ from .base import HfQuantizer
 if TYPE_CHECKING:
     from ..modeling_utils import PreTrainedModel
 
-from ..utils import is_accelerate_available, is_eetq_available, is_torch_available, logging
+from ..utils import is_accelerate_available, is_kernels_available, is_torch_available, logging
 from .quantizers_utils import get_module_from_name
 
 
@@ -32,40 +32,17 @@ logger = logging.get_logger(__name__)
 
 class EetqHfQuantizer(HfQuantizer):
     """
-    8-bit quantization from EETQ quantization method:
-        before loading: converts transformer layers into W8A16Linear during loading: load 16bit weight and pass to the
-        layer object after: quantizes individual weights in Linear8bitLt into 8bit at first .cuda() call
+    8-bit quantization from EETQ quantization method
     """
 
-    requires_parameters_quantization = True
     requires_calibration = False
 
-    required_packages = ["eetq", "accelerate"]
-
     def __init__(self, quantization_config, **kwargs):
         super().__init__(quantization_config, **kwargs)
-        self.quantization_config = quantization_config
 
     def validate_environment(self, *args, **kwargs):
-        if not is_eetq_available():
-            raise ImportError(
-                "Using `eetq` 8-bit quantization requires eetq."
-                "Please install the latest version of eetq from : https://github.com/NetEase-FuXi/EETQ"
-            )
-
-        try:
-            import eetq  # noqa: F401
-        except ImportError as exc:
-            if "shard_checkpoint" in str(exc):
-                # EETQ 1.0.0 is currently broken with the latest transformers because it tries to import the removed
-                # shard_checkpoint function, see https://github.com/NetEase-FuXi/EETQ/issues/34.
-                # TODO: Update message once eetq releases a fix
-                raise ImportError(
-                    "You are using a version of EETQ that is incompatible with the current transformers version. "
-                    "Either downgrade transformers to <= v4.46.3 or, if available, upgrade EETQ to > v1.0.0."
-                ) from exc
-            else:
-                raise
+        if not is_kernels_available():
+            raise ImportError("Loading an EETQ quantized model requires kernels (`pip install kernels`)")
 
         if not is_accelerate_available():
             raise ImportError("Loading an EETQ quantized model requires accelerate (`pip install accelerate`)")
@@ -79,8 +56,8 @@ class EetqHfQuantizer(HfQuantizer):
                 "You have loaded an EETQ model on CPU and have a CUDA device available, make sure to set "
                 "your model on a GPU device in order to run your model."
             )
-        elif device_map is not None:
-            if isinstance(device_map, dict) and ("cpu" in device_map.values() or "disk" in device_map.values()):
+        elif isinstance(device_map, dict):
+            if len(device_map) > 1 and "cpu" in device_map.values() or "disk" in device_map.values():
                 raise ValueError(
                     "You are attempting to load an EETQ model with a device_map that contains a CPU or disk device."
                     " This is not supported. Please remove the CPU or disk device from the device_map."
@@ -101,7 +78,7 @@ class EetqHfQuantizer(HfQuantizer):
         return dtype
 
     def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool:
-        from eetq import EetqLinear
+        from ..integrations.eetq import EetqLinear
 
         module, tensor_name = get_module_from_name(model, param_name)
 
@@ -112,31 +89,6 @@ class EetqHfQuantizer(HfQuantizer):
                 return True
         return False
 
-    def create_quantized_param(
-        self,
-        model: "PreTrainedModel",
-        param_value: "torch.Tensor",
-        param_name: str,
-        target_device: "torch.device",
-        **kwargs,
-    ):
-        from eetq import EetqLinear, quantize_and_preprocess_weights
-
-        module, tensor_name = get_module_from_name(model, param_name)
-        new_value, weight_scale = quantize_and_preprocess_weights(param_value)
-
-        # Samity check
-        if isinstance(module, EetqLinear):
-            if self.pre_quantized or tensor_name == "bias":
-                if tensor_name == "weight" and param_value.dtype != torch.int8:
-                    raise ValueError("Expect quantized weights but got an unquantized weight")
-            else:
-                if tensor_name == "weight_scale":
-                    raise ValueError("Expect unquantized weights but got a quantized weight_scale")
-
-        module._buffers[tensor_name] = new_value.to(target_device)
-        module.register("weight_scales", weight_scale.to(target_device))
-
     def _process_model_before_weight_loading(
         self,
         model: "PreTrainedModel",
@@ -150,17 +102,17 @@ class EetqHfQuantizer(HfQuantizer):
         )
 
         model = replace_with_eetq_linear(
-            model,
-            modules_to_not_convert=self.modules_to_not_convert,
-            quantization_config=self.quantization_config,
-            pre_quantized=self.pre_quantized,
+            model, modules_to_not_convert=self.modules_to_not_convert, pre_quantized=self.pre_quantized
         )
 
-        model.config.quantization_config = self.quantization_config
-
-    def is_serializable(self, safe_serialization=None):
+    def is_serializable(self):
         return True
 
     @property
     def is_trainable(self) -> bool:
         return True
+
+    def get_quantize_ops(self):
+        from ..integrations.eetq import EetqQuantize
+
+        return EetqQuantize(self)

transformers/quantizers/quantizer_fbgemm_fp8.py

@@ -19,14 +19,21 @@ from .base import HfQuantizer
 if TYPE_CHECKING:
     from ..modeling_utils import PreTrainedModel
 
-from ..utils import is_accelerate_available, is_fbgemm_gpu_available, is_torch_available, logging
+from ..utils import (
+    is_accelerate_available,
+    is_fbgemm_gpu_available,
+    is_kernels_available,
+    is_torch_available,
+    is_torch_cuda_available,
+    is_torch_xpu_available,
+    logging,
+)
 from .quantizers_utils import get_module_from_name
 
 
 if is_torch_available():
     import torch
 
-
 logger = logging.get_logger(__name__)
 
 
@@ -35,54 +42,41 @@ class FbgemmFp8HfQuantizer(HfQuantizer):
     FP8 quantization using fbgemm kernels
     """
 
-    requires_parameters_quantization = True
     requires_calibration = False
 
-    required_packages = ["fbgemm-gpu", "accelerate"]
-
     def __init__(self, quantization_config, **kwargs):
         super().__init__(quantization_config, **kwargs)
-        self.quantization_config = quantization_config
 
     def validate_environment(self, *args, **kwargs):
-        if not is_torch_available():
-            raise ImportError(
-                "Using fbgemm fp8 quantization requires torch >= 2.1.0"
-                "Please install the latest version of torch ( pip install --upgrade torch )"
-            )
-        if not is_fbgemm_gpu_available():
+        if not is_torch_cuda_available() and not is_torch_xpu_available():
+            raise ImportError("Using fbgemm fp8 quantization requires a GPU or XPU")
+        if is_torch_xpu_available() and not is_kernels_available():
+            raise ImportError("Using FP8 fbgemm on XPU requires kernels (`pip install kernels`)")
+        if is_torch_cuda_available() and not is_fbgemm_gpu_available():
             raise ImportError(
-                "Using fbgemm fp8 quantization requires fbgemm-gpu library"
+                "Loading an FP8 fbgemm quantized model on CUDA requires fbgemm-gpu library"
                 "Please install the latest version of fbgemm-gpu library by following : https://pytorch.org/FBGEMM/fbgemm_gpu-development/InstallationInstructions.html#fbgemm-gpu-install-libraries"
             )
-
         if not is_accelerate_available():
             raise ImportError(
                 "Loading an FP8 quantized model requires accelerate (`pip install --upgrade accelerate`)"
             )
-
-        if not torch.cuda.is_available():
-            raise RuntimeError("Using FP8 quantized models with fbgemm kernels requires a GPU")
-
-        compute_capability = torch.cuda.get_device_capability()
-        major, minor = compute_capability
-        if major < 9:
-            raise ValueError(
-                "FP8 quantized models is only supported on GPUs with compute capability >= 9.0 (e.g H100)"
-            )
+        if is_torch_cuda_available():
+            compute_capability = torch.cuda.get_device_capability()
+            major, _ = compute_capability
+            if major < 9:
+                raise ValueError(
+                    "FP8 quantized models is only supported on GPUs with compute capability >= 9.0 (e.g H100)"
+                )
 
         device_map = kwargs.get("device_map")
         if device_map is None:
             logger.warning_once(
-                "You have loaded an FP8 model on CPU and have a CUDA device available, make sure to set "
-                "your model on a GPU device in order to run your model. To remove this warning, pass device_map = 'cuda'. "
+                "You have loaded an FP8 model on CPU and have a CUDA/XPU device available, make sure to set "
+                "your model on a GPU/XPU device in order to run your model. To remove this warning, pass device_map = 'cuda' or 'xpu' or 'auto'. "
             )
-        elif device_map is not None:
-            if (
-                not self.pre_quantized
-                and isinstance(device_map, dict)
-                and ("cpu" in device_map.values() or "disk" in device_map.values())
-            ):
+        elif isinstance(device_map, dict):
+            if not self.pre_quantized and ("cpu" in device_map.values() or "disk" in device_map.values()):
                 raise ValueError(
                     "You are attempting to load an FP8 model with a device_map that contains a CPU or disk device."
                     "This is not supported when the model is quantized on the fly. "
@@ -101,7 +95,7 @@ class FbgemmFp8HfQuantizer(HfQuantizer):
             )
         elif dtype == torch.float16:
             raise ValueError(
-                "You cannot use FP8 with dtype=torch.float16.We recommend you passing dtype=torch.bfloat16"
+                "You cannot use FP8 with dtype=torch.float16. We recommend you passing dtype=torch.bfloat16"
             )
         return dtype
 
@@ -122,76 +116,6 @@ class FbgemmFp8HfQuantizer(HfQuantizer):
                 return True
         return False
 
-    def create_quantized_param(
-        self,
-        model: "PreTrainedModel",
-        param_value: "torch.Tensor",
-        param_name: str,
-        target_device: "torch.device",
-        **kwargs,
-    ):
-        from ..integrations import FbgemmFp8Linear, FbgemmFp8Llama4TextExperts
-
-        module, tensor_name = get_module_from_name(model, param_name)
-
-        # Sanity checks
-        if isinstance(module, FbgemmFp8Linear):
-            if self.pre_quantized or tensor_name == "bias":
-                if tensor_name == "weight" and param_value.dtype != torch.float8_e4m3fn:
-                    raise ValueError("Expect quantized weights but got an unquantized weight")
-            else:
-                if tensor_name == "weight_scale":
-                    raise ValueError("Expect unquantized weights but got a quantized weight_scale")
-        if isinstance(module, FbgemmFp8Llama4TextExperts):
-            if not (self.pre_quantized or tensor_name == "bias"):
-                if tensor_name == "gate_up_proj_scale" or tensor_name == "down_proj_scale":
-                    raise ValueError("Expect unquantized weights but got a quantized weight_scale")
-
-        if isinstance(module, FbgemmFp8Llama4TextExperts):
-            if tensor_name == "gate_up_proj":
-                # Process each expert separately
-                # Transpose the second and third dimension
-                transposed_param = param_value.transpose(1, 2)
-
-                # Reshape to 2D for quantization
-                original_shape = transposed_param.shape
-                flattened_param = transposed_param.reshape(-1, original_shape[-1])
-
-                # Quantize using per row instead of per column
-                new_value_flat, weight_scale_flat = torch.ops.fbgemm.quantize_fp8_per_row(flattened_param)
-
-                # Reshape back to original dimensions
-                new_value = new_value_flat.reshape(original_shape)
-                new_value = new_value.transpose(1, 2)
-                weight_scale = weight_scale_flat.reshape(original_shape[0], 1, original_shape[1])
-            elif tensor_name == "down_proj":
-                # Process each expert separately
-                # Transpose the weights for proper quantization
-                transposed_param = param_value.transpose(1, 2)
-
-                # Reshape to 2D for quantization
-                original_shape = transposed_param.shape
-                flattened_param = transposed_param.reshape(-1, original_shape[-1])
-
-                # Quantize using per column
-                new_value_flat, weight_scale_flat = torch.ops.fbgemm.quantize_fp8_per_row(flattened_param)
-
-                # Reshape back to original dimensions
-                new_value = new_value_flat.reshape(original_shape)
-                new_value = new_value.transpose(1, 2)
-                weight_scale = weight_scale_flat.reshape(original_shape[0], original_shape[1], 1)
-
-            module._parameters[f"{tensor_name}_scale"] = torch.nn.Parameter(weight_scale.to(target_device))
-        else:
-            new_value, weight_scale = torch.ops.fbgemm.quantize_fp8_per_row(param_value)
-            module._parameters[f"{tensor_name}_scale"] = torch.nn.Parameter(
-                weight_scale.view(weight_scale.shape[0], 1).to(target_device)
-            )
-
-        module._parameters[tensor_name] = torch.nn.Parameter(new_value.to(target_device))
-
-        del param_name
-
     def _process_model_before_weight_loading(
         self,
         model: "PreTrainedModel",
@@ -200,38 +124,18 @@ class FbgemmFp8HfQuantizer(HfQuantizer):
     ):
         from ..integrations import replace_with_fbgemm_fp8_linear
 
-        tp_plan = model._tp_plan
         self.modules_to_not_convert = self.get_modules_to_not_convert(
             model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules
         )
 
-        config = model.config
         model = replace_with_fbgemm_fp8_linear(
             model,
             modules_to_not_convert=self.modules_to_not_convert,
             quantization_config=self.quantization_config,
             pre_quantized=self.pre_quantized,
-            config=config,
-            tp_plan=tp_plan,
+            tp_plan=model._tp_plan,
         )
 
-        model.config.quantization_config = self.quantization_config
-
-    def update_missing_keys(self, model, missing_keys: list[str], prefix: str) -> list[str]:
-        from ..integrations import FbgemmFp8Linear, FbgemmFp8Llama4TextExperts
-
-        not_missing_keys = []
-        for name, module in model.named_modules():
-            if isinstance(module, (FbgemmFp8Linear, FbgemmFp8Llama4TextExperts)):
-                for missing in missing_keys:
-                    if (
-                        (name in missing or name in f"{prefix}.{missing}")
-                        and not missing.endswith(".weight")
-                        and not missing.endswith(".bias")
-                    ):
-                        not_missing_keys.append(missing)
-        return [k for k in missing_keys if k not in not_missing_keys]
-
     def update_tp_plan(self, config):
         if "Llama4" in config.__class__.__name__:
             text_plan = {
@@ -279,9 +183,14 @@ class FbgemmFp8HfQuantizer(HfQuantizer):
 
         return config
 
-    def is_serializable(self, safe_serialization=None):
+    def is_serializable(self):
         return True
 
     @property
     def is_trainable(self) -> bool:
         return False
+
+    def get_quantize_ops(self):
+        from ..integrations.fbgemm_fp8 import FbgemmFp8Quantize
+
+        return FbgemmFp8Quantize(self)