transformers 5.0.0rc0__py3-none-any.whl → 5.0.0rc1__py3-none-any.whl

transformers/quantizers/quantizer_finegrained_fp8.py

@@ -20,26 +20,20 @@ class FineGrainedFP8HfQuantizer(HfQuantizer):
     Supports both e4m3fn formats based on platform.
     """
 
-    requires_parameters_quantization = True
     requires_calibration = False
-    required_packages = ["accelerate"]
 
     def __init__(self, quantization_config, **kwargs):
         super().__init__(quantization_config, **kwargs)
-        self.quantization_config = quantization_config
 
     def validate_environment(self, *args, **kwargs):
-        if not is_torch_available():
-            raise ImportError(
-                "Using fp8 quantization requires torch >= 2.1.0"
-                "Please install the latest version of torch ( pip install --upgrade torch )"
-            )
-
         if not is_accelerate_available():
             raise ImportError("Loading an FP8 quantized model requires accelerate (`pip install accelerate`)")
 
-        if (not (torch.cuda.is_available() or is_torch_xpu_available())) and not self.quantization_config.dequantize:
-            if self.pre_quantized:
+        if self.quantization_config.dequantize:
+            return
+
+        if not torch.cuda.is_available() and not is_torch_xpu_available():
+            if self.pre_quantized and not self.quantization_config.dequantize:
                 logger.warning_once(
                     "Using FP8 quantized models requires a GPU or XPU, we will default to dequantizing the model to bf16 since no GPU or XPU is available"
                 )
@@ -64,11 +58,12 @@ class FineGrainedFP8HfQuantizer(HfQuantizer):
                 "your model on a GPU or XPU device in order to run your model. To remove this warning, "
                 "pass device_map = 'cuda' or 'xpu'. "
             )
-        elif device_map is not None:
+        elif isinstance(device_map, dict):
             if (
                 not self.pre_quantized
-                and isinstance(device_map, dict)
-                and ("cpu" in device_map.values() or "disk" in device_map.values())
+                and len(device_map) > 1
+                and "cpu" in device_map.values()
+                or "disk" in device_map.values()
             ):
                 raise ValueError(
                     "You are attempting to load an FP8 model with a device_map that contains a cpu/disk device."
@@ -76,76 +71,6 @@ class FineGrainedFP8HfQuantizer(HfQuantizer):
                     "Please use a quantized checkpoint or remove the cpu/disk device from the device_map."
                 )
 
-    def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype":
-        if dtype is None:
-            logger.info("Setting dtype to torch.float32 as no dtype was specified in from_pretrained")
-            dtype = torch.float32
-        return dtype
-
-    # TODO: make this into a `ConversionType` ops -> potentially requires all weights on all ranks
-    # depending on the layer type (moe -> no if ep)
-    def create_quantized_param(
-        self,
-        model: "PreTrainedModel",
-        param_value: "torch.Tensor",
-        param_name: str,
-        target_device: "torch.device",
-        **kwargs,
-    ):
-        from ..integrations.finegrained_fp8 import FP8Linear
-        from ..modeling_utils import _load_parameter_into_model
-
-        # Sanity checks
-        module, tensor_name = get_module_from_name(model, param_name)
-        if isinstance(module, FP8Linear):
-            if self.pre_quantized or tensor_name == "bias":
-                if tensor_name == "weight" and param_value.dtype != torch.float8_e4m3fn:
-                    raise ValueError("Expect quantized weights but got an unquantized weight")
-            else:
-                return
-                # if tensor_name == "weight_scale_inv":
-                #     raise ValueError("Expect unquantized weights but got a quantized weight_scale")
-
-        param_value = param_value.to(target_device)
-
-        # Get FP8 min/max values
-        fp8_min = torch.finfo(torch.float8_e4m3fn).min
-        fp8_max = torch.finfo(torch.float8_e4m3fn).max
-
-        block_size_m, block_size_n = self.quantization_config.weight_block_size
-
-        rows, cols = param_value.shape[-2:]
-
-        if rows % block_size_m != 0 or cols % block_size_n != 0:
-            raise ValueError(
-                f"Matrix dimensions ({rows}, {cols}) must be divisible by block sizes ({block_size_m}, {block_size_n})"
-            )
-        param_value_orig_shape = param_value.shape
-
-        param_value = param_value.reshape(
-            -1, rows // block_size_m, block_size_m, cols // block_size_n, block_size_n
-        ).permute(0, 1, 3, 2, 4)
-
-        # Calculate scaling factor for each block
-        max_abs = torch.amax(torch.abs(param_value), dim=(-1, -2))
-        scale = fp8_max / max_abs
-        scale_orig_shape = scale.shape
-        scale = scale.unsqueeze(-1).unsqueeze(-1)
-
-        # Quantize the weights
-        quantized_param = torch.clamp(param_value * scale, min=fp8_min, max=fp8_max).to(torch.float8_e4m3fn)
-
-        quantized_param = quantized_param.permute(0, 1, 3, 2, 4)
-        # Reshape back to matrix shape
-        quantized_param = quantized_param.reshape(param_value_orig_shape)
-
-        # Reshape scale to match the number of blocks
-        scale = scale.reshape(scale_orig_shape).squeeze().reciprocal()
-
-        # Load into the model
-        _load_parameter_into_model(model, param_name, quantized_param)
-        _load_parameter_into_model(model, param_name.rsplit(".", 1)[0] + ".weight_scale_inv", scale)
-
     def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool:
         from ..integrations.finegrained_fp8 import FP8Expert, FP8Linear
 
@@ -165,35 +90,17 @@ class FineGrainedFP8HfQuantizer(HfQuantizer):
     ):
         from ..integrations.finegrained_fp8 import replace_with_fp8_linear
 
-        # takes 2 fucking seconds
         self.modules_to_not_convert = self.get_modules_to_not_convert(
             model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules
         )
 
-        # while this one is 81ms :)
         model = replace_with_fp8_linear(
             model,
             modules_to_not_convert=self.modules_to_not_convert,
             quantization_config=self.quantization_config,
+            pre_quantized=self.pre_quantized,
         )
 
-        model.config.quantization_config = self.quantization_config
-
-    def update_missing_keys(self, model, missing_keys: list[str], prefix: str) -> list[str]:
-        from ..integrations import FP8Linear
-
-        not_missing_keys = []
-        for name, module in model.named_modules():
-            if isinstance(module, FP8Linear):
-                for missing in missing_keys:
-                    if (
-                        (name in missing or name in f"{prefix}.{missing}")
-                        and not missing.endswith(".weight")
-                        and not missing.endswith(".bias")
-                    ):
-                        not_missing_keys.append(missing)
-        return [k for k in missing_keys if k not in not_missing_keys]
-
     # NOTE: TP is applied before quantization so this is only to add hooks.
     # Quantization is incompatible with DTensors, so we have to anyway have
     # gathers! But it should be model independant -> figure out where to put
@@ -223,7 +130,7 @@ class FineGrainedFP8HfQuantizer(HfQuantizer):
 
         return config
 
-    def is_serializable(self, safe_serialization=None):
+    def is_serializable(self):
         return True
 
     @property
@@ -246,8 +153,9 @@ class FineGrainedFP8HfQuantizer(HfQuantizer):
         if self.pre_quantized and self.quantization_config.dequantize:
             return [
                 # either use the dollar sign, or permute the source patterns to start matching against the scales first
+                # We also collect the activation scales, they will not be used
                 WeightConverter(
-                    source_patterns=["weight$", "weight_scale_inv"],
+                    source_patterns=["weight$", "weight_scale_inv", "activation_scale"],
                     target_patterns="weight",
                     operations=[Fp8Dequantize(self)],
                 )

transformers/quantizers/quantizer_fp_quant.py

@@ -36,13 +36,10 @@ class FPQuantHfQuantizer(HfQuantizer):
     """
 
     requires_calibration = False
-    requires_parameters_quantization = True
     is_qat_trainable = True
-    required_packages = ["fp_quant"]
 
     def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):
         super().__init__(quantization_config, **kwargs)
-        self.quantization_config = quantization_config
 
     def validate_environment(self, device_map, **kwargs):
         if not torch.cuda.is_available() and not is_torch_xpu_available():
@@ -68,15 +65,17 @@ class FPQuantHfQuantizer(HfQuantizer):
                 "You are attempting to load a FPQuant model without setting device_map."
                 " Please set device_map comprised of 'cuda' devices."
             )
-        elif (
-            isinstance(device_map, dict)
-            and ("cpu" in device_map.values() or "disk" in device_map.values())
-            and not self.quantization_config.pseudoquantization
-        ):
-            raise ValueError(
-                "You are attempting to load a FPQuant model with a device_map that contains a CPU or disk device."
-                " This is not supported. Please remove the CPU or disk device from the device_map."
-            )
+        elif isinstance(device_map, dict):
+            if (
+                not self.quantization_config.pseudoquantization
+                and len(device_map) > 1
+                and "cpu" in device_map.values()
+                or "disk" in device_map.values()
+            ):
+                raise ValueError(
+                    "You are attempting to load a FPQuant model with a device_map that contains a CPU or disk device."
+                    " This is not supported. Please remove the CPU or disk device from the device_map."
+                )
 
     def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype":
         if dtype is None:
@@ -84,50 +83,17 @@ class FPQuantHfQuantizer(HfQuantizer):
             dtype = torch.bfloat16
         elif dtype != torch.bfloat16:
             raise ValueError(f"Invalid `dtype` {dtype}. fp_quant quantization only supports `dtype=torch.bfloat16`.")
-
         return dtype
 
-    def create_quantized_param(
-        self,
-        model: "PreTrainedModel",
-        param_value: "torch.Tensor",
-        param_name: str,
-        target_device: "torch.device",
-        **kwargs,
-    ):
-        module, _ = get_module_from_name(model, param_name)
-
-        if target_device == "cpu" and param_name.endswith("weight"):
-            # Works agains hard-coded missing key dispatch to CPU
-            return
-
-        # The module holds either:
-        #  * `weight` when `store_master_weights=True`
-        #  * `qweight` and `scales` when `store_master_weights=False` and `pseudoquantization=False`
-        #  * `dqweight` when `store_master_weights=False` and `pseudoquantization=True`
-
-        if param_name.endswith(".qweight"):
-            # Loading a real quantized checkpoint without master weights
-            module.qweight = torch.nn.Parameter(
-                param_value.to(target_device),
-                requires_grad=False,
-            )
-            module.weight = None
-            module.dqweight = None
-            return
-
-        if param_name.endswith(".dqweight"):
-            # Loading a pseudo-quantized checkpoint without master weights
-            module.dqweight = torch.nn.Parameter(param_value.to(target_device))
-            module.weight = None
-            module.qweight = None
-            module.scales = None
-            return
-
-        # Loading master weights or an unquantized checkpoint
-        module.weight = torch.nn.Parameter(param_value.to(target_device))
-        # Let pre-forward handle the quantization and set None where necessary
-        module.pre_forward()
+    def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool:
+        from fp_quant import FPQuantLinear
+
+        module, tensor_name = get_module_from_name(model, param_name)
+        if isinstance(module, FPQuantLinear) and tensor_name in ["weight", "qweight", "dqweight"]:
+            # Only quantize weights of FPQuantLinear modules that are not already quantized
+            return True
+        else:
+            return False
 
     def _process_model_before_weight_loading(
         self,
@@ -142,20 +108,6 @@ class FPQuantHfQuantizer(HfQuantizer):
             model,
             fp_quant_linear_config=adapt_fp_quant_config(self.quantization_config),
         )
-        model.config.quantization_config = self.quantization_config
-
-    def update_missing_keys(self, model, missing_keys: list[str], prefix: str) -> list[str]:
-        from fp_quant import FPQuantLinear
-
-        fp_quant_names = {name for name, module in model.named_modules() if isinstance(module, FPQuantLinear)}
-
-        def should_exclude(key: str) -> bool:
-            if key.endswith(".weight") or key.endswith(".bias"):
-                return False
-            full_key = f"{prefix}.{key}"
-            return any(name in key or name in full_key for name in fp_quant_names)
-
-        return [key for key in missing_keys if not should_exclude(key)]
 
     @property
     def is_trainable(self, model: Optional["PreTrainedModel"] = None):
@@ -166,15 +118,33 @@ class FPQuantHfQuantizer(HfQuantizer):
             )
         return trainable
 
-    def is_serializable(self, safe_serialization=None):
+    def is_serializable(self):
         return True
 
-    def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool:
-        from fp_quant import FPQuantLinear
-
-        module, tensor_name = get_module_from_name(model, param_name)
-        if isinstance(module, FPQuantLinear) and tensor_name in ["weight", "qweight", "dqweight"]:
-            # Only quantize weights of FPQuantLinear modules that are not already quantized
-            return True
-        else:
-            return False
+    def get_quantize_ops(self):
+        from ..integrations.fp_quant import FpQuantQuantize
+
+        return FpQuantQuantize(self)
+
+    def get_weight_conversions(self):
+        from ..core_model_loading import WeightConverter
+        from ..integrations.fp_quant import FpQuantDeserialize
+
+        if self.pre_quantized:
+            if self.quantization_config.pseudoquantization:
+                return [
+                    WeightConverter(
+                        source_patterns=[".dqweight"],
+                        target_patterns=".dqweight",
+                        operations=[FpQuantDeserialize(self)],
+                    ),
+                ]
+            else:
+                return [
+                    WeightConverter(
+                        source_patterns=[".qweight"],
+                        target_patterns=".qweight",
+                        operations=[FpQuantDeserialize(self)],
+                    ),
+                ]
+        return []

transformers/quantizers/quantizer_gptq.py

@@ -22,7 +22,7 @@ from .base import HfQuantizer
 if TYPE_CHECKING:
     from ..modeling_utils import PreTrainedModel
 
-from ..utils import is_auto_gptq_available, is_gptqmodel_available, is_optimum_available, is_torch_available, logging
+from ..utils import is_gptqmodel_available, is_optimum_available, is_torch_available, logging
 from ..utils.quantization_config import GPTQConfig, QuantizationConfigMixin
 
 
@@ -35,12 +35,11 @@ logger = logging.get_logger(__name__)
 class GptqHfQuantizer(HfQuantizer):
     """
     Quantizer of the GPTQ method - for GPTQ the quantizer support calibration of the model through
-    `auto_gptq` or `gptqmodel` package. Quantization is done under the hood for users if they load a non-prequantized model.
+    the GPT-QModel package (Python import name `gptqmodel`). Quantization is done under the hood for users if they
+    load a non-prequantized model.
     """
 
     requires_calibration = False
-    required_packages = ["optimum", "auto_gptq", "gptqmodel"]
-    optimum_quantizer = None
 
     def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):
         super().__init__(quantization_config, **kwargs)
@@ -54,25 +53,12 @@ class GptqHfQuantizer(HfQuantizer):
     def validate_environment(self, *args, **kwargs):
         if not is_optimum_available():
             raise ImportError("Loading a GPTQ quantized model requires optimum (`pip install optimum`)")
-        if is_auto_gptq_available() and is_gptqmodel_available():
-            logger.warning("Detected gptqmodel and auto-gptq, will use gptqmodel")
 
-        gptq_supports_cpu = (
-            is_auto_gptq_available()
-            and version.parse(importlib.metadata.version("auto-gptq")) > version.parse("0.4.2")
-        ) or is_gptqmodel_available()
+        gptq_supports_cpu = is_gptqmodel_available()
         if not gptq_supports_cpu and not torch.cuda.is_available():
             raise RuntimeError("GPU is required to quantize or run quantize model.")
-        elif not (is_auto_gptq_available() or is_gptqmodel_available()):
-            raise ImportError(
-                "Loading a GPTQ quantized model requires gptqmodel (`pip install gptqmodel`) or auto-gptq (`pip install auto-gptq`) library. "
-            )
-        elif is_auto_gptq_available() and version.parse(importlib.metadata.version("auto_gptq")) < version.parse(
-            "0.4.2"
-        ):
-            raise ImportError(
-                "You need a version of auto_gptq >= 0.4.2 to use GPTQ: `pip install --upgrade auto-gptq` or use gptqmodel by `pip install gptqmodel>=1.4.3`."
-            )
+        elif not is_gptqmodel_available():
+            raise ImportError("Loading a GPTQ quantized model requires gptqmodel (`pip install gptqmodel`) library.")
         elif is_gptqmodel_available() and (
             version.parse(importlib.metadata.version("gptqmodel")) < version.parse("1.4.3")
             or version.parse(importlib.metadata.version("optimum")) < version.parse("1.23.99")
@@ -90,9 +76,6 @@ class GptqHfQuantizer(HfQuantizer):
     def update_device_map(self, device_map):
         if device_map is None:
             device_map = {"": torch.device("cpu")}
-        # Only with auto-gptq do not support CPU, we should move the model to cuda if available.
-        if not is_gptqmodel_available() and device_map in ("cpu", {"": torch.device("cpu")}):
-            device_map = {"": 0}
         return device_map
 
     def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):
@@ -120,5 +103,5 @@ class GptqHfQuantizer(HfQuantizer):
     def is_trainable(self) -> bool:
         return True
 
-    def is_serializable(self, safe_serialization=None):
+    def is_serializable(self):
         return True

transformers/quantizers/quantizer_higgs.py

@@ -37,12 +37,9 @@ class HiggsHfQuantizer(HfQuantizer):
     """
 
     requires_calibration = False
-    requires_parameters_quantization = True
-    required_packages = ["flute-kernel", "fast_hadamard_transform"]
 
     def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):
         super().__init__(quantization_config, **kwargs)
-        self.quantization_config = quantization_config
 
     def validate_environment(self, device_map, **kwargs):
         if not torch.cuda.is_available():
@@ -64,11 +61,12 @@ class HiggsHfQuantizer(HfQuantizer):
                 "You are attempting to load a HIGGS model without setting device_map."
                 " Please set device_map comprised of 'cuda' devices."
             )
-        elif isinstance(device_map, dict) and ("cpu" in device_map.values() or "disk" in device_map.values()):
-            raise ValueError(
-                "You are attempting to load a HIGGS model with a device_map that contains a CPU or disk device."
-                " This is not supported. Please remove the CPU or disk device from the device_map."
-            )
+        elif isinstance(device_map, dict):
+            if "cpu" in device_map.values() or "disk" in device_map.values():
+                raise ValueError(
+                    "You are attempting to load a HIGGS model with a device_map that contains a CPU or disk device."
+                    " This is not supported. Please remove the CPU or disk device from the device_map."
+                )
 
     def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype":
         if dtype is None:
@@ -81,37 +79,39 @@ class HiggsHfQuantizer(HfQuantizer):
 
         return dtype
 
-    def create_quantized_param(
-        self,
-        model: "PreTrainedModel",
-        param_value: "torch.Tensor",
-        param_name: str,
-        target_device: "torch.device",
-        **kwargs,
-    ):
-        from ..integrations import quantize_with_higgs
-
-        flute_dict = quantize_with_higgs(
-            param_value.to(target_device),
-            self.quantization_config.bits,
-            self.quantization_config.p,
-            self.quantization_config.group_size,
-            self.quantization_config.hadamard_size,
-        )
-        del param_value
-
-        module, _ = get_module_from_name(model, param_name)
-        module_name = ".".join(param_name.split(".")[:-1])
-        for key, value in flute_dict.items():
-            if key in module._parameters:
-                module._parameters[key] = torch.nn.Parameter(value, requires_grad=False)
-            elif key in module._buffers:
-                module._buffers[key] = torch.nn.Buffer(value)
-            elif key == "tune_metadata":
-                module.tune_metadata = value
-                self.quantization_config.tune_metadata[module_name] = value.to_dict()
-            else:
-                raise ValueError(f"Unexpected key {key} in module {module}")
+    # TODO: to remove
+    # Kept here in case we see some interest in adding support for it
+    # def create_quantized_param(
+    #     self,
+    #     model: "PreTrainedModel",
+    #     param_value: "torch.Tensor",
+    #     param_name: str,
+    #     target_device: "torch.device",
+    #     **kwargs,
+    # ):
+    #     from ..integrations import quantize_with_higgs
+
+    #     flute_dict = quantize_with_higgs(
+    #         param_value.to(target_device),
+    #         self.quantization_config.bits,
+    #         self.quantization_config.p,
+    #         self.quantization_config.group_size,
+    #         self.quantization_config.hadamard_size,
+    #     )
+    #     del param_value
+
+    #     module, _ = get_module_from_name(model, param_name)
+    #     module_name = ".".join(param_name.split(".")[:-1])
+    #     for key, value in flute_dict.items():
+    #         if key in module._parameters:
+    #             module._parameters[key] = torch.nn.Parameter(value, requires_grad=False)
+    #         elif key in module._buffers:
+    #             module._buffers[key] = torch.nn.Buffer(value)
+    #         elif key == "tune_metadata":
+    #             module.tune_metadata = value
+    #             self.quantization_config.tune_metadata[module_name] = value.to_dict()
+    #         else:
+    #             raise ValueError(f"Unexpected key {key} in module {module}")
 
     def _process_model_before_weight_loading(
         self,
@@ -130,7 +130,6 @@ class HiggsHfQuantizer(HfQuantizer):
             quantization_config=self.quantization_config,
             modules_to_not_convert=self.modules_to_not_convert,
         )
-        model.config.quantization_config = self.quantization_config
 
     def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs):
         from flute.tune import TuneMetaData, maybe_tune_and_repack
@@ -157,24 +156,11 @@ class HiggsHfQuantizer(HfQuantizer):
             )
             self.quantization_config.tune_metadata[name] = module.tune_metadata.to_dict()
 
-    def update_missing_keys(self, model, missing_keys: list[str], prefix: str) -> list[str]:
-        from ..integrations import HiggsLinear
-
-        higgs_names = {name for name, module in model.named_modules() if isinstance(module, HiggsLinear)}
-
-        def should_update(key: str) -> bool:
-            if key.endswith(".weight") or key.endswith(".bias"):
-                return False
-            full_key = f"{prefix}.{key}"
-            return any(name in key or name in full_key for name in higgs_names)
-
-        return [key for key in missing_keys if not should_update(key)]
-
     @property
     def is_trainable(self) -> bool:
         return False
 
-    def is_serializable(self, safe_serialization=None):
+    def is_serializable(self):
         return True
 
     def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool: