transformers 5.0.0rc0__py3-none-any.whl → 5.0.0rc2__py3-none-any.whl

transformers/quantizers/quantizer_gptq.py

@@ -22,7 +22,7 @@ from .base import HfQuantizer
 if TYPE_CHECKING:
     from ..modeling_utils import PreTrainedModel
 
-from ..utils import is_auto_gptq_available, is_gptqmodel_available, is_optimum_available, is_torch_available, logging
+from ..utils import is_gptqmodel_available, is_optimum_available, is_torch_available, logging
 from ..utils.quantization_config import GPTQConfig, QuantizationConfigMixin
 
 
@@ -35,12 +35,11 @@ logger = logging.get_logger(__name__)
 class GptqHfQuantizer(HfQuantizer):
     """
     Quantizer of the GPTQ method - for GPTQ the quantizer support calibration of the model through
-    `auto_gptq` or `gptqmodel` package. Quantization is done under the hood for users if they load a non-prequantized model.
+    the GPT-QModel package (Python import name `gptqmodel`). Quantization is done under the hood for users if they
+    load a non-prequantized model.
     """
 
     requires_calibration = False
-    required_packages = ["optimum", "auto_gptq", "gptqmodel"]
-    optimum_quantizer = None
 
     def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):
         super().__init__(quantization_config, **kwargs)
@@ -54,25 +53,12 @@ class GptqHfQuantizer(HfQuantizer):
     def validate_environment(self, *args, **kwargs):
         if not is_optimum_available():
             raise ImportError("Loading a GPTQ quantized model requires optimum (`pip install optimum`)")
-        if is_auto_gptq_available() and is_gptqmodel_available():
-            logger.warning("Detected gptqmodel and auto-gptq, will use gptqmodel")
 
-        gptq_supports_cpu = (
-            is_auto_gptq_available()
-            and version.parse(importlib.metadata.version("auto-gptq")) > version.parse("0.4.2")
-        ) or is_gptqmodel_available()
+        gptq_supports_cpu = is_gptqmodel_available()
         if not gptq_supports_cpu and not torch.cuda.is_available():
             raise RuntimeError("GPU is required to quantize or run quantize model.")
-        elif not (is_auto_gptq_available() or is_gptqmodel_available()):
-            raise ImportError(
-                "Loading a GPTQ quantized model requires gptqmodel (`pip install gptqmodel`) or auto-gptq (`pip install auto-gptq`) library. "
-            )
-        elif is_auto_gptq_available() and version.parse(importlib.metadata.version("auto_gptq")) < version.parse(
-            "0.4.2"
-        ):
-            raise ImportError(
-                "You need a version of auto_gptq >= 0.4.2 to use GPTQ: `pip install --upgrade auto-gptq` or use gptqmodel by `pip install gptqmodel>=1.4.3`."
-            )
+        elif not is_gptqmodel_available():
+            raise ImportError("Loading a GPTQ quantized model requires gptqmodel (`pip install gptqmodel`) library.")
         elif is_gptqmodel_available() and (
             version.parse(importlib.metadata.version("gptqmodel")) < version.parse("1.4.3")
             or version.parse(importlib.metadata.version("optimum")) < version.parse("1.23.99")
@@ -80,19 +66,13 @@ class GptqHfQuantizer(HfQuantizer):
             raise ImportError("The gptqmodel version should be >= 1.4.3, optimum version should >= 1.24.0")
 
     def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype":
-        if dtype is None:
-            dtype = torch.float16
-            logger.info("Loading the model in `torch.float16`. To overwrite it, set `dtype` manually.")
-        elif dtype != torch.float16:
+        if dtype != torch.float16:
             logger.info("We suggest you to set `dtype=torch.float16` for better efficiency with GPTQ.")
         return dtype
 
     def update_device_map(self, device_map):
         if device_map is None:
             device_map = {"": torch.device("cpu")}
-        # Only with auto-gptq do not support CPU, we should move the model to cuda if available.
-        if not is_gptqmodel_available() and device_map in ("cpu", {"": torch.device("cpu")}):
-            device_map = {"": 0}
         return device_map
 
     def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):
@@ -120,5 +100,5 @@ class GptqHfQuantizer(HfQuantizer):
     def is_trainable(self) -> bool:
         return True
 
-    def is_serializable(self, safe_serialization=None):
+    def is_serializable(self):
         return True

transformers/quantizers/quantizer_higgs.py

@@ -37,12 +37,9 @@ class HiggsHfQuantizer(HfQuantizer):
     """
 
     requires_calibration = False
-    requires_parameters_quantization = True
-    required_packages = ["flute-kernel", "fast_hadamard_transform"]
 
     def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):
         super().__init__(quantization_config, **kwargs)
-        self.quantization_config = quantization_config
 
     def validate_environment(self, device_map, **kwargs):
         if not torch.cuda.is_available():
@@ -64,65 +61,64 @@ class HiggsHfQuantizer(HfQuantizer):
                 "You are attempting to load a HIGGS model without setting device_map."
                 " Please set device_map comprised of 'cuda' devices."
             )
-        elif isinstance(device_map, dict) and ("cpu" in device_map.values() or "disk" in device_map.values()):
-            raise ValueError(
-                "You are attempting to load a HIGGS model with a device_map that contains a CPU or disk device."
-                " This is not supported. Please remove the CPU or disk device from the device_map."
-            )
+        elif isinstance(device_map, dict):
+            if "cpu" in device_map.values() or "disk" in device_map.values():
+                raise ValueError(
+                    "You are attempting to load a HIGGS model with a device_map that contains a CPU or disk device."
+                    " This is not supported. Please remove the CPU or disk device from the device_map."
+                )
 
     def update_dtype(self, dtype: "torch.dtype") -> "torch.dtype":
-        if dtype is None:
-            logger.info("`dtype` is None. Setting `dtype=torch.float16` for FLUTE compatibility.")
-            dtype = torch.float16
-        elif dtype != torch.float16 and dtype != torch.bfloat16:
+        if dtype != torch.float16 and dtype != torch.bfloat16:
             raise ValueError(
                 f"Invalid `dtype` {dtype}. HIGGS quantization only supports `dtype=torch.float16` or `dtype=torch.bfloat16`."
             )
 
         return dtype
 
-    def create_quantized_param(
-        self,
-        model: "PreTrainedModel",
-        param_value: "torch.Tensor",
-        param_name: str,
-        target_device: "torch.device",
-        **kwargs,
-    ):
-        from ..integrations import quantize_with_higgs
-
-        flute_dict = quantize_with_higgs(
-            param_value.to(target_device),
-            self.quantization_config.bits,
-            self.quantization_config.p,
-            self.quantization_config.group_size,
-            self.quantization_config.hadamard_size,
-        )
-        del param_value
-
-        module, _ = get_module_from_name(model, param_name)
-        module_name = ".".join(param_name.split(".")[:-1])
-        for key, value in flute_dict.items():
-            if key in module._parameters:
-                module._parameters[key] = torch.nn.Parameter(value, requires_grad=False)
-            elif key in module._buffers:
-                module._buffers[key] = torch.nn.Buffer(value)
-            elif key == "tune_metadata":
-                module.tune_metadata = value
-                self.quantization_config.tune_metadata[module_name] = value.to_dict()
-            else:
-                raise ValueError(f"Unexpected key {key} in module {module}")
+    # TODO: to remove
+    # Kept here in case we see some interest in adding support for it
+    # def create_quantized_param(
+    #     self,
+    #     model: "PreTrainedModel",
+    #     param_value: "torch.Tensor",
+    #     param_name: str,
+    #     target_device: "torch.device",
+    #     **kwargs,
+    # ):
+    #     from ..integrations import quantize_with_higgs
+
+    #     flute_dict = quantize_with_higgs(
+    #         param_value.to(target_device),
+    #         self.quantization_config.bits,
+    #         self.quantization_config.p,
+    #         self.quantization_config.group_size,
+    #         self.quantization_config.hadamard_size,
+    #     )
+    #     del param_value
+
+    #     module, _ = get_module_from_name(model, param_name)
+    #     module_name = ".".join(param_name.split(".")[:-1])
+    #     for key, value in flute_dict.items():
+    #         if key in module._parameters:
+    #             module._parameters[key] = torch.nn.Parameter(value, requires_grad=False)
+    #         elif key in module._buffers:
+    #             module._buffers[key] = torch.nn.Buffer(value)
+    #         elif key == "tune_metadata":
+    #             module.tune_metadata = value
+    #             self.quantization_config.tune_metadata[module_name] = value.to_dict()
+    #         else:
+    #             raise ValueError(f"Unexpected key {key} in module {module}")
 
     def _process_model_before_weight_loading(
         self,
         model: "PreTrainedModel",
-        keep_in_fp32_modules: list[str] | None = None,
         **kwargs,
     ):
         from ..integrations import replace_with_higgs_linear
 
         self.modules_to_not_convert = self.get_modules_to_not_convert(
-            model, self.quantization_config.modules_to_not_convert, keep_in_fp32_modules
+            model, self.quantization_config.modules_to_not_convert, model._keep_in_fp32_modules
         )
 
         replace_with_higgs_linear(
@@ -130,7 +126,6 @@ class HiggsHfQuantizer(HfQuantizer):
             quantization_config=self.quantization_config,
             modules_to_not_convert=self.modules_to_not_convert,
         )
-        model.config.quantization_config = self.quantization_config
 
     def _process_model_after_weight_loading(self, model: "PreTrainedModel", **kwargs):
         from flute.tune import TuneMetaData, maybe_tune_and_repack
@@ -157,24 +152,11 @@ class HiggsHfQuantizer(HfQuantizer):
             )
             self.quantization_config.tune_metadata[name] = module.tune_metadata.to_dict()
 
-    def update_missing_keys(self, model, missing_keys: list[str], prefix: str) -> list[str]:
-        from ..integrations import HiggsLinear
-
-        higgs_names = {name for name, module in model.named_modules() if isinstance(module, HiggsLinear)}
-
-        def should_update(key: str) -> bool:
-            if key.endswith(".weight") or key.endswith(".bias"):
-                return False
-            full_key = f"{prefix}.{key}"
-            return any(name in key or name in full_key for name in higgs_names)
-
-        return [key for key in missing_keys if not should_update(key)]
-
     @property
     def is_trainable(self) -> bool:
         return False
 
-    def is_serializable(self, safe_serialization=None):
+    def is_serializable(self):
         return True
 
     def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool:

transformers/quantizers/quantizer_hqq.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from collections import defaultdict
 from typing import TYPE_CHECKING
 
 from ..integrations import prepare_for_hqq_linear
@@ -49,10 +48,7 @@ class HqqHfQuantizer(HfQuantizer):
     nn.Linear modules are first tagged with quant_config in _process_model_before_weight_loading().
     """
 
-    use_keep_in_fp32_modules = False
-    requires_parameters_quantization = True
     requires_calibration = False
-    required_packages = ["hqq"]
 
     def __init__(self, quantization_config, **kwargs):
         if not is_hqq_available():
@@ -83,73 +79,67 @@ class HqqHfQuantizer(HfQuantizer):
             else:
                 self.using_multi_gpu = len(set(device_map.values())) > 1
 
-    def update_missing_keys(
-        self, model: "PreTrainedModel", missing_keys: list[str], prefix: str, **kwargs
-    ) -> list[str]:
-        if self.pre_quantized:
-            return [key for key in missing_keys if ("weight" not in key)]
-        else:
-            return missing_keys
-
-    # Adds missing keys for HQQLinear modules that are loaded but the model with initialized with torch.nn.Linear
-    def update_expected_keys(
-        self, model: "PreTrainedModel", expected_keys: list[str], loaded_keys: list[str]
-    ) -> list[str]:
-        if not self.pre_quantized:
-            return expected_keys
-
-        # Collects all quantizable (linear) layers
-        def _find_hqq_quantizable_layers(model, layers):
-            for name, module in model.named_children():
-                if isinstance(module, (torch.nn.Linear)):
-                    layers.add(module.name)
-                _find_hqq_quantizable_layers(module, layers)
-
-        new_keys = set(expected_keys)
-
-        # Name modules
-        for name, module in model.named_modules():
-            module.name = name
-
-        # valid modules are Linear layers that have HQQLinear state_dict. We ignore skip_modules and any layers with Linear state_dict() params
-        _valid_modules = set()
-        _find_hqq_quantizable_layers(model, _valid_modules)
-
-        # Remove skipped modules
-        _skipped_modules = set()
-        for _module in _valid_modules:
-            for _skip_module in model.config.quantization_config["skip_modules"]:
-                if _skip_module in _module:
-                    _skipped_modules.add(_module)
-        _valid_modules -= _skipped_modules
-
-        # Append new expected layers based on _ref_keys
-        _ref_keys = HQQLinear(
-            linear_layer=None,
-            quant_config=None,
-            compute_dtype=torch.float16,
-            device="cpu",
-            del_orig=False,
-        ).state_dict_keys() - {"bias"}
-
-        # Clean-up
-        _rm_keys = set()
-        for key in new_keys:
-            if any(_module in key for _module in _valid_modules):
-                _rm_keys.add(key)
-        new_keys -= _rm_keys
-        # At this point, new_keys contains all the keys of the layers that are NOT HQQLinear or torch.nn.Linear
-
-        # Re-populate Linear/HQQLinear
-        for _module in _valid_modules:
-            if _module + ".weight" in loaded_keys:
-                new_keys.add(_module + ".weight")
-            else:
-                new_keys.update({_module + "." + _ref_key for _ref_key in _ref_keys})
-            if _module + ".bias" in loaded_keys:
-                new_keys.add(_module + ".bias")
-
-        return list(new_keys)
+    # TODO: to remove
+    # Kept here in case we see some interest in adding support for it
+    # # Adds missing keys for HQQLinear modules that are loaded but the model with initialized with torch.nn.Linear
+    # def update_expected_keys(
+    #     self, model: "PreTrainedModel", expected_keys: list[str], loaded_keys: list[str]
+    # ) -> list[str]:
+    #     if not self.pre_quantized:
+    #         return expected_keys
+
+    #     # Collects all quantizable (linear) layers
+    #     def _find_hqq_quantizable_layers(model, layers):
+    #         for name, module in model.named_children():
+    #             if isinstance(module, (torch.nn.Linear)):
+    #                 layers.add(module.name)
+    #             _find_hqq_quantizable_layers(module, layers)
+
+    #     new_keys = set(expected_keys)
+
+    #     # Name modules
+    #     for name, module in model.named_modules():
+    #         module.name = name
+
+    #     # valid modules are Linear layers that have HQQLinear state_dict. We ignore skip_modules and any layers with Linear state_dict() params
+    #     _valid_modules = set()
+    #     _find_hqq_quantizable_layers(model, _valid_modules)
+
+    #     # Remove skipped modules
+    #     _skipped_modules = set()
+    #     for _module in _valid_modules:
+    #         for _skip_module in model.config.quantization_config["skip_modules"]:
+    #             if _skip_module in _module:
+    #                 _skipped_modules.add(_module)
+    #     _valid_modules -= _skipped_modules
+
+    #     # Append new expected layers based on _ref_keys
+    #     _ref_keys = HQQLinear(
+    #         linear_layer=None,
+    #         quant_config=None,
+    #         compute_dtype=torch.float16,
+    #         device="cpu",
+    #         del_orig=False,
+    #     ).state_dict_keys() - {"bias"}
+
+    #     # Clean-up
+    #     _rm_keys = set()
+    #     for key in new_keys:
+    #         if any(_module in key for _module in _valid_modules):
+    #             _rm_keys.add(key)
+    #     new_keys -= _rm_keys
+    #     # At this point, new_keys contains all the keys of the layers that are NOT HQQLinear or torch.nn.Linear
+
+    #     # Re-populate Linear/HQQLinear
+    #     for _module in _valid_modules:
+    #         if _module + ".weight" in loaded_keys:
+    #             new_keys.add(_module + ".weight")
+    #         else:
+    #             new_keys.update({_module + "." + _ref_key for _ref_key in _ref_keys})
+    #         if _module + ".bias" in loaded_keys:
+    #             new_keys.add(_module + ".bias")
+
+    #     return list(new_keys)
 
     def param_needs_quantization(self, model: "PreTrainedModel", param_name: str, **kwargs) -> bool:
         module, _ = get_module_from_name(model, param_name)
@@ -157,87 +147,88 @@ class HqqHfQuantizer(HfQuantizer):
         # `create_quantized_param`, even when `self.is_quantized == True`
         return isinstance(module, torch.nn.Linear)
 
-    def create_quantized_param(
-        self,
-        model: "PreTrainedModel",
-        param_value: "torch.Tensor",
-        param_name: str,
-        target_device: "torch.device",
-        **kwargs,
-    ):
-        module, tensor_name = get_module_from_name(model, param_name)
-        module_name = param_name.rsplit(".", 1)[0]
-        parent_module, node = get_module_from_name(model, module_name)
-
-        quant_config = model.config.quantization_config["quant_config"]
-        skip_modules = model.config.quantization_config["skip_modules"]
-
-        # In this case we do not quantize this layer (it's explicitly skipped) -> simply load param
-        if any(skip_module in module.name for skip_module in skip_modules):
-            module.load_state_dict(
-                {tensor_name: param_value.to(device=target_device, dtype=self.dtype)}, strict=False, assign=True
-            )
-            return
-
-        # We need this hack as the model is not pre-prepared as an empty skeleton on meta device
-        if self.pre_quantized:
-            # Save them for later
-            if not hasattr(self, "hqq_params"):
-                self.hqq_params = defaultdict(dict)
-            self.hqq_params[module_name].update({tensor_name: param_value})
-            hqq_params = self.hqq_params[module_name]
-
-            # If they are all present and saved, make it a HQQLinear layer! (we cannot do it param after param because
-            # hqq does not support it...)
-            if all(k in hqq_params for k in self.hqq_keys) and ("bias" in hqq_params or module.bias is None):
-                hqq_layer = HQQLinear(
-                    linear_layer=None,
-                    quant_config=None,
-                    compute_dtype=self.dtype,
-                    device=target_device,
-                    del_orig=False,
-                )
-                hqq_layer.load_state_dict(hqq_params)
-
-                if hqq_layer.bias is not None and isinstance(hqq_layer.bias, torch.Tensor):
-                    hqq_layer.bias = torch.nn.Parameter(hqq_layer.bias)
-                if self.using_multi_gpu:
-                    hqq_layer = self._patch_layer_for_multigpu(hqq_layer)
-
-                setattr(parent_module, node, hqq_layer)
-                del self.hqq_params[module_name], module
-            return
-
-        # Load param in the module (without caring about device or dtype, it will be changed later)
-        module.load_state_dict({tensor_name: param_value}, strict=False, assign=True)
-
-        # If both the weight and bias have already been loaded, time to quantize!
-        module_is_ready = module.weight.device.type != "meta" and (
-            module.bias is None or module.bias.device.type != "meta"
-        )
-
-        if module_is_ready:
-            module_tag = ".".join(module.name.split(".")[-2:])
-            if "weight_quant_params" in quant_config:
-                module_quant_config = quant_config
-            elif module_tag in quant_config:
-                module_quant_config = quant_config[module_tag]
-
-            hqq_layer = HQQLinear(
-                module,
-                quant_config=module_quant_config,
-                compute_dtype=self.dtype,
-                device=target_device,
-                del_orig=True,
-            )
-
-            if hqq_layer.bias is not None and isinstance(hqq_layer.bias, torch.Tensor):
-                hqq_layer.bias = torch.nn.Parameter(hqq_layer.bias)
-
-            if self.using_multi_gpu:
-                hqq_layer = self._patch_layer_for_multigpu(hqq_layer)
-
-            setattr(parent_module, node, hqq_layer)
+    # TODO: to remove
+    # def create_quantized_param(
+    #     self,
+    #     model: "PreTrainedModel",
+    #     param_value: "torch.Tensor",
+    #     param_name: str,
+    #     target_device: "torch.device",
+    #     **kwargs,
+    # ):
+    #     module, tensor_name = get_module_from_name(model, param_name)
+    #     module_name = param_name.rsplit(".", 1)[0]
+    #     parent_module, node = get_module_from_name(model, module_name)
+
+    #     quant_config = model.config.quantization_config["quant_config"]
+    #     skip_modules = model.config.quantization_config["skip_modules"]
+
+    #     # In this case we do not quantize this layer (it's explicitly skipped) -> simply load param
+    #     if any(skip_module in module.name for skip_module in skip_modules):
+    #         module.load_state_dict(
+    #             {tensor_name: param_value.to(device=target_device, dtype=self.dtype)}, strict=False, assign=True
+    #         )
+    #         return
+
+    #     # We need this hack as the model is not pre-prepared as an empty skeleton on meta device
+    #     if self.pre_quantized:
+    #         # Save them for later
+    #         if not hasattr(self, "hqq_params"):
+    #             self.hqq_params = defaultdict(dict)
+    #         self.hqq_params[module_name].update({tensor_name: param_value})
+    #         hqq_params = self.hqq_params[module_name]
+
+    #         # If they are all present and saved, make it a HQQLinear layer! (we cannot do it param after param because
+    #         # hqq does not support it...)
+    #         if all(k in hqq_params for k in self.hqq_keys) and ("bias" in hqq_params or module.bias is None):
+    #             hqq_layer = HQQLinear(
+    #                 linear_layer=None,
+    #                 quant_config=None,
+    #                 compute_dtype=self.dtype,
+    #                 device=target_device,
+    #                 del_orig=False,
+    #             )
+    #             hqq_layer.load_state_dict(hqq_params)
+
+    #             if hqq_layer.bias is not None and isinstance(hqq_layer.bias, torch.Tensor):
+    #                 hqq_layer.bias = torch.nn.Parameter(hqq_layer.bias)
+    #             if self.using_multi_gpu:
+    #                 hqq_layer = self._patch_layer_for_multigpu(hqq_layer)
+
+    #             setattr(parent_module, node, hqq_layer)
+    #             del self.hqq_params[module_name], module
+    #         return
+
+    #     # Load param in the module (without caring about device or dtype, it will be changed later)
+    #     module.load_state_dict({tensor_name: param_value}, strict=False, assign=True)
+
+    #     # If both the weight and bias have already been loaded, time to quantize!
+    #     module_is_ready = module.weight.device.type != "meta" and (
+    #         module.bias is None or module.bias.device.type != "meta"
+    #     )
+
+    #     if module_is_ready:
+    #         module_tag = ".".join(module.name.split(".")[-2:])
+    #         if "weight_quant_params" in quant_config:
+    #             module_quant_config = quant_config
+    #         elif module_tag in quant_config:
+    #             module_quant_config = quant_config[module_tag]
+
+    #         hqq_layer = HQQLinear(
+    #             module,
+    #             quant_config=module_quant_config,
+    #             compute_dtype=self.dtype,
+    #             device=target_device,
+    #             del_orig=True,
+    #         )
+
+    #         if hqq_layer.bias is not None and isinstance(hqq_layer.bias, torch.Tensor):
+    #             hqq_layer.bias = torch.nn.Parameter(hqq_layer.bias)
+
+    #         if self.using_multi_gpu:
+    #             hqq_layer = self._patch_layer_for_multigpu(hqq_layer)
+
+    #         setattr(parent_module, node, hqq_layer)
 
     def _patch_layer_for_multigpu(self, hqq_layer):
         def forward_with_device(self, x):
@@ -263,7 +254,7 @@ class HqqHfQuantizer(HfQuantizer):
         model.is_hqq_serializable = self.is_serializable()
         return model
 
-    def is_serializable(self, safe_serialization=None):
+    def is_serializable(self):
         return True
 
     @property