transformers 5.0.0__py3-none-any.whl → 5.0.0rc0__py3-none-any.whl

transformers/utils/quantization_config.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# coding=utf-8
 
 # Copyright 2023 The HuggingFace Inc. team. All rights reserved.
 # Modifications Copyright (C) 2025, Advanced Micro Devices, Inc. All rights reserved.
@@ -27,13 +28,16 @@ from typing import Any, Optional, Union
 from packaging import version
 
 from ..utils import (
+    is_auto_awq_available,
     is_compressed_tensors_available,
+    is_gptqmodel_available,
     is_hqq_available,
     is_quark_available,
     is_torch_available,
     is_torchao_available,
     logging,
 )
+from .import_utils import is_auto_gptq_available
 
 
 if is_torch_available():
@@ -64,27 +68,30 @@ class QuantizationMethod(str, Enum):
     MXFP4 = "mxfp4"
 
 
-class AwqFormat(str, Enum):
+class AWQLinearVersion(str, Enum):
     GEMM = "gemm"
     GEMV = "gemv"
-    GEMV_FAST = "gemv_fast"
-    LLM_AWQ = "llm-awq"
-
-
-class AwqBackend(str, Enum):
-    LEGACY_AWQ = "autoawq"
-    AUTO = "auto"
-    AUTO_TRAINABLE = "auto_trainable"
-    MACHETE = "machete"
-    MARLIN = "marlin"
-    EXLLAMA_V2 = "exllama_v2"
-    EXLLAMA_V1 = "exllama_v1"
-    GEMM = "gemm"
-    GEMM_TRITON = "gemm_triton"
-    GEMV = "gemv"
-    GEMV_FAST = "gemv_fast"
-    TORCH_AWQ = "torch_awq"
-    TORCH_FUSED_AWQ = "torch_fused_awq"
+    EXLLAMA = "exllama"
+    IPEX = "ipex"
+
+    @staticmethod
+    def from_str(version: str):
+        version = version.lower()
+        if version == "gemm":
+            return AWQLinearVersion.GEMM
+        elif version == "gemv":
+            return AWQLinearVersion.GEMV
+        elif version == "exllama":
+            return AWQLinearVersion.EXLLAMA
+        elif version == "ipex":
+            return AWQLinearVersion.IPEX
+        else:
+            raise ValueError(f"Unknown AWQLinearVersion {version}")
+
+
+class AwqBackendPackingMethod(str, Enum):
+    AUTOAWQ = "autoawq"
+    LLMAWQ = "llm-awq"
 
 
 @dataclass
@@ -153,7 +160,8 @@ class QuantizationConfigMixin:
 
     def __iter__(self):
         """allows `dict(obj)` for situations where obj may be a dict or QuantizationConfigMixin"""
-        yield from copy.deepcopy(self.__dict__).items()
+        for attr, value in copy.deepcopy(self.__dict__).items():
+            yield attr, value
 
     def __repr__(self):
         return f"{self.__class__.__name__} {self.to_json_string()}"
@@ -612,7 +620,7 @@ class ExllamaVersion(int, Enum):
 class GPTQConfig(QuantizationConfigMixin):
     """
     This is a wrapper class about all possible attributes and features that you can play with a model that has been
-    loaded using `optimum` api for GPTQ quantization relying on the gptqmodel backend.
+    loaded using `optimum` api for gptq quantization relying on auto_gptq backend.
 
     Args:
         bits (`int`):
@@ -633,23 +641,22 @@ class GPTQConfig(QuantizationConfigMixin):
         desc_act (`bool`, *optional*, defaults to `False`):
             Whether to quantize columns in order of decreasing activation size. Setting it to False can significantly
             speed up inference but the perplexity may become slightly worse. Also known as act-order.
-        act_group_aware (`bool`, *optional*, defaults to `True`):
-            Use GAR (group aware activation order) during quantization. Has measurable positive impact on quantization
-            quality. Only applicable when `desc_act = False`. Will forced to be `False` when `desc_act = True`.
         sym (`bool`, *optional*, defaults to `True`):
             Whether to use symmetric quantization.
         true_sequential (`bool`, *optional*, defaults to `True`):
             Whether to perform sequential quantization even within a single Transformer block. Instead of quantizing
             the entire block at once, we perform layer-wise quantization. As a result, each layer undergoes
             quantization using inputs that have passed through the previously quantized layers.
-        format (`str`, *optional*, defaults to `"gptq"`):
-            GPTQ weight format. `gptq` (v1) is supported by gptqmodel. `gptq_v2` is gptqmodel only.
+        checkpoint_format (`str`, *optional*, defaults to `"gptq"`):
+            GPTQ weight format. `gptq`(v1) is supported by both gptqmodel and auto-gptq. `gptq_v2` is gptqmodel only.
         meta (`dict[str, any]`, *optional*):
             Properties, such as tooling:version, that do not directly contributes to quantization or quant inference are stored in meta.
             i.e. `meta.quantizer`: ["optimum:_version_", "gptqmodel:_version_"]
         backend (`str`, *optional*):
-            Controls which kernel to use. Valid values for gptqmodel are `auto`, `auto_trainable` and more. Ref gptqmodel backends:
-            https://github.com/ModelCloud/GPTQModel/blob/main/gptqmodel/utils/backend.py
+            Controls which gptq kernel to be used. Valid values for gptqmodel are `auto`, `auto_trainable` and more. For auto-gptq, only
+            valid value is None and `auto_trainable`. Ref gptqmodel backends: https://github.com/ModelCloud/GPTQModel/blob/main/gptqmodel/utils/backend.py
+        use_cuda_fp16 (`bool`, *optional*, defaults to `False`):
+            Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16. Auto-gptq only.
         model_seqlen (`int`, *optional*):
             The maximum sequence length that the model can take.
         block_name_to_quantize (`str`, *optional*):
@@ -660,9 +667,14 @@ class GPTQConfig(QuantizationConfigMixin):
             The batch size used when processing the dataset
         pad_token_id (`int`, *optional*):
             The pad token id. Needed to prepare the dataset when `batch_size` > 1.
+        use_exllama (`bool`, *optional*):
+            Whether to use exllama backend. Defaults to `True` if unset. Only works with `bits` = 4.
         max_input_length (`int`, *optional*):
             The maximum input length. This is needed to initialize a buffer that depends on the maximum expected input
             length. It is specific to the exllama backend with act-order.
+        exllama_config (`dict[str, Any]`, *optional*):
+            The exllama config. You can specify the version of the exllama kernel through the `version` key. Defaults
+            to `{"version": 1}` if unset.
         cache_block_outputs (`bool`, *optional*, defaults to `True`):
             Whether to cache block outputs to reuse as inputs for the succeeding block.
         modules_in_block_to_quantize (`list[list[str]]`, *optional*):
@@ -682,18 +694,20 @@ class GPTQConfig(QuantizationConfigMixin):
         group_size: int = 128,
         damp_percent: float = 0.1,
         desc_act: bool = False,
-        act_group_aware: bool = True,
         sym: bool = True,
         true_sequential: bool = True,
-        format: str = "gptq",
+        checkpoint_format: str = "gptq",
         meta: dict[str, Any] | None = None,
         backend: str | None = None,
+        use_cuda_fp16: bool = False,
         model_seqlen: int | None = None,
         block_name_to_quantize: str | None = None,
         module_name_preceding_first_block: list[str] | None = None,
         batch_size: int = 1,
         pad_token_id: int | None = None,
+        use_exllama: bool | None = None,
         max_input_length: int | None = None,
+        exllama_config: dict[str, Any] | None = None,
         cache_block_outputs: bool = True,
         modules_in_block_to_quantize: list[list[str]] | None = None,
         **kwargs,
@@ -705,28 +719,33 @@ class GPTQConfig(QuantizationConfigMixin):
         self.group_size = group_size
         self.damp_percent = damp_percent
         self.desc_act = desc_act
-        self.act_group_aware = act_group_aware
         self.sym = sym
         self.true_sequential = true_sequential
-        self.format = format.lower()
-        # Compatible with legacy field: checkpoint_format
-        if kwargs.get("checkpoint_format") is not None:
-            self.format = kwargs.pop("checkpoint_format").lower()
+        self.checkpoint_format = checkpoint_format.lower()
         self.meta = meta
         self.backend = backend.lower() if isinstance(backend, str) else backend
+        self.use_cuda_fp16 = use_cuda_fp16
         self.model_seqlen = model_seqlen
         self.block_name_to_quantize = block_name_to_quantize
         self.module_name_preceding_first_block = module_name_preceding_first_block
         self.batch_size = batch_size
         self.pad_token_id = pad_token_id
+        self.use_exllama = use_exllama
         self.max_input_length = max_input_length
+        self.exllama_config = exllama_config
         self.cache_block_outputs = cache_block_outputs
         self.modules_in_block_to_quantize = modules_in_block_to_quantize
         self.post_init()
 
     def get_loading_attributes(self):
         attributes_dict = copy.deepcopy(self.__dict__)
-        loading_attributes = ["max_input_length", "backend"]
+        loading_attributes = [
+            "use_exllama",
+            "exllama_config",
+            "use_cuda_fp16",
+            "max_input_length",
+            "backend",
+        ]
         loading_attributes_dict = {i: j for i, j in attributes_dict.items() if i in loading_attributes}
         return loading_attributes_dict
 
@@ -753,14 +772,46 @@ class GPTQConfig(QuantizationConfigMixin):
                     ['wikitext2','c4','c4-new'], but we found {self.dataset}"""
                 )
 
-        # act_group_order is only applicable when `desc_act = False`
-        if self.desc_act and self.act_group_aware:
-            self.act_group_aware = False
-            logger.warning("`act_group_aware` has been auto-disabled as it is not compatible with `desc_act = True`.")
+        # make sure backend is back/forward compatible with both gptqmodel (full) and auto-gptq (partial)
+        if is_gptqmodel_available():
+            # convert auto-gptq control into gptqmodel backend
+            if self.backend is None:
+                self.backend = "auto_trainable" if self.use_exllama is not None and not self.use_exllama else "auto"
+        else:
+            # convert gptqmodel backend `auto_trainable` into auto-gptq control
+            if self.backend == "auto_trainable":
+                self.use_exllama = False
 
-        # make sure backend default stays consistent with gptqmodel expectations
-        if self.backend is None:
-            self.backend = "auto"
+        # auto-gptq specific kernel control logic
+        if self.use_exllama is None:
+            # New default behaviour
+            self.use_exllama = True
+
+        if self.exllama_config is None:
+            self.exllama_config = {"version": ExllamaVersion.ONE}
+        else:
+            if "version" not in self.exllama_config:
+                raise ValueError("`exllama_config` needs to have a `version` key.")
+            elif self.exllama_config["version"] not in [ExllamaVersion.ONE, ExllamaVersion.TWO]:
+                exllama_version = self.exllama_config["version"]
+                raise ValueError(
+                    f"Only supported versions are in [ExllamaVersion.ONE, ExllamaVersion.TWO] - not recognized version {exllama_version}"
+                )
+
+        if self.bits == 4 and self.use_exllama:
+            if self.exllama_config["version"] == ExllamaVersion.ONE:
+                logger.info(
+                    "You have activated exllama backend. Note that you can get better inference "
+                    "speed using exllamav2 kernel by setting `exllama_config`."
+                )
+            elif self.exllama_config["version"] == ExllamaVersion.TWO:
+                if is_auto_gptq_available():
+                    optimum_version = version.parse(importlib.metadata.version("optimum"))
+                    autogptq_version = version.parse(importlib.metadata.version("auto_gptq"))
+                    if optimum_version <= version.parse("1.13.2") or autogptq_version <= version.parse("0.4.2"):
+                        raise ValueError(
+                            f"You need optimum > 1.13.2 and auto-gptq > 0.4.2 . Make sure to have that version installed - detected version : optimum {optimum_version} and autogptq {autogptq_version}"
+                        )
         if self.modules_in_block_to_quantize is not None:
             optimum_version = version.parse(importlib.metadata.version("optimum"))
             if optimum_version < version.parse("1.15.0"):
@@ -770,15 +821,17 @@ class GPTQConfig(QuantizationConfigMixin):
 
     def to_dict(self) -> dict[str, Any]:
         config_dict = super().to_dict()
-        # Compatible with legacy field: checkpoint_format
-        config_dict["checkpoint_format"] = self.format
+        config_dict.pop("disable_exllama", None)
         return config_dict
 
     def to_dict_optimum(self):
         """
         Get compatible dict for optimum gptq config
         """
-        return self.to_dict()
+        quant_dict = self.to_dict()
+        # make it compatible with optimum config
+        quant_dict["disable_exllama"] = not self.use_exllama
+        return quant_dict
 
     @classmethod
     def from_dict_optimum(cls, config_dict):
@@ -786,12 +839,17 @@ class GPTQConfig(QuantizationConfigMixin):
         Get compatible class with optimum gptq config dict
         """
 
+        if "disable_exllama" in config_dict:
+            config_dict["use_exllama"] = not config_dict["disable_exllama"]
+            # switch to None to not trigger the warning
+            config_dict.pop("disable_exllama")
+
         config = cls(**config_dict)
         return config
 
 
 @dataclass
-class AwqConfig(GPTQConfig):
+class AwqConfig(QuantizationConfigMixin):
     """
     This is a wrapper class about all possible attributes and features that you can play with a model that has been
     loaded using `auto-awq` library awq quantization relying on auto_awq backend.
@@ -803,12 +861,26 @@ class AwqConfig(GPTQConfig):
             The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization.
         zero_point (`bool`, *optional*, defaults to `True`):
             Whether to use zero point quantization.
-        backend (`AwqBackend`, *optional*, defaults to `AwqBackend.AUTO`):
-            The quantization backend.
+        version (`AWQLinearVersion`, *optional*, defaults to `AWQLinearVersion.GEMM`):
+            The version of the quantization algorithm to use. GEMM is better for big batch_size (e.g. >= 8) otherwise,
+            GEMV is better (e.g. < 8 ). GEMM models are compatible with Exllama kernels.
+        backend (`AwqBackendPackingMethod`, *optional*, defaults to `AwqBackendPackingMethod.AUTOAWQ`):
+            The quantization backend. Some models might be quantized using `llm-awq` backend. This is useful for users
+            that quantize their own models using `llm-awq` library.
+        do_fuse (`bool`, *optional*, defaults to `False`):
+            Whether to fuse attention and mlp layers together for faster inference
+        fuse_max_seq_len (`int`, *optional*):
+            The Maximum sequence length to generate when using fusing.
+        modules_to_fuse (`dict`, *optional*, default to `None`):
+            Overwrite the natively supported fusing scheme with the one specified by the users.
         modules_to_not_convert (`list`, *optional*, default to `None`):
             The list of modules to not quantize, useful for quantizing models that explicitly require to have
             some modules left in their original precision (e.g. Whisper encoder, Llava encoder, Mixtral gate layers).
             Note you cannot quantize directly with transformers, please refer to `AutoAWQ` documentation for quantizing HF models.
+        exllama_config (`dict[str, Any]`, *optional*):
+            You can specify the version of the exllama kernel through the `version` key, the maximum sequence
+            length through the `max_input_len` key, and the maximum batch size through the `max_batch_size` key.
+            Defaults to `{"version": 2, "max_input_len": 2048, "max_batch_size": 8}` if unset.
     """
 
     def __init__(
@@ -816,44 +888,141 @@ class AwqConfig(GPTQConfig):
         bits: int = 4,
         group_size: int = 128,
         zero_point: bool = True,
-        backend: AwqBackend = AwqBackend.AUTO,
+        version: AWQLinearVersion = AWQLinearVersion.GEMM,
+        backend: AwqBackendPackingMethod = AwqBackendPackingMethod.AUTOAWQ,
+        do_fuse: bool | None = None,
+        fuse_max_seq_len: int | None = None,
+        modules_to_fuse: dict | None = None,
         modules_to_not_convert: list | None = None,
+        exllama_config: dict[str, int] | None = None,
         **kwargs,
     ):
-        format = kwargs.pop("format", AwqFormat.GEMM)
-        # Compatible with legacy field: version
-        if kwargs.get("version") is not None:
-            format = kwargs.pop("version").lower()
-        # Compatible with legacy backend
-        if backend == AwqBackend.LEGACY_AWQ:
-            backend = AwqBackend.AUTO
+        self.quant_method = QuantizationMethod.AWQ
+
+        self.bits = bits
+        self.group_size = group_size
         self.zero_point = zero_point
+        self.version = version
+        self.backend = backend
+        self.fuse_max_seq_len = fuse_max_seq_len
         self.modules_to_not_convert = modules_to_not_convert
+        self.exllama_config = exllama_config
 
-        super().__init__(bits=bits, group_size=group_size, backend=backend, format=format, **kwargs)
-        self.quant_method = QuantizationMethod.AWQ
+        self.modules_to_fuse = modules_to_fuse
+        if do_fuse is None:
+            self.do_fuse = modules_to_fuse is not None and len(modules_to_fuse) > 0
+        else:
+            self.do_fuse = do_fuse
+        self.fuse_max_seq_len = fuse_max_seq_len
+
+        self.post_init()
 
     def post_init(self):
         r"""
         Safety checker that arguments are correct
         """
+        if self.backend not in [AwqBackendPackingMethod.AUTOAWQ, AwqBackendPackingMethod.LLMAWQ]:
+            raise ValueError(
+                f"Only supported quantization backends in {AwqBackendPackingMethod.AUTOAWQ} and {AwqBackendPackingMethod.LLMAWQ} - not recognized backend {self.backend}"
+            )
 
-        if self.backend == "llm-awq":
-            self.format = AwqFormat.LLM_AWQ
-            self.backend = AwqBackend.AUTO
+        self.version = AWQLinearVersion.from_str(self.version)
+        if self.version not in [
+            AWQLinearVersion.GEMM,
+            AWQLinearVersion.GEMV,
+            AWQLinearVersion.EXLLAMA,
+            AWQLinearVersion.IPEX,
+        ]:
+            raise ValueError(
+                f"Only supported versions are in [AWQLinearVersion.GEMM, AWQLinearVersion.GEMV, AWQLinearVersion.EXLLAMA, AWQLinearVersion.IPEX] - not recognized version {self.version}"
+            )
 
-        if self.format not in AwqFormat.__members__.values():
-            raise ValueError(f"Invalid format '{self.format}'. Must be one of: {[b.value for b in AwqFormat]}")
+        if self.backend == AwqBackendPackingMethod.LLMAWQ:
+            # Only cuda device can run this function
+            if not (torch.cuda.is_available() or torch.xpu.is_available()):
+                raise ValueError("LLM-AWQ backend is only supported on CUDA and XPU")
+            if torch.cuda.is_available():
+                compute_capability = torch.cuda.get_device_capability()
+                major, minor = compute_capability
+                if major < 8:
+                    raise ValueError("LLM-AWQ backend is only supported on CUDA GPUs with compute capability >= 8.0")
+
+        if self.do_fuse and self.fuse_max_seq_len is None:
+            raise ValueError(
+                "You cannot enable fused modules without specifying a `fuse_max_seq_len`, make sure to pass a valid `fuse_max_seq_len` for your usecase"
+            )
 
-        if self.backend not in AwqBackend.__members__.values():
-            raise ValueError(f"Invalid backend '{self.backend}'. Must be one of: {[b.value for b in AwqBackend]}")
+        if self.do_fuse:
+            awq_version_supports_fusing = False
+            MIN_AWQ_VERSION = "0.1.7"
+            if is_auto_awq_available():
+                awq_version_supports_fusing = version.parse(importlib.metadata.version("autoawq")) >= version.parse(
+                    MIN_AWQ_VERSION
+                )
 
-    def to_dict(self) -> dict[str, Any]:
-        config_dict = super().to_dict()
-        config_dict.pop("checkpoint_format")
-        # Compatible with legacy field: version
-        config_dict["version"] = self.format
-        return config_dict
+            if not awq_version_supports_fusing:
+                raise ValueError(
+                    f"You current version of `autoawq` does not support module fusing, please upgrade `autoawq` package to at least {MIN_AWQ_VERSION}."
+                )
+
+        if self.modules_to_not_convert is not None:
+            awq_version_supports_non_conversion = False
+            MIN_AWQ_VERSION = "0.1.8"
+            if is_auto_awq_available():
+                awq_version_supports_non_conversion = version.parse(
+                    importlib.metadata.version("autoawq")
+                ) >= version.parse(MIN_AWQ_VERSION)
+
+            if not awq_version_supports_non_conversion:
+                raise ValueError(
+                    f"You current version of `autoawq` does not support module quantization skipping, please upgrade `autoawq` package to at least {MIN_AWQ_VERSION}."
+                )
+
+        if self.do_fuse and self.modules_to_fuse is not None:
+            required_keys = [
+                "hidden_size",
+                "num_attention_heads",
+                "num_key_value_heads",
+                "mlp",
+                "attention",
+                "layernorm",
+                "use_alibi",
+            ]
+            if not all(key in self.modules_to_fuse for key in required_keys):
+                raise ValueError(
+                    f"Required fields are missing in the fusing mapping, required fields are {required_keys}"
+                )
+
+        if self.version == AWQLinearVersion.EXLLAMA:
+            awq_version_supports_exllama = False
+            MIN_AWQ_VERSION = "0.2.0"
+            if is_auto_awq_available():
+                awq_version_supports_exllama = version.parse(importlib.metadata.version("autoawq")) >= version.parse(
+                    MIN_AWQ_VERSION
+                )
+
+            if not awq_version_supports_exllama:
+                raise ValueError(
+                    f"You current version of `autoawq` does not support exllama backend, "
+                    f"please upgrade `autoawq` package to at least {MIN_AWQ_VERSION}."
+                )
+
+            if self.exllama_config is None:
+                self.exllama_config = {"version": ExllamaVersion.TWO, "max_input_len": 2048, "max_batch_size": 8}
+            else:
+                if "version" not in self.exllama_config:
+                    raise ValueError("`exllama_config` needs to have a `version` key.")
+                elif self.exllama_config["version"] not in [ExllamaVersion.ONE, ExllamaVersion.TWO]:
+                    exllama_version = self.exllama_config["version"]
+                    raise ValueError(
+                        f"Only supported versions are in [ExllamaVersion.ONE, ExllamaVersion.TWO] - not recognized version {exllama_version}"
+                    )
+
+    def get_loading_attributes(self):
+        attributes_dict = copy.deepcopy(self.__dict__)
+        loading_attributes = ["version", "do_fuse", "modules_to_fuse", "fuse_max_seq_len", "exllama_config"]
+        loading_attributes_dict = {i: j for i, j in attributes_dict.items() if i in loading_attributes}
+        return loading_attributes_dict
 
 
 @dataclass