transformers 5.0.0rc0__py3-none-any.whl → 5.0.0rc1__py3-none-any.whl

transformers/integrations/quark.py

@@ -0,0 +1,55 @@
+# coding=utf-8
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+from ..core_model_loading import ConversionOps
+from ..utils import is_torch_available
+
+
+if is_torch_available():
+    import torch
+
+
+class QuarkDeserialize(ConversionOps):
+    def __init__(self, hf_quantizer):
+        self.hf_quantizer = hf_quantizer
+
+    def convert(
+        self,
+        input_dict: torch.Tensor,
+        model: Optional[torch.nn.Module] = None,
+        missing_keys: Optional[list[str]] = None,
+        full_layer_name: str | None = None,
+        **kwargs,
+    ) -> dict[str, torch.Tensor]:
+        # target_key should be in the form of weight_scale, bias_scale, input_scale, output_scale, weight_zero_point, bias_zero_point, input_zero_point, output_zero_point
+        target_key, value = tuple(input_dict.items())[0]
+        value = value[0] if isinstance(value, list) else value
+        # this will get the param name : weight, input, bias or output
+        param = target_key.split("_", 1)[0]
+        # quant_state should be in the form of scale, or zero_point
+        quant_state = target_key.split("_", 1)[-1]
+
+        # here we change the name for example from the form of :
+        # model.layers.0.mlp.down_proj.weight_scale to model.layers.0.mlp.down_proj.weight_quantizer.scale to fit within
+        # the QParamsLinear module of quark
+        sub_module_state = full_layer_name.rsplit(".", 1)[0] + "." + param + "_quantizer" + "." + quant_state
+
+        # since quark module was expecting keys in the form of model.layers.0.mlp.down_proj.weight_scale
+        # we need to remove it from the missing_keys list
+        missing_keys.discard(full_layer_name)
+
+        return {sub_module_state: value}

transformers/integrations/spqr.py

@@ -13,110 +13,64 @@
 # limitations under the License.
 "SpQR (Sparse-Quantized Representation) integration file"
 
-from ..utils import is_accelerate_available, is_spqr_available, is_torch_available
+from ..quantizers.quantizers_utils import should_convert_module
+from ..utils import is_accelerate_available, is_spqr_available, is_torch_available, logging
 
 
+if is_accelerate_available():
+    from accelerate import init_empty_weights
+
 if is_torch_available():
     import torch.nn as nn
 
+logger = logging.get_logger(__name__)
+
 
-def replace_with_spqr_linear(
-    model,
-    quantization_config=None,
-    modules_to_not_convert=None,
-    current_key_name=None,
-    has_been_replaced=False,
-):
+def replace_with_spqr_linear(model, modules_to_not_convert: list[str] | None = None, quantization_config=None):
     """
-    Public method that recursively replaces the Linear layers of the given model with SpQR quantized layers.
-    `accelerate` is needed to use this method. Returns the converted model and a boolean that indicates if the
-    conversion has been successful or not.
+    Public method that replaces the Linear layers of the given model with SPQR quantized layers.
 
     Args:
         model (`torch.nn.Module`):
             The model to convert, can be any `torch.nn.Module` instance.
-        quantization_config (`SpQRConfig`):
-            The quantization config object that contains the quantization parameters.
-        modules_to_not_convert (`list[str]`, *optional*):
+        modules_to_not_convert (`list[str]`, *optional*, defaults to `None`):
             A list of nn.Linear weights to not convert. If a parameter path is in the list (e.g. `lm_head.weight`), the corresponding module will not be
             converted.
-        current_key_name (`list`, *optional*):
-            A list that contains the current key name. This is used for recursion and should not be passed by the user.
-        has_been_replaced (`bool`, *optional*):
-            A boolean that indicates if the conversion has been successful or not. This is used for recursion and
-            should not be passed by the user.
+        quantization_config (`SpQRConfig`):
+            The quantization config object that contains the quantization parameters.
     """
-    if modules_to_not_convert is None:
-        modules_to_not_convert = []
-
-    if is_accelerate_available():
-        from accelerate import init_empty_weights
     if is_spqr_available():
         from spqr_quant import QuantizedLinear
 
-    for name, module in model.named_children():
-        if current_key_name is None:
-            current_key_name = []
-        current_key_name.append(name)
-
-        if isinstance(module, nn.Linear):
-            # Check if the current key is not in the `modules_to_not_convert`
-            if ".".join(current_key_name) + ".weight" not in modules_to_not_convert:
-                with init_empty_weights():
-                    tensor_name = ".".join(current_key_name)
-
-                    shapes = quantization_config.shapes
-                    shapes_keys = shapes.keys()
-
-                    shapes_valid = (
-                        f"{tensor_name}.dense_weights.shape" in shapes_keys
-                        and f"{tensor_name}.row_offsets.shape" in shapes_keys
-                        and f"{tensor_name}.col_vals.shape" in shapes_keys
-                        and f"{tensor_name}.in_perm.shape" in shapes_keys
-                    )
-
-                    if not shapes_valid:
-                        raise ValueError(
-                            f"The SpQR quantization config does not contain the shape "
-                            f"configuration for {tensor_name}. This indicates that the "
-                            f"configuration is either invalid or corrupted."
-                        )
-
-                    dense_weights_shape = shapes[f"{tensor_name}.dense_weights.shape"]
-                    row_offsets_shape = shapes[f"{tensor_name}.row_offsets.shape"]
-                    col_vals_shape = shapes[f"{tensor_name}.col_vals.shape"]
-                    in_perm_shape = shapes[f"{tensor_name}.in_perm.shape"]
-
-                    in_features = module.in_features
-                    out_features = module.out_features
-
-                    model._modules[name] = QuantizedLinear.create_placehodler(
-                        rows=out_features,
-                        cols=in_features,
-                        bits=quantization_config.bits,
-                        beta1=quantization_config.beta1,
-                        beta2=quantization_config.beta2,
-                        dense_weights_shape=dense_weights_shape,
-                        row_offsets_shape=row_offsets_shape,
-                        col_vals_shape=col_vals_shape,
-                        in_perm_shape=in_perm_shape,
-                    )
-                    has_been_replaced = True
-
-                    # Store the module class in case we need to transpose the weight later
-                    model._modules[name].source_cls = type(module)
-                    # Force requires grad to False to avoid unexpected errors
-                    model._modules[name].requires_grad_(False)
-            else:
-                pass
-        if len(list(module.children())) > 0:
-            _, has_been_replaced = replace_with_spqr_linear(
-                module,
-                quantization_config=quantization_config,
-                modules_to_not_convert=modules_to_not_convert,
-                current_key_name=current_key_name,
-                has_been_replaced=has_been_replaced,
-            )
-        # Remove the last key for recursion
-        current_key_name.pop(-1)
-    return model, has_been_replaced
+    has_been_replaced = False
+    # we need this to correctly materialize the weights during quantization
+    for module_name, module in model.named_modules():
+        if not should_convert_module(module_name, modules_to_not_convert):
+            continue
+        with init_empty_weights():
+            if isinstance(module, nn.Linear):
+                shapes = quantization_config.shapes
+
+                new_module = QuantizedLinear.create_placehodler(
+                    rows=module.out_features,
+                    cols=module.in_features,
+                    bits=quantization_config.bits,
+                    beta1=quantization_config.beta1,
+                    beta2=quantization_config.beta2,
+                    dense_weights_shape=shapes[f"{module_name}.dense_weights.shape"],
+                    row_offsets_shape=shapes[f"{module_name}.row_offsets.shape"],
+                    col_vals_shape=shapes[f"{module_name}.col_vals.shape"],
+                    in_perm_shape=shapes[f"{module_name}.in_perm.shape"],
+                )
+                # Force requires grad to False to avoid unexpected errors
+                model._modules[module_name].requires_grad_(False)
+                model.set_submodule(module_name, new_module)
+                has_been_replaced = True
+    if not has_been_replaced:
+        logger.warning(
+            "You are loading your model using eetq but no linear modules were found in your model."
+            " Please double check your model architecture, or submit an issue on github if you think this is"
+            " a bug."
+        )
+
+    return model

transformers/integrations/torchao.py

@@ -32,7 +32,7 @@ from ..quantizers.quantizers_utils import get_module_from_name
 
 if is_torchao_available():
     TORCHAO_VERSION = version.parse(importlib.metadata.version("torchao"))
-    if version.parse(importlib.metadata.version("torchao")) >= version.parse("0.14.0"):
+    if version.parse(importlib.metadata.version("torchao")) >= version.parse("0.15.0"):
         from torchao.prototype.safetensors.safetensors_support import (
             unflatten_tensor_state_dict,
         )
@@ -210,61 +210,55 @@ class TorchAoDeserialize(ConversionOps):
     def convert(
         self,
         input_dict: dict[str, torch.Tensor],
+        source_patterns: list[str] | None = None,
         model: Optional[torch.nn.Module] = None,
         full_layer_name: str | None = None,
         missing_keys=None,
         **kwargs,
     ) -> dict[str, torch.Tensor]:
-        if isinstance(self.hf_quantizer.quantization_config.quant_type, str):
-            is_int_4 = "int4" in self.hf_quantizer.quantization_config.quant_type
-        else:
-            config_name = self.hf_quantizer.quantization_config.quant_type.__class__.__name__
-            is_int_4 = fuzzy_match_size(config_name) == "4"
-
-        # Simple case if we gather layermsnorm weights, we can just return the value since they are not quantized
-        if "weight:_data" in input_dict.keys():
-            value = (
-                input_dict["weight:_data"][0]
-                if isinstance(input_dict["weight:_data"], list)
-                else input_dict["weight:_data"]
-            )
-            return {full_layer_name: value}
-
-        is_unsafe_serialization = ":" not in list(input_dict.keys())[0]
+        """
+        Consolidates tensor subclass components before reconstructing the object
+
+        For example:
+            input_dict: {
+                "_weight_qdata": torch.Tensor,
+                "_weight_scale": torch.Tensor,
+            }
+            full_layer_name: "model.layers.0.self_attn.k_proj.weight"
+
+            Given this, we reconstruct a Float8Tensor instance using the qdata and scale
+            and return it as a dictionary with the full_layer_name as the key and the recovered
+            Float8Tensor instance as the value.
+        """
+        is_unsafe_serialization = list(input_dict.keys())[0] not in source_patterns
 
         param_data = {}
+        layer_name = ".".join(full_layer_name.split(".")[:-1])
         if is_unsafe_serialization:
             if isinstance(input_dict["weight"], list):
                 weight = input_dict["weight"][0]
             else:
                 weight = input_dict["weight"]
         else:
-            if isinstance(input_dict["weight:qdata"], list):
-                param_data[f"{full_layer_name}:qdata"] = input_dict["weight:qdata"][0]
-            else:
-                param_data[f"{full_layer_name}:qdata"] = input_dict["weight:qdata"]
-
-            if isinstance(input_dict["weight:scale"], list):
-                param_data[f"{full_layer_name}:scale"] = input_dict["weight:scale"][0]
-            else:
-                param_data[f"{full_layer_name}:scale"] = input_dict["weight:scale"]
-
-            if is_int_4:
-                if isinstance(input_dict["weight:zero_point"], list):
-                    param_data[f"{full_layer_name}:zero_point"] = input_dict["weight:zero_point"][0]
-                else:
-                    param_data[f"{full_layer_name}:zero_point"] = input_dict["weight:zero_point"]
+            for suffix in input_dict.keys():
+                if len(input_dict[suffix]) != 1:
+                    raise ValueError(
+                        f"Expected a single tensor for {suffix} but got {len(input_dict[suffix])} tensors instead"
+                    )
+                param_data[f"{layer_name}.{suffix}"] = input_dict[suffix][0]
 
-        # If it's a bias, no need to do anything special (except removing the ":_data" part of the key, but was
-        # already done) - if it's unsafe-serialized (i.e. not safetensors), not need for anything either
+        # If it's unsafe-serialized (i.e. not safetensors), no need for anything
         if is_unsafe_serialization:
             return {full_layer_name: weight}
         # Sanity check for the new serialization format
-        elif not (TORCHAO_VERSION >= version.parse("0.14.0") and is_metadata_torchao(self.hf_quantizer.metadata)):
-            # print("metadata", self.hf_quantizer.metadata)
-            raise ValueError("To use `safetensors` serialization, you should have `torchao>=0.14.0` installed")
+        elif not (TORCHAO_VERSION >= version.parse("0.15.0") and is_metadata_torchao(self.hf_quantizer.metadata)):
+            raise ValueError("To use `safetensors` serialization, you should have `torchao>=0.15.0` installed")
 
-        new_param = unflatten_tensor_state_dict(param_data, self.hf_quantizer.metadata)[full_layer_name]
+        unflattened_state_dict, leftover_state_dict = unflatten_tensor_state_dict(
+            param_data, self.hf_quantizer.metadata
+        )
+        assert not leftover_state_dict  # there should be no unprocessed tensors
+        new_param = unflattened_state_dict[full_layer_name]
 
         module, _ = get_module_from_name(model, full_layer_name)
         # Add repr to the module

transformers/integrations/vptq.py

@@ -13,64 +13,49 @@
 # limitations under the License.
 "VPTQ (Vector Post-Training Quantization) integration file"
 
-import torch.nn as nn
-from accelerate import init_empty_weights
-from vptq import VQuantLinear
+from ..quantizers.quantizers_utils import should_convert_module
+from ..utils import is_accelerate_available, is_torch_available, logging
 
 
-def replace_with_vptq_linear(
-    model,
-    quantization_config=None,
-    modules_to_not_convert=None,
-    current_key_name=None,
-    has_been_replaced=False,
-):
+if is_accelerate_available():
+    from accelerate import init_empty_weights
+
+if is_torch_available():
+    import torch.nn as nn
+
+logger = logging.get_logger(__name__)
+
+
+def replace_with_vptq_linear(model, modules_to_not_convert: list[str] | None = None, quantization_config=None):
     """
-    Public method that recursively replaces the Linear layers of the given model with VPTQ quantized layers.
-    `accelerate` is needed to use this method. Returns the converted model and a boolean that indicates if the
-    conversion has been successful or not.
+    Public method that replaces the Linear layers of the given model with SPQR quantized layers.
 
     Args:
         model (`torch.nn.Module`):
             The model to convert, can be any `torch.nn.Module` instance.
+        modules_to_not_convert (`list[str]`, *optional*, defaults to `None`):
+            A list of nn.Linear weights to not convert. If a parameter path is in the list (e.g. `lm_head.weight`), the corresponding module will not be
+            converted.
         quantization_config (`VptqConfig`):
             The quantization config object that contains the quantization parameters.
-        modules_to_not_convert (`list[`str`]`, *optional*, defaults to `["lm_head"]`):
-            Names of the modules to not convert in `VQuantLinear`. In practice we keep the `lm_head` in full precision
-            for numerical stability reasons.
-        current_key_name (`list`, *optional*):
-            A list that contains the current key name. This is used for recursion and should not be passed by the user.
-        has_been_replaced (`bool`, *optional*):
-            A boolean that indicates if the conversion has been successful or not. This is used for recursion and
-            should not be passed by the user.
     """
+    from vptq import VQuantLinear
 
-    modules_to_not_convert = modules_to_not_convert if modules_to_not_convert else ["lm_head"]
-
-    for name, module in model.named_children():
-        if current_key_name is None:
-            current_key_name = []
-        current_key_name.append(name)
-        layer_name = ".".join(current_key_name)
-        shared_layer_config = quantization_config.shared_layer_config
-        config_for_layers = quantization_config.config_for_layers
-
-        if (
-            isinstance(module, nn.Linear)
-            and layer_name not in modules_to_not_convert
-            and ((layer_name in config_for_layers) or (current_key_name[-1] in shared_layer_config))
-        ):
-            layer_params = config_for_layers.get(layer_name, None) or shared_layer_config.get(
-                current_key_name[-1], None
-            )
+    has_been_replaced = False
+    shared_layer_config = quantization_config.shared_layer_config
+    config_for_layers = quantization_config.config_for_layers
 
-            with init_empty_weights():
-                in_features = module.in_features
-                out_features = module.out_features
-
-                model._modules[name] = VQuantLinear(
-                    in_features,
-                    out_features,
+    for module_name, module in model.named_modules():
+        if not should_convert_module(module_name, modules_to_not_convert):
+            continue
+        with init_empty_weights():
+            if isinstance(module, nn.Linear):
+                layer_params = config_for_layers.get(module_name, None) or shared_layer_config.get(
+                    module_name.rsplit(".")[1], None
+                )
+                new_module = VQuantLinear(
+                    module.in_features,
+                    module.out_features,
                     vector_lens=layer_params["vector_lens"],
                     num_centroids=layer_params["num_centroids"],
                     num_res_centroids=layer_params["num_res_centroids"],
@@ -84,18 +69,16 @@ def replace_with_vptq_linear(
                     enable_proxy_error=False,
                     bias=module.bias is not None,
                 )
+                # Force requires grad to False to avoid unexpected errors
+                model._modules[module_name].requires_grad_(False)
+                model.set_submodule(module_name, new_module)
                 has_been_replaced = True
 
-                # Force requires grad to False to avoid unexpected errors
-                model._modules[name].requires_grad_(False)
-        if len(list(module.children())) > 0:
-            _, has_been_replaced = replace_with_vptq_linear(
-                module,
-                quantization_config=quantization_config,
-                modules_to_not_convert=modules_to_not_convert,
-                current_key_name=current_key_name,
-                has_been_replaced=has_been_replaced,
-            )
-        # Remove the last key for recursion
-        current_key_name.pop(-1)
-    return model, has_been_replaced
+    if not has_been_replaced:
+        logger.warning(
+            "You are loading your model using eetq but no linear modules were found in your model."
+            " Please double check your model architecture, or submit an issue on github if you think this is"
+            " a bug."
+        )
+
+    return model

transformers/modelcard.py

@@ -23,7 +23,7 @@ from typing import Any, Optional, Union
 
 import httpx
 import yaml
-from huggingface_hub import model_info
+from huggingface_hub import is_offline_mode, model_info
 from huggingface_hub.errors import OfflineModeIsEnabled
 from huggingface_hub.utils import HFValidationError
 
@@ -50,7 +50,6 @@ from .utils import (
     MODEL_CARD_NAME,
     cached_file,
     is_datasets_available,
-    is_offline_mode,
     is_tokenizers_available,
     is_torch_available,
     logging,

transformers/modeling_gguf_pytorch_utils.py

@@ -20,6 +20,7 @@ import numpy as np
 from tqdm.auto import tqdm
 
 from .integrations import (
+    GGUF_CONFIG_DEFAULTS_MAPPING,
     GGUF_CONFIG_MAPPING,
     GGUF_TOKENIZER_MAPPING,
     _gguf_parse_value,
@@ -437,6 +438,13 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo
         all("output.weight" != tensor.name for tensor in reader.tensors) or architecture in exceptions
     )
 
+    # Set GGUF-specific default values
+    config_defaults = GGUF_CONFIG_DEFAULTS_MAPPING.get(
+        updated_architecture, GGUF_CONFIG_DEFAULTS_MAPPING.get(architecture) or {}
+    )
+    for key, value in config_defaults.items():
+        parsed_parameters["config"].setdefault(key, value)
+
     # List all key-value pairs in a columnized format
     for gguf_key, field in reader.fields.items():
         gguf_key = gguf_key.replace(architecture, updated_architecture)

transformers/modeling_rope_utils.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import math
+import warnings
 from functools import wraps
 from typing import TYPE_CHECKING, Optional, TypedDict
 
@@ -653,20 +654,26 @@ class RotaryEmbeddingConfigMixin:
         Helper to standardize the config's rope params field by ensuring the params are defined for each
         later type. For old model the fn will duplicate a single rope param in each layer type (backward compatibility)
         """
-        # Move `rope_theta` and `partial_rotary_factor` to the params dict, if not there yet
+        # Move `rope_theta` and `partial_rotary_factor` to the `rope_parameters`, if not there yet
         rope_theta = getattr(self, "rope_theta", None)
         partial_rotary_factor = getattr(self, "partial_rotary_factor", None)
-        rope_parameters = self.rope_parameters
+        rope_parameters = getattr(self, "rope_parameters", None) or {}
+        layer_types = getattr(self, "layer_types", None)
 
+        # Case 0: no RoPE params defined
+        if not (rope_parameters or rope_theta):
+            # partial_rotary_factor without rope_theta is invalid, so we don't check for it here
+            logger.warning("`standardize_rope_params` was called but no RoPE parameters were found.")
+            return
         # Case 1: RoPE param keys do not intersect with possible `layer_types` -> one global dict
-        if getattr(self, "layer_types", None) is None or not set(rope_parameters.keys()).issubset(self.layer_types):
+        elif layer_types is None or rope_parameters == {} or not set(rope_parameters.keys()).issubset(layer_types):
             rope_parameters.setdefault("rope_type", rope_parameters.get("type", "default"))
             rope_parameters.setdefault("rope_theta", rope_theta)
             if partial_rotary_factor is not None:
                 rope_parameters["partial_rotary_factor"] = partial_rotary_factor
         # Case 2: different RoPE for each layer -> several params as nested dict
         else:
-            for layer_type in self.layer_types:
+            for layer_type in layer_types:
                 rope_parameters[layer_type].setdefault("rope_type", rope_parameters[layer_type].get("type", "default"))
                 rope_parameters[layer_type].setdefault("rope_theta", rope_theta)
                 if partial_rotary_factor is not None:
@@ -691,14 +698,14 @@ class RotaryEmbeddingConfigMixin:
 
         for rope_parameters in rope_parameters_dict.values():
             rope_type = rope_parameters.get("rope_type", rope_parameters.get("type", "default"))
-            validation_fn = getattr(self, f"_validate_{rope_type}_rope_parameters")
+            validation_fn = getattr(self, f"_validate_{rope_type}_rope_parameters", None)
             rope_parameters["rope_type"] = rope_type
 
             if validation_fn is not None:
                 validation_fn(rope_parameters, ignore_keys=ignore_keys)
             else:
                 logger.warning(
-                    f"Missing validation function mapping in `ROPE_VALIDATION_FUNCTIONS` for 'rope_type'='{rope_type}'"
+                    f"Missing validation function in 'RotaryEmbeddingConfigMixin' for 'rope_type'='{rope_type}'"
                 )
 
     def _validate_default_rope_parameters(self, rope_parameters: dict, ignore_keys: Optional[set] = None):
@@ -913,3 +920,20 @@ class RotaryEmbeddingConfigMixin:
             unused_keys = received_keys - required_keys
         if unused_keys:
             logger.warning(f"Unrecognized keys in `rope_parameters` for 'rope_type'='{rope_type}': {unused_keys}")
+
+
+def rope_config_validation(config: RotaryEmbeddingConfigMixin, ignore_keys: Optional[set] = None):
+    """
+    This is a deprecated function.
+    It has been kept for backward compatibility with custom code models.
+    """
+    warnings.warn(
+        "`rope_config_validation` is deprecated and has been removed. "
+        "Its functionality has been moved to RotaryEmbeddingConfigMixin.validate_rope method. "
+        "PreTrainedConfig inherits this class, so please call self.validate_rope() instead. "
+        "Also, make sure to use the new rope_parameters syntax. "
+        "You can call self.standardize_rope_params() in the meantime.",
+        FutureWarning,
+    )
+    config.standardize_rope_params()
+    config.validate_rope(ignore_keys=ignore_keys)