transformers 5.0.0rc0__py3-none-any.whl → 5.0.0rc2__py3-none-any.whl

transformers/integrations/torchao.py

@@ -32,7 +32,7 @@ from ..quantizers.quantizers_utils import get_module_from_name
 
 if is_torchao_available():
     TORCHAO_VERSION = version.parse(importlib.metadata.version("torchao"))
-    if version.parse(importlib.metadata.version("torchao")) >= version.parse("0.14.0"):
+    if version.parse(importlib.metadata.version("torchao")) >= version.parse("0.15.0"):
         from torchao.prototype.safetensors.safetensors_support import (
             unflatten_tensor_state_dict,
         )
@@ -210,61 +210,55 @@ class TorchAoDeserialize(ConversionOps):
     def convert(
         self,
         input_dict: dict[str, torch.Tensor],
+        source_patterns: list[str] | None = None,
         model: Optional[torch.nn.Module] = None,
         full_layer_name: str | None = None,
         missing_keys=None,
         **kwargs,
     ) -> dict[str, torch.Tensor]:
-        if isinstance(self.hf_quantizer.quantization_config.quant_type, str):
-            is_int_4 = "int4" in self.hf_quantizer.quantization_config.quant_type
-        else:
-            config_name = self.hf_quantizer.quantization_config.quant_type.__class__.__name__
-            is_int_4 = fuzzy_match_size(config_name) == "4"
-
-        # Simple case if we gather layermsnorm weights, we can just return the value since they are not quantized
-        if "weight:_data" in input_dict.keys():
-            value = (
-                input_dict["weight:_data"][0]
-                if isinstance(input_dict["weight:_data"], list)
-                else input_dict["weight:_data"]
-            )
-            return {full_layer_name: value}
-
-        is_unsafe_serialization = ":" not in list(input_dict.keys())[0]
+        """
+        Consolidates tensor subclass components before reconstructing the object
+
+        For example:
+            input_dict: {
+                "_weight_qdata": torch.Tensor,
+                "_weight_scale": torch.Tensor,
+            }
+            full_layer_name: "model.layers.0.self_attn.k_proj.weight"
+
+            Given this, we reconstruct a Float8Tensor instance using the qdata and scale
+            and return it as a dictionary with the full_layer_name as the key and the recovered
+            Float8Tensor instance as the value.
+        """
+        is_unsafe_serialization = list(input_dict.keys())[0] not in source_patterns
 
         param_data = {}
+        layer_name = ".".join(full_layer_name.split(".")[:-1])
         if is_unsafe_serialization:
             if isinstance(input_dict["weight"], list):
                 weight = input_dict["weight"][0]
             else:
                 weight = input_dict["weight"]
         else:
-            if isinstance(input_dict["weight:qdata"], list):
-                param_data[f"{full_layer_name}:qdata"] = input_dict["weight:qdata"][0]
-            else:
-                param_data[f"{full_layer_name}:qdata"] = input_dict["weight:qdata"]
-
-            if isinstance(input_dict["weight:scale"], list):
-                param_data[f"{full_layer_name}:scale"] = input_dict["weight:scale"][0]
-            else:
-                param_data[f"{full_layer_name}:scale"] = input_dict["weight:scale"]
-
-            if is_int_4:
-                if isinstance(input_dict["weight:zero_point"], list):
-                    param_data[f"{full_layer_name}:zero_point"] = input_dict["weight:zero_point"][0]
-                else:
-                    param_data[f"{full_layer_name}:zero_point"] = input_dict["weight:zero_point"]
+            for suffix in input_dict.keys():
+                if len(input_dict[suffix]) != 1:
+                    raise ValueError(
+                        f"Expected a single tensor for {suffix} but got {len(input_dict[suffix])} tensors instead"
+                    )
+                param_data[f"{layer_name}.{suffix}"] = input_dict[suffix][0]
 
-        # If it's a bias, no need to do anything special (except removing the ":_data" part of the key, but was
-        # already done) - if it's unsafe-serialized (i.e. not safetensors), not need for anything either
+        # If it's unsafe-serialized (i.e. not safetensors), no need for anything
         if is_unsafe_serialization:
             return {full_layer_name: weight}
         # Sanity check for the new serialization format
-        elif not (TORCHAO_VERSION >= version.parse("0.14.0") and is_metadata_torchao(self.hf_quantizer.metadata)):
-            # print("metadata", self.hf_quantizer.metadata)
-            raise ValueError("To use `safetensors` serialization, you should have `torchao>=0.14.0` installed")
+        elif not (TORCHAO_VERSION >= version.parse("0.15.0") and is_metadata_torchao(self.hf_quantizer.metadata)):
+            raise ValueError("To use `safetensors` serialization, you should have `torchao>=0.15.0` installed")
 
-        new_param = unflatten_tensor_state_dict(param_data, self.hf_quantizer.metadata)[full_layer_name]
+        unflattened_state_dict, leftover_state_dict = unflatten_tensor_state_dict(
+            param_data, self.hf_quantizer.metadata
+        )
+        assert not leftover_state_dict  # there should be no unprocessed tensors
+        new_param = unflattened_state_dict[full_layer_name]
 
         module, _ = get_module_from_name(model, full_layer_name)
         # Add repr to the module

transformers/integrations/vptq.py

@@ -13,64 +13,47 @@
 # limitations under the License.
 "VPTQ (Vector Post-Training Quantization) integration file"
 
-import torch.nn as nn
-from accelerate import init_empty_weights
-from vptq import VQuantLinear
+from ..quantizers.quantizers_utils import should_convert_module
+from ..utils import is_torch_available, logging
 
 
-def replace_with_vptq_linear(
-    model,
-    quantization_config=None,
-    modules_to_not_convert=None,
-    current_key_name=None,
-    has_been_replaced=False,
-):
+if is_torch_available():
+    import torch
+    import torch.nn as nn
+
+logger = logging.get_logger(__name__)
+
+
+def replace_with_vptq_linear(model, modules_to_not_convert: list[str] | None = None, quantization_config=None):
     """
-    Public method that recursively replaces the Linear layers of the given model with VPTQ quantized layers.
-    `accelerate` is needed to use this method. Returns the converted model and a boolean that indicates if the
-    conversion has been successful or not.
+    Public method that replaces the Linear layers of the given model with SPQR quantized layers.
 
     Args:
         model (`torch.nn.Module`):
             The model to convert, can be any `torch.nn.Module` instance.
+        modules_to_not_convert (`list[str]`, *optional*, defaults to `None`):
+            A list of nn.Linear weights to not convert. If a parameter path is in the list (e.g. `lm_head.weight`), the corresponding module will not be
+            converted.
         quantization_config (`VptqConfig`):
             The quantization config object that contains the quantization parameters.
-        modules_to_not_convert (`list[`str`]`, *optional*, defaults to `["lm_head"]`):
-            Names of the modules to not convert in `VQuantLinear`. In practice we keep the `lm_head` in full precision
-            for numerical stability reasons.
-        current_key_name (`list`, *optional*):
-            A list that contains the current key name. This is used for recursion and should not be passed by the user.
-        has_been_replaced (`bool`, *optional*):
-            A boolean that indicates if the conversion has been successful or not. This is used for recursion and
-            should not be passed by the user.
     """
+    from vptq import VQuantLinear
 
-    modules_to_not_convert = modules_to_not_convert if modules_to_not_convert else ["lm_head"]
-
-    for name, module in model.named_children():
-        if current_key_name is None:
-            current_key_name = []
-        current_key_name.append(name)
-        layer_name = ".".join(current_key_name)
-        shared_layer_config = quantization_config.shared_layer_config
-        config_for_layers = quantization_config.config_for_layers
-
-        if (
-            isinstance(module, nn.Linear)
-            and layer_name not in modules_to_not_convert
-            and ((layer_name in config_for_layers) or (current_key_name[-1] in shared_layer_config))
-        ):
-            layer_params = config_for_layers.get(layer_name, None) or shared_layer_config.get(
-                current_key_name[-1], None
-            )
+    has_been_replaced = False
+    shared_layer_config = quantization_config.shared_layer_config
+    config_for_layers = quantization_config.config_for_layers
 
-            with init_empty_weights():
-                in_features = module.in_features
-                out_features = module.out_features
-
-                model._modules[name] = VQuantLinear(
-                    in_features,
-                    out_features,
+    for module_name, module in model.named_modules():
+        if not should_convert_module(module_name, modules_to_not_convert):
+            continue
+        with torch.device("meta"):
+            if isinstance(module, nn.Linear):
+                layer_params = config_for_layers.get(module_name, None) or shared_layer_config.get(
+                    module_name.rsplit(".")[1], None
+                )
+                new_module = VQuantLinear(
+                    module.in_features,
+                    module.out_features,
                     vector_lens=layer_params["vector_lens"],
                     num_centroids=layer_params["num_centroids"],
                     num_res_centroids=layer_params["num_res_centroids"],
@@ -84,18 +67,16 @@ def replace_with_vptq_linear(
                     enable_proxy_error=False,
                     bias=module.bias is not None,
                 )
+                # Force requires grad to False to avoid unexpected errors
+                model._modules[module_name].requires_grad_(False)
+                model.set_submodule(module_name, new_module)
                 has_been_replaced = True
 
-                # Force requires grad to False to avoid unexpected errors
-                model._modules[name].requires_grad_(False)
-        if len(list(module.children())) > 0:
-            _, has_been_replaced = replace_with_vptq_linear(
-                module,
-                quantization_config=quantization_config,
-                modules_to_not_convert=modules_to_not_convert,
-                current_key_name=current_key_name,
-                has_been_replaced=has_been_replaced,
-            )
-        # Remove the last key for recursion
-        current_key_name.pop(-1)
-    return model, has_been_replaced
+    if not has_been_replaced:
+        logger.warning(
+            "You are loading your model using eetq but no linear modules were found in your model."
+            " Please double check your model architecture, or submit an issue on github if you think this is"
+            " a bug."
+        )
+
+    return model

transformers/modelcard.py

@@ -23,7 +23,7 @@ from typing import Any, Optional, Union
 
 import httpx
 import yaml
-from huggingface_hub import model_info
+from huggingface_hub import is_offline_mode, model_info
 from huggingface_hub.errors import OfflineModeIsEnabled
 from huggingface_hub.utils import HFValidationError
 
@@ -50,7 +50,6 @@ from .utils import (
     MODEL_CARD_NAME,
     cached_file,
     is_datasets_available,
-    is_offline_mode,
     is_tokenizers_available,
     is_torch_available,
     logging,

transformers/modeling_gguf_pytorch_utils.py

@@ -20,6 +20,7 @@ import numpy as np
 from tqdm.auto import tqdm
 
 from .integrations import (
+    GGUF_CONFIG_DEFAULTS_MAPPING,
     GGUF_CONFIG_MAPPING,
     GGUF_TOKENIZER_MAPPING,
     _gguf_parse_value,
@@ -62,6 +63,24 @@ class TensorProcessor:
     def __init__(self, config=None):
         self.config = config or {}
 
+    def preprocess_name(self, hf_name: str) -> str:
+        """
+        Preprocesses the tensor name to ease loading the GGUF tensors.
+        """
+        return hf_name
+
+    def perform_fallback_tensor_mapping(
+        self, gguf_to_hf_name_map: dict[str, str], suffix: str, qual_name: str, hf_name: str
+    ):
+        """
+        Called when get_gguf_hf_weights_map fails to map a HF parameter
+        (tensor) and corresponding GGUF one.
+
+        This is particularly useful to resolve one-to-many
+        HF-GGUF mappings sometimes appear in some MoE models.
+        """
+        pass
+
     def process(self, weights, name, **kwargs):
         return GGUFTensor(weights, name, {})
 
@@ -97,15 +116,31 @@ class LlamaTensorProcessor(TensorProcessor):
 
 
 class Qwen2MoeTensorProcessor(TensorProcessor):
+    HF_EXPERT_RENAME_PATTERN = re.compile(r"mlp.experts.\d+.")
+    HF_MOE_W13_PATTERN = re.compile(r"model\.layers\.(?P<bid>\d+)\.mlp\.experts\.gate_up_proj")
+    GGUF_MOE_WEIGHTS_PATTERN = re.compile(r"(?P<name>.*\.ffn_(?P<w>gate|down|up)_exps)\.weight$")
+
     def __init__(self, config=None):
         super().__init__(config=config)
 
-    def process(self, weights, name, **kwargs):
-        if "_exp" in name:
+    def preprocess_name(self, hf_name: str) -> str:
+        return re.sub(self.HF_EXPERT_RENAME_PATTERN, "mlp.experts.", hf_name)
+
+    def perform_fallback_tensor_mapping(
+        self, gguf_to_hf_name_map: dict[str, str], suffix: str, qual_name: str, hf_name: str
+    ):
+        # Map merged MoE weights (w1 (gate) and w3 (up)) separately.
+        if m := re.fullmatch(self.HF_MOE_W13_PATTERN, hf_name):
+            full_hf_name = qual_name + hf_name
+            gguf_to_hf_name_map[f"blk.{m['bid']}.ffn_gate_exps{suffix}"] = full_hf_name
+            gguf_to_hf_name_map[f"blk.{m['bid']}.ffn_up_exps{suffix}"] = full_hf_name
+
+    def process(self, weights, name: str, **kwargs):
+        if m := re.fullmatch(self.GGUF_MOE_WEIGHTS_PATTERN, name):
             tensor_key_mapping = kwargs.get("tensor_key_mapping")
             parsed_parameters = kwargs.get("parsed_parameters")
             if tensor_key_mapping:
-                self._split_moe_expert_tensor(weights, parsed_parameters, name, tensor_key_mapping)
+                self._set_moe_expert_tensor(weights, parsed_parameters, tensor_key_mapping[m["name"]], m["w"])
                 return GGUFTensor(weights, None, {})
         if "ffn_gate_inp_shexp" in name:
             # for compatibility tensor shared_expert_gate must be (1, 2048) dim,
@@ -113,17 +148,27 @@ class Qwen2MoeTensorProcessor(TensorProcessor):
             weights = np.expand_dims(weights, axis=0)
         return GGUFTensor(weights, name, {})
 
-    def _split_moe_expert_tensor(
-        self, weights: np.ndarray, parsed_parameters: dict[str, dict], name: str, tensor_key_mapping: dict
-    ):
-        # Original merge implementation
-        # https://github.com/ggerganov/llama.cpp/blob/master/convert_hf_to_gguf.py#L1994-L2022
-        name = tensor_key_mapping[name]
-        w_counter = self.config.get("num_experts", 60)
-        for i in range(0, w_counter):
-            temp_name = name.replace("mlp.experts.", f"mlp.experts.{i}.")
-            exp_weight = weights[i]
-            parsed_parameters["tensors"][temp_name] = torch.from_numpy(np.copy(exp_weight))
+    def _set_moe_expert_tensor(self, weights: np.ndarray, parsed_parameters: dict[str, dict], hf_name: str, w: str):
+        torch_weights = torch.from_numpy(np.copy(weights))
+        if w == "down":
+            parsed_parameters["tensors"][hf_name] = torch_weights
+        else:
+            # Double the size of the second dimension to interleave w1 (gate) and w3 (up)
+            # weights per expert (which is the first dimension).
+            # w1 (gate) comes first and w3 (up) comes second.
+            # ref: https://github.com/vllm-project/vllm/blob/8f8fda261a620234fdeea338f44093d5d8072879/vllm/model_executor/layers/fused_moe/layer.py#L988-L1015
+            shape = list(weights.shape)
+            shard_dim = 1
+            shard_size = shape[shard_dim]
+            shape[shard_dim] = shard_size * 2
+            if hf_name not in parsed_parameters["tensors"]:
+                parsed_parameters["tensors"][hf_name] = torch.zeros(shape, dtype=torch_weights.dtype)
+            out: torch.Tensor = parsed_parameters["tensors"][hf_name]
+            if w == "gate":
+                out = out.narrow(shard_dim, 0, shard_size)
+            else:  # w == "up"
+                out = out.narrow(shard_dim, shard_size, shard_size)
+            out.copy_(torch_weights)
 
 
 class BloomTensorProcessor(TensorProcessor):
@@ -280,6 +325,7 @@ def read_field(reader, field):
 # modified from https://github.com/vllm-project/vllm/blob/v0.6.4.post1/vllm/model_executor/model_loader/loader.py#L1115-L1147
 def get_gguf_hf_weights_map(
     hf_model,
+    processor: TensorProcessor,
     model_type: Optional[str] = None,
     num_layers: Optional[int] = None,
     qual_name: str = "",
@@ -333,9 +379,7 @@ def get_gguf_hf_weights_map(
     gguf_to_hf_name_map = {}
     state_dict = hf_model.state_dict()
     for hf_name in state_dict:
-        # An exception for qwen2moe/qwen3moe model, where the expert layers are packed
-        if model_type in ("qwen2moe", "qwen3moe") and "mlp.experts." in hf_name:
-            hf_name = re.sub(r"mlp.experts.\d+.", "mlp.experts.", hf_name)
+        hf_name = processor.preprocess_name(hf_name)
 
         name, suffix = hf_name, ""
         if hf_name.endswith(".weight") or hf_name.endswith(".bias"):
@@ -344,6 +388,7 @@ def get_gguf_hf_weights_map(
 
         gguf_name = name_map.get_name(name)
         if gguf_name is None:
+            processor.perform_fallback_tensor_mapping(gguf_to_hf_name_map, suffix, qual_name, hf_name)
             continue
 
         gguf_to_hf_name_map[gguf_name + suffix] = qual_name + hf_name
@@ -352,7 +397,9 @@ def get_gguf_hf_weights_map(
     # Therefore, we need to check submodule as well to get a correct mapping
     if named_children := hf_model.named_children():
         for name, child in named_children:
-            sub_map = get_gguf_hf_weights_map(child, model_type, num_layers, qual_name=f"{qual_name}{name}.")
+            sub_map = get_gguf_hf_weights_map(
+                child, processor, model_type, num_layers, qual_name=f"{qual_name}{name}."
+            )
             # Ignore the keys that are already in the main map to avoid overwriting
             sub_map = {k: v for k, v in sub_map.items() if k not in gguf_to_hf_name_map}
             gguf_to_hf_name_map.update(sub_map)
@@ -437,6 +484,13 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo
         all("output.weight" != tensor.name for tensor in reader.tensors) or architecture in exceptions
     )
 
+    # Set GGUF-specific default values
+    config_defaults = GGUF_CONFIG_DEFAULTS_MAPPING.get(
+        updated_architecture, GGUF_CONFIG_DEFAULTS_MAPPING.get(architecture) or {}
+    )
+    for key, value in config_defaults.items():
+        parsed_parameters["config"].setdefault(key, value)
+
     # List all key-value pairs in a columnized format
     for gguf_key, field in reader.fields.items():
         gguf_key = gguf_key.replace(architecture, updated_architecture)
@@ -499,12 +553,13 @@ def load_gguf_checkpoint(gguf_checkpoint_path, return_tensors=False, model_to_lo
     if return_tensors:
         parsed_parameters["tensors"] = {}
 
-        tensor_key_mapping = get_gguf_hf_weights_map(model_to_load)
         config = parsed_parameters.get("config", {})
 
         ProcessorClass = TENSOR_PROCESSORS.get(architecture, TensorProcessor)
         processor = ProcessorClass(config=config)
 
+        tensor_key_mapping = get_gguf_hf_weights_map(model_to_load, processor)
+
         for tensor in tqdm(reader.tensors, desc="Converting and de-quantizing GGUF tensors..."):
             name = tensor.name
             weights = dequantize(tensor.data, tensor.tensor_type)