transformers 5.0.0rc3__py3-none-any.whl → 5.1.0__py3-none-any.whl

transformers/conversion_mapping.py

@@ -21,6 +21,7 @@ from .core_model_loading import (
     Chunk,
     Concatenate,
     ErnieFuseAndSplitTextVisionExperts,
+    Force16BytesAlignment,
     MergeModulelist,
     Transpose,
     WeightConverter,
@@ -38,16 +39,72 @@ if TYPE_CHECKING:
     from .quantizers import HfQuantizer
 
 
+_MODEL_TO_CONVERSION_PATTERN = {
+    # Mixtral-style MoE
+    "mixtral": "mixtral",
+    "minimax": "mixtral",
+    "minimax_m2": "mixtral",
+    # Qwen2-style MoE
+    "qwen2_moe": "qwen2_moe",
+    "deepseek_v2": "qwen2_moe",
+    "deepseek_v3": "qwen2_moe",
+    "dots1": "qwen2_moe",
+    "ernie4_5_moe": "qwen2_moe",
+    "glm4_moe": "qwen2_moe",
+    "glm4_moe_lite": "qwen2_moe",
+    "glm4v_moe": "qwen2_moe",
+    "longcat_flash": "qwen2_moe",
+    "solar_open": "qwen2_moe",
+    "qwen3_moe": "qwen2_moe",
+    "qwen3_omni_moe": "qwen2_moe",
+    "qwen3_omni_moe_thinker": "qwen2_moe",
+    "qwen3_next": "qwen2_moe",
+    "hunyuan_v1_moe": "qwen2_moe",
+    "flex_olmo": "qwen2_moe",
+    "olmoe": "qwen2_moe",
+    "exaone_moe": "qwen2_moe",
+    "rt_detr_v2": "rt_detr",
+    "pp_doclayout_v3": "rt_detr",
+}
+
+
 def _build_checkpoint_conversion_mapping():
     mapping = {
+        "t5gemma2": [
+            WeightRenaming(r"(?<!vision_model\.)encoder.embed_tokens.", "encoder.text_model.embed_tokens."),
+            WeightRenaming(r"(?<!vision_model\.)encoder.norm.", "encoder.text_model.norm."),
+            WeightRenaming(r"(?<!vision_model\.)encoder.layers.", "encoder.text_model.layers."),
+        ],
+        "t5gemma2_encoder": [
+            WeightRenaming("^embed_tokens.", "text_model.embed_tokens."),
+            WeightRenaming("^norm.", "text_model.norm."),
+            WeightRenaming("^layers.", "text_model.layers."),
+        ],
+        "gpt_oss": [
+            # NOTE: These converters are only applied if the model is being loaded from pre-dequantized checkpoint.
+            # If you are dequantizing the model on the fly, these converters will be ignored because the tensors
+            # that match these patterns are only created after dequantization.
+            # That's not an issue for now since the dequantization converters already ensure 16 bytes alignment
+            # by enforcing contiguity.
+            WeightConverter(
+                source_patterns="mlp.experts.gate_up_proj$",
+                target_patterns="mlp.experts.gate_up_proj",
+                operations=[Force16BytesAlignment()],
+            ),
+            WeightConverter(
+                source_patterns="mlp.experts.down_proj$",
+                target_patterns="mlp.experts.down_proj",
+                operations=[Force16BytesAlignment()],
+            ),
+        ],
         "mixtral": [
-            WeightRenaming(".block_sparse_moe.gate", ".mlp.gate"),
+            WeightRenaming(".block_sparse_moe.", ".mlp."),
             WeightConverter(
                 source_patterns=[
-                    "block_sparse_moe.experts.*.w1.weight",
-                    "block_sparse_moe.experts.*.w3.weight",
+                    ".experts.*.w1.weight",
+                    ".experts.*.w3.weight",
                 ],  # you give me a list of 2 keys, I collect a list of a list of tensors
-                target_patterns="mlp.experts.gate_up_proj",  # target key gets the list of two tensors
+                target_patterns=".experts.gate_up_proj",  # target key gets the list of two tensors
                 operations=[
                     MergeModulelist(
                         dim=0
@@ -57,9 +114,9 @@ def _build_checkpoint_conversion_mapping():
             ),
             WeightConverter(
                 source_patterns=[
-                    "block_sparse_moe.experts.*.w2.weight",
+                    ".experts.*.w2.weight",
                 ],
-                target_patterns="mlp.experts.down_proj",  # target key gets the list of two tensors
+                target_patterns=".experts.down_proj",  # target key gets the list of two tensors
                 operations=[
                     MergeModulelist(
                         dim=0
@@ -82,18 +139,35 @@ def _build_checkpoint_conversion_mapping():
                 operations=[MergeModulelist(dim=0)],
             ),
         ],
-        "phimoe": [
+        "qwen3_vl_moe": [
             WeightConverter(
                 source_patterns=[
-                    "mlp.experts.*.w1.weight",
-                    "mlp.experts.*.w3.weight",
+                    "mlp.experts.*.gate_proj.weight",
+                    "mlp.experts.*.up_proj.weight",
                 ],
                 target_patterns="mlp.experts.gate_up_proj",
-                operations=[MergeModulelist(dim=0), Concatenate(dim=1)],
+                operations=[MergeModulelist(dim=0), Concatenate(dim=1), Transpose(1, 2)],
             ),
             WeightConverter(
-                source_patterns="mlp.experts.*.w2.weight",
+                source_patterns="mlp.experts.*.down_proj.weight",
                 target_patterns="mlp.experts.down_proj",
+                operations=[MergeModulelist(dim=0), Transpose(1, 2)],
+            ),
+        ],
+        "phimoe": [
+            WeightRenaming(".block_sparse_moe.", ".mlp."),
+            WeightRenaming(".gate.weight", ".router.weight"),
+            WeightConverter(
+                source_patterns=[
+                    ".experts.*.w1.weight",
+                    ".experts.*.w3.weight",
+                ],
+                target_patterns=".experts.gate_up_proj",
+                operations=[MergeModulelist(dim=0), Concatenate(dim=1)],
+            ),
+            WeightConverter(
+                source_patterns=".experts.*.w2.weight",
+                target_patterns=".experts.down_proj",
                 operations=[MergeModulelist(dim=0)],
             ),
         ],
@@ -163,6 +237,52 @@ def _build_checkpoint_conversion_mapping():
                 operations=[ErnieFuseAndSplitTextVisionExperts(stack_dim=0, concat_dim=1)],
             ),
         ],
+        "detr": [
+            WeightRenaming("backbone.conv_encoder", "backbone"),
+            WeightRenaming("out_proj", "o_proj"),
+            WeightRenaming(r"layers.(\d+).fc1", r"layers.\1.mlp.fc1"),
+            WeightRenaming(r"layers.(\d+).fc2", r"layers.\1.mlp.fc2"),
+        ],
+        "rt_detr": [
+            WeightRenaming("out_proj", "o_proj"),
+            WeightRenaming(r"layers.(\d+).fc1", r"layers.\1.mlp.fc1"),
+            WeightRenaming(r"layers.(\d+).fc2", r"layers.\1.mlp.fc2"),
+            WeightRenaming(r"encoder.encoder.(\d+).layers", r"encoder.aifi.\1.layers"),
+        ],
+        "conditional_detr": [
+            WeightRenaming("backbone.conv_encoder", "backbone"),
+            WeightRenaming("self_attn.out_proj", "self_attn.o_proj"),
+            WeightRenaming("encoder_attn.out_proj", "encoder_attn.o_proj"),
+            WeightRenaming(r"layers.(\d+).fc1", r"layers.\1.mlp.fc1"),
+            WeightRenaming(r"layers.(\d+).fc2", r"layers.\1.mlp.fc2"),
+            # Decoder self-attention projections moved into self_attn module
+            WeightRenaming(r"decoder.layers.(\d+).sa_qcontent_proj", r"decoder.layers.\1.self_attn.q_content_proj"),
+            WeightRenaming(r"decoder.layers.(\d+).sa_qpos_proj", r"decoder.layers.\1.self_attn.q_pos_proj"),
+            WeightRenaming(r"decoder.layers.(\d+).sa_kcontent_proj", r"decoder.layers.\1.self_attn.k_content_proj"),
+            WeightRenaming(r"decoder.layers.(\d+).sa_kpos_proj", r"decoder.layers.\1.self_attn.k_pos_proj"),
+            WeightRenaming(r"decoder.layers.(\d+).sa_v_proj", r"decoder.layers.\1.self_attn.v_proj"),
+            # Decoder cross-attention projections moved into encoder_attn module
+            WeightRenaming(r"decoder.layers.(\d+).ca_qcontent_proj", r"decoder.layers.\1.encoder_attn.q_content_proj"),
+            WeightRenaming(r"decoder.layers.(\d+).ca_qpos_proj", r"decoder.layers.\1.encoder_attn.q_pos_proj"),
+            WeightRenaming(r"decoder.layers.(\d+).ca_kcontent_proj", r"decoder.layers.\1.encoder_attn.k_content_proj"),
+            WeightRenaming(r"decoder.layers.(\d+).ca_kpos_proj", r"decoder.layers.\1.encoder_attn.k_pos_proj"),
+            WeightRenaming(r"decoder.layers.(\d+).ca_v_proj", r"decoder.layers.\1.encoder_attn.v_proj"),
+            WeightRenaming(
+                r"decoder.layers.(\d+).ca_qpos_sine_proj", r"decoder.layers.\1.encoder_attn.q_pos_sine_proj"
+            ),
+        ],
+        "deformable_detr": [
+            WeightRenaming("backbone.conv_encoder", "backbone"),
+            WeightRenaming("self_attn.out_proj", "self_attn.o_proj"),
+            WeightRenaming(r"layers.(\d+).fc1", r"layers.\1.mlp.fc1"),
+            WeightRenaming(r"layers.(\d+).fc2", r"layers.\1.mlp.fc2"),
+        ],
+        "d_fine": [
+            WeightRenaming("out_proj", "o_proj"),
+            WeightRenaming(r"layers.(\d+).fc1", r"layers.\1.mlp.layers.0"),
+            WeightRenaming(r"layers.(\d+).fc2", r"layers.\1.mlp.layers.1"),
+            WeightRenaming(r"encoder.encoder.(\d+).layers", r"encoder.aifi.\1.layers"),
+        ],
         "jamba": [
             WeightConverter(
                 source_patterns=[
@@ -220,29 +340,21 @@ def _build_checkpoint_conversion_mapping():
             ),
         ]
 
-    mapping["deepseek_v2"] = mapping["qwen2_moe"].copy()
-    mapping["deepseek_v3"] = mapping["qwen2_moe"].copy()
-    mapping["dots1"] = mapping["qwen2_moe"].copy()
     mapping["ernie4_5_moe"] = mapping["qwen2_moe"].copy()
     mapping["ernie4_5_moe"] += [
         WeightRenaming("mlp.moe_statics.e_score_correction_bias", "mlp.gate.moe_statics.e_score_correction_bias")
     ]
-    mapping["glm4_moe"] = mapping["qwen2_moe"].copy()
-    mapping["glm4_moe_lite"] = mapping["qwen2_moe"].copy()
-    mapping["glm4v_moe"] = mapping["qwen2_moe"].copy()
-    mapping["longcat_flash"] = mapping["qwen2_moe"].copy()
-    mapping["qwen3_moe"] = mapping["qwen2_moe"].copy()
-    mapping["qwen3_omni_moe"] = mapping["qwen2_moe"].copy()
-    mapping["qwen3_next"] = mapping["qwen2_moe"].copy()
-    mapping["qwen3_vl_moe"] = mapping["qwen2_moe"].copy()
-    mapping["hunyuan_v1_moe"] = mapping["qwen2_moe"].copy()
-    mapping["minimax"] = mapping["mixtral"].copy()
     mapping["minimax_m2"] = mapping["mixtral"].copy()
     mapping["minimax_m2"] += [
         WeightRenaming(".block_sparse_moe.e_score_correction_bias", ".mlp.e_score_correction_bias"),
     ]
-    mapping["flex_olmo"] = mapping["qwen2_moe"].copy()
-    mapping["olmoe"] = mapping["qwen2_moe"].copy()
+    mapping["exaone_moe"] = mapping["qwen2_moe"].copy()
+    mapping["exaone_moe"] += [WeightRenaming("mlp.e_score_correction_bias", "mlp.gate.e_score_correction_bias")]
+
+    for model_type, base_pattern in _MODEL_TO_CONVERSION_PATTERN.items():
+        if model_type in mapping:
+            continue
+        mapping[model_type] = mapping[base_pattern].copy()
 
     return mapping
 
@@ -293,6 +405,7 @@ VLMS = [
     "sam3_tracker_video",
     "paddleocrvl",
     "ernie4_5_vl_moe",
+    "detr",
 ]
 
 
@@ -333,6 +446,13 @@ def get_model_conversion_mapping(
 
     # Add the ones from the quantizer as well if provided
     if hf_quantizer is not None:
+        # NOTE: Since get_weight_conversions() only serve to dequantize, we would normally want to apply them first.
+        # However, for now it's not possible to cascade converters (i.e., applying model-specific conversions on top
+        # of tensors created by the dequantization conversions)
+        # This means that if a model has model-specific conversions and is being dequantized, the model-specific conversion
+        # that relies on tensors created by dequantization conversions will not be applied.
+        # GptOss example: with Mxfp4Config(dequantize=True), Force16BytesAlignment converters are ignored because the tensors
+        # "mlp.experts.gate_up_proj$" and "mlp.experts.down_proj$" are only created after dequantization conversions are applied.
         weight_conversions.extend(hf_quantizer.get_weight_conversions())
 
     return weight_conversions

transformers/convert_slow_tokenizer.py

@@ -1892,9 +1892,10 @@ class TikTokenConverter:
         )
         tokenizer.decoder = decoders.ByteLevel()
 
-        tokenizer.add_special_tokens(
-            [AddedToken(token, normalized=False, special=True) for token in self.extra_special_tokens]
-        )
+        if self.extra_special_tokens is not None:
+            tokenizer.add_special_tokens(
+                [AddedToken(token, normalized=False, special=True) for token in self.extra_special_tokens]
+            )
 
         tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
 
@@ -1942,6 +1943,7 @@ class MistralConverter:
             vocab[token.content] = idx
         bpe_ranks = [base64.b64decode(k["token_bytes"]) for k in bpe_ranks]
         rank_set = set(bpe_ranks)
+        token_to_rank = {token: rank for rank, token in enumerate(bpe_ranks)}
         for rank, token in enumerate(tqdm(bpe_ranks, desc="Converting tekken.json to tokenizer.json")):
             vocab[token_bytes_to_string(token)] = rank
             if len(token) == 1:
@@ -1951,7 +1953,7 @@ class MistralConverter:
                 piece_l, piece_r = token[:index], token[index:]
                 if piece_l in rank_set and piece_r in rank_set and (piece_l + piece_r) in rank_set:
                     local.append((piece_l, piece_r, rank))
-            local = sorted(local, key=lambda x: (bpe_ranks.index(x[0]), bpe_ranks.index(x[1])), reverse=False)
+            local = sorted(local, key=lambda x: (token_to_rank[x[0]], token_to_rank[x[1]]), reverse=False)
             merges.extend(local)
         merges = sorted(merges, key=lambda val: val[2], reverse=False)
         merges = [(token_bytes_to_string(val[0]), token_bytes_to_string(val[1])) for val in merges]