Defuser 0.0.25__tar.gz → 0.0.27__tar.gz

{defuser-0.0.25 → defuser-0.0.27}/Defuser.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Defuser
-Version: 0.0.25
+Version: 0.0.27
 Summary: Model defuser helper for HF Transformers.
 Author-email: ModelCloud <qubitium@modelcloud.ai>
 License-Expression: Apache-2.0
@@ -93,7 +93,7 @@ Defuser currently supports the following `transformers>=5.3.0` `model_type` valu
 | Pattern | Supported model types                                                                                                                                                                                                               | Defused op performed ⚙️ |
 | --- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --- |
 | Standard routed expert tensors 🧱 | `deepseek_v2`, `dots1`, `ernie4_5_moe`, `ernie4_5_vl_moe`, `exaone_moe`, `flex_olmo`, `glm4_moe_lite`, `glm4v_moe`, `hunyuan_v1_moe`, `jamba`, `laguna`, `lfm2_moe`, `minimax`, `minimax_m2`, `olmoe`, `qwen3_vl_moe`, `solar_open`, `solar_open2` | Splits fused expert tensors or registered expert buffers into numbered expert `nn.Linear` modules with per-expert `gate_proj`, `up_proj`, and `down_proj`. |
-| Mixed sparse and shared experts | `deepseek_v3`, deepseek_v4`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text`                                                                                                                                                       | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. |
+| Mixed sparse and shared experts | `deepseek_v3`, `deepseek_v32`, `deepseek_v4`, `glm5_next`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text`, `qwen4_exp`, `qwen4_exp_text`                                                                                                              | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. |
 | Transposed or packed expert tensors | `gpt_oss`, `phimoe`                                                                                                                                                                                                                 | Splits transposed fused expert `gate_up_proj` tensors into per-expert `gate_proj` + `up_proj`, preserves expert bias when present, and converts expert tensors into numbered expert `nn.Linear` modules. |
 | Flattened expert layout | `dbrx`                                                                                                                                                                                                                              | Rebuilds the flattened DBRX expert FFN weights into numbered expert `gate_proj`, `up_proj`, and `down_proj` `nn.Linear` modules. |
 | Batched expert-input execution | `llama4`                                                                                                                                                                                                                            | Runtime expert tensor defusion plus preservation of the llama4 batched expert-input execution contract. |

{defuser-0.0.25 → defuser-0.0.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Defuser
-Version: 0.0.25
+Version: 0.0.27
 Summary: Model defuser helper for HF Transformers.
 Author-email: ModelCloud <qubitium@modelcloud.ai>
 License-Expression: Apache-2.0
@@ -93,7 +93,7 @@ Defuser currently supports the following `transformers>=5.3.0` `model_type` valu
 | Pattern | Supported model types                                                                                                                                                                                                               | Defused op performed ⚙️ |
 | --- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --- |
 | Standard routed expert tensors 🧱 | `deepseek_v2`, `dots1`, `ernie4_5_moe`, `ernie4_5_vl_moe`, `exaone_moe`, `flex_olmo`, `glm4_moe_lite`, `glm4v_moe`, `hunyuan_v1_moe`, `jamba`, `laguna`, `lfm2_moe`, `minimax`, `minimax_m2`, `olmoe`, `qwen3_vl_moe`, `solar_open`, `solar_open2` | Splits fused expert tensors or registered expert buffers into numbered expert `nn.Linear` modules with per-expert `gate_proj`, `up_proj`, and `down_proj`. |
-| Mixed sparse and shared experts | `deepseek_v3`, deepseek_v4`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text`                                                                                                                                                       | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. |
+| Mixed sparse and shared experts | `deepseek_v3`, `deepseek_v32`, `deepseek_v4`, `glm5_next`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text`, `qwen4_exp`, `qwen4_exp_text`                                                                                                              | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. |
 | Transposed or packed expert tensors | `gpt_oss`, `phimoe`                                                                                                                                                                                                                 | Splits transposed fused expert `gate_up_proj` tensors into per-expert `gate_proj` + `up_proj`, preserves expert bias when present, and converts expert tensors into numbered expert `nn.Linear` modules. |
 | Flattened expert layout | `dbrx`                                                                                                                                                                                                                              | Rebuilds the flattened DBRX expert FFN weights into numbered expert `gate_proj`, `up_proj`, and `down_proj` `nn.Linear` modules. |
 | Batched expert-input execution | `llama4`                                                                                                                                                                                                                            | Runtime expert tensor defusion plus preservation of the llama4 batched expert-input execution contract. |

{defuser-0.0.25 → defuser-0.0.27}/README.md

@@ -67,7 +67,7 @@ Defuser currently supports the following `transformers>=5.3.0` `model_type` valu
 | Pattern | Supported model types                                                                                                                                                                                                               | Defused op performed ⚙️ |
 | --- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --- |
 | Standard routed expert tensors 🧱 | `deepseek_v2`, `dots1`, `ernie4_5_moe`, `ernie4_5_vl_moe`, `exaone_moe`, `flex_olmo`, `glm4_moe_lite`, `glm4v_moe`, `hunyuan_v1_moe`, `jamba`, `laguna`, `lfm2_moe`, `minimax`, `minimax_m2`, `olmoe`, `qwen3_vl_moe`, `solar_open`, `solar_open2` | Splits fused expert tensors or registered expert buffers into numbered expert `nn.Linear` modules with per-expert `gate_proj`, `up_proj`, and `down_proj`. |
-| Mixed sparse and shared experts | `deepseek_v3`, deepseek_v4`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text`                                                                                                                                                       | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. |
+| Mixed sparse and shared experts | `deepseek_v3`, `deepseek_v32`, `deepseek_v4`, `glm5_next`, `glm_moe_dsa`, `qwen3_5_moe`, `qwen3_5_moe_text`, `qwen4_exp`, `qwen4_exp_text`                                                                                                              | Runtime expert tensor defusion for routed experts while preserving the model's shared-expert path. |
 | Transposed or packed expert tensors | `gpt_oss`, `phimoe`                                                                                                                                                                                                                 | Splits transposed fused expert `gate_up_proj` tensors into per-expert `gate_proj` + `up_proj`, preserves expert bias when present, and converts expert tensors into numbered expert `nn.Linear` modules. |
 | Flattened expert layout | `dbrx`                                                                                                                                                                                                                              | Rebuilds the flattened DBRX expert FFN weights into numbered expert `gate_proj`, `up_proj`, and `down_proj` `nn.Linear` modules. |
 | Batched expert-input execution | `llama4`                                                                                                                                                                                                                            | Runtime expert tensor defusion plus preservation of the llama4 batched expert-input execution contract. |

{defuser-0.0.25 → defuser-0.0.27}/defuser/model_registry.py

@@ -37,6 +37,9 @@ MODEL_CONFIG = {
     "deepseek_v3": {
         "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION,
     },
+    "deepseek_v32": {
+        "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION,
+    },
     "deepseek_v4": {
         "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION,
     },
@@ -143,6 +146,12 @@ MODEL_CONFIG = {
             ),
         ],
     },
+    "qwen4_exp": {
+        "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION,
+    },
+    "qwen4_exp_text": {
+        "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION,
+    },
     "glm4_moe": {
         "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION,
         PATCH.REPLACE_MODULE: [
@@ -190,6 +199,15 @@ MODEL_CONFIG = {
     "glm_image": {
         "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION,
     },
+    "glm5_next": {
+        "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION,
+        PATCH.EXPERTS_DEFUSE: [
+            {
+                "module_class": "transformers.models.glm5_next.modeling_glm5_next.Glm5NextTextExperts",
+                "forward_impl": "linear_loop",
+            }
+        ],
+    },
     "glm_moe_dsa": {
         "min_transformers_version": MIN_SUPPORTED_TRANSFORMERS_VERSION,
     },

{defuser-0.0.25 → defuser-0.0.27}/defuser/modeling/moe_experts_interface.py

@@ -282,6 +282,10 @@ def _matching_experts_defuse_spec(module: nn.Module, specs: list[dict]) -> dict
 
 def _install_instance_forward(module: nn.Module, implementation: str) -> None:
     """Attach a generic forward implementation directly to one experts module."""
+    if implementation == LINEAR_LOOP_IMPL:
+        module.forward = MethodType(linear_loop_experts_forward, module)
+        return
+
     if implementation == BATCHED_INPUT_IMPL:
         module.forward = MethodType(batched_input_experts_forward, module)
         return

{defuser-0.0.25 → defuser-0.0.27}/pyproject.toml

@@ -4,12 +4,12 @@
 # Contact: qubitium@modelcloud.ai, x.com/qubitium
 
 [build-system]
-requires = ["setuptools>=77.0.1,<84"]
+requires = ["setuptools>=77.0.1,<85"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "Defuser"
-version = "0.0.25"
+version = "0.0.27"
 description = "Model defuser helper for HF Transformers."
 readme = "README.md"
 requires-python = ">=3.9"

{defuser-0.0.25 → defuser-0.0.27}/tests/test_candidate_coverage.py

@@ -161,6 +161,40 @@ STANDARD_MOE_CASES = [
         "route_indices": [[0], [1], [2], [3], [0]],
         "expert_attrs": ("gate_proj", "up_proj", "down_proj"),
     },
+    {
+        "model_type": "deepseek_v32",
+        "module_path": "transformers.models.deepseek_v32.modeling_deepseek_v32",
+        "class_name": "DeepseekV32Experts",
+        "config_module": "transformers.models.deepseek_v32.configuration_deepseek_v32",
+        "config_name": "DeepseekV32Config",
+        "config_updates": {
+            "hidden_size": 64,
+            "moe_intermediate_size": 32,
+            "n_routed_experts": 4,
+            "hidden_act": "silu",
+        },
+        "input_dim": 64,
+        "route_indices": [[0], [1], [2], [3], [0]],
+        "expert_attrs": ("gate_proj", "up_proj", "down_proj"),
+    },
+    {
+        "model_type": "glm5_next",
+        "module_path": "transformers.models.glm5_next.modeling_glm5_next",
+        "class_name": "Glm5NextTextExperts",
+        "config_module": "transformers.models.glm5_next.configuration_glm5_next",
+        "config_name": "Glm5NextTextConfig",
+        "config_updates": {
+            "hidden_size": 64,
+            "moe_intermediate_size": 32,
+            "n_routed_experts": 4,
+            "hidden_act": "silu",
+            "swiglu_limit": 1.0,
+            "_experts_implementation": "grouped_mm",
+        },
+        "input_dim": 64,
+        "route_indices": [[0], [1], [2], [3], [0]],
+        "expert_attrs": ("gate_proj", "up_proj", "down_proj"),
+    },
     {
         "model_type": "dots1",
         "module_path": "transformers.models.dots1.modeling_dots1",
@@ -547,6 +581,7 @@ ALL_CANDIDATE_MODEL_TYPES = {
     "dbrx",
     "deepseek_v2",
     "deepseek_v3",
+    "deepseek_v32",
     "dia",
     "dots1",
     "ernie4_5_moe",
@@ -559,6 +594,7 @@ ALL_CANDIDATE_MODEL_TYPES = {
     "glm4_moe_lite",
     "glm4v",
     "glm4v_moe",
+    "glm5_next",
     "glm_image",
     "glm_moe_dsa",
     "glm_ocr",
@@ -586,6 +622,8 @@ ALL_CANDIDATE_MODEL_TYPES = {
     "qwen3_next",
     "qwen3_omni_moe",
     "qwen3_vl_moe",
+    "qwen4_exp",
+    "qwen4_exp_text",
     "solar_open",
     "solar_open2",
     "zamba2",

{defuser-0.0.25 → defuser-0.0.27}/tests/test_convert_model.py

@@ -38,6 +38,8 @@ from transformers.models.qwen3_next.modeling_qwen3_next import (
     Qwen3NextSparseMoeBlock,
 )
 from transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe import Qwen3OmniMoeConfig
+from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig
+from transformers.models.qwen4_exp.modeling_qwen4_exp import Qwen4ExpForCausalLM
 from transformers.models.gpt_oss.modeling_gpt_oss import GptOssConfig, GptOssForCausalLM
 from transformers.models.phimoe.modeling_phimoe import PhimoeConfig, PhimoeForCausalLM
 from transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4ForConditionalGeneration
@@ -176,6 +178,39 @@ def _tiny_qwen3_5_moe_config():
     )
 
 
+def _tiny_qwen4_exp_text_config():
+    return Qwen4ExpTextConfig(
+        vocab_size=64,
+        hidden_size=16,
+        num_hidden_layers=2,
+        num_attention_heads=4,
+        num_key_value_heads=2,
+        head_dim=4,
+        moe_intermediate_size=8,
+        shared_expert_intermediate_size=8,
+        num_experts=3,
+        num_experts_per_tok=2,
+        max_position_embeddings=32,
+        layer_types=["linear_attention", "qwen_sparse_attention"],
+        linear_num_key_heads=2,
+        linear_num_value_heads=4,
+        linear_key_head_dim=4,
+        linear_value_head_dim=4,
+        linear_conv_kernel_dim=2,
+        hc_count=2,
+        hc_lowrank=4,
+        ple_layer_ids=[],
+        bos_token_id=1,
+        eos_token_id=2,
+        pad_token_id=0,
+        indexer_n_heads=2,
+        indexer_kv_heads=1,
+        indexer_head_dim=4,
+        indexer_budget=8,
+        indexer_compress_ratio=2,
+    )
+
+
 def _tiny_mixtral_config():
     return MixtralConfig(
         vocab_size=128,
@@ -739,6 +774,24 @@ def test_qwen3_5_moe():
     torch.testing.assert_close(expert0.down_proj.weight, expected_down)
 
 
+def test_qwen4_exp_text_preserves_forward():
+    model = Qwen4ExpForCausalLM(_tiny_qwen4_exp_text_config()).eval()
+    input_ids = torch.tensor([[1, 7, 8, 2]])
+
+    with torch.inference_mode():
+        expected = model(input_ids=input_ids, use_cache=False).logits
+
+    assert convert_model(model, cleanup_original=False)
+    assert model.config.model_type == "qwen4_exp_text"
+    experts = model.model.layers[0].mlp.experts
+    assert hasattr(experts, "0")
+    assert not hasattr(experts, "gate_up_proj")
+
+    with torch.inference_mode():
+        actual = model(input_ids=input_ids, use_cache=False).logits
+    torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-7)
+
+
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
 def test_qwen3_5_moe_runtime_defusion_preserves_cuda_device():
     model = Qwen3_5MoeForConditionalGeneration(_tiny_qwen3_5_moe_config()).cuda().eval()

{defuser-0.0.25 → defuser-0.0.27}/tests/test_meta_model_defusion.py

@@ -155,6 +155,8 @@ def _build_model_config(case: dict):
         config.ffn_config.ffn_act_fn = {"name": "silu"}
     elif model_type == "deepseek_v3":
         config.first_k_dense_replace = 0
+    elif model_type == "deepseek_v32":
+        config.mlp_layer_types = ["sparse"] * config.num_hidden_layers
     elif model_type == "ernie4_5_vl_moe":
         config.text_config.moe_intermediate_size = [32, 32]
         config.text_config.rope_parameters = {
@@ -164,6 +166,8 @@ def _build_model_config(case: dict):
         }
     elif model_type == "glm4_moe":
         config.first_k_dense_replace = -1
+    elif model_type == "glm5_next":
+        config.text_config.mlp_layer_types = ["sparse"] * config.text_config.num_hidden_layers
     elif model_type == "glm_moe_dsa":
         config.mlp_layer_types = ["sparse"] * config.num_hidden_layers
     elif model_type == "granitemoehybrid":
@@ -342,6 +346,19 @@ META_MODEL_CASES = [
         "validator": "experts",
         "min_targets": 2,
     },
+    {
+        "model_type": "deepseek_v32",
+        "mode": "convert",
+        "model_module": "transformers.models.deepseek_v32.modeling_deepseek_v32",
+        "model_class": "DeepseekV32ForCausalLM",
+        "config_module": "transformers.models.deepseek_v32.configuration_deepseek_v32",
+        "config_class": "DeepseekV32Config",
+        "target_class_paths": (
+            "transformers.models.deepseek_v32.modeling_deepseek_v32.DeepseekV32Experts",
+        ),
+        "validator": "experts",
+        "min_targets": 2,
+    },
     {
         "model_type": "deepseek_v4",
         "mode": "convert",
@@ -485,6 +502,19 @@ META_MODEL_CASES = [
         "target_class_paths": ("transformers.models.glm_image.modeling_glm_image.GlmImageTextMLP",),
         "validator": "dense_split",
     },
+    {
+        "model_type": "glm5_next",
+        "mode": "convert",
+        "model_module": "transformers.models.glm5_next.modeling_glm5_next",
+        "model_class": "Glm5NextForConditionalGeneration",
+        "config_module": "transformers.models.glm5_next.configuration_glm5_next",
+        "config_class": "Glm5NextConfig",
+        "target_class_paths": (
+            "transformers.models.glm5_next.modeling_glm5_next.Glm5NextTextExperts",
+        ),
+        "validator": "experts",
+        "min_targets": 2,
+    },
     {
         "model_type": "glm_moe_dsa",
         "mode": "convert",
@@ -735,6 +765,28 @@ META_MODEL_CASES = [
         "validator": "experts",
         "min_targets": 2,
     },
+    {
+        "model_type": "qwen4_exp",
+        "mode": "convert",
+        "model_module": "transformers.models.qwen4_exp.modeling_qwen4_exp",
+        "model_class": "Qwen4ExpForConditionalGeneration",
+        "config_module": "transformers.models.qwen4_exp.configuration_qwen4_exp",
+        "config_class": "Qwen4ExpConfig",
+        "target_class_paths": ("transformers.models.qwen4_exp.modeling_qwen4_exp.Qwen4ExpTextExperts",),
+        "validator": "experts",
+        "min_targets": 2,
+    },
+    {
+        "model_type": "qwen4_exp_text",
+        "mode": "convert",
+        "model_module": "transformers.models.qwen4_exp.modeling_qwen4_exp",
+        "model_class": "Qwen4ExpForCausalLM",
+        "config_module": "transformers.models.qwen4_exp.configuration_qwen4_exp",
+        "config_class": "Qwen4ExpTextConfig",
+        "target_class_paths": ("transformers.models.qwen4_exp.modeling_qwen4_exp.Qwen4ExpTextExperts",),
+        "validator": "experts",
+        "min_targets": 2,
+    },
     {
         "model_type": "qwen3_moe",
         "mode": "replace",