keras-hub-nightly 0.21.0.dev202505050407__py3-none-any.whl → 0.21.0.dev202505070407__py3-none-any.whl

keras_hub/src/utils/transformers/convert_mixtral.py

@@ -0,0 +1,139 @@
+import numpy as np
+
+from keras_hub.src.models.mixtral.mixtral_backbone import MixtralBackbone
+from keras_hub.src.utils.preset_utils import get_file
+
+backbone_cls = MixtralBackbone
+
+
+def convert_backbone_config(transformers_config):
+    return {
+        "vocabulary_size": transformers_config["vocab_size"],
+        "num_layers": transformers_config["num_hidden_layers"],
+        "num_query_heads": transformers_config["num_attention_heads"],
+        "hidden_dim": transformers_config["hidden_size"],
+        "intermediate_dim": transformers_config["intermediate_size"],
+        "num_key_value_heads": transformers_config["num_key_value_heads"],
+        "num_experts": transformers_config["num_local_experts"],
+        "top_k": transformers_config["num_experts_per_tok"],
+        "rope_max_wavelength": transformers_config["rope_theta"],
+        "layer_norm_epsilon": transformers_config["rms_norm_eps"],
+        "sliding_window": transformers_config["sliding_window"],
+        "output_router_logits": transformers_config["output_router_logits"],
+    }
+
+
+def convert_weights(backbone, loader, transformers_config):
+    # Embeddings
+    loader.port_weight(
+        keras_variable=backbone.get_layer("token_embedding").embeddings,
+        hf_weight_key="model.embed_tokens.weight",
+    )
+    loader.port_weight(
+        keras_variable=backbone.get_layer("token_embedding").reverse_embeddings,
+        hf_weight_key="lm_head.weight",
+        hook_fn=lambda hf_tensor, _: np.transpose(hf_tensor, axes=(1, 0)),
+    )
+
+    def transpose_and_reshape(x, shape):
+        return np.reshape(np.transpose(x), shape)
+
+    for i in range(backbone.num_layers):
+        decoder_layer = backbone.get_layer(f"transformer_layer_{i}")
+
+        # Input layernorm
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layernorm.scale,
+            hf_weight_key=f"model.layers.{i}.input_layernorm.weight",
+        )
+
+        # Attention layers
+        ## Query
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer.query_dense.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.q_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+        ## Key
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer.key_dense.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.k_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+        ## Value
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer.value_dense.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.v_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+        ## Output
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer._output_dense.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.o_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+
+        # MoE layers
+        # Router gate
+        loader.port_weight(
+            keras_variable=decoder_layer._sparse_moe_block._sparse_feedforward_gate_dense.kernel,
+            hf_weight_key=f"model.layers.{i}.block_sparse_moe.gate.weight",
+            hook_fn=lambda hf_tensor, _: np.transpose(hf_tensor, axes=(1, 0)),
+        )
+
+        # Batched experts: w1 (gate), w3 (intermediate), and w2 (output) weights
+        gate_weights_list = []
+        intermediate_weights_list = []
+        output_weights_list = []
+        for expert_idx in range(backbone.num_experts):
+            # Load w1 (gate dense) for each expert
+            w1 = loader.get_tensor(
+                f"model.layers.{i}.block_sparse_moe.experts.{expert_idx}.w1.weight"
+            )
+            w1_transposed = np.transpose(w1, axes=(1, 0))
+            gate_weights_list.append(w1_transposed)
+
+            w3 = loader.get_tensor(
+                f"model.layers.{i}.block_sparse_moe.experts.{expert_idx}.w3.weight"
+            )
+            w3_transposed = np.transpose(w3, axes=(1, 0))
+            intermediate_weights_list.append(w3_transposed)
+
+            w2 = loader.get_tensor(
+                f"model.layers.{i}.block_sparse_moe.experts.{expert_idx}.w2.weight"
+            )
+            w2_transposed = np.transpose(w2, axes=(1, 0))
+            output_weights_list.append(w2_transposed)
+
+        gate_batched = np.stack(gate_weights_list, axis=0)
+        intermediate_batched = np.stack(intermediate_weights_list, axis=0)
+        output_batched = np.stack(output_weights_list, axis=0)
+
+        # Assign batched weights to expert_bank
+        decoder_layer._sparse_moe_block.expert_bank._expert_feedforward_gate_dense.assign(
+            gate_batched
+        )
+        decoder_layer._sparse_moe_block.expert_bank._expert_feedforward_intermediate_dense.assign(
+            intermediate_batched
+        )
+        decoder_layer._sparse_moe_block.expert_bank._expert_feedforward_output_dense.assign(
+            output_batched
+        )
+
+        # Feedforward layernorm
+        loader.port_weight(
+            keras_variable=decoder_layer._feedforward_layernorm.scale,
+            hf_weight_key=f"model.layers.{i}.post_attention_layernorm.weight",
+        )
+
+    # Final normalization layer
+    loader.port_weight(
+        keras_variable=backbone.get_layer("sequence_output_layernorm").scale,
+        hf_weight_key="model.norm.weight",
+    )
+
+    return backbone
+
+
+def convert_tokenizer(cls, preset, **kwargs):
+    return cls(get_file(preset, "tokenizer.model"), **kwargs)

keras_hub/src/utils/transformers/convert_qwen_moe.py

@@ -0,0 +1,253 @@
+import numpy as np
+
+from keras_hub.src.models.qwen_moe.qwen_moe_backbone import QwenMoeBackbone
+from keras_hub.src.utils.preset_utils import load_json
+
+backbone_cls = QwenMoeBackbone
+
+
+def convert_backbone_config(transformers_config):
+    return {
+        "vocabulary_size": transformers_config["vocab_size"],
+        "hidden_dim": transformers_config["hidden_size"],
+        "num_layers": transformers_config["num_hidden_layers"],
+        "num_query_heads": transformers_config["num_attention_heads"],
+        "num_key_value_heads": transformers_config["num_key_value_heads"],
+        "intermediate_dim": transformers_config["intermediate_size"],
+        "moe_intermediate_dim": transformers_config["moe_intermediate_size"],
+        "shared_expert_intermediate_dim": transformers_config[
+            "shared_expert_intermediate_size"
+        ],
+        "num_experts": transformers_config["num_experts"],
+        "top_k": transformers_config["num_experts_per_tok"],
+        "norm_top_k_prob": transformers_config["norm_topk_prob"],
+        "decoder_sparse_step": transformers_config["decoder_sparse_step"],
+        "layer_norm_epsilon": transformers_config["rms_norm_eps"],
+        "rope_max_wavelength": transformers_config["rope_theta"],
+        "use_sliding_window": transformers_config["use_sliding_window"],
+        "sliding_window_size": transformers_config["sliding_window"],
+        "output_router_logits": transformers_config["output_router_logits"],
+        "router_aux_loss_coefficient": transformers_config[
+            "router_aux_loss_coef"
+        ],
+    }
+
+
+def convert_weights(backbone, loader, transformers_config):
+    loader.port_weight(
+        keras_variable=backbone.get_layer("token_embedding").embeddings,
+        hf_weight_key="model.embed_tokens.weight",
+    )
+    if not backbone.tie_word_embeddings:
+        loader.port_weight(
+            keras_variable=backbone.get_layer(
+                "token_embedding"
+            ).reverse_embeddings,
+            hf_weight_key="lm_head.weight",
+            # rearrange_pattern="b a -> a b",
+            hook_fn=lambda hf_tensor, _: np.transpose(hf_tensor, axes=(1, 0)),
+        )
+
+    def transpose_and_reshape(x, shape):
+        return np.reshape(np.transpose(x), shape)
+
+    for i in range(backbone.num_layers):
+        decoder_layer = backbone.get_layer(f"transformer_layer_{i}")
+
+        # Input layernorm
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layernorm.scale,
+            hf_weight_key=f"model.layers.{i}.input_layernorm.weight",
+        )
+
+        # Attention layers
+
+        ## Query
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer.query_dense.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.q_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer.query_dense.bias,
+            hf_weight_key=f"model.layers.{i}.self_attn.q_proj.bias",
+            hook_fn=transpose_and_reshape,
+        )
+        ## Key
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer.key_dense.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.k_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer.key_dense.bias,
+            hf_weight_key=f"model.layers.{i}.self_attn.k_proj.bias",
+            hook_fn=transpose_and_reshape,
+        )
+        ## Value
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer.value_dense.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.v_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer.value_dense.bias,
+            hf_weight_key=f"model.layers.{i}.self_attn.v_proj.bias",
+            hook_fn=transpose_and_reshape,
+        )
+        ## Output
+        loader.port_weight(
+            keras_variable=decoder_layer._self_attention_layer._output_dense.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.o_proj.weight",
+            # rearrange_patterns="c (a b) -> a b c",
+            # rearrange_dims={"a": backbone.num_query_heads},
+            hook_fn=transpose_and_reshape,
+        )
+
+        # MLP layers
+        if (
+            (i not in backbone.mlp_only_layers)
+            and backbone.num_experts > 0
+            and ((i + 1) % backbone.decoder_sparse_step == 0)
+        ):
+            # MoE layers
+            loader.port_weight(
+                keras_variable=decoder_layer.mlp._sparse_feedforward_gate_dense.kernel,
+                hf_weight_key=f"model.layers.{i}.mlp.gate.weight",
+                # rearrange_patterns="b a -> a b",
+                hook_fn=lambda hf_tensor, _: np.transpose(
+                    hf_tensor, axes=(1, 0)
+                ),
+            )
+            # Batched experts: gate_up_proj and down_proj
+            gate_up_proj_list = []
+            down_proj_list = []
+            for expert_idx in range(backbone.num_experts):
+                # Load gate_proj and up_proj for each expert
+                gate_proj = loader.get_tensor(
+                    f"model.layers.{i}.mlp.experts.{expert_idx}.gate_proj.weight"
+                )
+                up_proj = loader.get_tensor(
+                    f"model.layers.{i}.mlp.experts.{expert_idx}.up_proj.weight"
+                )
+                # Transpose to (hidden_dim, intermediate_dim)
+                gate_proj = np.transpose(gate_proj, axes=(1, 0))
+                up_proj = np.transpose(up_proj, axes=(1, 0))
+                # Concatenate gate_proj and up_proj along the last dimension
+                gate_up_proj = np.concatenate([gate_proj, up_proj], axis=-1)
+                gate_up_proj_list.append(gate_up_proj)
+
+                # Load down_proj for each expert
+                down_proj = loader.get_tensor(
+                    f"model.layers.{i}.mlp.experts.{expert_idx}.down_proj.weight"
+                )
+                down_proj = np.transpose(
+                    down_proj, axes=(1, 0)
+                )  # (intermediate_dim, hidden_dim)
+                down_proj_list.append(down_proj)
+
+            # Stack the lists to create batched weights
+            gate_up_proj_batched = np.stack(
+                gate_up_proj_list, axis=0
+            )  # (num_experts, hidden_dim, 2 * intermediate_dim)
+            down_proj_batched = np.stack(
+                down_proj_list, axis=0
+            )  # (num_experts, intermediate_dim, hidden_dim)
+
+            # Assign batched weights to expert_bank
+            decoder_layer.mlp.expert_bank._expert_feedforward_gate_dense.assign(
+                gate_up_proj_batched
+            )
+            decoder_layer.mlp.expert_bank._expert_feedforward_output_dense.assign(
+                down_proj_batched
+            )
+
+            loader.port_weight(
+                keras_variable=decoder_layer.mlp.shared_expert_dense._feedforward_intermediate_dense.kernel,
+                hf_weight_key=f"model.layers.{i}.mlp.shared_expert.up_proj.weight",
+                hook_fn=lambda hf_tensor, _: np.transpose(
+                    hf_tensor, axes=(1, 0)
+                ),
+            )
+            loader.port_weight(
+                keras_variable=decoder_layer.mlp.shared_expert_dense._feedforward_output_dense.kernel,
+                hf_weight_key=f"model.layers.{i}.mlp.shared_expert.down_proj.weight",
+                hook_fn=lambda hf_tensor, _: np.transpose(
+                    hf_tensor, axes=(1, 0)
+                ),
+            )
+            loader.port_weight(
+                keras_variable=decoder_layer.mlp.shared_expert_dense._feedforward_gate_dense.kernel,
+                hf_weight_key=f"model.layers.{i}.mlp.shared_expert.gate_proj.weight",
+                hook_fn=lambda hf_tensor, _: np.transpose(
+                    hf_tensor, axes=(1, 0)
+                ),
+            )
+
+            loader.port_weight(
+                keras_variable=decoder_layer.mlp.shared_expert_gate_dense.kernel,
+                hf_weight_key=f"model.layers.{i}.mlp.shared_expert_gate.weight",
+                hook_fn=lambda hf_tensor, _: np.transpose(
+                    hf_tensor, axes=(1, 0)
+                ),
+            )
+        else:
+            loader.port_weight(
+                keras_variable=decoder_layer._feedforward_intermediate_dense.kernel,
+                hf_weight_key=f"model.layers.{i}.mlp.up_proj.weight",
+                # rearrange_patterns="b a -> a b",
+                hook_fn=lambda hf_tensor, _: np.transpose(
+                    hf_tensor, axes=(1, 0)
+                ),
+            )
+            loader.port_weight(
+                keras_variable=decoder_layer._feedforward_output_dense.kernel,
+                hf_weight_key=f"model.layers.{i}.mlp.down_proj.weight",
+                # rearrange_patterns="b a -> a b",
+                hook_fn=lambda hf_tensor, _: np.transpose(
+                    hf_tensor, axes=(1, 0)
+                ),
+            )
+            loader.port_weight(
+                keras_variable=decoder_layer._feedforward_gate_dense.kernel,
+                hf_weight_key=f"model.layers.{i}.mlp.gate_proj.weight",
+                # rearrange_patterns="b a -> a b",
+                hook_fn=lambda hf_tensor, _: np.transpose(
+                    hf_tensor, axes=(1, 0)
+                ),
+            )
+
+        # Feedforward layernorm
+        loader.port_weight(
+            keras_variable=decoder_layer._feedforward_layernorm.scale,
+            hf_weight_key=f"model.layers.{i}.post_attention_layernorm.weight",
+        )
+
+    # Final normalization layer
+    loader.port_weight(
+        keras_variable=backbone.get_layer("sequence_output_layernorm").scale,
+        hf_weight_key="model.norm.weight",
+    )
+
+    return backbone
+
+
+def convert_tokenizer(cls, preset, **kwargs):
+    tokenizer_config = load_json(preset, "tokenizer.json")
+    vocab = tokenizer_config["model"]["vocab"]
+    merges = tokenizer_config["model"]["merges"]
+
+    # Load all special tokens with the exception of "reserved" ones.
+    special_tokens = set()
+    for token in tokenizer_config["added_tokens"]:
+        if not token["content"].startswith("<|reserved_special_token_"):
+            vocab[token["content"]] = token["id"]
+            special_tokens.add(token["content"])
+
+    kwargs.update(
+        {
+            "unsplittable_tokens": list(special_tokens),
+        }
+    )
+
+    return cls(vocabulary=vocab, merges=merges, **kwargs)

keras_hub/src/utils/transformers/preset_loader.py

@@ -11,8 +11,10 @@ from keras_hub.src.utils.transformers import convert_gemma
 from keras_hub.src.utils.transformers import convert_gpt2
 from keras_hub.src.utils.transformers import convert_llama3
 from keras_hub.src.utils.transformers import convert_mistral
+from keras_hub.src.utils.transformers import convert_mixtral
 from keras_hub.src.utils.transformers import convert_pali_gemma
 from keras_hub.src.utils.transformers import convert_qwen
+from keras_hub.src.utils.transformers import convert_qwen_moe
 from keras_hub.src.utils.transformers import convert_vit
 from keras_hub.src.utils.transformers.safetensor_utils import SafetensorLoader
 
@@ -44,6 +46,10 @@ class TransformersPresetLoader(PresetLoader):
             self.converter = convert_vit
         elif model_type == "qwen2":
             self.converter = convert_qwen
+        elif model_type == "mixtral":
+            self.converter = convert_mixtral
+        elif model_type == "qwen2_moe":
+            self.converter = convert_qwen_moe
         else:
             raise ValueError(
                 "KerasHub has no converter for huggingface/transformers models "

keras_hub/src/version.py

@@ -1,7 +1,7 @@
 from keras_hub.src.api_export import keras_hub_export
 
 # Unique source of truth for the version number.
-__version__ = "0.21.0.dev202505050407"
+__version__ = "0.21.0.dev202505070407"
 
 
 @keras_hub_export("keras_hub.version")

keras_hub/tokenizers/__init__.py

@@ -55,6 +55,9 @@ from keras_hub.src.models.llama3.llama3_tokenizer import (
 from keras_hub.src.models.mistral.mistral_tokenizer import (
     MistralTokenizer as MistralTokenizer,
 )
+from keras_hub.src.models.mixtral.mixtral_tokenizer import (
+    MixtralTokenizer as MixtralTokenizer,
+)
 from keras_hub.src.models.opt.opt_tokenizer import OPTTokenizer as OPTTokenizer
 from keras_hub.src.models.pali_gemma.pali_gemma_tokenizer import (
     PaliGemmaTokenizer as PaliGemmaTokenizer,
@@ -68,6 +71,9 @@ from keras_hub.src.models.qwen.qwen_tokenizer import (
 from keras_hub.src.models.qwen.qwen_tokenizer import (
     QwenTokenizer as QwenTokenizer,
 )
+from keras_hub.src.models.qwen_moe.qwen_moe_tokenizer import (
+    QwenMoeTokenizer as QwenMoeTokenizer,
+)
 from keras_hub.src.models.roberta.roberta_tokenizer import (
     RobertaTokenizer as RobertaTokenizer,
 )

{keras_hub_nightly-0.21.0.dev202505050407.dist-info → keras_hub_nightly-0.21.0.dev202505070407.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: keras-hub-nightly
-Version: 0.21.0.dev202505050407
+Version: 0.21.0.dev202505070407
 Summary: Pretrained models for Keras.
 Author-email: Keras team <keras-users@googlegroups.com>
 License-Expression: Apache-2.0

{keras_hub_nightly-0.21.0.dev202505050407.dist-info → keras_hub_nightly-0.21.0.dev202505070407.dist-info}/RECORD

@@ -1,11 +1,11 @@
 keras_hub/__init__.py,sha256=bJbUZkqwhZvTb1Tqx1fbkq6mzBYiEyq-Hin3oQIkhdE,558
 keras_hub/layers/__init__.py,sha256=LhMUEcl3xJwqr0XphTgRZ5Ayz5SsBAKV19c0XwSzj1I,4952
 keras_hub/metrics/__init__.py,sha256=KYalsMPBnfwim9BdGHFfJ5WxUKFXOQ1QoKIMT_0lwlM,439
-keras_hub/models/__init__.py,sha256=Np-V3YdZcIa0xwqhjmgm5NWnsCj647aClYW0Uhi3eSI,25108
+keras_hub/models/__init__.py,sha256=kFWNpjemQ8FLzDlFfMdAOOXJKtxuVHFxyZm7-1mH4Gc,25909
 keras_hub/samplers/__init__.py,sha256=aFQIkiqbZpi8vjrPp2MVII4QUfE-eQjra5fMeHsoy7k,886
 keras_hub/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/api_export.py,sha256=9pQZK27JObxWZ96QPLBp1OBsjWigh1iuV6RglPGMRk0,1499
-keras_hub/src/version.py,sha256=GU1P_7eS7irSsw27d_6GiQJtuMQe1eApQnOwemKRH6E,222
+keras_hub/src/version.py,sha256=aaWmSzLN-AkMZRmAoGmZ2N0SrrxAlh7bwjgP-M3xnCs,222
 keras_hub/src/layers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/layers/modeling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/layers/modeling/alibi_bias.py,sha256=1XBTHI52L_iJDhN_w5ydu_iMhCuTgQAxEPwcLA6BPuk,4411
@@ -41,7 +41,7 @@ keras_hub/src/metrics/rouge_base.py,sha256=Pt2DUznhTTeR-fX1nQ_wSbPtmuTgxQTvrGpu8
 keras_hub/src/metrics/rouge_l.py,sha256=JlZhMBV6wS_6zMd57pkTc6yxHkEJT9fVQMlPZKekQzQ,2729
 keras_hub/src/metrics/rouge_n.py,sha256=JoFtmgjF4Ic263ny6bfD6vMHKreH9le3HnOOxemupRc,3620
 keras_hub/src/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-keras_hub/src/models/backbone.py,sha256=TwfJOO7lk50BNO36gg8m_DvgPiBxAhHK0XSbab2qpSA,11309
+keras_hub/src/models/backbone.py,sha256=KS2x3HFWKhEYhroUFT3uZgSkeW_48zPGqUNvxCDDIQQ,11534
 keras_hub/src/models/causal_lm.py,sha256=ReaF-i3SHsCkHh4c28jM72QjMQ8x7yiCwG39FRb-7KE,16786
 keras_hub/src/models/causal_lm_preprocessor.py,sha256=YY7VJZicdmnjDSWi9g4_pEpd5bdJK166GlWcapvokF0,6663
 keras_hub/src/models/feature_pyramid_backbone.py,sha256=clEW-TTQSVJ_5qFNdDF0iABkin1p_xlBUFjJrC7T0IA,2247
@@ -58,7 +58,7 @@ keras_hub/src/models/object_detector_preprocessor.py,sha256=kOSVRNFAg-UjtrCEVBdH
 keras_hub/src/models/preprocessor.py,sha256=kBlahgVST3L6vKeWDM4fXuDoXa6pwaJW2A5__L85wFU,8487
 keras_hub/src/models/seq_2_seq_lm.py,sha256=w0gX-5YZjatfvAJmFAgSHyqS_BLqc8FF8DPLGK8mrgI,1864
 keras_hub/src/models/seq_2_seq_lm_preprocessor.py,sha256=DJmm4VTt8AdLtq1k9YKl_VR31cKUHaYjfSbrk7-fJqA,9667
-keras_hub/src/models/task.py,sha256=a7eW8ylmsSY4opI5bEo_AQQOXBgsGU5CihQWm1DWfnw,14592
+keras_hub/src/models/task.py,sha256=e9zK2zHgeOkjNACcCmAf-lGuEGF_eRoP_lKlirdIXuk,14817
 keras_hub/src/models/text_classifier.py,sha256=B6cTYDbDZW8vRvenXrLwgMMVIYMb7Pr14GvX8C_wclQ,4159
 keras_hub/src/models/text_classifier_preprocessor.py,sha256=EoWp-GHnaLnAKTdAzDmC-soAV92ATF3QozdubdV2WXI,4722
 keras_hub/src/models/text_to_image.py,sha256=NIy4S6Fh8MsbNiskAFhjmFXgRiiFqn_rOvpGOO6LlF0,13390
@@ -110,11 +110,11 @@ keras_hub/src/models/clip/clip_tokenizer.py,sha256=6gIm_LWRbCeBQUI9M2gA8-OXb4tXG
 keras_hub/src/models/clip/clip_vision_embedding.py,sha256=6_qC7T1dqKd-39EreGmHZj-YfjOLEDDKjWnEKcKIyuY,3667
 keras_hub/src/models/clip/clip_vision_encoder.py,sha256=q62MXySZN38uCsjqq8cttfBxD7P5abaKQV2i8_u4N6E,6385
 keras_hub/src/models/cspnet/__init__.py,sha256=TOpvk2cfOVv1bPA1BOGZj0mhmhc6E98zZmW9e0PIvhk,257
-keras_hub/src/models/cspnet/cspnet_backbone.py,sha256=xCeu8BpQSpf-EgCrQehQDg4jNKRAWm0h8paWBfN2DGE,41381
+keras_hub/src/models/cspnet/cspnet_backbone.py,sha256=meHzxubG_9vHQHSelDfrROaQERkDiWkjTtk_gKaWsDc,42457
 keras_hub/src/models/cspnet/cspnet_image_classifier.py,sha256=JqfBHIBTFxaLOyAWx6TdXs0aAOMbcCx1oo47RoQnytc,510
 keras_hub/src/models/cspnet/cspnet_image_classifier_preprocessor.py,sha256=ACRnOhjslk2ZZhpPfJioW4um4RLYa-Suk59z9wa5vfo,543
 keras_hub/src/models/cspnet/cspnet_image_converter.py,sha256=f-ICTY2T-RlCykU6qOHDxg0fY7ECfZ_xpSJzIVmbvpc,342
-keras_hub/src/models/cspnet/cspnet_presets.py,sha256=fWzPz3eZuhFNxxPn9MJHabcXiyJA2PRRVlzNmoFBwWg,533
+keras_hub/src/models/cspnet/cspnet_presets.py,sha256=n01_7DTvbmaA_qs2GWiNLkBXNrrEvigPXSGc2NDTot8,1870
 keras_hub/src/models/deberta_v3/__init__.py,sha256=6E-QtAD1uvTBobrn5bUoyB1qtaCJU-t73TtbAEH6i9g,288
 keras_hub/src/models/deberta_v3/deberta_v3_backbone.py,sha256=oXdV7naTiMowuU3GsXEUo5K0GXiKbPKxdo27o5fXWjc,7258
 keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py,sha256=ADBktf1DdiP9T6LCaMhdFiZ_mUbBRKMekY5mGwAeJIo,4186
@@ -250,6 +250,13 @@ keras_hub/src/models/mit/mit_image_classifier_preprocessor.py,sha256=oNYs-pUK8Vn
 keras_hub/src/models/mit/mit_image_converter.py,sha256=Mw7nV-OzyBveGuZUNFsPPKyq9jXJVW2_cVH024CNkXM,311
 keras_hub/src/models/mit/mit_layers.py,sha256=HUJO5uhJ6jgwANpwbQdPlEVwLRVb3BZQ-Ftjg3B9XvY,9734
 keras_hub/src/models/mit/mit_presets.py,sha256=ooLrh2OoGZKxnCGnhB6BynYJtVCXH7nDDFhgQRWt36U,4528
+keras_hub/src/models/mixtral/mixtral_attention.py,sha256=rdUBjIFQZKBpyCXlXMDgmB8gLCk0ngnhdhNs_twFE_c,9089
+keras_hub/src/models/mixtral/mixtral_backbone.py,sha256=vUAFXvqwVBgKxYbOsqIHzPN59bhaDrGWwOnBCzeUtt0,8034
+keras_hub/src/models/mixtral/mixtral_causal_lm.py,sha256=JA1t6xTeaYX_fNo9ftRyvzdRDG3vndC-Rlwn5fnsbQo,12001
+keras_hub/src/models/mixtral/mixtral_causal_lm_preprocessor.py,sha256=q2qXa9QAUWBvOWv9DeNvwsBNXSORJAbQFoQsWQ7e8V8,3079
+keras_hub/src/models/mixtral/mixtral_decoder.py,sha256=CvOjhTxPnGQ_HNknZXRI6Cx1kpuHG99_TiOh-mNcsDw,18190
+keras_hub/src/models/mixtral/mixtral_layer_norm.py,sha256=zfbDKZEb45FTwP0zQd7WPPp8tuiGoSNfS-DRYWkZyWw,1031
+keras_hub/src/models/mixtral/mixtral_tokenizer.py,sha256=Kc233k879QMyX164X_CzWbqpnqEkKWNqa648guTGkBk,661
 keras_hub/src/models/mobilenet/__init__.py,sha256=hxkNGGj_iAMu62iooUDEPA818sNOIgjG7pXMLEMOsAE,275
 keras_hub/src/models/mobilenet/mobilenet_backbone.py,sha256=aZBSFeLUObYYoi3od9DI1KfgPCqh5GHTcAI8Y2ZHShA,29536
 keras_hub/src/models/mobilenet/mobilenet_image_classifier.py,sha256=rgPVJeSRqyp3-Fgf5ERbg_97c4cSawRmAtoJpdBN8WA,2437
@@ -283,13 +290,22 @@ keras_hub/src/models/phi3/phi3_presets.py,sha256=sb2ce7Gq1OikFEf2KIYG69rFKHYKj8q
 keras_hub/src/models/phi3/phi3_rotary_embedding.py,sha256=wqiRn8nETNcLc5Vsm_d_8s11Ro6ibWZbWvODdLqIOo4,5013
 keras_hub/src/models/phi3/phi3_tokenizer.py,sha256=bOPH14wTVVHJHq8mgzXLjsgvKMNhfO8eayevAPpjYVA,1992
 keras_hub/src/models/qwen/__init__.py,sha256=hskG3tZUY_AYZPp0WVzbCtw37AIYENyp3DOnqHmdRBw,65
-keras_hub/src/models/qwen/qwen_attention.py,sha256=FL_09-eCFugktDNzFPm6beZLD04pNg9TFKgfXdhWUwk,12953
+keras_hub/src/models/qwen/qwen_attention.py,sha256=SrUYESCg27ksuDKZHKJ5Wmnkbr6WZdF7nHv0AHFfWR8,13014
 keras_hub/src/models/qwen/qwen_backbone.py,sha256=i39_LoKu6hcYWV6KFh2OzUDaXjV7g1WLNGF2-JD_tqI,13015
 keras_hub/src/models/qwen/qwen_causal_lm.py,sha256=_f-UHaKHp0ncxknpkpEJiW3jlng3E4CmddjQfz2QzJo,12249
 keras_hub/src/models/qwen/qwen_causal_lm_preprocessor.py,sha256=Va-4TLJD3ycEnkS41rF3dVj4_6K0j-gxLTrREFRcyr0,609
 keras_hub/src/models/qwen/qwen_decoder.py,sha256=utmAvZlU7_nP-6pjGPDinK4JaMzsQSwOARG0ote-jAg,11771
 keras_hub/src/models/qwen/qwen_layernorm.py,sha256=DS35r3qd6g5ocL7Nhf_vNzLLMo1aI9VCSmL64dgNOYI,924
+keras_hub/src/models/qwen/qwen_presets.py,sha256=_jRG7bB4yBGWteBLbK2elc1e9doRl8zdzQRZgxFvnfc,1988
 keras_hub/src/models/qwen/qwen_tokenizer.py,sha256=LCv3IyiDDHqVnM9N3lf5-BE3iwicIh0nKS1hjoPw9lE,1532
+keras_hub/src/models/qwen_moe/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+keras_hub/src/models/qwen_moe/qwen_moe_attention.py,sha256=mXc4uGkUSK3FHdJ5_77xiX7Gm0eO1GWTF40ei_68pvU,13472
+keras_hub/src/models/qwen_moe/qwen_moe_backbone.py,sha256=nrfELvIvRLmrgKrUNXci2CrecmeI6bWzJj7HH-RcWJA,15341
+keras_hub/src/models/qwen_moe/qwen_moe_causal_lm.py,sha256=MeP60v7GcN_SmH5_ULRpqgmFVgaYAosSecZiSQVlJvU,13256
+keras_hub/src/models/qwen_moe/qwen_moe_causal_lm_preprocessor.py,sha256=uKaXRrJs02vkVudjdehzJPp0B84tPMkxNHlp166kceE,589
+keras_hub/src/models/qwen_moe/qwen_moe_decoder.py,sha256=kmUjLpYTbJQ3J_31qWhLOd0Dg2_9cl_JX_zM8ZMH1Qo,23130
+keras_hub/src/models/qwen_moe/qwen_moe_layernorm.py,sha256=DbkWJo7U0-cwdZwHPeAnFznYwtao6o0fjpoDJ9UWnpc,927
+keras_hub/src/models/qwen_moe/qwen_moe_tokenizer.py,sha256=2c3X8jNGO0q0UL5NtUqSgHWLqhyJGi2ohNcTeOGhd84,1407
 keras_hub/src/models/resnet/__init__.py,sha256=C5UqlQ6apm8WSp1bnrxB6Bi3BGaknxRQs-r3b2wpaGA,257
 keras_hub/src/models/resnet/resnet_backbone.py,sha256=Q7nlqcTXZzjqd0e-DsjHC4ok58yOX7qxseotym3uZpM,31276
 keras_hub/src/models/resnet/resnet_image_classifier.py,sha256=nf35EKDzvBkfhHsK-s6Ks0nbhvKO7HEOYZm94YckyWE,510
@@ -300,11 +316,11 @@ keras_hub/src/models/retinanet/__init__.py,sha256=veWIFvMN6151M69l7FvTcI-IIEe_8d
 keras_hub/src/models/retinanet/feature_pyramid.py,sha256=hbdrj6X-D2SlwOp2h1WcBlTdSAlLmFK43X7OrkJRoMA,17614
 keras_hub/src/models/retinanet/prediction_head.py,sha256=xWHt21-SS2t7vCmTONlR1lSbJXhml5jx68V8MGbGybg,7863
 keras_hub/src/models/retinanet/retinanet_backbone.py,sha256=BJBPJLxpOCOU0Br7b4JsgCZBHQHLAhxLqo9BHNIsl1g,5659
-keras_hub/src/models/retinanet/retinanet_image_converter.py,sha256=Yr1ACzrPXzX1equjDqkrzRQv5nL5TARICc55Gnhwx7o,785
+keras_hub/src/models/retinanet/retinanet_image_converter.py,sha256=jnVAqQ3zem0JNk5iaIdrMGKyGv_ulAcePpM5t1lulWI,360
 keras_hub/src/models/retinanet/retinanet_label_encoder.py,sha256=Vowhs4uOZAevmVg1a19efIPfvjxkckXwsJDTX3VPDxs,10967
 keras_hub/src/models/retinanet/retinanet_object_detector.py,sha256=WJ3YLnnC4mcCLLoE7uUFA0cOSVuFgnx9Cr47If50Aig,15595
 keras_hub/src/models/retinanet/retinanet_object_detector_preprocessor.py,sha256=RnJkdqv4zYVcGx50sHoA7j9G1AKwEN-RNtyMQg-MMbo,568
-keras_hub/src/models/retinanet/retinanet_presets.py,sha256=qzs568Me0bSoXwgoG8wQrGbY_WuS2t1qgGU2wL8R5Hs,950
+keras_hub/src/models/retinanet/retinanet_presets.py,sha256=75_Gnxt84MBjTDd4xQVSsIa2sDQ-KnQ4_Hw9nZ90ljE,950
 keras_hub/src/models/roberta/__init__.py,sha256=3ouSnKdLlMwoDDLVKD9cNtxam6f8XWgCyc0pwWJ0Zjo,263
 keras_hub/src/models/roberta/roberta_backbone.py,sha256=q16dylXbgWshT-elCA08lS_b_IZNphsBrrXiv3eJksM,6339
 keras_hub/src/models/roberta/roberta_masked_lm.py,sha256=j2dFANRFHd1MNFP_REchljGWOcpOjCpdSya-WGdRzPA,4176
@@ -337,8 +353,8 @@ keras_hub/src/models/segformer/__init__.py,sha256=ERgxA8tyeG2l4G6ywHisn6Oo0Iu7_9
 keras_hub/src/models/segformer/segformer_backbone.py,sha256=T61WQ50T6IwSeiK1NfUKJu3eqbj_m5gz9cpUPtqMfcc,5666
 keras_hub/src/models/segformer/segformer_image_converter.py,sha256=zePZ1cYZl-2TaEF82lj3y7kXjDao5Hgw8c7qfKI2Jd8,360
 keras_hub/src/models/segformer/segformer_image_segmenter.py,sha256=JzX8oJASWdkw8wbm8cohjPnumIvBvj7GGEpbK7ex-6w,5926
-keras_hub/src/models/segformer/segformer_image_segmenter_preprocessor.py,sha256=4xj6_E-JlYpXv064VtEewxaQuD8aXw5egoUKlr_fLPg,1125
-keras_hub/src/models/segformer/segformer_presets.py,sha256=ET39ospixkTaCsjoMLdJrr3wlGvTAQu5prleVC5lMZI,4793
+keras_hub/src/models/segformer/segformer_image_segmenter_preprocessor.py,sha256=Jlsy41n7IymjvU3ENKQJtsQ4xkJEGRtxEm-defz2Nbc,568
+keras_hub/src/models/segformer/segformer_presets.py,sha256=4fPkGTP_jjd3Qcd1KbWYZ7-ze0wdJskMwKG4GZ-UwEg,4793
 keras_hub/src/models/siglip/__init__.py,sha256=uImQYl06pioLwla6c_tiF2PSJKHtq0aSxDPsynQbXcA,257
 keras_hub/src/models/siglip/siglip_backbone.py,sha256=dXp7BU7mqKWthl70KFZ2AMILjNIu5A6itQFu6XD22Qs,8372
 keras_hub/src/models/siglip/siglip_image_converter.py,sha256=yjYc0XOyL37WLlr-X6V4QXI1FCyiKngbHXrZK0hNB1U,342
@@ -435,23 +451,23 @@ keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py,sha256=hRv_XxoPIPDpHfO0Z
 keras_hub/src/tokenizers/word_piece_tokenizer.py,sha256=vP6AZgbzsRiuPCt3W_n94nsF7XiERnagWcH_rqJHtVU,19943
 keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py,sha256=cylrs02ZrYQ1TuZr9oyS3NrVbDwGctA3VXbIh1pFJMQ,6743
 keras_hub/src/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-keras_hub/src/utils/keras_utils.py,sha256=mtj5Kr9EROso10SafmQ-C9uCLbIId4cXAuJSNDRqHb8,4290
+keras_hub/src/utils/keras_utils.py,sha256=2qrh4F-rqceVFSx0-cbsFBfWae5hBXFb_sEtPPcImf4,4628
 keras_hub/src/utils/pipeline_model.py,sha256=jgzB6NQPSl0KOu08N-TazfOnXnUJbZjH2EXXhx25Ftg,9084
-keras_hub/src/utils/preset_utils.py,sha256=Zhc2xIHUagBajmdvuz-91gRnKqJA0CGPr_yIHI_UXEY,32006
+keras_hub/src/utils/preset_utils.py,sha256=fx0gNqOTdvW-ZdP0Y3ZaCGE7frYBhwi3lG_GO0swG4w,34602
 keras_hub/src/utils/python_utils.py,sha256=N8nWeO3san4YnGkffRXG3Ix7VEIMTKSN21FX5TuL7G8,202
-keras_hub/src/utils/tensor_utils.py,sha256=1Y9E2psWzN5hJqWz_r2kOoFU7kIp7mPBTX5xZbTsE_A,15144
+keras_hub/src/utils/tensor_utils.py,sha256=vRbvvnFwA6FutJ7InC1w60HDTVNi87CniDGOLQ3hKPA,15855
 keras_hub/src/utils/coco/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/utils/coco/coco_utils.py,sha256=x_QnUUvZ92zoFzMJugiInHORc4NrMdWVBkpp8BAYF6s,2586
 keras_hub/src/utils/imagenet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/utils/imagenet/imagenet_utils.py,sha256=07ilM5feeD7Ut6YSbVj99RXAZOQONSC1IeKa3I9U6UQ,40161
 keras_hub/src/utils/timm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-keras_hub/src/utils/timm/convert_cspnet.py,sha256=O5HCdeKcSFWOoFr8_wIUQb4Noc0tBEo5Aogk2d6SEes,5676
+keras_hub/src/utils/timm/convert_cspnet.py,sha256=9p1IF0B4UPbDTruQQXR6mJEUdhvQvHx9E0SKNn_Lbw4,8047
 keras_hub/src/utils/timm/convert_densenet.py,sha256=fu8HBIQis5o3ib2tyI2qnmYScVrVIQySok8vTfa1qJ8,3393
 keras_hub/src/utils/timm/convert_efficientnet.py,sha256=SgEIlyyinS04qoQpEgh3WazHq544zNUCCpfmWh3EjSs,17100
 keras_hub/src/utils/timm/convert_mobilenet.py,sha256=XTqHOK4nJwigKefsw7ktWJtOgRpEVMO9MtRhuP5qP_k,9219
 keras_hub/src/utils/timm/convert_resnet.py,sha256=8JFkVtdpy5z9h83LJ97rD-a8FRejXPZvMNksNuStqjM,5834
 keras_hub/src/utils/timm/convert_vgg.py,sha256=MT5jGnLrzenPpe66Af_Lp1IdR9KGtsSrcmn6_UPqHvQ,2419
-keras_hub/src/utils/timm/preset_loader.py,sha256=j2HYi61Zbt0CGd33evFJ8j2fraXl0Zardf4qqAb82K0,3841
+keras_hub/src/utils/timm/preset_loader.py,sha256=4hULdq2K2hgPYTZR71PGV4YNDHLG1zcoxF9TXpg6fGE,3905
 keras_hub/src/utils/transformers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/utils/transformers/convert_albert.py,sha256=VdKclZpCxtDWq3UbUUQZf4fR9DJK_JYZ73B4O_G9skg,7695
 keras_hub/src/utils/transformers/convert_bart.py,sha256=Tk4h9Md9rwN5wjQbGIVrC7qzDpF8kI8qm-FKL8HlUok,14411
@@ -461,14 +477,16 @@ keras_hub/src/utils/transformers/convert_gemma.py,sha256=ElCgwBpSN5Q7rV5PJawTsoy
 keras_hub/src/utils/transformers/convert_gpt2.py,sha256=HCeHN_-GiQJRxLCM9OCJJ1watPVpIBF8ujS8pGbBOWc,5703
 keras_hub/src/utils/transformers/convert_llama3.py,sha256=c5phNl-QayQ_BS0s-lenbu6oHxqfwDShKJoh9DluxUU,6146
 keras_hub/src/utils/transformers/convert_mistral.py,sha256=kVhN9h1ZFVhwkNW8p3wnS7eANJUXIsNy1RxWXy20Gqw,4760
+keras_hub/src/utils/transformers/convert_mixtral.py,sha256=_esUzVRYABR5pHHSALqUieSuAeBg4te1JnlGQENqECU,5509
 keras_hub/src/utils/transformers/convert_pali_gemma.py,sha256=B1leeDw96Yvu81hYumf66hIid07k5NLqoeWAJgPnaLs,10649
 keras_hub/src/utils/transformers/convert_qwen.py,sha256=WUxMAEFVqRs7TRw7QU5TH3_ev4yf02R1xFVliMvTQqg,5886
+keras_hub/src/utils/transformers/convert_qwen_moe.py,sha256=a7R28aln-PdAcNuKAXdrtzvslho2Co6GypChxLMKPpc,10618
 keras_hub/src/utils/transformers/convert_vit.py,sha256=9SUZ9utNJhW_5cj3acMn9cRy47u2eIcDsrhmzj77o9k,5187
-keras_hub/src/utils/transformers/preset_loader.py,sha256=0Hi7R8HnATcwFVLsJwMMIMWTCXHNfep4IPiRpQXqM-w,3933
+keras_hub/src/utils/transformers/preset_loader.py,sha256=1nfS5xVsl-JROGXJXltTqV1fQdcUlZbGGcbf-n79pXM,4225
 keras_hub/src/utils/transformers/safetensor_utils.py,sha256=CYUHyA4y-B61r7NDnCsFb4t_UmSwZ1k9L-8gzEd6KRg,3339
-keras_hub/tokenizers/__init__.py,sha256=1X8s88lzi6zM9XaMHbpACa4kpwjDILgmUV0tl1jXeeo,3839
+keras_hub/tokenizers/__init__.py,sha256=4etC--bzhczJrRcvCmxZmOC9hJJcIVOUCgmqMLB3bp0,4051
 keras_hub/utils/__init__.py,sha256=jXPqVGBpJr_PpYmqD8aDG-fRMlxH-ulqCR2SZMn288Y,646
-keras_hub_nightly-0.21.0.dev202505050407.dist-info/METADATA,sha256=5tCCNXZ3oRWimz5briR2ot1Erf4kZczTmg_nhYya0P4,7393
-keras_hub_nightly-0.21.0.dev202505050407.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-keras_hub_nightly-0.21.0.dev202505050407.dist-info/top_level.txt,sha256=N4J6piIWBKa38A4uV-CnIopnOEf8mHAbkNXafXm_CuA,10
-keras_hub_nightly-0.21.0.dev202505050407.dist-info/RECORD,,
+keras_hub_nightly-0.21.0.dev202505070407.dist-info/METADATA,sha256=qZV9GqHYenWgLpKBXdil_rhFnetmHPFV-OgVFjxPKTs,7393
+keras_hub_nightly-0.21.0.dev202505070407.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+keras_hub_nightly-0.21.0.dev202505070407.dist-info/top_level.txt,sha256=N4J6piIWBKa38A4uV-CnIopnOEf8mHAbkNXafXm_CuA,10
+keras_hub_nightly-0.21.0.dev202505070407.dist-info/RECORD,,