keras-hub-nightly 0.23.0.dev202510170417__py3-none-any.whl → 0.23.0.dev202510190425__py3-none-any.whl

keras_hub/src/models/smollm3/smollm3_tokenizer.py

@@ -0,0 +1,60 @@
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.smollm3.smollm3_backbone import SmolLM3Backbone
+from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
+
+
+@keras_hub_export(
+    [
+        "keras_hub.tokenizers.SmolLM3Tokenizer",
+        "keras_hub.tokenizers.SmolLMTokenizer",
+        "keras_hub.models.SmolLM3Tokenizer",
+        "keras_hub.models.SmolLMTokenizer",
+    ]
+)
+class SmolLM3Tokenizer(BytePairTokenizer):
+    """Tokenizer for SmolLM3 models.
+
+    This tokenizer implements byte-pair encoding (BPE) for SmolLM3 models,
+    handling special tokens like BOS (beginning of sequence) and EOS (end of
+    sequence).
+
+    Args:
+        vocabulary: Dictionary mapping tokens to token IDs, or path to
+            vocabulary file.
+        merges: List of BPE merges, or path to merges file.
+        bos_token: Beginning of sequence token. Defaults to None.
+        eos_token: End of sequence token. Defaults to "<|endoftext|>".
+        misc_special_tokens: Set of additional special tokens. Defaults to
+            empty set.
+    """
+
+    backbone_cls = SmolLM3Backbone
+
+    def __init__(
+        self,
+        vocabulary=None,
+        merges=None,
+        **kwargs,
+    ):
+        # Add EOS token
+        eos_token = "<|end_of_text|>"
+        self._add_special_token(eos_token, "end_token")
+
+        bos_token = "<|begin_of_text|>"
+        self._add_special_token(bos_token, "bos_token")
+
+        start_think_token = "<think>"
+        self._add_special_token(start_think_token, "start_think_token")
+
+        end_think_token = "</think>"
+        self._add_special_token(end_think_token, "end_think_token")
+
+        self.start_token_id = None
+        self.start_token = None
+        self.pad_token_id = 0
+
+        super().__init__(
+            vocabulary=vocabulary,
+            merges=merges,
+            **kwargs,
+        )

keras_hub/src/models/smollm3/smollm3_utils.py

@@ -0,0 +1,56 @@
+from keras import ops
+
+
+def rotate_half(x):
+    x1 = x[..., : ops.shape(x)[-1] // 2]
+    x2 = x[..., ops.shape(x)[-1] // 2 :]
+    return ops.concatenate((-x2, x1), axis=-1)
+
+
+def apply_rotary_pos_emb(q, k, cos, sin, expansion_axis=1):
+    cos = ops.expand_dims(cos, expansion_axis)
+    sin = ops.expand_dims(sin, expansion_axis)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+def apply_rotary_pos_single(tensor, cos, sin, expansion_axis=1):
+    cos = ops.expand_dims(cos, expansion_axis)
+    sin = ops.expand_dims(sin, expansion_axis)
+    tensor_embed = (tensor * cos) + (rotate_half(tensor) * sin)
+    return tensor_embed
+
+
+def repeat_kv(hidden_states, n_rep):
+    batch, num_key_value_heads, slen, head_dim = ops.shape(hidden_states)
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = ops.expand_dims(hidden_states, axis=2)
+    target_shape = (batch, num_key_value_heads, n_rep, slen, head_dim)
+    hidden_states = ops.broadcast_to(hidden_states, target_shape)
+    return ops.reshape(
+        hidden_states, [batch, num_key_value_heads * n_rep, slen, head_dim]
+    )
+
+
+def rope_init(rope_theta, partial_rotary_factor, head_dim):
+    """Initialize RoPE (Rotary Position Embedding) parameters.
+
+    Args:
+        rope_theta: float. The theta value for RoPE.
+        partial_rotary_factor: float. The factor for partial rotary embedding.
+        head_dim: int. The dimension of each attention head.
+
+    Returns:
+        A tuple of (inv_freq, attention_scaling) where inv_freq is the inverse
+        frequency tensor and attention_scaling is the scaling factor.
+    """
+    base = rope_theta
+    dim = int(head_dim * partial_rotary_factor)
+
+    inv_freq = 1.0 / (
+        ops.power(base, ops.arange(0, dim, 2, dtype="float32") / dim)
+    )
+    attention_scaling = 1.0
+    return inv_freq, attention_scaling

keras_hub/src/utils/transformers/convert_smollm3.py

@@ -0,0 +1,139 @@
+import numpy as np
+
+from keras_hub.src.models.smollm3.smollm3_backbone import SmolLM3Backbone
+from keras_hub.src.utils.preset_utils import load_json
+
+backbone_cls = SmolLM3Backbone
+
+
+def convert_backbone_config(transformers_config):
+    return {
+        "vocabulary_size": transformers_config["vocab_size"],
+        "hidden_dim": transformers_config["hidden_size"],
+        "num_layers": transformers_config["num_hidden_layers"],
+        "num_attention_heads": transformers_config["num_attention_heads"],
+        "num_key_value_heads": transformers_config["num_key_value_heads"],
+        "intermediate_dim": transformers_config["intermediate_size"],
+        "layer_norm_epsilon": transformers_config[
+            "rms_norm_eps"
+        ],  # Using rms_norm_eps as layer_norm_epsilon
+        "max_position_embeddings": transformers_config[
+            "max_position_embeddings"
+        ],
+        "rope_theta": transformers_config["rope_theta"],
+        # partial_rotary_factor is not explicitly in config.json
+        # but is inherited from the default value in the
+        # `_compute_default_rope_parameters()` function
+        "partial_rotary_factor": 1.0,
+        "attention_bias": transformers_config["attention_bias"],
+        "attention_dropout": transformers_config["attention_dropout"],
+        # Despite the name, no_rope_layers: 1 = HAS RoPE, 0 = NO RoPE
+        "rope_layer_enabled_list": [
+            bool(x) for x in transformers_config["no_rope_layers"]
+        ],
+        "layer_types": transformers_config["layer_types"],
+        "mlp_bias": transformers_config["mlp_bias"],
+    }
+
+
+def convert_weights(backbone, loader, transformers_config):
+    loader.port_weight(
+        keras_variable=backbone.get_layer("token_embedding").embeddings,
+        hf_weight_key="model.embed_tokens.weight",
+    )
+
+    def transpose_and_reshape(x, shape):
+        return np.reshape(np.transpose(x), shape)
+
+    for i in range(backbone.num_layers):
+        decoder_layer = backbone.get_layer(f"transformer_layer_{i}")
+
+        # Input layernorm
+        loader.port_weight(
+            keras_variable=decoder_layer.input_layernorm.scale,
+            hf_weight_key=f"model.layers.{i}.input_layernorm.weight",
+        )
+
+        # Attention layers
+        ## Query
+        loader.port_weight(
+            keras_variable=decoder_layer.self_attn.q_proj.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.q_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+        ## Key
+        loader.port_weight(
+            keras_variable=decoder_layer.self_attn.k_proj.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.k_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+        ## Value
+        loader.port_weight(
+            keras_variable=decoder_layer.self_attn.v_proj.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.v_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+        ## Output
+        loader.port_weight(
+            keras_variable=decoder_layer.self_attn.o_proj.kernel,
+            hf_weight_key=f"model.layers.{i}.self_attn.o_proj.weight",
+            hook_fn=transpose_and_reshape,
+        )
+
+        # MLP layers
+        loader.port_weight(
+            keras_variable=decoder_layer.mlp.up_proj.kernel,
+            hf_weight_key=f"model.layers.{i}.mlp.up_proj.weight",
+            # rearrange_patterns="b a -> a b",
+            hook_fn=lambda hf_tensor, _: np.transpose(hf_tensor, axes=(1, 0)),
+        )
+        loader.port_weight(
+            keras_variable=decoder_layer.mlp.down_proj.kernel,
+            hf_weight_key=f"model.layers.{i}.mlp.down_proj.weight",
+            # rearrange_patterns="b a -> a b",
+            hook_fn=lambda hf_tensor, _: np.transpose(hf_tensor, axes=(1, 0)),
+        )
+        loader.port_weight(
+            keras_variable=decoder_layer.mlp.gate_proj.kernel,
+            hf_weight_key=f"model.layers.{i}.mlp.gate_proj.weight",
+            # rearrange_patterns="b a -> a b",
+            hook_fn=lambda hf_tensor, _: np.transpose(hf_tensor, axes=(1, 0)),
+        )
+
+        # Feedforward layernorm
+        loader.port_weight(
+            keras_variable=decoder_layer.post_attention_layernorm.scale,
+            hf_weight_key=f"model.layers.{i}.post_attention_layernorm.weight",
+        )
+
+    # Final normalization layer
+    loader.port_weight(
+        keras_variable=backbone.get_layer("sequence_output_layernorm").scale,
+        hf_weight_key="model.norm.weight",
+    )
+
+    backbone.training = False
+
+    return backbone
+
+
+def convert_tokenizer(cls, preset, **kwargs):
+    tokenizer_config = load_json(preset, "tokenizer.json")
+    vocab = tokenizer_config["model"]["vocab"]
+    merges = tokenizer_config["model"]["merges"]
+    merges = [" ".join(item) for item in merges]
+
+    # Load all special tokens with the exception of "reserved" ones.
+    special_tokens = set()
+    for token in tokenizer_config["added_tokens"]:
+        if not token["content"].startswith("<|reserved_special_token_"):
+            vocab[token["content"]] = token["id"]
+            special_tokens.add(token["content"])
+
+    kwargs.update(
+        {
+            "unsplittable_tokens": list(special_tokens),
+        }
+    )
+
+    return cls(vocabulary=vocab, merges=merges, **kwargs)

keras_hub/src/utils/transformers/preset_loader.py

@@ -20,6 +20,7 @@ from keras_hub.src.utils.transformers import convert_qwen
 from keras_hub.src.utils.transformers import convert_qwen3
 from keras_hub.src.utils.transformers import convert_qwen3_moe
 from keras_hub.src.utils.transformers import convert_qwen_moe
+from keras_hub.src.utils.transformers import convert_smollm3
 from keras_hub.src.utils.transformers import convert_t5gemma
 from keras_hub.src.utils.transformers import convert_vit
 from keras_hub.src.utils.transformers.safetensor_utils import SafetensorLoader
@@ -66,6 +67,8 @@ class TransformersPresetLoader(PresetLoader):
             self.converter = convert_qwen3_moe
         elif model_type == "qwen3":
             self.converter = convert_qwen3
+        elif model_type == "smollm3":
+            self.converter = convert_smollm3
         elif model_type == "t5gemma":
             self.converter = convert_t5gemma
         else:

keras_hub/src/version.py

@@ -1,7 +1,7 @@
 from keras_hub.src.api_export import keras_hub_export
 
 # Unique source of truth for the version number.
-__version__ = "0.23.0.dev202510170417"
+__version__ = "0.23.0.dev202510190425"
 
 
 @keras_hub_export("keras_hub.version")

keras_hub/tokenizers/__init__.py

@@ -93,6 +93,12 @@ from keras_hub.src.models.roformer_v2.roformer_v2_tokenizer import (
 from keras_hub.src.models.siglip.siglip_tokenizer import (
     SigLIPTokenizer as SigLIPTokenizer,
 )
+from keras_hub.src.models.smollm3.smollm3_tokenizer import (
+    SmolLM3Tokenizer as SmolLM3Tokenizer,
+)
+from keras_hub.src.models.smollm3.smollm3_tokenizer import (
+    SmolLM3Tokenizer as SmolLMTokenizer,
+)
 from keras_hub.src.models.t5.t5_tokenizer import T5Tokenizer as T5Tokenizer
 from keras_hub.src.models.t5gemma.t5gemma_tokenizer import (
     T5GemmaTokenizer as T5GemmaTokenizer,

{keras_hub_nightly-0.23.0.dev202510170417.dist-info → keras_hub_nightly-0.23.0.dev202510190425.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: keras-hub-nightly
-Version: 0.23.0.dev202510170417
+Version: 0.23.0.dev202510190425
 Summary: Pretrained models for Keras.
 Author-email: Keras team <keras-users@googlegroups.com>
 License-Expression: Apache-2.0

{keras_hub_nightly-0.23.0.dev202510170417.dist-info → keras_hub_nightly-0.23.0.dev202510190425.dist-info}/RECORD

@@ -1,11 +1,11 @@
 keras_hub/__init__.py,sha256=bJbUZkqwhZvTb1Tqx1fbkq6mzBYiEyq-Hin3oQIkhdE,558
 keras_hub/layers/__init__.py,sha256=ufJKHxMTFhwp--E3ixfGCZqq89pZOUOxCQYgI5pEUA8,5944
 keras_hub/metrics/__init__.py,sha256=KYalsMPBnfwim9BdGHFfJ5WxUKFXOQ1QoKIMT_0lwlM,439
-keras_hub/models/__init__.py,sha256=E9_kQFlM75iKRiSUFEF7F8e9aqMo89-88XSSLy2sbtY,31172
+keras_hub/models/__init__.py,sha256=yazrEg57HafE6Fgr-bfDXR3iFrArx6RytOEnV1CJJS8,32068
 keras_hub/samplers/__init__.py,sha256=aFQIkiqbZpi8vjrPp2MVII4QUfE-eQjra5fMeHsoy7k,886
 keras_hub/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/api_export.py,sha256=9pQZK27JObxWZ96QPLBp1OBsjWigh1iuV6RglPGMRk0,1499
-keras_hub/src/version.py,sha256=QhHRETV8gMH-iFFSUoHNFk78w1l9wnlAvjWBJYCdzaU,222
+keras_hub/src/version.py,sha256=prwvwrYrxmvIFIyWOgIskLKJZj33Mluw4iVKP1XNOcM,222
 keras_hub/src/layers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/layers/modeling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/layers/modeling/alibi_bias.py,sha256=1XBTHI52L_iJDhN_w5ydu_iMhCuTgQAxEPwcLA6BPuk,4411
@@ -236,7 +236,7 @@ keras_hub/src/models/gemma/gemma_backbone.py,sha256=pAAVaVKB6nlA0PncVnFXvNgJV7Se
 keras_hub/src/models/gemma/gemma_causal_lm.py,sha256=3OXaIXlrKqMIuUnBk-bUz-0SYFL-XkkQTWm8qRY2YII,16770
 keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py,sha256=bpKkEurWIfa6Kp9s4pz84-sBDSA6ZFNHP8nXG1fFQrg,2912
 keras_hub/src/models/gemma/gemma_decoder_block.py,sha256=f5UsRO-VNsKJfm_WHVJWK4UahhzYm3sKprJ8jjr-zm4,7628
-keras_hub/src/models/gemma/gemma_presets.py,sha256=6Okl8USxHhp8EvVebNgo7JDkO527fx0aHdcRy9WNm9k,7467
+keras_hub/src/models/gemma/gemma_presets.py,sha256=wAH7mjz9tbQqqdwajU2dilGytnWK1qc-aTIVLtjpTWg,8263
 keras_hub/src/models/gemma/gemma_tokenizer.py,sha256=FhcyNL4lo63MqOhTQPFr07-u3BddL0fVM4TmOm8ku-I,2622
 keras_hub/src/models/gemma/rms_normalization.py,sha256=fku-JEo2sNy-ytX7ySD1sRzdhRAPmYex_z8oFk1NiG8,833
 keras_hub/src/models/gemma3/__init__.py,sha256=oPFadkdK5DRLD6sYx83iTetY5daWuSzmJilLjokHcbU,257
@@ -469,6 +469,12 @@ keras_hub/src/models/siglip/siglip_presets.py,sha256=gOzSVhLskAthfzq8jWOtQWv14eu
 keras_hub/src/models/siglip/siglip_text_encoder.py,sha256=xOVvzyQHLX9ne30y4ussar99gNMXPXHYKlkbCX_On2Y,5380
 keras_hub/src/models/siglip/siglip_tokenizer.py,sha256=j_67JbIHJDRk-CbiemG2dgAO6lp3_0_JdnfroZ90G18,2579
 keras_hub/src/models/siglip/siglip_vision_encoder.py,sha256=CaNaFq5thBC3TUXXOf2qknk5vWsauM20ZoaDPYRnXcs,5927
+keras_hub/src/models/smollm3/smollm3_backbone.py,sha256=9e8ydwy7X0stVEWgIJYt69vt6JYSCiYpM73w6oLxyoQ,7949
+keras_hub/src/models/smollm3/smollm3_causal_lm.py,sha256=PWn2zPu0YS3uRvmjksmXKXpxehl8lvEHAXaO0u7nweM,12641
+keras_hub/src/models/smollm3/smollm3_causal_lm_preprocessor.py,sha256=gbj7IhDbgA30AM80uG6BeI1yZmGd5yQ2VEaPWWyS9M4,3121
+keras_hub/src/models/smollm3/smollm3_layers.py,sha256=lR33IynX-1G42L3hNzOBnnIx-INOzJguSQDAwIPaSIQ,26632
+keras_hub/src/models/smollm3/smollm3_tokenizer.py,sha256=evOVM8pgZUkWLoXAwWiYRSNNFZ7KBv1WtFdLqpHdCQU,1877
+keras_hub/src/models/smollm3/smollm3_utils.py,sha256=zAqtZTv1G--k-Dbjvk886OcmsuIxyYicRiUQXcpyud0,1904
 keras_hub/src/models/stable_diffusion_3/__init__.py,sha256=ZKYQuaRObyhKq8GVAHmoRvlXp6FpU8ChvutVCHyXKuc,343
 keras_hub/src/models/stable_diffusion_3/flow_match_euler_discrete_scheduler.py,sha256=1K_B3d3fNn50eY84OgxVHyIHHZhmlJY03b71pMSmE9s,3246
 keras_hub/src/models/stable_diffusion_3/mmdit.py,sha256=emyDmtpJiFU_9crSDBC5CaXoZnM1Eti8uAQtwv2v8B0,42794
@@ -602,15 +608,16 @@ keras_hub/src/utils/transformers/convert_qwen.py,sha256=WUxMAEFVqRs7TRw7QU5TH3_e
 keras_hub/src/utils/transformers/convert_qwen3.py,sha256=LIormvCMWPq6X9Wo2eNbADjtFZ0nI7tFGZFBxmo4GKw,5700
 keras_hub/src/utils/transformers/convert_qwen3_moe.py,sha256=4Cr2haS20VNHF1iqKRpKeZ47baV0TAXg87zkCfl-JTg,8876
 keras_hub/src/utils/transformers/convert_qwen_moe.py,sha256=a7R28aln-PdAcNuKAXdrtzvslho2Co6GypChxLMKPpc,10618
+keras_hub/src/utils/transformers/convert_smollm3.py,sha256=V2vWES85YSNXNx39I8OwAcOvSpb9KxUscrDr7ra-LPA,5281
 keras_hub/src/utils/transformers/convert_t5gemma.py,sha256=DPOwd61UhjspKuCsk3_EaNvSADGP_f8KLcZARHYVk5Y,9490
 keras_hub/src/utils/transformers/convert_vit.py,sha256=YAmXh519ecSgEO5B4g-aEQg1Bb_6ifFafLMqDTfLn_c,5259
-keras_hub/src/utils/transformers/preset_loader.py,sha256=_NX7T6ecEQ8I01FkaZApJf-u8JUymNplixR8vcriRlQ,5075
+keras_hub/src/utils/transformers/preset_loader.py,sha256=PmB4wcPaMlqMhrhk2bYt74TvRHRsZgpodfKlzixfr-Q,5219
 keras_hub/src/utils/transformers/safetensor_utils.py,sha256=CYUHyA4y-B61r7NDnCsFb4t_UmSwZ1k9L-8gzEd6KRg,3339
 keras_hub/src/utils/transformers/export/gemma.py,sha256=xX_vfQwvFZ_-lQX4kgMNOGKL7fL_1yk7QyGYV2Qyly4,4699
 keras_hub/src/utils/transformers/export/hf_exporter.py,sha256=Qk52c6LIA2eMHUNY9Vy4STJSpnhLMdJ_t-3ljqhSr4k,5081
-keras_hub/tokenizers/__init__.py,sha256=kyFWYm4mb--U4xYU-2Gb1COM8xEFWNK6LcKxr8h9Ivc,4561
+keras_hub/tokenizers/__init__.py,sha256=XFOxDmM1Mz9TxiE8ICZK_-yTTyRFywUUiVwRIzz2QZ8,4770
 keras_hub/utils/__init__.py,sha256=jXPqVGBpJr_PpYmqD8aDG-fRMlxH-ulqCR2SZMn288Y,646
-keras_hub_nightly-0.23.0.dev202510170417.dist-info/METADATA,sha256=7uTd-YS4ylZmQT0k7rfgGYmhJcoHSw56H1T923OsH6I,7395
-keras_hub_nightly-0.23.0.dev202510170417.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-keras_hub_nightly-0.23.0.dev202510170417.dist-info/top_level.txt,sha256=N4J6piIWBKa38A4uV-CnIopnOEf8mHAbkNXafXm_CuA,10
-keras_hub_nightly-0.23.0.dev202510170417.dist-info/RECORD,,
+keras_hub_nightly-0.23.0.dev202510190425.dist-info/METADATA,sha256=eK6H5tWE7LC1HvLcbo59rJMB8xdUJ29yHGP2MYHm_B4,7395
+keras_hub_nightly-0.23.0.dev202510190425.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+keras_hub_nightly-0.23.0.dev202510190425.dist-info/top_level.txt,sha256=N4J6piIWBKa38A4uV-CnIopnOEf8mHAbkNXafXm_CuA,10
+keras_hub_nightly-0.23.0.dev202510190425.dist-info/RECORD,,