keras-hub-nightly 0.21.0.dev202504170402__py3-none-any.whl → 0.21.0.dev202504190357__py3-none-any.whl

keras_hub/src/models/falcon/falcon_backbone.py

@@ -29,7 +29,7 @@ class FalconBackbone(Backbone):
         layer_norm_epsilon: float. Epsilon for the layer normalization layers in
             the transformer decoder.
         attention_dropout_rate: float. Dropout probability for the attention.
-        feedforward_dropout_rate: flaot. Dropout probability for the
+        feedforward_dropout_rate: float. Dropout probability for the
             feedforward.
         dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use
             for model computations and weights. Note that some computations,

keras_hub/src/models/gemma3/gemma3_causal_lm_preprocessor.py

@@ -512,6 +512,7 @@ class Gemma3CausalLMPreprocessor(CausalLMPreprocessor):
 
         # Extract text part of the input.
         prompts, responses = x["prompts"], x["responses"]
+        tf.debugging.assert_shapes([(prompts, ("N",)), (responses, ("N",))])
 
         # Find out if the input is batched/not batched. Uprank if not batched.
         # In other preprocessors, we don't have to do this, but here, all

keras_hub/src/models/llama/llama_attention.py

@@ -3,7 +3,9 @@ import math
 import keras
 from keras import ops
 
-from keras_hub.src.layers.modeling.rotary_embedding import RotaryEmbedding
+from keras_hub.src.models.llama.llama_rotary_embedding import (
+    LlamaRotaryEmbedding,
+)
 from keras_hub.src.utils.keras_utils import clone_initializer
 from keras_hub.src.utils.keras_utils import fused_attention_op_available
 
@@ -16,7 +18,11 @@ class LlamaAttention(keras.layers.Layer):
         num_query_heads,
         num_key_value_heads,
         rope_max_wavelength=10000,
-        rope_scaling_factor=1.0,
+        rope_position_scaling_factor=1.0,
+        rope_frequency_adjustment_factor=None,
+        rope_low_freq_factor=None,
+        rope_high_freq_factor=None,
+        rope_pretraining_sequence_length=None,
         kernel_initializer="glorot_uniform",
         dropout=0,
         **kwargs,
@@ -28,13 +34,16 @@ class LlamaAttention(keras.layers.Layer):
 
         self.num_key_value_groups = num_query_heads // num_key_value_heads
         self.rope_max_wavelength = rope_max_wavelength
+        self.rope_position_scaling_factor = rope_position_scaling_factor
+        self.rope_frequency_adjustment_factor = rope_frequency_adjustment_factor
+        self.rope_low_freq_factor = rope_low_freq_factor
+        self.rope_high_freq_factor = rope_high_freq_factor
+        self.rope_pretraining_sequence_length = rope_pretraining_sequence_length
 
         self.kernel_initializer = keras.initializers.get(
             clone_initializer(kernel_initializer)
         )
 
-        self.rope_scaling_factor = rope_scaling_factor
-
     def build(self, inputs_shape):
         # Einsum variables:
         # b = batch size
@@ -103,9 +112,13 @@ class LlamaAttention(keras.layers.Layer):
         )
         self._output_dense.build((None, None, self.num_query_heads, head_dim))
 
-        self.rotary_embedding_layer = RotaryEmbedding(
+        self.rotary_embedding_layer = LlamaRotaryEmbedding(
             max_wavelength=self.rope_max_wavelength,
-            scaling_factor=self.rope_scaling_factor,
+            position_scaling_factor=self.rope_position_scaling_factor,
+            frequency_adjustment_factor=self.rope_frequency_adjustment_factor,
+            low_freq_factor=self.rope_low_freq_factor,
+            high_freq_factor=self.rope_high_freq_factor,
+            pretraining_sequence_length=self.rope_pretraining_sequence_length,
             dtype=self.dtype_policy,
         )
 
@@ -224,6 +237,11 @@ class LlamaAttention(keras.layers.Layer):
                 "num_key_value_heads": self.num_key_value_heads,
                 "rope_max_wavelength": self.rope_max_wavelength,
                 "rope_scaling_factor": self.rope_scaling_factor,
+                "rope_low_freq_factor": self.rope_low_freq_factor,
+                "rope_high_freq_factor": self.rope_high_freq_factor,
+                "rope_pretraining_sequence_length": (
+                    self.rope_pretraining_sequence_length
+                ),
                 "kernel_initializer": keras.initializers.serialize(
                     self.kernel_initializer
                 ),

keras_hub/src/models/llama/llama_backbone.py

@@ -30,22 +30,30 @@ class LlamaBackbone(Backbone):
     constructor.
 
     Args:
-        vocabulary_size (int): The size of the token vocabulary.
-        num_layers (int): The number of transformer layers.
-        num_query_heads (int): The number of query attention heads for
+        vocabulary_size: int. The size of the token vocabulary.
+        num_layers: int. The number of transformer layers.
+        num_query_heads : int.  The number of query attention heads for
             each transformer.
-        hidden_dim (int): The size of the transformer encoding and pooling
+        hidden_dim : int.  The size of the transformer encoding and pooling
             layers.
-        intermediate_dim (int): The output dimension of the first Dense layer in
+        intermediate_dim : int. The output dimension of the first Dense layer in
             a three-layer feedforward network for each transformer.
-        num_key_value_heads (int): The number of key and value attention heads
+        num_key_value_heads : int. The number of key and value attention heads
             for each transformer.
-        rope_max_wavelength (int, optional): The maximum angular wavelength of
+        rope_max_wavelength : int. The maximum angular wavelength of
             the sine/cosine curves, for rotary embeddings. Defaults to `10000`.
-        rope_scaling_factor (float, optional): The scaling factor for
-            calculation of roatary embedding. Defaults to `1.0`.
-        layer_norm_epsilon (float, optional): Epsilon for the layer
-            normalization layers in the transformer decoder. Defaults to `1e-6`.
+        rope_position_scaling_factor: float. The scaling factor for
+            calculation of rotary embedding. Defaults to `1.0`
+        rope_frequency_adjustment_factor: float. The scaling factor
+            used to scale the inverse frequencies.  Defaults to `None`.
+        rope_low_freq_factor: float. The low frequency scaling
+            factor. Defaults to `None`.
+        rope_high_freq_factor: float. Used for Llama3.1+. The high
+            frequency scaling factor. Defaults to `None`.
+        rope_pretraining_sequence_length: int. Used for Llama3.1+.
+            Defaults to `None`.
+        layer_norm_epsilon : float. Epsilon for the layer normalization layers
+            in the transformer decoder. Defaults to `1e-6`.
         dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use
             for model computations and weights. Note that some computations,
             such as softmax and layer normalization, will always be done at
@@ -87,7 +95,11 @@ class LlamaBackbone(Backbone):
         intermediate_dim,
         num_key_value_heads,
         rope_max_wavelength=10000,
-        rope_scaling_factor=1.0,
+        rope_position_scaling_factor=1.0,
+        rope_frequency_adjustment_factor=None,
+        rope_low_freq_factor=None,
+        rope_high_freq_factor=None,
+        rope_pretraining_sequence_length=None,
         layer_norm_epsilon=1e-6,
         dropout=0,
         dtype=None,
@@ -110,7 +122,15 @@ class LlamaBackbone(Backbone):
                 num_query_heads=num_query_heads,
                 num_key_value_heads=num_key_value_heads,
                 rope_max_wavelength=rope_max_wavelength,
-                rope_scaling_factor=rope_scaling_factor,
+                rope_position_scaling_factor=rope_position_scaling_factor,
+                rope_frequency_adjustment_factor=(
+                    rope_frequency_adjustment_factor
+                ),
+                rope_low_freq_factor=rope_low_freq_factor,
+                rope_high_freq_factor=rope_high_freq_factor,
+                rope_pretraining_sequence_length=(
+                    rope_pretraining_sequence_length
+                ),
                 layer_norm_epsilon=layer_norm_epsilon,
                 activation=ops.silu,
                 kernel_initializer=_llama_kernel_initializer(stddev=0.02),
@@ -152,9 +172,13 @@ class LlamaBackbone(Backbone):
         self.num_query_heads = num_query_heads
         self.hidden_dim = hidden_dim
         self.intermediate_dim = intermediate_dim
-        self.rope_max_wavelength = rope_max_wavelength
         self.num_key_value_heads = num_key_value_heads
-        self.rope_scaling_factor = rope_scaling_factor
+        self.rope_max_wavelength = rope_max_wavelength
+        self.rope_position_scaling_factor = rope_position_scaling_factor
+        self.rope_frequency_adjustment_factor = rope_frequency_adjustment_factor
+        self.rope_low_freq_factor = rope_low_freq_factor
+        self.rope_high_freq_factor = rope_high_freq_factor
+        self.rope_pretraining_sequence_length = rope_pretraining_sequence_length
         self.layer_norm_epsilon = layer_norm_epsilon
         self.dropout = dropout
         self.tie_word_embeddings = tie_word_embeddings
@@ -169,7 +193,17 @@ class LlamaBackbone(Backbone):
                 "hidden_dim": self.hidden_dim,
                 "intermediate_dim": self.intermediate_dim,
                 "rope_max_wavelength": self.rope_max_wavelength,
-                "rope_scaling_factor": self.rope_scaling_factor,
+                "rope_position_scaling_factor": (
+                    self.rope_position_scaling_factor
+                ),
+                "rope_frequency_adjustment_factor": (
+                    self.rope_frequency_adjustment_factor
+                ),
+                "rope_low_freq_factor": self.rope_low_freq_factor,
+                "rope_high_freq_factor": self.rope_high_freq_factor,
+                "rope_pretraining_sequence_length": (
+                    self.rope_pretraining_sequence_length
+                ),
                 "num_key_value_heads": self.num_key_value_heads,
                 "layer_norm_epsilon": self.layer_norm_epsilon,
                 "dropout": self.dropout,

keras_hub/src/models/llama/llama_decoder.py

@@ -21,7 +21,11 @@ class LlamaTransformerDecoder(keras.layers.Layer):
         num_query_heads,
         num_key_value_heads,
         rope_max_wavelength=10000,
-        rope_scaling_factor=1.0,
+        rope_position_scaling_factor=1.0,
+        rope_frequency_adjustment_factor=None,
+        rope_low_freq_factor=None,
+        rope_high_freq_factor=None,
+        rope_pretraining_sequence_length=None,
         activation="silu",
         layer_norm_epsilon=1e-5,
         kernel_initializer="glorot_uniform",
@@ -34,7 +38,11 @@ class LlamaTransformerDecoder(keras.layers.Layer):
         self.num_key_value_heads = num_key_value_heads
 
         self.rope_max_wavelength = rope_max_wavelength
-        self.rope_scaling_factor = rope_scaling_factor
+        self.rope_position_scaling_factor = rope_position_scaling_factor
+        self.rope_frequency_adjustment_factor = rope_frequency_adjustment_factor
+        self.rope_low_freq_factor = rope_low_freq_factor
+        self.rope_high_freq_factor = rope_high_freq_factor
+        self.rope_pretraining_sequence_length = rope_pretraining_sequence_length
 
         self.dropout = dropout
 
@@ -53,7 +61,11 @@ class LlamaTransformerDecoder(keras.layers.Layer):
             num_query_heads=self.num_query_heads,
             num_key_value_heads=self.num_key_value_heads,
             rope_max_wavelength=self.rope_max_wavelength,
-            rope_scaling_factor=self.rope_scaling_factor,
+            rope_position_scaling_factor=self.rope_position_scaling_factor,
+            rope_frequency_adjustment_factor=self.rope_frequency_adjustment_factor,
+            rope_low_freq_factor=self.rope_low_freq_factor,
+            rope_high_freq_factor=self.rope_high_freq_factor,
+            rope_pretraining_sequence_length=self.rope_pretraining_sequence_length,
             kernel_initializer=clone_initializer(self.kernel_initializer),
             dropout=self.dropout,
             dtype=self.dtype_policy,
@@ -221,6 +233,11 @@ class LlamaTransformerDecoder(keras.layers.Layer):
                 "num_query_heads": self.num_query_heads,
                 "rope_max_wavelength": self.rope_max_wavelength,
                 "rope_scaling_factor": self.rope_scaling_factor,
+                "rope_low_freq_factor": self.rope_low_freq_factor,
+                "rope_high_freq_factor": self.rope_high_freq_factor,
+                "rope_pretraining_sequence_length": (
+                    self.rope_pretraining_sequence_length
+                ),
                 "num_key_value_heads": self.num_key_value_heads,
                 "activation": keras.activations.serialize(self.activation),
                 "layer_norm_epsilon": self.layer_norm_epsilon,

keras_hub/src/models/llama/llama_rotary_embedding.py

@@ -0,0 +1,180 @@
+import math
+
+from keras import ops
+
+from keras_hub.src.layers.modeling.rotary_embedding import RotaryEmbedding
+
+
+class LlamaRotaryEmbedding(RotaryEmbedding):
+    """Rotary positional encoding layer.
+
+    This layer encodes absolute positional information with a rotation
+    matrix. It calculates the rotary encoding with a mix of sine and
+    cosine functions with geometrically increasing wavelengths.
+    Defined and formulated in
+    [RoFormer: Enhanced Transformer with Rotary Position Embedding](https://arxiv.org/abs/2104.09864v4).
+    The input must be a tensor with shape a sequence dimension and a feature
+    dimension. Typically, this will either an input with shape
+    `(batch_size, sequence_length, feature_length)` or
+    `(batch_size, sequence_length, num_heads, feature_length)`.
+    This layer will return a new tensor with the rotary embedding applied to
+    the input tensor.
+    It is extended from `RotaryEmbedding` layer in `keras_hub.layers`.
+    It has additional smoothening and interpolation for some frequency ranges.
+
+    Args:
+        max_wavelength: int. The maximum angular wavelength of the sine/cosine
+            curves. Defaults to `10000`.
+        position_scaling_factor: float. The scaling factor used to scale
+            positions of the tokens. Defaults to `1.0`.
+        frequency_adjustment_factor: float. The scaling factor used to scale the
+            inverse frequencies. Defaults to `None`.
+        low_freq_factor: float. The low frequency scaling factor.
+            Defaults to `None`.
+        high_freq_factor: float. The high frequency scaling factor.
+            Defaults to `None`.
+        pretraining_sequence_length: int. Used for Llama3.1+, the original
+            context length at time of pretraining. Defaults to `None`.
+        sequence_axis: int. Sequence axis in the input tensor.
+        feature_axis: int. Feature axis in the input tensor.
+        **kwargs: other keyword arguments passed to `keras.layers.Layer`,
+            including `name`, `trainable`, `dtype` etc.
+
+    Call arguments:
+        inputs: The tensor inputs to apply the embedding to. This can have
+            any shape, but must contain both a sequence and feature axis. The
+            rotary embedding will be applied to `inputs` and returned.
+        start_index: An integer or integer tensor. The starting position to
+            compute the rotary embedding from. This is useful during cached
+            decoding, where each position is predicted separately in a loop.
+
+    Examples:
+
+    ```python
+    batch_size = 16
+    feature_length = 18
+    sequence_length = 256
+    num_heads = 8
+
+    # No multi-head dimension.
+    tensor = np.ones((batch_size, sequence_length, feature_length))
+    rot_emb_layer = RotaryEmbedding()
+    tensor_rot = rot_emb_layer(tensor)
+
+    # With multi-head dimension.
+    tensor = np.ones((batch_size, sequence_length, num_heads, feature_length))
+    tensor_rot = rot_emb_layer(tensor)
+    ```
+
+    References:
+     - [RoFormer: Enhanced Transformer with Rotary Position Embedding](https://arxiv.org/abs/2104.09864v4)
+    """
+
+    def __init__(
+        self,
+        max_wavelength=10000,
+        position_scaling_factor=1.0,
+        sequence_axis=1,
+        feature_axis=-1,
+        frequency_adjustment_factor=None,
+        low_freq_factor=None,
+        high_freq_factor=None,
+        pretraining_sequence_length=None,
+        **kwargs,
+    ):
+        super().__init__(
+            max_wavelength=max_wavelength,
+            scaling_factor=position_scaling_factor,
+            sequence_axis=sequence_axis,
+            feature_axis=feature_axis,
+            **kwargs,
+        )
+        self.max_wavelength = max_wavelength
+        self.sequence_axis = sequence_axis
+        self.feature_axis = feature_axis
+        self.position_scaling_factor = position_scaling_factor
+        self.frequency_adjustment_factor = frequency_adjustment_factor
+        self.low_freq_factor = low_freq_factor
+        self.high_freq_factor = high_freq_factor
+        self.pretraining_sequence_length = pretraining_sequence_length
+
+        grouped_args = [
+            low_freq_factor,
+            high_freq_factor,
+            frequency_adjustment_factor,
+            pretraining_sequence_length,
+        ]
+        args_none = [x is None for x in grouped_args]
+        if any(args_none) and not all(args_none):
+            raise ValueError(
+                "Either all of `low_freq_factor`,`high_freq_factor`, "
+                "`frequency_adjustment_factor` and "
+                "`pretraining_sequence_length` should be set, or all of should"
+                " be set `None`."
+            )
+        self.built = True
+
+    def _get_inverse_freq(self, rotary_dim):
+        freq_range = ops.divide(
+            ops.arange(0, rotary_dim, 2, dtype="float32"),
+            ops.cast(rotary_dim, "float32"),
+        )
+        inverse_freq = 1.0 / (self.max_wavelength**freq_range)
+
+        # From llama3.1+ we have additional smoothening and interpolation.
+        # low_freq_factor, high_freq_factor, pretraining_sequence_length,
+        # frequency_adjustment_factor are all set at once so it is fine.
+        if self.low_freq_factor is not None:
+            low_freq_wavelen = (
+                self.pretraining_sequence_length / self.low_freq_factor
+            )
+            high_freq_wavelen = (
+                self.pretraining_sequence_length / self.high_freq_factor
+            )
+            wavelen = 2 * math.pi / inverse_freq
+
+            # wavelen < high_freq_wavelen: do nothing
+            # wavelen > low_freq_wavelen: divide by factor
+            inverse_freq = ops.where(
+                ops.greater(wavelen, low_freq_wavelen),
+                (inverse_freq / self.frequency_adjustment_factor),
+                inverse_freq,
+            )
+
+            # otherwise: interpolate between the two, using a smooth factor
+            smooth_factor = (
+                (self.pretraining_sequence_length / wavelen)
+                - self.low_freq_factor
+            ) / (self.high_freq_factor - self.low_freq_factor)
+            smoothed_inv_freq = (1 - smooth_factor) * (
+                inverse_freq / self.frequency_adjustment_factor
+            ) + (smooth_factor * inverse_freq)
+            is_medium_freq = ops.logical_and(
+                ops.greater_equal(wavelen, high_freq_wavelen),
+                ops.less_equal(wavelen, low_freq_wavelen),
+            )
+
+            inverse_freq = ops.where(
+                is_medium_freq, smoothed_inv_freq, inverse_freq
+            )
+
+        return inverse_freq
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "max_wavelength": self.max_wavelength,
+                "sequence_axis": self.sequence_axis,
+                "feature_axis": self.feature_axis,
+                "position_scaling_factor": self.position_scaling_factor,
+                "frequency_adjustment_factor": self.frequency_adjustment_factor,
+                "low_freq_factor": self.low_freq_factor,
+                "high_freq_factor": self.high_freq_factor,
+                "original_max_embeddings": self.pretraining_sequence_length,
+            }
+        )
+        return config
+
+    def compute_output_shape(self, input_shape):
+        return input_shape

keras_hub/src/models/llama3/llama3_backbone.py

@@ -32,8 +32,16 @@ class Llama3Backbone(LlamaBackbone):
             fo each transformer.
         rope_max_wavelength (int, optional): The maximum angular wavelength of
             the sine/cosine curves, for rotary embeddings. Defaults to `10000`.
-        rope_scaling_factor (float, optional): The scaling factor for
-            calculation of roatary embedding. Defaults to `1.0`.
+        rope_position_scaling_factor (float, optional): The scaling factor for
+            calculation of roatary embedding. Defaults to `1.0`
+        rope_requency_adjustment_factor (float, optional): The scaling factor
+            used to scale the inverse frequencies.
+        rope_low_freq_factor (float, optional): The low frequency factor.
+            Defaults to None.
+        rope_high_freq_factor: (float, optional) Used for Llama3.1+. The high
+            frequency factor. Defaults to None.
+        rope_pretraining_sequence_length: (int, optional) Sequence length during
+            original pretraining. Defaults to None.
         layer_norm_epsilon (float, optional): Epsilon for the layer
             normalization layers in the transformer decoder. Defaults to `1e-6`.
         dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use

keras_hub/src/models/llama3/llama3_presets.py

@@ -45,4 +45,86 @@ backbone_presets = {
             "kaggle://keras/llama3/keras/llama3_instruct_8b_en_int8/2"
         ),
     },
+    "llama3.1_8b": {
+        "metadata": {
+            "description": (
+                "8 billion parameter, 32-layer, based LLaMA 3.1 model. "
+            ),
+            "params": 8030261248,
+            "path": "llama3",
+        },
+        "kaggle_handle": ("kaggle://keras/llama3/keras/llama3.1_8b/1"),
+    },
+    "llama3.1_instruct_8b": {
+        "metadata": {
+            "description": (
+                "8 billion parameter, 32-layer, instruction tuned LLaMA 3.1. "
+            ),
+            "params": 8030261248,
+            "path": "llama3",
+        },
+        "kaggle_handle": ("kaggle://keras/llama3/keras/lama3.1_instruct_8b/1"),
+    },
+    "llama3.1_guard_8b": {
+        "metadata": {
+            "description": (
+                "8 billion parameter, 32-layer, LLaMA 3.1 fine-tuned for "
+                "consent safety classification. "
+            ),
+            "params": 8030261248,
+            "path": "llama3",
+        },
+        "kaggle_handle": ("kaggle://keras/llama3/keras/llama3.1_guard_8b/1"),
+    },
+    "llama3.2_1b": {
+        "metadata": {
+            "description": (
+                "1 billion parameter, 16-layer, based LLaMA 3.2 model. "
+            ),
+            "params": 1498482688,
+            "path": "llama3",
+        },
+        "kaggle_handle": ("kaggle://keras/llama3/keras/llama3.2_1b/1"),
+    },
+    "llama3.2_instruct_1b": {
+        "metadata": {
+            "description": (
+                "1 billion parameter, 16-layer, instruction tuned LLaMA 3.2. "
+            ),
+            "params": 1498482688,
+            "path": "llama3",
+        },
+        "kaggle_handle": ("kaggle://keras/llama3/keras/llama3.2_instruct_1b/1"),
+    },
+    "llama3.2_3b": {
+        "metadata": {
+            "description": (
+                "3 billion parameter, 26-layer, based LLaMA 3.2 model. "
+            ),
+            "params": 3606752256,
+            "path": "llama3",
+        },
+        "kaggle_handle": ("kaggle://keras/llama3/keras/llama3.2_3b/1"),
+    },
+    "llama3.2_instruct_3b": {
+        "metadata": {
+            "description": (
+                "3 billion parameter, 28-layer, instruction tuned LLaMA 3.2. "
+            ),
+            "params": 3606752256,
+            "path": "llama3",
+        },
+        "kaggle_handle": ("kaggle://keras/llama3/keras/llama3.2_instruct_3b/1"),
+    },
+    "llama3.2_guard_1b": {
+        "metadata": {
+            "description": (
+                "1 billion parameter, 16-layer, based LLaMA 3.2 model "
+                "fine-tuned for consent safety classification. "
+            ),
+            "params": 1498482688,
+            "path": "llama3",
+        },
+        "kaggle_handle": ("kaggle://keras/llama3/keras/llama3.2_guard_1b/1"),
+    },
 }

keras_hub/src/utils/transformers/convert_llama3.py

@@ -7,7 +7,7 @@ backbone_cls = Llama3Backbone
 
 
 def convert_backbone_config(transformers_config):
-    return {
+    backbone_config = {
         "vocabulary_size": transformers_config["vocab_size"],
         "num_layers": transformers_config["num_hidden_layers"],
         "num_query_heads": transformers_config["num_attention_heads"],
@@ -15,8 +15,28 @@ def convert_backbone_config(transformers_config):
         "intermediate_dim": transformers_config["intermediate_size"],
         "num_key_value_heads": transformers_config["num_key_value_heads"],
         "tie_word_embeddings": transformers_config["tie_word_embeddings"],
+        "rope_max_wavelength": transformers_config["rope_theta"],
     }
 
+    if transformers_config.get("rope_scaling", None) is not None:
+        if transformers_config["rope_scaling"]["rope_type"] != "llama3":
+            raise ValueError("The config should be a valid llama3 config.")
+        backbone_config["rope_frequency_adjustment_factor"] = (
+            transformers_config["rope_scaling"]["factor"]
+        )
+        backbone_config["rope_low_freq_factor"] = transformers_config[
+            "rope_scaling"
+        ]["low_freq_factor"]
+        backbone_config["rope_high_freq_factor"] = transformers_config[
+            "rope_scaling"
+        ]["high_freq_factor"]
+        backbone_config["rope_pretraining_sequence_length"] = (
+            transformers_config["rope_scaling"][
+                "original_max_position_embeddings"
+            ]
+        )
+    return backbone_config
+
 
 def convert_weights(backbone, loader, transformers_config):
     loader.port_weight(

keras_hub/src/version_utils.py

@@ -1,7 +1,7 @@
 from keras_hub.src.api_export import keras_hub_export
 
 # Unique source of truth for the version number.
-__version__ = "0.21.0.dev202504170402"
+__version__ = "0.21.0.dev202504190357"
 
 
 @keras_hub_export("keras_hub.version")

{keras_hub_nightly-0.21.0.dev202504170402.dist-info → keras_hub_nightly-0.21.0.dev202504190357.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: keras-hub-nightly
-Version: 0.21.0.dev202504170402
+Version: 0.21.0.dev202504190357
 Summary: Industry-strength Natural Language Processing extensions for Keras.
 Home-page: https://github.com/keras-team/keras-hub
 Author: Keras team

{keras_hub_nightly-0.21.0.dev202504170402.dist-info → keras_hub_nightly-0.21.0.dev202504190357.dist-info}/RECORD

@@ -8,7 +8,7 @@ keras_hub/api/tokenizers/__init__.py,sha256=NCQSOg3vf3KlM2YBsxApcJUVu9MH2jV0NQrM
 keras_hub/api/utils/__init__.py,sha256=Gp1E6gG-RtKQS3PBEQEOz9PQvXkXaJ0ySGMqZ7myN7A,215
 keras_hub/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/api_export.py,sha256=9pQZK27JObxWZ96QPLBp1OBsjWigh1iuV6RglPGMRk0,1499
-keras_hub/src/version_utils.py,sha256=um5nImV3kQfkhp9f7hoNHS8pkeGqPkhA4xKqbhBdupQ,222
+keras_hub/src/version_utils.py,sha256=JHxB700m8f2SDoTYZCedVnki2N4OQO1V_32-jRHk4tU,222
 keras_hub/src/layers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/layers/modeling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/layers/modeling/alibi_bias.py,sha256=1XBTHI52L_iJDhN_w5ydu_iMhCuTgQAxEPwcLA6BPuk,4411
@@ -173,7 +173,7 @@ keras_hub/src/models/f_net/f_net_text_classifier_preprocessor.py,sha256=UUa7RKyl
 keras_hub/src/models/f_net/f_net_tokenizer.py,sha256=ZRTaSfgZnYLTVXgM51303LpryRsSL5GaC2Cl_D7g27A,2285
 keras_hub/src/models/falcon/__init__.py,sha256=IVwPgPbw0l8XJRPQETmeQNvpdn_SneXhe_3oRMOvdx8,257
 keras_hub/src/models/falcon/falcon_attention.py,sha256=fRHuK_y_w64hrqq0XYfcsycs3KD1_3RmeKP7j8LEjGU,4559
-keras_hub/src/models/falcon/falcon_backbone.py,sha256=nGJcHnbqncZRTPERRi4ZuYGcODpkH2Mu0-Db59vH5io,5451
+keras_hub/src/models/falcon/falcon_backbone.py,sha256=hRwomKH_GIKJ0KMfccpHVU43HVN0WQy1n9PldvlUaTM,5451
 keras_hub/src/models/falcon/falcon_causal_lm.py,sha256=2UEIeju5Tg-FstVuusejJ-MbHZ6vsNfsSJzzBM89fnU,10908
 keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py,sha256=nI9E8N9enx5DppDHpLwGslb65rqGorL2sEz1jzet4gA,3033
 keras_hub/src/models/falcon/falcon_presets.py,sha256=PDghkND0-7le4W-atm4BitzA127z-5ZyQguCnCChSBo,463
@@ -199,7 +199,7 @@ keras_hub/src/models/gemma3/__init__.py,sha256=oPFadkdK5DRLD6sYx83iTetY5daWuSzmJ
 keras_hub/src/models/gemma3/gemma3_attention.py,sha256=VstFCTVsplcDNSgnyBcSpLgKn-pktJ39D5Ri-Bb7BQA,13628
 keras_hub/src/models/gemma3/gemma3_backbone.py,sha256=xw6gbFZWZuREcN1iyPj-1Hm-3EmRglgFD5fQSzDp3zA,16439
 keras_hub/src/models/gemma3/gemma3_causal_lm.py,sha256=U3C9TWlIz8VefAxQ0wJ6bDz18wqHBie8B26Ub_nFZs4,13843
-keras_hub/src/models/gemma3/gemma3_causal_lm_preprocessor.py,sha256=HPIkSRAevePLEWx-t6oqtaOdxF0FjeBQKAg2Ey9axLA,29524
+keras_hub/src/models/gemma3/gemma3_causal_lm_preprocessor.py,sha256=vjt4N-zr0Eb5kvkOR-WUgskDTNe64L_6tYnhyNb6xaE,29601
 keras_hub/src/models/gemma3/gemma3_decoder_block.py,sha256=6PLlpDxxF67stDv74fw9nNgUHBWmTLx6qGygJwyu5FY,10819
 keras_hub/src/models/gemma3/gemma3_image_converter.py,sha256=czi5JrTyKiK0nFzvonviBIX8jjvLHqvGNA9RyheB31k,536
 keras_hub/src/models/gemma3/gemma3_interleave_embeddings.py,sha256=_Q5hvhA93HAJe-A2IBRKVu0_RDVht61lFQiYse_9Rm4,4597
@@ -222,19 +222,20 @@ keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py,sha256=YiVz9q
 keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py,sha256=hmB81V0SuI6bEsxEuFkYgq58wbcrv1YLvmXGin5T3E0,9732
 keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py,sha256=aKso-8yGrynn3tZ5xm2egcXIBQo3__sWZDBtjmS3ZgU,1991
 keras_hub/src/models/llama/__init__.py,sha256=svVZjGi71R3lVbq0AdbqlXj909mr3Rp9EPXdiO0w0G0,251
-keras_hub/src/models/llama/llama_attention.py,sha256=Q5N37sAESAjdFg9GNlanvNbD-dHS3mNNtt3vMXAFKMs,7931
-keras_hub/src/models/llama/llama_backbone.py,sha256=tjNEIKIL9ncoEL5KNFE5i0oTUkysjmJmh3mHmCz4RCw,11861
+keras_hub/src/models/llama/llama_attention.py,sha256=UFHOWr69vTkOxLdgSUckGaSuUUyqlJ_xYoswWHVnTOU,8977
+keras_hub/src/models/llama/llama_backbone.py,sha256=AT8kUPHEn6DT-aGY838_sZkBhByIdh82DWW8y-Sp3mE,13614
 keras_hub/src/models/llama/llama_causal_lm.py,sha256=9bP4-XDCMgsZuH1ILIMzmwq2Fyy6vkk1Vsht-lMGCNo,13258
 keras_hub/src/models/llama/llama_causal_lm_preprocessor.py,sha256=VTboOMiRBoxHrwP343upLUTsv3AG65r2H8h_PNPVphE,3047
-keras_hub/src/models/llama/llama_decoder.py,sha256=6iERIblED0ZB5w_EUlHks4UvMnsrWONdO_Xdz2OzhWM,8623
+keras_hub/src/models/llama/llama_decoder.py,sha256=CfWI8ru1-uWjDs0sL6H7g8ElYXWu6h7c5XIx-2Y8lX8,9668
 keras_hub/src/models/llama/llama_layernorm.py,sha256=LfRbePHUJs00Ptf7dvNaw3Aj9n1xBMBpE_rS5zzsYMo,1050
 keras_hub/src/models/llama/llama_presets.py,sha256=k0JPQggSQ0XUkhiPlfM0gTqHXGOt39InVLglPUi4AJU,1902
+keras_hub/src/models/llama/llama_rotary_embedding.py,sha256=nqQGl7lFXJq7xGBfoONx2-wuuvKdoydnzUjy6FGQjwo,7300
 keras_hub/src/models/llama/llama_tokenizer.py,sha256=NKWhxTutQ2jd6sd3NSTy9plQyKGCmuNG7U6kVxhZU4Y,1981
 keras_hub/src/models/llama3/__init__.py,sha256=Vqvr2E10cnANkrRQGNBJtVLNAu-Bg9Lx6sqKOZWFy_8,257
-keras_hub/src/models/llama3/llama3_backbone.py,sha256=g_IkHys5cr0gBXhDiqgIICO93RdGAm6WS5NK2SPhFvM,2866
+keras_hub/src/models/llama3/llama3_backbone.py,sha256=TEocD8X7GihQFGJAz3jPwLCqDb86nyeZ1DqBF7RgQLE,3366
 keras_hub/src/models/llama3/llama3_causal_lm.py,sha256=qk_onuf7S6d7rxAntilq2Q2orggMbPEJbNHJNVe2G0U,1541
 keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py,sha256=twbXel9hsQgGxDAoQhEQuVm2udnEybI4fAQTJzXAuBs,3064
-keras_hub/src/models/llama3/llama3_presets.py,sha256=PWEW_hLMCD9SIYm3QLhRVIcwjrPuqv-KDebXACXRNbM,1579
+keras_hub/src/models/llama3/llama3_presets.py,sha256=--_6Uao-fK4xD4ShgsqzKmlyQPyO9tRkF0VDYKjGpNw,4302
 keras_hub/src/models/llama3/llama3_tokenizer.py,sha256=J-KxRc08vGs4olFw_4mtJs0W_dTeUyj_XxMycazBmxI,1934
 keras_hub/src/models/mistral/__init__.py,sha256=vjBlzcrIsFSwJKnfwfTNMKstIEKGFTE3kVcdAdfwlnE,263
 keras_hub/src/models/mistral/mistral_attention.py,sha256=nGDlD4NcIwIGlfbt3ArxdT5QAvamY7yiNEGDlTgWirU,8609
@@ -458,14 +459,14 @@ keras_hub/src/utils/transformers/convert_bert.py,sha256=4gQqXCJzC9QWdLPDUAq741K8
 keras_hub/src/utils/transformers/convert_distilbert.py,sha256=SlfIRhSRk5c1ir2HGiDPiXa5XdOId_DbcnZO9lbwyZ8,6498
 keras_hub/src/utils/transformers/convert_gemma.py,sha256=ElCgwBpSN5Q7rV5PJawTsoytPzs5ZjuwoY60YAe8y_A,6533
 keras_hub/src/utils/transformers/convert_gpt2.py,sha256=HCeHN_-GiQJRxLCM9OCJJ1watPVpIBF8ujS8pGbBOWc,5703
-keras_hub/src/utils/transformers/convert_llama3.py,sha256=zlg0yFscjytyOFymDwqnbuXkmYvb88qqYzAROKcpaPU,5250
+keras_hub/src/utils/transformers/convert_llama3.py,sha256=c5phNl-QayQ_BS0s-lenbu6oHxqfwDShKJoh9DluxUU,6146
 keras_hub/src/utils/transformers/convert_mistral.py,sha256=kVhN9h1ZFVhwkNW8p3wnS7eANJUXIsNy1RxWXy20Gqw,4760
 keras_hub/src/utils/transformers/convert_pali_gemma.py,sha256=B1leeDw96Yvu81hYumf66hIid07k5NLqoeWAJgPnaLs,10649
 keras_hub/src/utils/transformers/convert_qwen.py,sha256=WUxMAEFVqRs7TRw7QU5TH3_ev4yf02R1xFVliMvTQqg,5886
 keras_hub/src/utils/transformers/convert_vit.py,sha256=9SUZ9utNJhW_5cj3acMn9cRy47u2eIcDsrhmzj77o9k,5187
 keras_hub/src/utils/transformers/preset_loader.py,sha256=0Hi7R8HnATcwFVLsJwMMIMWTCXHNfep4IPiRpQXqM-w,3933
 keras_hub/src/utils/transformers/safetensor_utils.py,sha256=CYUHyA4y-B61r7NDnCsFb4t_UmSwZ1k9L-8gzEd6KRg,3339
-keras_hub_nightly-0.21.0.dev202504170402.dist-info/METADATA,sha256=ADjI3HBVo2_uW8UYGR2aA6Uuetr05sEOpr2BcAR9SY0,7715
-keras_hub_nightly-0.21.0.dev202504170402.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-keras_hub_nightly-0.21.0.dev202504170402.dist-info/top_level.txt,sha256=N4J6piIWBKa38A4uV-CnIopnOEf8mHAbkNXafXm_CuA,10
-keras_hub_nightly-0.21.0.dev202504170402.dist-info/RECORD,,
+keras_hub_nightly-0.21.0.dev202504190357.dist-info/METADATA,sha256=upX5yCkjjUPOo3UnzvGqlIQVsM8hYfO5VlOKWL7aOI4,7715
+keras_hub_nightly-0.21.0.dev202504190357.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+keras_hub_nightly-0.21.0.dev202504190357.dist-info/top_level.txt,sha256=N4J6piIWBKa38A4uV-CnIopnOEf8mHAbkNXafXm_CuA,10
+keras_hub_nightly-0.21.0.dev202504190357.dist-info/RECORD,,