keras-hub-nightly 0.20.0.dev202504010407__py3-none-any.whl → 0.20.0.dev202504020401__py3-none-any.whl

keras_hub/api/models/__init__.py

@@ -323,6 +323,24 @@ from keras_hub.src.models.roberta.roberta_text_classifier_preprocessor import (
     RobertaTextClassifierPreprocessor as RobertaPreprocessor,
 )
 from keras_hub.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone as RorformerV2Backbone,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_masked_lm import (
+    RoformerV2MaskedLM,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_masked_lm_preprocessor import (
+    RoformerV2MaskedLMPreprocessor,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_text_classifier import (
+    RorformerV2TextClassifier,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_text_classifier_preprocessor import (
+    RoformerV2TextClassifierPreprocessor,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_tokenizer import (
+    RoformerV2Tokenizer,
+)
 from keras_hub.src.models.sam.sam_backbone import SAMBackbone
 from keras_hub.src.models.sam.sam_image_segmenter import SAMImageSegmenter
 from keras_hub.src.models.sam.sam_image_segmenter_preprocessor import (

keras_hub/api/tokenizers/__init__.py

@@ -35,6 +35,9 @@ from keras_hub.src.models.qwen.qwen_tokenizer import (
     QwenTokenizer as Qwen2Tokenizer,
 )
 from keras_hub.src.models.roberta.roberta_tokenizer import RobertaTokenizer
+from keras_hub.src.models.roformer_v2.roformer_v2_tokenizer import (
+    RoformerV2Tokenizer,
+)
 from keras_hub.src.models.siglip.siglip_tokenizer import SigLIPTokenizer
 from keras_hub.src.models.t5.t5_tokenizer import T5Tokenizer
 from keras_hub.src.models.whisper.whisper_tokenizer import WhisperTokenizer

keras_hub/src/models/gemma/gemma_attention.py

@@ -133,6 +133,13 @@ class CachedGemmaAttention(keras.layers.Layer):
             query_normalization = 1 / np.sqrt(
                 self.hidden_dim // self.num_query_heads
             )
+
+        if self.use_sliding_window_attention and attention_mask is not None:
+            attention_mask = self._mask_sliding_window(
+                attention_mask,
+                cache_update_index=cache_update_index,
+            )
+
         if self._can_use_flash_attention():
             if attention_mask is not None:
                 attention_mask = ops.expand_dims(attention_mask, axis=1)
@@ -172,13 +179,8 @@ class CachedGemmaAttention(keras.layers.Layer):
                 ops.tanh(attention_logits), self.logit_soft_cap
             )
 
-        if self.use_sliding_window_attention:
-            attention_mask = self._mask_sliding_window(
-                attention_mask,
-                cache_update_index=cache_update_index,
-            )
-
-        attention_mask = attention_mask[:, None, None, :, :]
+        if attention_mask is not None:
+            attention_mask = attention_mask[:, None, None, :, :]
         orig_dtype = attention_logits.dtype
         attention_softmax = self.softmax(attention_logits, mask=attention_mask)
         attention_softmax = ops.cast(attention_softmax, orig_dtype)

keras_hub/src/models/roformer_v2/roformer_v2_attention.py

@@ -0,0 +1,212 @@
+import keras
+from keras import initializers
+from keras import ops
+
+
+class RoformerNorm(keras.layers.Layer):
+    """A normalization layer for Roformer that implements RMS normalization."""
+
+    def __init__(self, epsilon=1e-6, **kwargs):
+        super().__init__(**kwargs)
+        self.epsilon = epsilon
+
+    def build(self, input_shape):
+        dim = input_shape[-1]
+        self.scale = self.add_weight(
+            name="scale",
+            trainable=True,
+            shape=(dim,),
+            initializer="ones",
+            dtype=self.variable_dtype,
+        )
+        self.built = True
+
+    def call(self, x):
+        x = ops.cast(x, "float32")
+        var = ops.mean(ops.power(x, 2), axis=-1, keepdims=True)
+        x = x * ops.rsqrt(var + self.epsilon)
+        return ops.cast(x * self.scale, self.compute_dtype)
+
+    def get_config(self):
+        config = super().get_config()
+        config.update({"epsilon": self.epsilon})
+        return config
+
+
+class RoformrPositionalEmbedding(keras.layers.Layer):
+    """Native rotary implement by jianlin su
+    from native implement
+    https://github.com/bojone/bert4keras
+
+    """
+
+    def __init__(self, output_dim, max_wavelength=10000, **kwargs):
+        super().__init__(**kwargs)
+        self.max_wavelength = max_wavelength
+        self.output_dim = output_dim
+
+    def call(self, tensors):
+        input_shape = ops.shape(tensors[0])
+        seq_len = input_shape[1]
+        position_ids = ops.arange(0, seq_len, dtype=tensors[0].dtype)[None]
+        embeddings = self.sinusoidal_embeddings(
+            position_ids, self.output_dim, self.max_wavelength
+        )
+        embeddings = ops.cast(embeddings, self.compute_dtype)
+
+        ndim = ops.ndim(tensors[0])
+        sinusoidal = self.align(embeddings, [0, 1, -1], ndim)
+        cos_pos = ops.repeat(sinusoidal[..., 1::2], 2, -1)
+        sin_pos = ops.repeat(sinusoidal[..., ::2], 2, -1)
+        outputs = []
+        for tensor in tensors:
+            tensor2 = ops.stack([-tensor[..., 1::2], tensor[..., ::2]], ndim)
+            tensor2 = ops.reshape(tensor2, ops.shape(tensor))
+            outputs.append(tensor * cos_pos + tensor2 * sin_pos)
+        return outputs[0] if len(outputs) == 1 else outputs
+
+    def align(self, tensor, axes, ndim=None):
+        ndim = ndim or max(axes) + 1
+        indices = [None] * ndim
+        for i in axes:
+            indices[i] = slice(None)
+        if keras.config.backend() == "jax":
+            return tensor[tuple(indices)]
+        return tensor[indices]
+
+    def sinusoidal_embeddings(self, pos, dim, base=10000):
+        if dim % 2 != 0:
+            raise ("Dimension must be even")
+
+        indices = ops.arange(0, dim // 2, dtype="float32")
+        indices = ops.power(ops.cast(base, dtype="float32"), -2 * indices / dim)
+        embeddings = ops.einsum("...,d->...d", pos, indices)
+        embeddings = ops.stack(
+            [ops.sin(embeddings), ops.cos(embeddings)], axis=-1
+        )
+        shape = list(ops.shape(embeddings))
+        embeddings = ops.reshape(embeddings, shape[:-2] + [-1])
+        return embeddings
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "out_dim": self.out_dim,
+                "max_wavelength": self.max_wavelength,
+            }
+        )
+        return config
+
+
+@keras.saving.register_keras_serializable(package="keras_hub")
+class RoformerAttention(keras.layers.Layer):
+    """MultiHeadAttention by roformerV2
+
+    modifity from native implement
+    https://github.com/bojone/bert4keras
+    """
+
+    def __init__(
+        self,
+        heads,
+        head_size,
+        out_dim=None,
+        use_bias=False,
+        max_wavelength=10000,
+        kernel_initializer="glorot_uniform",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.heads = heads
+        self.head_size = head_size
+        self.out_dim = out_dim or heads * head_size
+        self.use_bias = use_bias
+        self.kernel_initializer = initializers.get(kernel_initializer)
+        self.max_wavelength = max_wavelength
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        self.q_dense = keras.layers.Dense(
+            units=self.head_size * self.heads,
+            use_bias=self.use_bias,
+            kernel_initializer=self.kernel_initializer,
+            name="q_dense_layer",
+            dtype=self.dtype_policy,
+        )
+        self.q_dense.build(input_shape)
+
+        self.k_dense = keras.layers.Dense(
+            units=self.head_size * self.heads,
+            use_bias=self.use_bias,
+            kernel_initializer=self.kernel_initializer,
+            name="k_dense_layer",
+            dtype=self.dtype_policy,
+        )
+        self.k_dense.build(input_shape)
+
+        self.v_dense = keras.layers.Dense(
+            units=self.head_size * self.heads,
+            use_bias=self.use_bias,
+            kernel_initializer=self.kernel_initializer,
+            name="v_dense_layer",
+            dtype=self.dtype_policy,
+        )
+        self.v_dense.build(input_shape)
+
+        self.o_dense = keras.layers.Dense(
+            units=self.out_dim,
+            use_bias=self.use_bias,
+            kernel_initializer=self.kernel_initializer,
+            name="o_dense_layer",
+            dtype=self.dtype_policy,
+        )
+        self.o_dense.build([None, None, self.head_size * self.heads])
+
+        self.rotary_embedding_layer = RoformrPositionalEmbedding(
+            self.head_size, self.max_wavelength, dtype=self.dtype_policy
+        )
+        self.rotary_embedding_layer.build([])
+
+    def call(self, x, attention_mask=None):
+        qw = self.q_dense(x)
+        kw = self.k_dense(x)
+        vw = self.v_dense(x)
+
+        b, s = ops.shape(qw)[:2]
+        qw = ops.reshape(qw, (b, s, self.heads, self.head_size))
+        kw = ops.reshape(kw, (b, s, self.heads, self.head_size))
+        vw = ops.reshape(vw, (b, s, self.heads, self.head_size))
+
+        qw, kw = self.rotary_embedding_layer([qw, kw])
+        if keras.__version__ < "3.6":
+            raise ("Please make sure your Keras version is >=3.6.")
+        flash_attention = keras.config.is_flash_attention_enabled()
+        attention_mask = ops.reshape(attention_mask, [b, 1, s, 1])
+        if keras.config.backend() == "torch":
+            attention_mask = ops.repeat(attention_mask, s, -1)
+            attention_mask = ops.transpose(attention_mask, [0, 1, 3, 2])
+        o = ops.dot_product_attention(
+            qw, kw, vw, mask=attention_mask, flash_attention=flash_attention
+        )
+
+        return self.o_dense(ops.reshape(o, [b, s, -1]))
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "heads": self.heads,
+                "head_size": self.head_size,
+                "out_dim": self.out_dim,
+                "use_bias": self.use_bias,
+                "max_wavelength": self.max_wavelength,
+                "kernel_initializer": initializers.serialize(
+                    self.kernel_initializer
+                ),
+            }
+        )
+        return config

keras_hub/src/models/roformer_v2/roformer_v2_backbone.py

@@ -0,0 +1,198 @@
+import keras
+from keras import activations
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.layers.modeling.reversible_embedding import (
+    ReversibleEmbedding,
+)
+from keras_hub.src.models.backbone import Backbone
+from keras_hub.src.models.roformer_v2.roformer_v2_attention import RoformerNorm
+from keras_hub.src.models.roformer_v2.roformer_v2_encoder import (
+    RoformerV2Encoder,
+)
+
+
+def roformer_kernel_initializer(stddev=0.02):
+    return keras.initializers.TruncatedNormal(stddev=stddev)
+
+
+@keras_hub_export("keras_hub.models.RorformerV2Backbone")
+class RoformerV2Backbone(Backbone):
+    """A RoformerV2 encoder network.
+
+    This class implements a bi-directional Transformer-based encoder as
+    described in ["Roformer"](https://github.com/ZhuiyiTechnology/roformer).
+    It includes the
+    embedding lookups and transformer layers, but not the masked language model
+    or next sentence prediction heads.
+
+    The default constructor gives a fully customizable, randomly initialized
+    RoformerV2 encoder with any number of layers, heads, and embed dim.To
+    load preset architectures and weights, use the `from_preset()` constructor.
+
+    Disclaimer: Pre-trained models are provided on an "as is" basis, without
+    warranties or conditions of any kind.
+
+    Args:
+        vocabulary_size: int. The size of the token vocabulary.
+        num_layers: int. The number of transformer layers.
+        num_heads: int. The number of attention heads for each transformer.
+            The hidden size must be divisible by the number of attention heads.
+        hidden_dim: int. The size of the transformer encoding and pooler layers.
+        intermediate_dim: int. The output dimension of the first Dense layer in
+            a two-layer feedforward network for each transformer.
+        dropout: float. Dropout probability for the Transformer encoder.
+        num_segments: int. The number of types that the 'segment_ids' input can
+            take.
+        dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use
+            for model computations and weights. Note that some computations,
+            such as softmax and layer normalization, will always be done at
+            float32 precision regardless of dtype.
+
+    Examples:
+    ```python
+    input_data = {
+        "token_ids": np.ones(shape=(1, 12), dtype="int32"),
+        "segment_ids": np.array([[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0]]),
+        "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]]),
+    }
+
+    # Pretrained RoformerV2 encoder.
+    model = keras_hub.models.RoformerV2Backbone.from_preset("roformer_v2_base")
+    model(input_data)
+
+    # Randomly initialized RoformerV2 encoder with a custom config.
+    model = keras_hub.models.RoformerV2Backbone(
+        vocabulary_size=30552,
+        num_layers=4,
+        num_heads=4,
+        hidden_dim=256,
+        intermediate_dim=512,
+        head_size = 64,
+    )
+    model(input_data)
+    ```
+    """
+
+    def __init__(
+        self,
+        vocabulary_size,
+        num_layers,
+        num_heads,
+        hidden_dim,
+        intermediate_dim,
+        head_size,
+        use_bias=False,
+        activation="relu",
+        dropout=0.1,
+        num_segments=2,
+        dtype=None,
+        max_wavelength=10000,
+        **kwargs,
+    ):
+        # === Layers ===
+        self.token_embedding = ReversibleEmbedding(
+            input_dim=vocabulary_size,
+            output_dim=hidden_dim,
+            embeddings_initializer=roformer_kernel_initializer(),
+            dtype=dtype,
+            name="token_embedding",
+        )
+        self.segment_embedding = keras.layers.Embedding(
+            input_dim=num_segments,
+            output_dim=hidden_dim,
+            embeddings_initializer=roformer_kernel_initializer(),
+            dtype=dtype,
+            name="segment_embedding",
+        )
+        self.embeddings_add = keras.layers.Add(
+            dtype=dtype,
+            name="embeddings_add",
+        )
+        self.embeddings_layer_norm = RoformerNorm(
+            epsilon=keras.backend.epsilon(),
+            dtype=dtype,
+            name="embeddings_layer_norm",
+        )
+        self.embeddings_dropout = keras.layers.Dropout(
+            dropout,
+            dtype=dtype,
+            name="embeddings_dropout",
+        )
+        self.transformer_layers = []
+        for i in range(num_layers):
+            layer = RoformerV2Encoder(
+                heads=num_heads,
+                head_size=head_size,
+                intermediate_size=intermediate_dim,
+                use_bias=use_bias,
+                max_wavelength=max_wavelength,
+                dropout=dropout,
+                activation=activation,
+                kernel_initializer=roformer_kernel_initializer(),
+                dtype=dtype,
+                name=f"transformer_layer_{i}",
+            )
+            self.transformer_layers.append(layer)
+
+        # === Functional Model ===
+        token_id_input = keras.Input(
+            shape=(None,), dtype="int32", name="token_ids"
+        )
+        segment_id_input = keras.Input(
+            shape=(None,), dtype="int32", name="segment_ids"
+        )
+        attention_mask = keras.ops.not_equal(token_id_input, 0)
+        # Embed tokens, positions, and segment ids.
+        tokens = self.token_embedding(token_id_input)
+        segments = self.segment_embedding(segment_id_input)
+        # Sum, normalize and apply dropout to embeddings.
+        x = self.embeddings_add((tokens, segments))
+        x = self.embeddings_layer_norm(x)
+        x = self.embeddings_dropout(x)
+        for transformer_layer in self.transformer_layers:
+            x = transformer_layer(x, attention_mask=attention_mask)
+
+        super().__init__(
+            inputs={
+                "token_ids": token_id_input,
+                "segment_ids": segment_id_input,
+            },
+            outputs=x,
+            dtype=dtype,
+            **kwargs,
+        )
+
+        # === Config ===
+        self.vocabulary_size = vocabulary_size
+        self.num_layers = num_layers
+        self.num_heads = num_heads
+        self.hidden_dim = hidden_dim
+        self.intermediate_dim = intermediate_dim
+        self.dropout = dropout
+        self.num_segments = num_segments
+        self.max_wavelength = max_wavelength
+        self.head_size = head_size
+        self.dropout = dropout
+        self.activation = activations.get(activation)
+        self.use_bias = use_bias
+        self.start_token_index = 0
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "vocabulary_size": self.vocabulary_size,
+                "num_layers": self.num_layers,
+                "num_heads": self.num_heads,
+                "hidden_dim": self.hidden_dim,
+                "intermediate_dim": self.intermediate_dim,
+                "dropout": self.dropout,
+                "num_segments": self.num_segments,
+                "max_wavelength": self.max_wavelength,
+                "head_size": self.head_size,
+                "use_bias": self.use_bias,
+                "activation": activations.serialize(self.activation),
+            }
+        )
+        return config

keras_hub/src/models/roformer_v2/roformer_v2_encoder.py

@@ -0,0 +1,128 @@
+import keras
+from keras import activations
+from keras import initializers
+
+from keras_hub.src.models.roformer_v2.roformer_v2_attention import (
+    RoformerAttention,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_attention import RoformerNorm
+
+
+@keras.saving.register_keras_serializable(package="keras_hub")
+class RoformerV2Encoder(keras.layers.Layer):
+    """A Transformer Encoder layer for the Roformer backbone."""
+
+    def __init__(
+        self,
+        heads,
+        head_size,
+        intermediate_size=None,
+        max_wavelength=10000,
+        dropout=0,
+        activation="relu",
+        use_bias=False,
+        kernel_initializer="glorot_uniform",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.heads = heads
+        self.head_size = head_size
+        self.intermediate_size = intermediate_size
+        self.use_bias = use_bias
+        self.kernel_initializer = initializers.get(kernel_initializer)
+        self.max_wavelength = max_wavelength
+        self.dropout = dropout
+        self.activation = activations.get(activation)
+
+    def build(self, input_shape):
+        super().build(input_shape)
+        self.attention_layer = RoformerAttention(
+            heads=self.heads,
+            head_size=self.head_size,
+            use_bias=self.use_bias,
+            max_wavelength=self.max_wavelength,
+            kernel_initializer=self.kernel_initializer,
+            dtype=self.dtype_policy,
+            name="attention_layer",
+        )
+        self.attention_layer.build(input_shape)
+
+        self.dropout_layer = keras.layers.Dropout(
+            rate=self.dropout,
+            dtype=self.dtype_policy,
+            name="self_attention_dropout",
+        )
+        self.dropout_layer.build([])
+
+        # Feedforward layers.
+        self.feedforward_intermediate_dense = keras.layers.Dense(
+            self.intermediate_size,
+            kernel_initializer=self.kernel_initializer,
+            use_bias=self.use_bias,
+            dtype=self.dtype_policy,
+            activation=self.activation,
+            name="feedforward_intermediate_dense",
+        )
+        self.feedforward_intermediate_dense.build(input_shape)
+
+        self.feedforward_output_dense = keras.layers.Dense(
+            input_shape[-1],
+            kernel_initializer=self.kernel_initializer,
+            use_bias=self.use_bias,
+            dtype=self.dtype_policy,
+            name="feedforward_output_dense",
+        )
+
+        self.feedforward_output_dense.build(
+            [None, None, self.intermediate_size]
+        )
+
+        self.attention_norm = RoformerNorm(
+            epsilon=keras.backend.epsilon(),
+            name="attention_norm",
+            dtype=self.dtype_policy,
+        )
+        self.attention_norm.build(input_shape)
+
+        self.feedforward_norm = RoformerNorm(
+            epsilon=keras.backend.epsilon(),
+            name="feedforward_norm",
+            dtype=self.dtype_policy,
+        )
+        self.feedforward_norm.build(input_shape)
+
+    def call(self, x, attention_mask=None):
+        attention_output = self.attention_layer(
+            x,
+            attention_mask=attention_mask,
+        )
+
+        residual = x + self.dropout_layer(attention_output)
+        x = self.attention_norm(residual)
+
+        intermediate_output = self.feedforward_intermediate_dense(x)
+        feedroward_output = self.feedforward_output_dense(intermediate_output)
+
+        residual = x + self.dropout_layer(feedroward_output)
+        return self.feedforward_norm(residual)
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "heads": self.heads,
+                "head_size": self.head_size,
+                "intermediate_size": self.intermediate_size,
+                "max_wavelength": self.max_wavelength,
+                "use_bias": self.use_bias,
+                "activation": activations.serialize(self.activation),
+                "dropout": self.dropout,
+                "kernel_initializer": initializers.serialize(
+                    self.kernel_initializer
+                ),
+            }
+        )
+        return config

keras_hub/src/models/roformer_v2/roformer_v2_masked_lm.py

@@ -0,0 +1,173 @@
+import keras
+from keras import ops
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.masked_lm import MaskedLM
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_masked_lm_preprocessor import (  # noqa: E501
+    RoformerV2MaskedLMPreprocessor,
+)
+
+
+class RoformerV2MaskedLMHead(keras.layers.Layer):
+    def __init__(
+        self,
+        vocabulary_size=None,
+        token_embedding=None,
+        activation=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs, autocast=False)
+
+        self.token_embedding = token_embedding
+        self.activation = keras.activations.get(activation)
+
+        if vocabulary_size and vocabulary_size != token_embedding.input_dim:
+            raise ValueError(
+                "`vocabulary_size` should match the input dimension of the "
+                "of `token_embedding`. Received: "
+                f"`vocabulary_size={vocabulary_size}`, "
+                f"`token_embedding.input_dim={token_embedding.input_dim}`"
+            )
+        self.vocabulary_size = token_embedding.input_dim
+
+    def call(self, inputs, mask_positions):
+        if keras.config.backend() == "tensorflow":
+            import tensorflow as tf
+
+            # On the tf backend, we need to work around an issue with dynamic
+            # shape broadcasting in take_along_axis.
+            x = tf.gather(inputs, mask_positions, batch_dims=1)
+        else:
+            # Gather the encoded tokens at the masked indices.
+            mask_positions = ops.expand_dims(mask_positions, axis=-1)
+            x = ops.take_along_axis(inputs, mask_positions, axis=1)
+
+        outputs = self.token_embedding(x, reverse=True)
+
+        # Apply a final activation.
+        if self.activation is not None:
+            outputs = self.activation(outputs)
+
+        outputs = ops.cast(outputs, "float32")
+        return outputs
+
+    def get_config(self):
+        config = super().get_config()
+        embedding_config = None
+        if self.token_embedding:
+            embedding_config = keras.layers.serialize(self.token_embedding)
+        config.update(
+            {
+                "token_embedding": embedding_config,
+                "vocabulary_size": self.vocabulary_size,
+                "activation": keras.activations.serialize(self.activation),
+            }
+        )
+        return config
+
+
+@keras_hub_export("keras_hub.models.RoformerV2MaskedLM")
+class RoformerV2MaskedLM(MaskedLM):
+    """An end-to-end Roformer model for the masked language modeling task.
+
+    This model will train RoformerV2 on a masked language modeling task.
+    The model will predict labels for a number of masked tokens in the
+    input data. For usage of this model with pre-trained weights, see the
+    `from_preset()` constructor.
+
+    This model can optionally be configured with a `preprocessor` layer, in
+    which case inputs can be raw string features during `fit()`, `predict()`,
+    and `evaluate()`. Inputs will be tokenized and dynamically masked during
+    training and evaluation. This is done by default when creating the model
+    with `from_preset()`.
+
+    Disclaimer: Pre-trained models are provided on an "as is" basis, without
+    warranties or conditions of any kind.
+
+    Args:
+        backbone: A `keras_hub.models.RoformerV2Backbone` instance.
+        preprocessor: A `keras_hub.models.RoformerV2MaskedLMPreprocessor` or
+            `None`. If `None`, this model will not apply preprocessing, and
+            inputs should be preprocessed before calling the model.
+
+    Examples:
+
+    Raw string data.
+    ```python
+    features = ["The quick brown fox jumped.", "I forgot my homework."]
+
+    # Pretrained language model.
+    masked_lm = keras_hub.models.RoformerV2MaskedLM.from_preset(
+        "roformer_v2_base_zh",
+    )
+    masked_lm.fit(x=features, batch_size=2)
+
+    # Re-compile (e.g., with a new learning rate).
+    masked_lm.compile(
+        loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
+        optimizer=keras.optimizers.Adam(5e-5),
+        jit_compile=True,
+    )
+    # Access backbone programmatically (e.g., to change `trainable`).
+    masked_lm.backbone.trainable = False
+    # Fit again.
+    masked_lm.fit(x=features, batch_size=2)
+    ```
+
+    Preprocessed integer data.
+    ```python
+    # Create preprocessed batch where 0 is the mask token.
+    features = {
+        "token_ids": np.array([[1, 2, 0, 4, 0, 6, 7, 8]] * 2),
+        "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 1]] * 2),
+        "mask_positions": np.array([[2, 4]] * 2),
+        "segment_ids": np.array([[0, 0, 0, 0, 0, 0, 0, 0]] * 2)
+    }
+    # Labels are the original masked values.
+    labels = [[3, 5]] * 2
+
+    masked_lm = keras_hub.models.RoformerV2MaskedLM.from_preset(
+        "roformer_v2_base_zh",
+        preprocessor=None,
+    )
+    masked_lm.fit(x=features, y=labels, batch_size=2)
+    ```
+    """
+
+    backbone_cls = RoformerV2Backbone
+    preprocessor_cls = RoformerV2MaskedLMPreprocessor
+
+    def __init__(
+        self,
+        backbone,
+        preprocessor=None,
+        **kwargs,
+    ):
+        # === Layers ===
+        self.backbone = backbone
+        self.preprocessor = preprocessor
+        self.masked_lm_head = RoformerV2MaskedLMHead(
+            vocabulary_size=backbone.vocabulary_size,
+            token_embedding=backbone.token_embedding,
+            name="mlm_head",
+        )
+
+        # === Functional Model ===
+        inputs = {
+            **backbone.input,
+            "mask_positions": keras.Input(
+                shape=(None,), dtype="int32", name="mask_positions"
+            ),
+        }
+        backbone_outputs = backbone(backbone.input)
+        outputs = self.masked_lm_head(
+            backbone_outputs, inputs["mask_positions"]
+        )
+        super().__init__(
+            inputs=inputs,
+            outputs=outputs,
+            **kwargs,
+        )

keras_hub/src/models/roformer_v2/roformer_v2_masked_lm_preprocessor.py

@@ -0,0 +1,125 @@
+import keras
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.masked_lm_preprocessor import MaskedLMPreprocessor
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_tokenizer import (
+    RoformerV2Tokenizer,
+)
+from keras_hub.src.utils.tensor_utils import preprocessing_function
+
+
+@keras_hub_export("keras_hub.models.RoformerV2MaskedLMPreprocessor")
+class RoformerV2MaskedLMPreprocessor(MaskedLMPreprocessor):
+    """RoformerV2 preprocessing for the masked language modeling task.
+
+    This preprocessing layer will prepare inputs for a masked language modeling
+    task. It is primarily intended for use with the
+    `keras_hub.models.RoformerV2MaskedLM` task model.
+    Preprocessing will occur in multiple steps.
+
+    1. Tokenize any number of input segments using the `tokenizer`.
+    2. Pack the inputs together with the appropriate `"[CLS]"`, `"[SEP]"` and
+      `"[PAD]"` tokens.
+    3. Randomly select non-special tokens to mask, controlled by
+      `mask_selection_rate`.
+    4. Construct a `(x, y, sample_weight)` tuple suitable for training with a
+      `keras_hub.models.RoformerV2MaskedLM` task model.
+
+    Args:
+        tokenizer: A `keras_hub.models.RoformerV2Tokenizer` instance.
+        sequence_length: int. The length of the packed inputs.
+        truncate: string. The algorithm to truncate a list of batched segments
+            to fit within `sequence_length`. The value can be either
+            `round_robin` or `waterfall`:
+            - `"round_robin"`: Available space is assigned one token at a
+                time in a round-robin fashion to the inputs that still need
+                some, until the limit is reached.
+            - `"waterfall"`: The allocation of the budget is done using a
+                "waterfall" algorithm that allocates quota in a
+                left-to-right manner and fills up the buckets until we run
+                out of budget. It supports an arbitrary number of segments.
+        mask_selection_rate: float. The probability an input token will be
+            dynamically masked.
+        mask_selection_length: int. The maximum number of masked tokens
+            in a given sample.
+        mask_token_rate: float. The probability the a selected token will be
+            replaced with the mask token.
+        random_token_rate: float. The probability the a selected token will be
+            replaced with a random token from the vocabulary. A selected token
+            will be left as is with probability
+            `1 - mask_token_rate - random_token_rate`.
+
+    Call arguments:
+        x: A tensor of single string sequences, or a tuple of multiple
+            tensor sequences to be packed together. Inputs may be batched or
+            unbatched. For single sequences, raw python inputs will be converted
+            to tensors. For multiple sequences, pass tensors directly.
+        y: Label data. Should always be `None` as the layer generates labels.
+        sample_weight: Label weights. Should always be `None` as the layer
+            generates label weights.
+
+    Examples:
+
+    Directly calling the layer on data.
+    ```python
+    preprocessor = keras_hub.models.RoformerV2MaskedLMPreprocessor.from_preset(
+        "roformer_v2_base_zh"
+    )
+
+    # Tokenize and mask a single sentence.
+    preprocessor("The quick brown fox jumped.")
+
+    # Tokenize and mask a batch of single sentences.
+    preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
+
+    # Tokenize and mask sentence pairs.
+    # In this case, always convert input to tensors before calling the layer.
+    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
+    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
+    preprocessor((first, second))
+    ```
+
+    Mapping with `tf.data.Dataset`.
+    ```python
+    preprocessor = keras_hub.models.RoformerV2MaskedLMPreprocessor.from_preset(
+        "roformer_v2_base_zh"
+    )
+
+    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
+    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
+
+    # Map single sentences.
+    ds = tf.data.Dataset.from_tensor_slices(first)
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+
+    # Map sentence pairs.
+    ds = tf.data.Dataset.from_tensor_slices((first, second))
+    # Watch out for tf.data's default unpacking of tuples here!
+    # Best to invoke the `preprocessor` directly in this case.
+    ds = ds.map(
+        lambda first, second: preprocessor(x=(first, second)),
+        num_parallel_calls=tf.data.AUTOTUNE,
+    )
+    ```
+    """
+
+    backbone_cls = RoformerV2Backbone
+    tokenizer_cls = RoformerV2Tokenizer
+
+    @preprocessing_function
+    def call(self, x, y=None, sample_weight=None):
+        x = x if isinstance(x, tuple) else (x,)
+        x = tuple(self.tokenizer(segment) for segment in x)
+        token_ids, segment_ids = self.packer(x)
+        masker_outputs = self.masker(token_ids)
+        x = {
+            "token_ids": masker_outputs["token_ids"],
+            "segment_ids": segment_ids,
+            "mask_positions": masker_outputs["mask_positions"],
+        }
+        y = masker_outputs["mask_ids"]
+        sample_weight = masker_outputs["mask_weights"]
+        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

keras_hub/src/models/roformer_v2/roformer_v2_text_classifier.py

@@ -0,0 +1,121 @@
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.roberta.roberta_text_classifier import (
+    RobertaTextClassifier,  # noqa: E501
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_text_classifier_preprocessor import (  # noqa: E501
+    RoformerV2TextClassifierPreprocessor,
+)
+
+
+@keras_hub_export("keras_hub.models.RorformerV2TextClassifier")
+class RorformerV2TextClassifier(RobertaTextClassifier):
+    """An end-to-end RoformerV2 model for classification tasks.
+
+    This model attaches a classification head to
+    `keras_hub.model.RoformerV2Backbone`, mapping from the backbone outputs
+    to logits suitable for a classification task. For usage of this model with
+    pre-trained weights, use the `from_preset()` constructor.
+
+    This model can optionally be configured with a `preprocessor` layer, in
+    which case it will automatically apply preprocessing to raw inputs during
+    `fit()`, `predict()`, and `evaluate()`. This is done by default when
+    creating the model with `from_preset()`.
+
+    Disclaimer: Pre-trained models are provided on an "as is" basis, without
+    warranties or conditions of any kind.
+
+    Args:
+        backbone: A `keras_hub.models.RoformerV2Backbone` instance.
+        num_classes: int. Number of classes to predict.
+        preprocessor: A `keras_hub.models.RoformerV2TextClassifierPreprocessor`
+            or `None`. If `None`, this model will not apply preprocessing, and
+            inputs should be preprocessed before calling the model.
+        activation: Optional `str` or callable. The
+            activation function to use on the model outputs. Set
+            `activation="softmax"` to return output probabilities.
+            Defaults to `None`.
+        dropout: float. The dropout probability value, applied after the dense
+            layer.
+
+    Examples:
+
+    Raw string data.
+    ```python
+    features = ["The quick brown fox jumped.", "I forgot my homework."]
+    labels = [0, 3]
+
+    # Pretrained classifier.
+    classifier = keras_hub.models.RoformerV2TextClassifier.from_preset(
+        "roformer_v2_base_zh",
+        num_classes=4,
+    )
+    classifier.fit(x=features, y=labels, batch_size=2)
+    classifier.predict(x=features, batch_size=2)
+
+    # Re-compile (e.g., with a new learning rate).
+    classifier.compile(
+        loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
+        optimizer=keras.optimizers.Adam(5e-5),
+        jit_compile=True,
+    )
+    # Access backbone programmatically (e.g., to change `trainable`).
+    classifier.backbone.trainable = False
+    # Fit again.
+    classifier.fit(x=features, y=labels, batch_size=2)
+    ```
+
+    Preprocessed integer data.
+    ```python
+    features = {
+        "token_ids": np.ones(shape=(2, 12), dtype="int32"),
+        "segment_ids": np.array([[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0]] * 2),
+        "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]] * 2),
+    }
+    labels = [0, 3]
+
+    # Pretrained classifier without preprocessing.
+    classifier = keras_hub.models.RoformerV2TextClassifier.from_preset(
+        "roformer_v2_base_zh",
+        num_classes=4,
+        preprocessor=None,
+    )
+    classifier.fit(x=features, y=labels, batch_size=2)
+    ```
+
+    Custom backbone and vocabulary.
+    ```python
+    features = ["The quick brown fox jumped.", "I forgot my homework."]
+    labels = [0, 3]
+
+    vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
+    vocab += ["The", "quick", "brown", "fox", "jumped", "."]
+    tokenizer = keras_hub.models.RoformerV2Tokenizer(
+        vocabulary=vocab,
+    )
+    preprocessor = keras_hub.models.RoformerV2TextClassifierPreprocessor(
+        tokenizer=tokenizer,
+        sequence_length=128,
+    )
+    backbone = keras_hub.models.RoformerV2Backbone(
+        vocabulary_size=30552,
+        num_layers=4,
+        num_heads=4,
+        hidden_dim=256,
+        intermediate_dim=512,
+        max_wavelength=128,
+        head_size=64,
+    )
+    classifier = keras_hub.models.RoformerV2TextClassifier(
+        backbone=backbone,
+        preprocessor=preprocessor,
+        num_classes=4,
+    )
+    classifier.fit(x=features, y=labels, batch_size=2)
+    ```
+    """
+
+    backbone_cls = RoformerV2Backbone
+    preprocessor_cls = RoformerV2TextClassifierPreprocessor

keras_hub/src/models/roformer_v2/roformer_v2_text_classifier_preprocessor.py

@@ -0,0 +1,128 @@
+import keras
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.bert.bert_text_classifier_preprocessor import (
+    BertTextClassifierPreprocessor,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_tokenizer import (
+    RoformerV2Tokenizer,
+)
+from keras_hub.src.utils.tensor_utils import preprocessing_function
+
+
+@keras_hub_export("keras_hub.models.RoformerV2TextClassifierPreprocessor")
+class RoformerV2TextClassifierPreprocessor(BertTextClassifierPreprocessor):
+    """A RoformerV2 preprocessing layer which tokenizes and packs inputs.
+
+    This preprocessing layer will do three things:
+
+    1. Tokenize any number of input segments using the `tokenizer`.
+    2. Pack the inputs together using a `keras_hub.layers.MultiSegmentPacker`.
+       with the appropriate `"[CLS]"`, `"[SEP]"` and `"[PAD]"` tokens.
+    3. Construct a dictionary with keys `"token_ids"`, `"segment_ids"`,
+       `"padding_mask"`, that can be passed directly to a RoformerV2 model.
+
+    This layer can be used directly with `tf.data.Dataset.map` to preprocess
+    string data in the `(x, y, sample_weight)` format used by
+    `keras.Model.fit`.
+
+    Args:
+        tokenizer: A `keras_hub.models.RoformerV2Tokenizer` instance.
+        sequence_length: The length of the packed inputs.
+        truncate: string. The algorithm to truncate a list of batched segments
+            to fit within `sequence_length`. The value can be either
+            `round_robin` or `waterfall`:
+            - `"round_robin"`: Available space is assigned one token at a
+                time in a round-robin fashion to the inputs that still need
+                some, until the limit is reached.
+            - `"waterfall"`: The allocation of the budget is done using a
+                "waterfall" algorithm that allocates quota in a
+                left-to-right manner and fills up the buckets until we run
+                out of budget. It supports an arbitrary number of segments.
+
+    Call arguments:
+        x: A tensor of single string sequences, or a tuple of multiple
+            tensor sequences to be packed together. Inputs may be batched or
+            unbatched. For single sequences, raw python inputs will be converted
+            to tensors. For multiple sequences, pass tensors directly.
+        y: Any label data. Will be passed through unaltered.
+        sample_weight: Any label weight data. Will be passed through unaltered.
+
+    Examples:
+
+    Directly calling the layer on data.
+    ```python
+    preprocessor = keras_hub.models.TextClassifierPreprocessor.from_preset(
+        "roformer_v2_base_zh"
+    )
+
+    # Tokenize and pack a single sentence.
+    preprocessor("The quick brown fox jumped.")
+
+    # Tokenize a batch of single sentences.
+    preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
+
+    # Preprocess a batch of sentence pairs.
+    # When handling multiple sequences, always convert to tensors first!
+    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
+    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
+    preprocessor((first, second))
+
+    # Custom vocabulary.
+    vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
+    vocab += ["The", "quick", "brown", "fox", "jumped", "."]
+    tokenizer = keras_hub.models.RoformerV2Tokenizer(vocabulary=vocab)
+    preprocessor =
+        keras_hub.models.RoformerV2TextClassifierPreprocessor(tokenizer)
+    preprocessor("The quick brown fox jumped.")
+    ```
+
+    Mapping with `tf.data.Dataset`.
+    ```python
+    preprocessor = keras_hub.models.TextClassifierPreprocessor.from_preset(
+        "roformer_v2_base_zh"
+    )
+
+    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
+    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
+    label = tf.constant([1, 1])
+
+    # Map labeled single sentences.
+    ds = tf.data.Dataset.from_tensor_slices((first, label))
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+
+    # Map unlabeled single sentences.
+    ds = tf.data.Dataset.from_tensor_slices(first)
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+
+    # Map labeled sentence pairs.
+    ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+
+    # Map unlabeled sentence pairs.
+    ds = tf.data.Dataset.from_tensor_slices((first, second))
+    # Watch out for tf.data's default unpacking of tuples here!
+    # Best to invoke the `preprocessor` directly in this case.
+    ds = ds.map(
+        lambda first, second: preprocessor(x=(first, second)),
+        num_parallel_calls=tf.data.AUTOTUNE,
+    )
+    ```
+    """
+
+    backbone_cls = RoformerV2Backbone
+    tokenizer_cls = RoformerV2Tokenizer
+
+    @preprocessing_function
+    def call(self, x, y=None, sample_weight=None):
+        x = x if isinstance(x, tuple) else (x,)
+        x = tuple(self.tokenizer(segment) for segment in x)
+        token_ids, segment_ids = self.packer(x)
+        x = {
+            "token_ids": token_ids,
+            "segment_ids": segment_ids,
+        }
+        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

keras_hub/src/models/roformer_v2/roformer_v2_tokenizer.py

@@ -0,0 +1,62 @@
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone,
+)
+
+
+@keras_hub_export(
+    [
+        "keras_hub.tokenizers.RoformerV2Tokenizer",
+        "keras_hub.models.RoformerV2Tokenizer",
+    ]
+)
+class RoformerV2Tokenizer(BertTokenizer):
+    """A RoformerV2 tokenizer using WordPiece subword segmentation.
+
+    This tokenizer class will tokenize raw strings into integer sequences and
+    is based on `keras_hub.tokenizers.WordPieceTokenizer`. Unlike the
+    underlying tokenizer, it will check for special tokens needed by RoformerV2
+    models and provides a `from_preset()` method to automatically download
+    a matching vocabulary for a RoformerV2 preset.
+
+    If input is a batch of strings (rank > 0), the layer will output a
+    `tf.RaggedTensor` where the last dimension of the output is ragged.
+
+    If input is a scalar string (rank == 0), the layer will output a dense
+    `tf.Tensor` with static shape `[None]`.
+
+    Args:
+        vocabulary: A list of strings or a string filename path. If
+            passing a list, each element of the list should be a single word
+            piece token string. If passing a filename, the file should be a
+            plain text file containing a single word piece token per line.
+        lowercase: If `True`, the input text will be first lowered before
+            tokenization.
+        special_tokens_in_strings: bool. A bool to indicate if the tokenizer
+            should expect special tokens in input strings that should be
+            tokenized and mapped correctly to their ids. Defaults to False.
+
+    Examples:
+    ```python
+    # Unbatched input.
+    tokenizer = keras_hub.models.RoformerV2Tokenizer.from_preset(
+        "roformer_v2_base_zh",
+    )
+    tokenizer("The quick brown fox jumped.")
+
+    # Batched input.
+    tokenizer(["The quick brown fox jumped.", "The fox slept."])
+
+    # Detokenization.
+    tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
+
+    # Custom vocabulary.
+    vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
+    vocab += ["The", "quick", "brown", "fox", "jumped", "."]
+    tokenizer = keras_hub.models.RoformerV2Tokenizer(vocabulary=vocab)
+    tokenizer("The quick brown fox jumped.")
+    ```
+    """
+
+    backbone_cls = RoformerV2Backbone

keras_hub/src/utils/preset_utils.py

@@ -238,7 +238,8 @@ def get_file(preset, path):
             "2) a Kaggle Models handle like "
             "`'kaggle://keras/bert/keras/bert_base_en'`\n"
             "3) a Hugging Face handle like `'hf://username/bert_base_en'`\n"
-            "4) a path to a local preset directory like `'./bert_base_en`\n"
+            "4) a Modelscope handle like `'modelscope://username/bert_base_en'`\n"
+            "5) a path to a local preset directory like `'./bert_base_en'`\n"
             "Use `print(cls.presets.keys())` to view all built-in presets for "
             "API symbol `cls`.\n"
             f"Received: preset='{preset}'"

keras_hub/src/version_utils.py

@@ -1,7 +1,7 @@
 from keras_hub.src.api_export import keras_hub_export
 
 # Unique source of truth for the version number.
-__version__ = "0.20.0.dev202504010407"
+__version__ = "0.20.0.dev202504020401"
 
 
 @keras_hub_export("keras_hub.version")

{keras_hub_nightly-0.20.0.dev202504010407.dist-info → keras_hub_nightly-0.20.0.dev202504020401.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: keras-hub-nightly
-Version: 0.20.0.dev202504010407
+Version: 0.20.0.dev202504020401
 Summary: Industry-strength Natural Language Processing extensions for Keras.
 Home-page: https://github.com/keras-team/keras-hub
 Author: Keras team

{keras_hub_nightly-0.20.0.dev202504010407.dist-info → keras_hub_nightly-0.20.0.dev202504020401.dist-info}/RECORD

@@ -2,13 +2,13 @@ keras_hub/__init__.py,sha256=QGdXyHgYt6cMUAP1ebxwc6oR86dE0dkMxNy2eOCQtFo,855
 keras_hub/api/__init__.py,sha256=EzR6D-XWsm_gDrX5LDwKEmrah_gu3ffpj8GKBudE0yI,485
 keras_hub/api/layers/__init__.py,sha256=cALDY20DHCVjJDKCD84zHvTZtAgz0xEvozk7a3Cl1Uk,3719
 keras_hub/api/metrics/__init__.py,sha256=So8Ec-lOcTzn_UUMmAdzDm8RKkPu2dbRUm2px8gpUEI,381
-keras_hub/api/models/__init__.py,sha256=Ml8AQyMz9VKuLGEjKKQzlIDNScIaoCYmlgH-Zfk4Jxo,18725
+keras_hub/api/models/__init__.py,sha256=cN3c2t8tldP79aVk4Z2CnMNLqwu0uOym7yuZ_niKZzE,19393
 keras_hub/api/samplers/__init__.py,sha256=n-_SEXxr2LNUzK2FqVFN7alsrkx1P_HOVTeLZKeGCdE,730
-keras_hub/api/tokenizers/__init__.py,sha256=TOTM3he5JWPME_4e29_S1nrWibTHomcuuKQ4KarqUNo,2831
+keras_hub/api/tokenizers/__init__.py,sha256=NCQSOg3vf3KlM2YBsxApcJUVu9MH2jV0NQrM3f4EhJ4,2927
 keras_hub/api/utils/__init__.py,sha256=Gp1E6gG-RtKQS3PBEQEOz9PQvXkXaJ0ySGMqZ7myN7A,215
 keras_hub/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/api_export.py,sha256=9pQZK27JObxWZ96QPLBp1OBsjWigh1iuV6RglPGMRk0,1499
-keras_hub/src/version_utils.py,sha256=OsUwPWPZpPCMjDiiDuMytzPQ_fmJixCqlxXypU5TQGg,222
+keras_hub/src/version_utils.py,sha256=Cu7L8D-l2tjToEgezwk_RlRVOFOvu5Fs_mSsbhAGc_4,222
 keras_hub/src/layers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/layers/modeling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/layers/modeling/alibi_bias.py,sha256=1XBTHI52L_iJDhN_w5ydu_iMhCuTgQAxEPwcLA6BPuk,4411
@@ -187,7 +187,7 @@ keras_hub/src/models/flux/flux_presets.py,sha256=z7C_FbI1_F5YETXuWpc7Yh_0w-5N0eB
 keras_hub/src/models/flux/flux_text_to_image.py,sha256=Rf5dD2EhG0bE8Gyg9sqaA8YEexS1kdraofIkxiZDjvc,4166
 keras_hub/src/models/flux/flux_text_to_image_preprocessor.py,sha256=Fs9jr97QtmRUbRRz1kITpkuhDM2GoV3n0XSFC-qQA14,2252
 keras_hub/src/models/gemma/__init__.py,sha256=rVzOJMJ39bgVlT8UdC0t8PlN2c237GKTBmfHIsbPuOQ,251
-keras_hub/src/models/gemma/gemma_attention.py,sha256=XShBTunOWQOOE4Aapy3HdV9uIWuMcdNdYS1k1P3ia60,9708
+keras_hub/src/models/gemma/gemma_attention.py,sha256=j-YjkcUIv2ZqQCrJ2GW1nMgpO2ZQsxNedSvxAyKaHNA,9783
 keras_hub/src/models/gemma/gemma_backbone.py,sha256=GzAUSArw_pN9dtWQzTVhWDbW-XyWt4GyMcFLn9hwmh0,13391
 keras_hub/src/models/gemma/gemma_causal_lm.py,sha256=3OXaIXlrKqMIuUnBk-bUz-0SYFL-XkkQTWm8qRY2YII,16770
 keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py,sha256=bpKkEurWIfa6Kp9s4pz84-sBDSA6ZFNHP8nXG1fFQrg,2912
@@ -315,6 +315,16 @@ keras_hub/src/models/roberta/roberta_presets.py,sha256=lu8_E888-YGlhMo1kE4LnsR0R
 keras_hub/src/models/roberta/roberta_text_classifier.py,sha256=x36hU84P-ROReZniUA8sMODzj2olrHvG0F5RTiz6Two,6681
 keras_hub/src/models/roberta/roberta_text_classifier_preprocessor.py,sha256=gAJa8JdPUmT1N7nxBqtaIbnfXV-xlNjTtkEevQhfjNU,5993
 keras_hub/src/models/roberta/roberta_tokenizer.py,sha256=VKPrgXVT9aMKP7et2DIWKlTN8g4tIzjya0MHqNz9BwQ,2712
+keras_hub/src/models/roformer_v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+keras_hub/src/models/roformer_v2/roformer_v2_attention.py,sha256=RvDxuh0eZ6QEhyU_SzVcCbadSFWtNtsbHfYfWtBU7r0,7166
+keras_hub/src/models/roformer_v2/roformer_v2_backbone.py,sha256=g1xCi9Zet0dHkiOtpjwsO1odc__HzLDn9cTHNykpYEE,7188
+keras_hub/src/models/roformer_v2/roformer_v2_encoder.py,sha256=o_M3dDtebBtXRAxwhiRmdWA59tu1_MNKLINf4GQYfeA,4218
+keras_hub/src/models/roformer_v2/roformer_v2_masked_lm.py,sha256=4uQ6DKFDdBOu0bHaL45bqtpL-CMZw59inXirD9zWFlI,5950
+keras_hub/src/models/roformer_v2/roformer_v2_masked_lm_preprocessor.py,sha256=1APlYMi_Be_aaEk4Ij6c16JH5OpDrGnvwtv8LY3fjrw,5403
+keras_hub/src/models/roformer_v2/roformer_v2_presets.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+keras_hub/src/models/roformer_v2/roformer_v2_text_classifier.py,sha256=QSRSSWt1T16BFlSynyJduUTd86AEV4NvqdJgIiw1wys,4380
+keras_hub/src/models/roformer_v2/roformer_v2_text_classifier_preprocessor.py,sha256=Md0ioE6OlEyhn6QDTrLVik8EVtiF2YvR5Bu5vAyGEHg,5270
+keras_hub/src/models/roformer_v2/roformer_v2_tokenizer.py,sha256=Pc4z7e5rSZH6qh_suORb-O89ZaFcCjde_4gs-odYook,2406
 keras_hub/src/models/sam/__init__.py,sha256=fp71Q288xeE81tIOZkkudec4Acs8v4aO5WdyzCD9x-c,239
 keras_hub/src/models/sam/sam_backbone.py,sha256=VtT-tjTaVW6v2u_JLe3vyKUoHASPDs5aetc3s0MDo6U,4302
 keras_hub/src/models/sam/sam_image_converter.py,sha256=5POp3aYFu6CK3R0NNfeUBbjhguBkincSMNvlcIJXarE,324
@@ -422,7 +432,7 @@ keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py,sha256=cylrs02ZrYQ1TuZr
 keras_hub/src/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 keras_hub/src/utils/keras_utils.py,sha256=IB_eIrln3N5sVyCapwv1jzLEmuBv8vBRwSVd3toSgyI,3097
 keras_hub/src/utils/pipeline_model.py,sha256=jgzB6NQPSl0KOu08N-TazfOnXnUJbZjH2EXXhx25Ftg,9084
-keras_hub/src/utils/preset_utils.py,sha256=5xEm6Uz1vfQkBqyENt97qaxWoq-P7mlPC0LIpXqDM70,31928
+keras_hub/src/utils/preset_utils.py,sha256=BqTqtLouPQrTKelup2og_yziRBcu9dXFbbAST0I-XlE,32012
 keras_hub/src/utils/python_utils.py,sha256=N8nWeO3san4YnGkffRXG3Ix7VEIMTKSN21FX5TuL7G8,202
 keras_hub/src/utils/tensor_utils.py,sha256=lczQWgPVJU09cLtNbo8MErVFNV9ne4gNlrzbNVQazg4,15042
 keras_hub/src/utils/imagenet/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -449,7 +459,7 @@ keras_hub/src/utils/transformers/convert_qwen.py,sha256=I2bfwo8AQd_JfwFpiAuCQ3k_
 keras_hub/src/utils/transformers/convert_vit.py,sha256=9SUZ9utNJhW_5cj3acMn9cRy47u2eIcDsrhmzj77o9k,5187
 keras_hub/src/utils/transformers/preset_loader.py,sha256=0Hi7R8HnATcwFVLsJwMMIMWTCXHNfep4IPiRpQXqM-w,3933
 keras_hub/src/utils/transformers/safetensor_utils.py,sha256=CYUHyA4y-B61r7NDnCsFb4t_UmSwZ1k9L-8gzEd6KRg,3339
-keras_hub_nightly-0.20.0.dev202504010407.dist-info/METADATA,sha256=77ncA8nPcUGWqFtGIA4EPqrK6BdhNskTR3vDrTVWaLU,7715
-keras_hub_nightly-0.20.0.dev202504010407.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-keras_hub_nightly-0.20.0.dev202504010407.dist-info/top_level.txt,sha256=N4J6piIWBKa38A4uV-CnIopnOEf8mHAbkNXafXm_CuA,10
-keras_hub_nightly-0.20.0.dev202504010407.dist-info/RECORD,,
+keras_hub_nightly-0.20.0.dev202504020401.dist-info/METADATA,sha256=UdVE9mpAcH4RH9b_-tSwYDep5ldoTNBEeDdj3uk1XnU,7715
+keras_hub_nightly-0.20.0.dev202504020401.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+keras_hub_nightly-0.20.0.dev202504020401.dist-info/top_level.txt,sha256=N4J6piIWBKa38A4uV-CnIopnOEf8mHAbkNXafXm_CuA,10
+keras_hub_nightly-0.20.0.dev202504020401.dist-info/RECORD,,