keras-hub-nightly 0.20.0.dev202503310403__py3-none-any.whl → 0.20.0.dev202504020401__py3-none-any.whl

keras_hub/src/models/roformer_v2/roformer_v2_masked_lm.py

@@ -0,0 +1,173 @@
+import keras
+from keras import ops
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.masked_lm import MaskedLM
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_masked_lm_preprocessor import (  # noqa: E501
+    RoformerV2MaskedLMPreprocessor,
+)
+
+
+class RoformerV2MaskedLMHead(keras.layers.Layer):
+    def __init__(
+        self,
+        vocabulary_size=None,
+        token_embedding=None,
+        activation=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs, autocast=False)
+
+        self.token_embedding = token_embedding
+        self.activation = keras.activations.get(activation)
+
+        if vocabulary_size and vocabulary_size != token_embedding.input_dim:
+            raise ValueError(
+                "`vocabulary_size` should match the input dimension of the "
+                "of `token_embedding`. Received: "
+                f"`vocabulary_size={vocabulary_size}`, "
+                f"`token_embedding.input_dim={token_embedding.input_dim}`"
+            )
+        self.vocabulary_size = token_embedding.input_dim
+
+    def call(self, inputs, mask_positions):
+        if keras.config.backend() == "tensorflow":
+            import tensorflow as tf
+
+            # On the tf backend, we need to work around an issue with dynamic
+            # shape broadcasting in take_along_axis.
+            x = tf.gather(inputs, mask_positions, batch_dims=1)
+        else:
+            # Gather the encoded tokens at the masked indices.
+            mask_positions = ops.expand_dims(mask_positions, axis=-1)
+            x = ops.take_along_axis(inputs, mask_positions, axis=1)
+
+        outputs = self.token_embedding(x, reverse=True)
+
+        # Apply a final activation.
+        if self.activation is not None:
+            outputs = self.activation(outputs)
+
+        outputs = ops.cast(outputs, "float32")
+        return outputs
+
+    def get_config(self):
+        config = super().get_config()
+        embedding_config = None
+        if self.token_embedding:
+            embedding_config = keras.layers.serialize(self.token_embedding)
+        config.update(
+            {
+                "token_embedding": embedding_config,
+                "vocabulary_size": self.vocabulary_size,
+                "activation": keras.activations.serialize(self.activation),
+            }
+        )
+        return config
+
+
+@keras_hub_export("keras_hub.models.RoformerV2MaskedLM")
+class RoformerV2MaskedLM(MaskedLM):
+    """An end-to-end Roformer model for the masked language modeling task.
+
+    This model will train RoformerV2 on a masked language modeling task.
+    The model will predict labels for a number of masked tokens in the
+    input data. For usage of this model with pre-trained weights, see the
+    `from_preset()` constructor.
+
+    This model can optionally be configured with a `preprocessor` layer, in
+    which case inputs can be raw string features during `fit()`, `predict()`,
+    and `evaluate()`. Inputs will be tokenized and dynamically masked during
+    training and evaluation. This is done by default when creating the model
+    with `from_preset()`.
+
+    Disclaimer: Pre-trained models are provided on an "as is" basis, without
+    warranties or conditions of any kind.
+
+    Args:
+        backbone: A `keras_hub.models.RoformerV2Backbone` instance.
+        preprocessor: A `keras_hub.models.RoformerV2MaskedLMPreprocessor` or
+            `None`. If `None`, this model will not apply preprocessing, and
+            inputs should be preprocessed before calling the model.
+
+    Examples:
+
+    Raw string data.
+    ```python
+    features = ["The quick brown fox jumped.", "I forgot my homework."]
+
+    # Pretrained language model.
+    masked_lm = keras_hub.models.RoformerV2MaskedLM.from_preset(
+        "roformer_v2_base_zh",
+    )
+    masked_lm.fit(x=features, batch_size=2)
+
+    # Re-compile (e.g., with a new learning rate).
+    masked_lm.compile(
+        loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
+        optimizer=keras.optimizers.Adam(5e-5),
+        jit_compile=True,
+    )
+    # Access backbone programmatically (e.g., to change `trainable`).
+    masked_lm.backbone.trainable = False
+    # Fit again.
+    masked_lm.fit(x=features, batch_size=2)
+    ```
+
+    Preprocessed integer data.
+    ```python
+    # Create preprocessed batch where 0 is the mask token.
+    features = {
+        "token_ids": np.array([[1, 2, 0, 4, 0, 6, 7, 8]] * 2),
+        "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 1]] * 2),
+        "mask_positions": np.array([[2, 4]] * 2),
+        "segment_ids": np.array([[0, 0, 0, 0, 0, 0, 0, 0]] * 2)
+    }
+    # Labels are the original masked values.
+    labels = [[3, 5]] * 2
+
+    masked_lm = keras_hub.models.RoformerV2MaskedLM.from_preset(
+        "roformer_v2_base_zh",
+        preprocessor=None,
+    )
+    masked_lm.fit(x=features, y=labels, batch_size=2)
+    ```
+    """
+
+    backbone_cls = RoformerV2Backbone
+    preprocessor_cls = RoformerV2MaskedLMPreprocessor
+
+    def __init__(
+        self,
+        backbone,
+        preprocessor=None,
+        **kwargs,
+    ):
+        # === Layers ===
+        self.backbone = backbone
+        self.preprocessor = preprocessor
+        self.masked_lm_head = RoformerV2MaskedLMHead(
+            vocabulary_size=backbone.vocabulary_size,
+            token_embedding=backbone.token_embedding,
+            name="mlm_head",
+        )
+
+        # === Functional Model ===
+        inputs = {
+            **backbone.input,
+            "mask_positions": keras.Input(
+                shape=(None,), dtype="int32", name="mask_positions"
+            ),
+        }
+        backbone_outputs = backbone(backbone.input)
+        outputs = self.masked_lm_head(
+            backbone_outputs, inputs["mask_positions"]
+        )
+        super().__init__(
+            inputs=inputs,
+            outputs=outputs,
+            **kwargs,
+        )

keras_hub/src/models/roformer_v2/roformer_v2_masked_lm_preprocessor.py

@@ -0,0 +1,125 @@
+import keras
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.masked_lm_preprocessor import MaskedLMPreprocessor
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_tokenizer import (
+    RoformerV2Tokenizer,
+)
+from keras_hub.src.utils.tensor_utils import preprocessing_function
+
+
+@keras_hub_export("keras_hub.models.RoformerV2MaskedLMPreprocessor")
+class RoformerV2MaskedLMPreprocessor(MaskedLMPreprocessor):
+    """RoformerV2 preprocessing for the masked language modeling task.
+
+    This preprocessing layer will prepare inputs for a masked language modeling
+    task. It is primarily intended for use with the
+    `keras_hub.models.RoformerV2MaskedLM` task model.
+    Preprocessing will occur in multiple steps.
+
+    1. Tokenize any number of input segments using the `tokenizer`.
+    2. Pack the inputs together with the appropriate `"[CLS]"`, `"[SEP]"` and
+      `"[PAD]"` tokens.
+    3. Randomly select non-special tokens to mask, controlled by
+      `mask_selection_rate`.
+    4. Construct a `(x, y, sample_weight)` tuple suitable for training with a
+      `keras_hub.models.RoformerV2MaskedLM` task model.
+
+    Args:
+        tokenizer: A `keras_hub.models.RoformerV2Tokenizer` instance.
+        sequence_length: int. The length of the packed inputs.
+        truncate: string. The algorithm to truncate a list of batched segments
+            to fit within `sequence_length`. The value can be either
+            `round_robin` or `waterfall`:
+            - `"round_robin"`: Available space is assigned one token at a
+                time in a round-robin fashion to the inputs that still need
+                some, until the limit is reached.
+            - `"waterfall"`: The allocation of the budget is done using a
+                "waterfall" algorithm that allocates quota in a
+                left-to-right manner and fills up the buckets until we run
+                out of budget. It supports an arbitrary number of segments.
+        mask_selection_rate: float. The probability an input token will be
+            dynamically masked.
+        mask_selection_length: int. The maximum number of masked tokens
+            in a given sample.
+        mask_token_rate: float. The probability the a selected token will be
+            replaced with the mask token.
+        random_token_rate: float. The probability the a selected token will be
+            replaced with a random token from the vocabulary. A selected token
+            will be left as is with probability
+            `1 - mask_token_rate - random_token_rate`.
+
+    Call arguments:
+        x: A tensor of single string sequences, or a tuple of multiple
+            tensor sequences to be packed together. Inputs may be batched or
+            unbatched. For single sequences, raw python inputs will be converted
+            to tensors. For multiple sequences, pass tensors directly.
+        y: Label data. Should always be `None` as the layer generates labels.
+        sample_weight: Label weights. Should always be `None` as the layer
+            generates label weights.
+
+    Examples:
+
+    Directly calling the layer on data.
+    ```python
+    preprocessor = keras_hub.models.RoformerV2MaskedLMPreprocessor.from_preset(
+        "roformer_v2_base_zh"
+    )
+
+    # Tokenize and mask a single sentence.
+    preprocessor("The quick brown fox jumped.")
+
+    # Tokenize and mask a batch of single sentences.
+    preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
+
+    # Tokenize and mask sentence pairs.
+    # In this case, always convert input to tensors before calling the layer.
+    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
+    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
+    preprocessor((first, second))
+    ```
+
+    Mapping with `tf.data.Dataset`.
+    ```python
+    preprocessor = keras_hub.models.RoformerV2MaskedLMPreprocessor.from_preset(
+        "roformer_v2_base_zh"
+    )
+
+    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
+    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
+
+    # Map single sentences.
+    ds = tf.data.Dataset.from_tensor_slices(first)
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+
+    # Map sentence pairs.
+    ds = tf.data.Dataset.from_tensor_slices((first, second))
+    # Watch out for tf.data's default unpacking of tuples here!
+    # Best to invoke the `preprocessor` directly in this case.
+    ds = ds.map(
+        lambda first, second: preprocessor(x=(first, second)),
+        num_parallel_calls=tf.data.AUTOTUNE,
+    )
+    ```
+    """
+
+    backbone_cls = RoformerV2Backbone
+    tokenizer_cls = RoformerV2Tokenizer
+
+    @preprocessing_function
+    def call(self, x, y=None, sample_weight=None):
+        x = x if isinstance(x, tuple) else (x,)
+        x = tuple(self.tokenizer(segment) for segment in x)
+        token_ids, segment_ids = self.packer(x)
+        masker_outputs = self.masker(token_ids)
+        x = {
+            "token_ids": masker_outputs["token_ids"],
+            "segment_ids": segment_ids,
+            "mask_positions": masker_outputs["mask_positions"],
+        }
+        y = masker_outputs["mask_ids"]
+        sample_weight = masker_outputs["mask_weights"]
+        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

keras_hub/src/models/roformer_v2/roformer_v2_text_classifier.py

@@ -0,0 +1,121 @@
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.roberta.roberta_text_classifier import (
+    RobertaTextClassifier,  # noqa: E501
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_text_classifier_preprocessor import (  # noqa: E501
+    RoformerV2TextClassifierPreprocessor,
+)
+
+
+@keras_hub_export("keras_hub.models.RorformerV2TextClassifier")
+class RorformerV2TextClassifier(RobertaTextClassifier):
+    """An end-to-end RoformerV2 model for classification tasks.
+
+    This model attaches a classification head to
+    `keras_hub.model.RoformerV2Backbone`, mapping from the backbone outputs
+    to logits suitable for a classification task. For usage of this model with
+    pre-trained weights, use the `from_preset()` constructor.
+
+    This model can optionally be configured with a `preprocessor` layer, in
+    which case it will automatically apply preprocessing to raw inputs during
+    `fit()`, `predict()`, and `evaluate()`. This is done by default when
+    creating the model with `from_preset()`.
+
+    Disclaimer: Pre-trained models are provided on an "as is" basis, without
+    warranties or conditions of any kind.
+
+    Args:
+        backbone: A `keras_hub.models.RoformerV2Backbone` instance.
+        num_classes: int. Number of classes to predict.
+        preprocessor: A `keras_hub.models.RoformerV2TextClassifierPreprocessor`
+            or `None`. If `None`, this model will not apply preprocessing, and
+            inputs should be preprocessed before calling the model.
+        activation: Optional `str` or callable. The
+            activation function to use on the model outputs. Set
+            `activation="softmax"` to return output probabilities.
+            Defaults to `None`.
+        dropout: float. The dropout probability value, applied after the dense
+            layer.
+
+    Examples:
+
+    Raw string data.
+    ```python
+    features = ["The quick brown fox jumped.", "I forgot my homework."]
+    labels = [0, 3]
+
+    # Pretrained classifier.
+    classifier = keras_hub.models.RoformerV2TextClassifier.from_preset(
+        "roformer_v2_base_zh",
+        num_classes=4,
+    )
+    classifier.fit(x=features, y=labels, batch_size=2)
+    classifier.predict(x=features, batch_size=2)
+
+    # Re-compile (e.g., with a new learning rate).
+    classifier.compile(
+        loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
+        optimizer=keras.optimizers.Adam(5e-5),
+        jit_compile=True,
+    )
+    # Access backbone programmatically (e.g., to change `trainable`).
+    classifier.backbone.trainable = False
+    # Fit again.
+    classifier.fit(x=features, y=labels, batch_size=2)
+    ```
+
+    Preprocessed integer data.
+    ```python
+    features = {
+        "token_ids": np.ones(shape=(2, 12), dtype="int32"),
+        "segment_ids": np.array([[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0]] * 2),
+        "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]] * 2),
+    }
+    labels = [0, 3]
+
+    # Pretrained classifier without preprocessing.
+    classifier = keras_hub.models.RoformerV2TextClassifier.from_preset(
+        "roformer_v2_base_zh",
+        num_classes=4,
+        preprocessor=None,
+    )
+    classifier.fit(x=features, y=labels, batch_size=2)
+    ```
+
+    Custom backbone and vocabulary.
+    ```python
+    features = ["The quick brown fox jumped.", "I forgot my homework."]
+    labels = [0, 3]
+
+    vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
+    vocab += ["The", "quick", "brown", "fox", "jumped", "."]
+    tokenizer = keras_hub.models.RoformerV2Tokenizer(
+        vocabulary=vocab,
+    )
+    preprocessor = keras_hub.models.RoformerV2TextClassifierPreprocessor(
+        tokenizer=tokenizer,
+        sequence_length=128,
+    )
+    backbone = keras_hub.models.RoformerV2Backbone(
+        vocabulary_size=30552,
+        num_layers=4,
+        num_heads=4,
+        hidden_dim=256,
+        intermediate_dim=512,
+        max_wavelength=128,
+        head_size=64,
+    )
+    classifier = keras_hub.models.RoformerV2TextClassifier(
+        backbone=backbone,
+        preprocessor=preprocessor,
+        num_classes=4,
+    )
+    classifier.fit(x=features, y=labels, batch_size=2)
+    ```
+    """
+
+    backbone_cls = RoformerV2Backbone
+    preprocessor_cls = RoformerV2TextClassifierPreprocessor

keras_hub/src/models/roformer_v2/roformer_v2_text_classifier_preprocessor.py

@@ -0,0 +1,128 @@
+import keras
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.bert.bert_text_classifier_preprocessor import (
+    BertTextClassifierPreprocessor,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone,
+)
+from keras_hub.src.models.roformer_v2.roformer_v2_tokenizer import (
+    RoformerV2Tokenizer,
+)
+from keras_hub.src.utils.tensor_utils import preprocessing_function
+
+
+@keras_hub_export("keras_hub.models.RoformerV2TextClassifierPreprocessor")
+class RoformerV2TextClassifierPreprocessor(BertTextClassifierPreprocessor):
+    """A RoformerV2 preprocessing layer which tokenizes and packs inputs.
+
+    This preprocessing layer will do three things:
+
+    1. Tokenize any number of input segments using the `tokenizer`.
+    2. Pack the inputs together using a `keras_hub.layers.MultiSegmentPacker`.
+       with the appropriate `"[CLS]"`, `"[SEP]"` and `"[PAD]"` tokens.
+    3. Construct a dictionary with keys `"token_ids"`, `"segment_ids"`,
+       `"padding_mask"`, that can be passed directly to a RoformerV2 model.
+
+    This layer can be used directly with `tf.data.Dataset.map` to preprocess
+    string data in the `(x, y, sample_weight)` format used by
+    `keras.Model.fit`.
+
+    Args:
+        tokenizer: A `keras_hub.models.RoformerV2Tokenizer` instance.
+        sequence_length: The length of the packed inputs.
+        truncate: string. The algorithm to truncate a list of batched segments
+            to fit within `sequence_length`. The value can be either
+            `round_robin` or `waterfall`:
+            - `"round_robin"`: Available space is assigned one token at a
+                time in a round-robin fashion to the inputs that still need
+                some, until the limit is reached.
+            - `"waterfall"`: The allocation of the budget is done using a
+                "waterfall" algorithm that allocates quota in a
+                left-to-right manner and fills up the buckets until we run
+                out of budget. It supports an arbitrary number of segments.
+
+    Call arguments:
+        x: A tensor of single string sequences, or a tuple of multiple
+            tensor sequences to be packed together. Inputs may be batched or
+            unbatched. For single sequences, raw python inputs will be converted
+            to tensors. For multiple sequences, pass tensors directly.
+        y: Any label data. Will be passed through unaltered.
+        sample_weight: Any label weight data. Will be passed through unaltered.
+
+    Examples:
+
+    Directly calling the layer on data.
+    ```python
+    preprocessor = keras_hub.models.TextClassifierPreprocessor.from_preset(
+        "roformer_v2_base_zh"
+    )
+
+    # Tokenize and pack a single sentence.
+    preprocessor("The quick brown fox jumped.")
+
+    # Tokenize a batch of single sentences.
+    preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
+
+    # Preprocess a batch of sentence pairs.
+    # When handling multiple sequences, always convert to tensors first!
+    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
+    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
+    preprocessor((first, second))
+
+    # Custom vocabulary.
+    vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
+    vocab += ["The", "quick", "brown", "fox", "jumped", "."]
+    tokenizer = keras_hub.models.RoformerV2Tokenizer(vocabulary=vocab)
+    preprocessor =
+        keras_hub.models.RoformerV2TextClassifierPreprocessor(tokenizer)
+    preprocessor("The quick brown fox jumped.")
+    ```
+
+    Mapping with `tf.data.Dataset`.
+    ```python
+    preprocessor = keras_hub.models.TextClassifierPreprocessor.from_preset(
+        "roformer_v2_base_zh"
+    )
+
+    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
+    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
+    label = tf.constant([1, 1])
+
+    # Map labeled single sentences.
+    ds = tf.data.Dataset.from_tensor_slices((first, label))
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+
+    # Map unlabeled single sentences.
+    ds = tf.data.Dataset.from_tensor_slices(first)
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+
+    # Map labeled sentence pairs.
+    ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+
+    # Map unlabeled sentence pairs.
+    ds = tf.data.Dataset.from_tensor_slices((first, second))
+    # Watch out for tf.data's default unpacking of tuples here!
+    # Best to invoke the `preprocessor` directly in this case.
+    ds = ds.map(
+        lambda first, second: preprocessor(x=(first, second)),
+        num_parallel_calls=tf.data.AUTOTUNE,
+    )
+    ```
+    """
+
+    backbone_cls = RoformerV2Backbone
+    tokenizer_cls = RoformerV2Tokenizer
+
+    @preprocessing_function
+    def call(self, x, y=None, sample_weight=None):
+        x = x if isinstance(x, tuple) else (x,)
+        x = tuple(self.tokenizer(segment) for segment in x)
+        token_ids, segment_ids = self.packer(x)
+        x = {
+            "token_ids": token_ids,
+            "segment_ids": segment_ids,
+        }
+        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

keras_hub/src/models/roformer_v2/roformer_v2_tokenizer.py

@@ -0,0 +1,62 @@
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.bert.bert_tokenizer import BertTokenizer
+from keras_hub.src.models.roformer_v2.roformer_v2_backbone import (
+    RoformerV2Backbone,
+)
+
+
+@keras_hub_export(
+    [
+        "keras_hub.tokenizers.RoformerV2Tokenizer",
+        "keras_hub.models.RoformerV2Tokenizer",
+    ]
+)
+class RoformerV2Tokenizer(BertTokenizer):
+    """A RoformerV2 tokenizer using WordPiece subword segmentation.
+
+    This tokenizer class will tokenize raw strings into integer sequences and
+    is based on `keras_hub.tokenizers.WordPieceTokenizer`. Unlike the
+    underlying tokenizer, it will check for special tokens needed by RoformerV2
+    models and provides a `from_preset()` method to automatically download
+    a matching vocabulary for a RoformerV2 preset.
+
+    If input is a batch of strings (rank > 0), the layer will output a
+    `tf.RaggedTensor` where the last dimension of the output is ragged.
+
+    If input is a scalar string (rank == 0), the layer will output a dense
+    `tf.Tensor` with static shape `[None]`.
+
+    Args:
+        vocabulary: A list of strings or a string filename path. If
+            passing a list, each element of the list should be a single word
+            piece token string. If passing a filename, the file should be a
+            plain text file containing a single word piece token per line.
+        lowercase: If `True`, the input text will be first lowered before
+            tokenization.
+        special_tokens_in_strings: bool. A bool to indicate if the tokenizer
+            should expect special tokens in input strings that should be
+            tokenized and mapped correctly to their ids. Defaults to False.
+
+    Examples:
+    ```python
+    # Unbatched input.
+    tokenizer = keras_hub.models.RoformerV2Tokenizer.from_preset(
+        "roformer_v2_base_zh",
+    )
+    tokenizer("The quick brown fox jumped.")
+
+    # Batched input.
+    tokenizer(["The quick brown fox jumped.", "The fox slept."])
+
+    # Detokenization.
+    tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
+
+    # Custom vocabulary.
+    vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
+    vocab += ["The", "quick", "brown", "fox", "jumped", "."]
+    tokenizer = keras_hub.models.RoformerV2Tokenizer(vocabulary=vocab)
+    tokenizer("The quick brown fox jumped.")
+    ```
+    """
+
+    backbone_cls = RoformerV2Backbone

keras_hub/src/models/stable_diffusion_3/stable_diffusion_3_text_to_image_preprocessor.py

@@ -2,14 +2,16 @@ import keras
 from keras import layers
 
 from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.models.preprocessor import Preprocessor
 from keras_hub.src.models.stable_diffusion_3.stable_diffusion_3_backbone import (  # noqa: E501
     StableDiffusion3Backbone,
 )
+from keras_hub.src.models.text_to_image_preprocessor import (
+    TextToImagePreprocessor,
+)
 
 
 @keras_hub_export("keras_hub.models.StableDiffusion3TextToImagePreprocessor")
-class StableDiffusion3TextToImagePreprocessor(Preprocessor):
+class StableDiffusion3TextToImagePreprocessor(TextToImagePreprocessor):
     """Stable Diffusion 3 text-to-image model preprocessor.
 
     This preprocessing layer is meant for use with

keras_hub/src/models/text_to_image_preprocessor.py

@@ -0,0 +1,35 @@
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.preprocessor import Preprocessor
+
+
+@keras_hub_export("keras_hub.models.TextToImagePreprocessor")
+class TextToImagePreprocessor(Preprocessor):
+    """Base class for text to image preprocessing layers.
+
+    `TextToImagePreprocessor` tasks wrap a `keras_hub.tokenizer.Tokenizer` to
+    create a preprocessing layer for text to image tasks. It is intended to be
+    paired with a `keras_hub.models.TextToImage` task.
+
+    The exact specifics of this layer will vary depending on the subclass
+    implementation per model architecture. Generally, it will take text input,
+    and tokenize, then pad/truncate so it is ready to be fed to a image
+    generation model (e.g. a diffusion model).
+
+    Examples.
+    ```python
+    preprocessor = keras_hub.models.TextToImagePreprocessor.from_preset(
+        "stable_diffusion_3_medium",
+        sequence_length=256, # Optional.
+    )
+
+    # Tokenize and pad/truncate a single sentence.
+    x = "The quick brown fox jumped."
+    x = preprocessor(x)
+
+    # Tokenize and pad/truncate a batch of sentences.
+    x = ["The quick brown fox jumped."]
+    x = preprocessor(x)
+    ```
+    """
+
+    pass

keras_hub/src/utils/preset_utils.py

@@ -238,7 +238,8 @@ def get_file(preset, path):
             "2) a Kaggle Models handle like "
             "`'kaggle://keras/bert/keras/bert_base_en'`\n"
             "3) a Hugging Face handle like `'hf://username/bert_base_en'`\n"
-            "4) a path to a local preset directory like `'./bert_base_en`\n"
+            "4) a Modelscope handle like `'modelscope://username/bert_base_en'`\n"
+            "5) a path to a local preset directory like `'./bert_base_en'`\n"
             "Use `print(cls.presets.keys())` to view all built-in presets for "
             "API symbol `cls`.\n"
             f"Received: preset='{preset}'"

keras_hub/src/version_utils.py

@@ -1,7 +1,7 @@
 from keras_hub.src.api_export import keras_hub_export
 
 # Unique source of truth for the version number.
-__version__ = "0.20.0.dev202503310403"
+__version__ = "0.20.0.dev202504020401"
 
 
 @keras_hub_export("keras_hub.version")