keras-hub-nightly 0.22.0.dev202508170419__py3-none-any.whl → 0.24.0.dev202511090424__py3-none-any.whl

keras_hub/src/models/smollm3/smollm3_causal_lm_preprocessor.py

@@ -0,0 +1,84 @@
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.causal_lm_preprocessor import CausalLMPreprocessor
+from keras_hub.src.models.smollm3.smollm3_backbone import SmolLM3Backbone
+from keras_hub.src.models.smollm3.smollm3_tokenizer import SmolLM3Tokenizer
+
+
+@keras_hub_export(
+    [
+        "keras_hub.models.SmolLMCausalLMPreprocessor",
+        "keras_hub.models.SmolLM3CausalLMPreprocessor",
+    ]
+)
+class SmolLM3CausalLMPreprocessor(CausalLMPreprocessor):
+    """SmolLM3 Causal LM preprocessor.
+
+    This preprocessing layer is meant for use with
+    `keras_hub.models.SmolLM3CausalLM`. By default, it will take in batches of
+    strings, and return outputs in a `(x, y, sample_weight)` format, where the
+    `y` label is the next token id in the `x` sequence.
+
+    For use with generation, the layer also exposes two methods
+    `generate_preprocess()` and `generate_postprocess()`. When this preprocessor
+    is attached to a `keras_hub.models.SmolLM3CausalLM` instance, these methods
+    will be called implicitly in `generate()`. They can also be called
+    standalone (e.g. to precompute preprocessing inputs for generation in a
+    separate process).
+
+    Args:
+        tokenizer: A `keras_hub.models.SmolLM3Tokenizer` instance.
+        sequence_length: The length of the packed inputs.
+        add_start_token: If `True`, the preprocessor will prepend the tokenizer
+            start token to each input sequence. Default is `True`.
+        add_end_token: If `True`, the preprocessor will append the tokenizer
+            end token to each input sequence. Default is `False`.
+
+    Call arguments:
+        x: A string, `tf.Tensor` or list of python strings.
+        y: Label data. Should always be `None` as the layer generates labels.
+        sample_weight: Label weights. Should always be `None` as the layer
+            generates label weights.
+        sequence_length: Pass to override the configured `sequence_length` of
+            the layer.
+
+    Examples:
+    ```python
+    # Load the preprocessor from a preset.
+    preprocessor = keras_hub.models.SmolLM3CausalLMPreprocessor.from_preset(
+        "..."
+    )
+
+    # Tokenize and pack a single sentence.
+    sentence = tf.constant("...")
+    preprocessor(sentence)
+    # Same output.
+    preprocessor("...")
+
+    # Tokenize a batch of sentences.
+    sentences = tf.constant(["...", "..."])
+    preprocessor(sentences)
+    # Same output.
+    preprocessor(["...", "..."])
+
+    # Map a dataset to preprocess a single sentence.
+    features = tf.constant(
+        [
+            "...",
+            "...",
+        ]
+    )
+    labels = tf.constant([1, 0])
+    ds = tf.data.Dataset.from_tensor_slices((features, labels))
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+
+    # Map a dataset to preprocess unlabled sentences.
+    ds = tf.data.Dataset.from_tensor_slices(features)
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+    ```
+    """
+
+    backbone_cls = SmolLM3Backbone
+    tokenizer_cls = SmolLM3Tokenizer
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)