keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl → 0.16.0.dev20240915160609__py3-none-any.whl

keras_hub/src/models/llama/llama_preprocessor.py

@@ -1,189 +0,0 @@
-# Copyright 2024 The KerasHub Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import keras
-
-from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_hub.src.models.llama.llama_tokenizer import LlamaTokenizer
-from keras_hub.src.models.preprocessor import Preprocessor
-from keras_hub.src.utils.keras_utils import (
-    convert_inputs_to_list_of_tensor_segments,
-)
-
-
-@keras_hub_export("keras_hub.models.LlamaPreprocessor")
-class LlamaPreprocessor(Preprocessor):
-    """A Llama preprocessing layer which tokenizes and packs inputs.
-
-    This preprocessing layer will do three things:
-
-     1. Tokenize any number of input segments using the `tokenizer`.
-     2. Pack the inputs together using a `keras_hub.layers.StartEndPacker`.
-       with the appropriate tokens.
-     3. Construct a dictionary with keys `"token_ids"`, and `"padding_mask"`
-       that can be passed directly to `keras_hub.models.LlamaBackbone`.
-
-    This layer can be used directly with `tf.data.Dataset.map` to preprocess
-    string data in the `(x, y, sample_weight)` format used by
-    `keras.Model.fit`.
-
-    Args:
-        tokenizer: A `keras_hub.models.LlamaTokenizer` instance.
-        sequence_length: The length of the packed inputs.
-        add_start_token: If `True`, the preprocessor will prepend the tokenizer
-            start token to each input sequence. Default is `True`.
-        add_end_token: If `True`, the preprocessor will append the tokenizer
-            end token to each input sequence. Default is `False`.
-
-    Call arguments:
-        x: A tensor of single string sequences, or a tuple of multiple
-            tensor sequences to be packed together. Inputs may be batched or
-            unbatched. For single sequences, raw python inputs will be converted
-            to tensors. For multiple sequences, pass tensors directly.
-        y: Any label data. Will be passed through unaltered.
-        sample_weight: Any label weight data. Will be passed through unaltered.
-        sequence_length: Pass to override the configured `sequence_length` of
-            the layer.
-
-    Examples:
-
-    Directly calling the from_preset().
-    ```python
-    preprocessor = keras_hub.models.LlamaPreprocessor.from_preset(
-        "llama_base_en"
-    )
-
-    # Tokenize and pack a single sentence.
-    preprocessor("The quick brown fox jumped.")
-
-    # Tokenize and a batch of single sentences.
-    preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
-
-    # Preprocess a batch of sentence pairs.
-    # When handling multiple sequences, always convert to tensors first!
-    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
-    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
-    preprocessor((first, second))
-    ```
-
-    Mapping with `tf.data.Dataset`.
-    ```python
-    preprocessor = keras_hub.models.LlamaPreprocessor.from_preset(
-        "llama_base_en"
-    )
-    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
-    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
-    label = tf.constant([1, 1])
-
-    # Map labeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices((first, label))
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map unlabeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices(first)
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map labeled sentence pairs.
-    ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map unlabeled sentence pairs.
-    ds = tf.data.Dataset.from_tensor_slices((first, second))
-
-    # Watch out for tf.data's default unpacking of tuples here!
-    # Best to invoke the `preprocessor` directly in this case.
-    ds = ds.map(
-        lambda first, second: preprocessor(x=(first, second)),
-        num_parallel_calls=tf.data.AUTOTUNE,
-    )
-    ```
-    """
-
-    tokenizer_cls = LlamaTokenizer
-
-    def __init__(
-        self,
-        tokenizer,
-        sequence_length=1024,
-        add_start_token=True,
-        add_end_token=False,
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-        self.tokenizer = tokenizer
-        self.packer = None
-        self.add_start_token = add_start_token
-        self.add_end_token = add_end_token
-        self.sequence_length = sequence_length
-
-    def build(self, input_shape):
-        # Defer packer creation to `build()` so that we can be sure tokenizer
-        # assets have loaded when restoring a saved model.
-        self.packer = StartEndPacker(
-            start_value=self.tokenizer.start_token_id,
-            end_value=self.tokenizer.end_token_id,
-            sequence_length=self.sequence_length,
-            return_padding_mask=True,
-        )
-        self.built = True
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.sequence_length,
-                "add_start_token": self.add_start_token,
-                "add_end_token": self.add_end_token,
-            }
-        )
-        return config
-
-    def call(
-        self,
-        x,
-        y=None,
-        sample_weight=None,
-        sequence_length=None,
-    ):
-        x = convert_inputs_to_list_of_tensor_segments(x)
-        if len(x) != 1:
-            raise ValueError(
-                "Llama requires each input feature to contain only "
-                f"one segment, but received {len(x)}. If you are using Llama"
-                " for a multi-segment classification task, please refer to "
-                "classification models like BERT or RoBERTa."
-            )
-        sequence_length = sequence_length or self.sequence_length
-        token_ids, padding_mask = self.packer(
-            self.tokenizer(x[0]),
-            sequence_length=sequence_length,
-            add_start_value=self.add_start_token,
-            add_end_value=self.add_end_token,
-        )
-        x = {
-            "token_ids": token_ids,
-            "padding_mask": padding_mask,
-        }
-        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
-
-    @property
-    def sequence_length(self):
-        """The padded length of model input sequences."""
-        return self._sequence_length
-
-    @sequence_length.setter
-    def sequence_length(self, value):
-        self._sequence_length = value
-        if self.packer is not None:
-            self.packer.sequence_length = value

keras_hub/src/models/mistral/mistral_preprocessor.py

@@ -1,190 +0,0 @@
-# Copyright 2024 The KerasHub Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import keras
-
-from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_hub.src.models.mistral.mistral_tokenizer import MistralTokenizer
-from keras_hub.src.models.preprocessor import Preprocessor
-from keras_hub.src.utils.keras_utils import (
-    convert_inputs_to_list_of_tensor_segments,
-)
-
-
-@keras_hub_export("keras_hub.models.MistralPreprocessor")
-class MistralPreprocessor(Preprocessor):
-    """A Mistral preprocessing layer which tokenizes and packs inputs.
-
-    This preprocessing layer will do three things:
-
-     1. Tokenize any number of input segments using the `tokenizer`.
-     2. Pack the inputs together using a `keras_hub.layers.StartEndPacker`.
-       with the appropriate tokens.
-     3. Construct a dictionary with keys `"token_ids"`, and `"padding_mask"`
-       that can be passed directly to `keras_hub.models.MistralBackbone`.
-
-    This layer can be used directly with `tf.data.Dataset.map` to preprocess
-    string data in the `(x, y, sample_weight)` format used by
-    `keras.Model.fit`.
-
-    Args:
-        tokenizer: A `keras_hub.models.MistralTokenizer` instance.
-        sequence_length: The length of the packed inputs.
-        add_start_token: If `True`, the preprocessor will prepend the tokenizer
-            start token to each input sequence. Default is `True`.
-        add_end_token: If `True`, the preprocessor will append the tokenizer
-            end token to each input sequence. Default is `False`.
-
-    Call arguments:
-        x: A tensor of single string sequences, or a tuple of multiple
-            tensor sequences to be packed together. Inputs may be batched or
-            unbatched. For single sequences, raw python inputs will be converted
-            to tensors. For multiple sequences, pass tensors directly.
-        y: Any label data. Will be passed through unaltered.
-        sample_weight: Any label weight data. Will be passed through unaltered.
-        sequence_length: Pass to override the configured `sequence_length` of
-            the layer.
-
-    Examples:
-
-    Directly calling the from_preset().
-    ```python
-    preprocessor = keras_hub.models.MistralPreprocessor.from_preset(
-        "mistral_base_en"
-    )
-
-    # Tokenize and pack a single sentence.
-    preprocessor("The quick brown fox jumped.")
-
-    # Tokenize and a batch of single sentences.
-    preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
-
-    # Preprocess a batch of sentence pairs.
-    # When handling multiple sequences, always convert to tensors first!
-    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
-    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
-    preprocessor((first, second))
-    ```
-
-    Mapping with `tf.data.Dataset`.
-    ```python
-    preprocessor = keras_hub.models.MistralPreprocessor.from_preset(
-        "mistral_base_en"
-    )
-    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
-    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
-    label = tf.constant([1, 1])
-
-    # Map labeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices((first, label))
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map unlabeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices(first)
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map labeled sentence pairs.
-    ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map unlabeled sentence pairs.
-    ds = tf.data.Dataset.from_tensor_slices((first, second))
-
-    # Watch out for tf.data's default unpacking of tuples here!
-    # Best to invoke the `preprocessor` directly in this case.
-    ds = ds.map(
-        lambda first, second: preprocessor(x=(first, second)),
-        num_parallel_calls=tf.data.AUTOTUNE,
-    )
-    ```
-    """
-
-    tokenizer_cls = MistralTokenizer
-
-    def __init__(
-        self,
-        tokenizer,
-        sequence_length=1024,
-        add_start_token=True,
-        add_end_token=False,
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-        self.tokenizer = tokenizer
-        self.packer = None
-        self.add_start_token = add_start_token
-        self.add_end_token = add_end_token
-        self.sequence_length = sequence_length
-
-    def build(self, input_shape):
-        # Defer packer creation to `build()` so that we can be sure tokenizer
-        # assets have loaded when restoring a saved model.
-        self.packer = StartEndPacker(
-            start_value=self.tokenizer.start_token_id,
-            end_value=self.tokenizer.end_token_id,
-            sequence_length=self.sequence_length,
-            return_padding_mask=True,
-        )
-        self.built = True
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.sequence_length,
-                "add_start_token": self.add_start_token,
-                "add_end_token": self.add_end_token,
-            }
-        )
-        return config
-
-    def call(
-        self,
-        x,
-        y=None,
-        sample_weight=None,
-        sequence_length=None,
-    ):
-        x = convert_inputs_to_list_of_tensor_segments(x)
-        if len(x) != 1:
-            raise ValueError(
-                "Mistral requires each input feature to contain only "
-                f"one segment, but received {len(x)}. If you are using Mistral"
-                " for a multi-segment classification task, please refer to "
-                "classification models like BERT or RoBERTa."
-            )
-        sequence_length = sequence_length or self.sequence_length
-        token_ids, padding_mask = self.packer(
-            self.tokenizer(x[0]),
-            sequence_length=sequence_length,
-            add_start_value=self.add_start_token,
-            add_end_value=self.add_end_token,
-        )
-        x = {
-            "token_ids": token_ids,
-            "padding_mask": padding_mask,
-        }
-        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
-
-    @property
-    def sequence_length(self):
-        """The padded length of model input sequences."""
-        return self._sequence_length
-
-    @sequence_length.setter
-    def sequence_length(self, value):
-        self._sequence_length = value
-        if self.packer is not None:
-            self.packer.sequence_length = value

keras_hub/src/models/opt/opt_preprocessor.py

@@ -1,188 +0,0 @@
-# Copyright 2024 The KerasHub Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import keras
-
-from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_hub.src.models.opt.opt_tokenizer import OPTTokenizer
-from keras_hub.src.models.preprocessor import Preprocessor
-from keras_hub.src.utils.keras_utils import (
-    convert_inputs_to_list_of_tensor_segments,
-)
-
-
-@keras_hub_export("keras_hub.models.OPTPreprocessor")
-class OPTPreprocessor(Preprocessor):
-    """OPT preprocessing layer which tokenizes and packs inputs.
-
-    This preprocessing layer will do 2 things:
-
-    - Tokenize the input using the `tokenizer`.
-    - Construct a dictionary with keys `"token_ids"`, `"padding_mask"`, that can
-        be passed directly to a `keras_hub.models.OPTBackbone`.
-
-    This layer can be used directly with `tf.data.Dataset.map` to preprocess
-    string data in the `(x, y, sample_weight)` format used by
-    `keras.Model.fit`.
-
-    The call method of this layer accepts three arguments, `x`, `y`, and
-    `sample_weight`. `x` can be a python string or tensor representing a single
-    segment, a list of python strings representing a batch of single segments,
-    or a list of tensors representing multiple segments to be packed together.
-    `y` and `sample_weight` are both optional, can have any format, and will be
-    passed through unaltered.
-
-    `OPTPreprocessor` forces the input to have only one segment, as OPT is
-    mainly used for generation tasks. For tasks having multi-segment inputs
-    like "glue/mnli", please use a model designed for classification purposes
-    such as BERT or RoBERTa.
-
-    Args:
-        tokenizer: A `keras_hub.models.OPTTokenizer` instance.
-        sequence_length: The length of the packed inputs.
-        add_start_token: If `True`, the preprocessor will append the tokenizer
-            start token to each input sequence.
-        add_end_token: If `True`, the preprocessor will append the tokenizer
-            end token to each input sequence.
-
-    Call arguments:
-        x: A string, `tf.Tensor` or list of python strings.
-        y: Any label data. Will be passed through unaltered.
-        sample_weight: Any label weight data. Will be passed through unaltered.
-        sequence_length: Pass to override the configured `sequence_length` of
-            the layer.
-
-    Examples:
-
-    Directly calling the layer on data.
-    ```python
-    preprocessor = keras_hub.models.OPTPreprocessor.from_preset("opt_125m_en")
-
-    # Tokenize and pack a single sentence.
-    preprocessor("The quick brown fox jumped.")
-
-    # Tokenize a batch of single sentences.
-    preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
-
-    # Custom vocabulary.
-    features = ["a quick fox.", "a fox quick."]
-    vocab = {"<|endoftext|>": 0, "a": 4, "Ġquick": 5, "Ġfox": 6}
-    merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"]
-    merges += ["Ġ f", "o x", "Ġf ox"]
-    tokenizer = keras_hub.models.OPTTokenizer(
-        vocabulary=vocab,
-        merges=merges,
-    )
-    preprocessor = keras_hub.models.OPTPreprocessor(tokenizer=tokenizer)
-    preprocessor("The quick brown fox jumped.")
-    ```
-
-    Mapping with `tf.data.Dataset`.
-    ```python
-    preprocessor = keras_hub.models.OPTPreprocessor.from_preset("opt_125m_en")
-
-    text = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
-    label = tf.constant([1, 1])
-
-    # Map labeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices((text, label))
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map unlabeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices(text)
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-    ```
-    """
-
-    tokenizer_cls = OPTTokenizer
-
-    def __init__(
-        self,
-        tokenizer,
-        sequence_length=1024,
-        add_start_token=True,
-        add_end_token=True,
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-
-        self.tokenizer = tokenizer
-        self.packer = None
-        self.sequence_length = sequence_length
-        self.add_start_token = add_start_token
-        self.add_end_token = add_end_token
-
-    def build(self, input_shape):
-        # Defer packer creation to `build()` so that we can be sure tokenizer
-        # assets have loaded when restoring a saved model.
-        self.packer = StartEndPacker(
-            start_value=self.tokenizer.start_token_id,
-            end_value=self.tokenizer.end_token_id,
-            pad_value=self.tokenizer.pad_token_id,
-            sequence_length=self.sequence_length,
-            return_padding_mask=True,
-        )
-        self.built = True
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.sequence_length,
-                "add_start_token": self.add_start_token,
-                "add_end_token": self.add_end_token,
-            }
-        )
-        return config
-
-    def call(
-        self,
-        x,
-        y=None,
-        sample_weight=None,
-        sequence_length=None,
-    ):
-        x = convert_inputs_to_list_of_tensor_segments(x)
-        if len(x) != 1:
-            raise ValueError(
-                "OPT requires each input feature to contain only "
-                f"one segment, but received {len(x)}. If you are using OPT "
-                "for a multi-segment classification task, please refer to "
-                "classification models like BERT or RoBERTa."
-            )
-        sequence_length = sequence_length or self.sequence_length
-        token_ids, padding_mask = self.packer(
-            self.tokenizer(x[0]),
-            sequence_length=sequence_length,
-            add_start_value=self.add_start_token,
-            add_end_value=self.add_end_token,
-        )
-        x = {
-            "token_ids": token_ids,
-            "padding_mask": padding_mask,
-        }
-        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
-
-    @property
-    def sequence_length(self):
-        """The padded length of model input sequences."""
-        return self._sequence_length
-
-    @sequence_length.setter
-    def sequence_length(self, value):
-        self._sequence_length = value
-        if self.packer is not None:
-            self.packer.sequence_length = value