keras-hub-nightly 0.15.0.dev20240911134614__py3-none-any.whl → 0.16.0.dev20240915160609__py3-none-any.whl

keras_hub/src/models/bart/bart_preprocessor.py

@@ -1,264 +0,0 @@
-# Copyright 2024 The KerasHub Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import keras
-
-from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_hub.src.models.bart.bart_backbone import BartBackbone
-from keras_hub.src.models.bart.bart_tokenizer import BartTokenizer
-from keras_hub.src.models.preprocessor import Preprocessor
-from keras_hub.src.utils.tensor_utils import preprocessing_function
-
-
-@keras_hub_export("keras_hub.models.BartPreprocessor")
-class BartPreprocessor(Preprocessor):
-    """A BART preprocessing layer which tokenizes and packs inputs.
-
-    This preprocessing layer will do three things:
-
-     1. Tokenize both encoder inputs and decoder inputs using the `tokenizer`.
-        Both inputs can contain only one segment.
-     2. Add the appropriate special tokens - `"<s>"`, `"</s>"` and `"<pad>"`.
-     3. Construct a dictionary with keys `"encoder_token_ids"`,
-        `"encoder_padding_mask"`, `"decoder_token_ids"`, `"decoder_padding_mask"`
-        that can be passed directly to a BART model.
-
-    Args:
-        tokenizer: A `keras_hub.models.BartTokenizer` instance.
-        encoder_sequence_length: The length of the packed encoder inputs.
-        decoder_sequence_length: The length of the packed decoder inputs.
-
-    Call arguments:
-        x: A dictionary with `encoder_text` and `decoder_text` as its keys.
-            Each value in the dictionary should be a tensor of single string
-            sequences. Inputs may be batched or unbatched. Raw python inputs
-            will be converted to tensors.
-        y: Any label data. Will be passed through unaltered.
-        sample_weight: Any label weight data. Will be passed through unaltered.
-
-    Examples:
-
-    Directly calling the layer on data.
-    ```python
-    preprocessor = keras_hub.models.BartPreprocessor.from_preset("bart_base_en")
-
-    # Preprocess unbatched inputs.
-    inputs = {
-        "encoder_text": "The fox was sleeping.",
-        "decoder_text": "The fox was awake."
-    }
-    preprocessor(inputs)
-
-    # Preprocess batched inputs.
-    inputs = {
-        "encoder_text": ["The fox was sleeping.", "The lion was quiet."],
-        "decoder_text": ["The fox was awake.", "The lion was roaring."]
-    }
-    preprocessor(inputs)
-
-    # Custom vocabulary.
-    vocab = {
-        "<s>": 0,
-        "<pad>": 1,
-        "</s>": 2,
-        "Ġafter": 5,
-        "noon": 6,
-        "Ġsun": 7,
-    }
-    merges = ["Ġ a", "Ġ s", "Ġ n", "e r", "n o", "o n", "Ġs u", "Ġa f", "no on"]
-    merges += ["Ġsu n", "Ġaf t", "Ġaft er"]
-
-    tokenizer = keras_hub.models.BartTokenizer(
-        vocabulary=vocab,
-        merges=merges,
-    )
-    preprocessor = keras_hub.models.BartPreprocessor(
-        tokenizer=tokenizer,
-        encoder_sequence_length=20,
-        decoder_sequence_length=10,
-    )
-    inputs = {
-        "encoder_text": "The fox was sleeping.",
-        "decoder_text": "The fox was awake."
-    }
-    preprocessor(inputs)
-    ```
-
-    Mapping with `tf.data.Dataset`.
-    ```python
-    preprocessor = keras_hub.models.BartPreprocessor.from_preset("bart_base_en")
-
-    # Map labeled single sentences.
-    features = {
-        "encoder_text": tf.constant(
-            ["The fox was sleeping.", "The lion was quiet."]
-        ),
-        "decoder_text": tf.constant(
-            ["The fox was awake.", "The lion was silent."]
-        )
-    }
-    labels = tf.constant(["True", "False"])
-    ds = tf.data.Dataset.from_tensor_slices((features, labels))
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map unlabeled single sentences.
-    features = {
-        "encoder_text": tf.constant(
-            ["The fox was sleeping.", "The lion was quiet."]
-        ),
-        "decoder_text": tf.constant(
-            ["The fox was awake.", "The lion was roaring."]
-        )
-    }
-    ds = tf.data.Dataset.from_tensor_slices(features)
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-    ```
-    """
-
-    backbone_cls = BartBackbone
-    tokenizer_cls = BartTokenizer
-
-    def __init__(
-        self,
-        tokenizer,
-        encoder_sequence_length=1024,
-        decoder_sequence_length=1024,
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-        self.tokenizer = tokenizer
-        self.encoder_packer = None
-        self.decoder_packer = None
-        self.encoder_sequence_length = encoder_sequence_length
-        self.decoder_sequence_length = decoder_sequence_length
-
-    def build(self, input_shape):
-        # Defer packer creation to `build()` so that we can be sure tokenizer
-        # assets have loaded when restoring a saved model.
-
-        # TODO: Use `MultiSegmentPacker` instead of `StartEndPacker` once we
-        # want to move to multi-segment packing and have improved
-        # `MultiSegmentPacker`'s performance.
-        self.encoder_packer = StartEndPacker(
-            start_value=self.tokenizer.start_token_id,
-            end_value=self.tokenizer.end_token_id,
-            pad_value=self.tokenizer.pad_token_id,
-            sequence_length=self.encoder_sequence_length,
-            return_padding_mask=True,
-        )
-
-        # The decoder is packed a bit differently; the format is as follows:
-        # `[end_token_id, start_token_id, tokens..., end_token_id, padding...]`.
-        self.decoder_packer = StartEndPacker(
-            start_value=[
-                self.tokenizer.end_token_id,
-                self.tokenizer.start_token_id,
-            ],
-            end_value=self.tokenizer.end_token_id,
-            pad_value=self.tokenizer.pad_token_id,
-            sequence_length=self.decoder_sequence_length,
-            return_padding_mask=True,
-        )
-        self.built = True
-
-    @preprocessing_function
-    def call(
-        self,
-        x,
-        y=None,
-        sample_weight=None,
-        *,
-        encoder_sequence_length=None,
-        decoder_sequence_length=None,
-        # `sequence_length` is an alias for `decoder_sequence_length`
-        sequence_length=None,
-    ):
-        if not (
-            isinstance(x, dict)
-            and all(k in x for k in ("encoder_text", "decoder_text"))
-        ):
-            raise ValueError(
-                '`x` must be a dictionary, containing the keys `"encoder_text"`'
-                f' and `"decoder_text"`. Received x={x}.'
-            )
-
-        if encoder_sequence_length is None:
-            encoder_sequence_length = self.encoder_sequence_length
-        decoder_sequence_length = decoder_sequence_length or sequence_length
-        if decoder_sequence_length is None:
-            decoder_sequence_length = self.decoder_sequence_length
-
-        encoder_inputs = self.tokenizer(x["encoder_text"])
-        encoder_token_ids, encoder_padding_mask = self.encoder_packer(
-            encoder_inputs,
-            sequence_length=encoder_sequence_length,
-        )
-
-        decoder_inputs = self.tokenizer(x["decoder_text"])
-        decoder_token_ids, decoder_padding_mask = self.decoder_packer(
-            decoder_inputs,
-            sequence_length=decoder_sequence_length,
-        )
-
-        x = {
-            "encoder_token_ids": encoder_token_ids,
-            "encoder_padding_mask": encoder_padding_mask,
-            "decoder_token_ids": decoder_token_ids,
-            "decoder_padding_mask": decoder_padding_mask,
-        }
-
-        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "encoder_sequence_length": self.encoder_sequence_length,
-                "decoder_sequence_length": self.decoder_sequence_length,
-            }
-        )
-        return config
-
-    @property
-    def encoder_sequence_length(self):
-        """The padded length of encoder input sequences."""
-        return self._encoder_sequence_length
-
-    @encoder_sequence_length.setter
-    def encoder_sequence_length(self, value):
-        self._encoder_sequence_length = value
-        if self.encoder_packer is not None:
-            self.encoder_packer.sequence_length = value
-
-    @property
-    def decoder_sequence_length(self):
-        """The padded length of decoder input sequences."""
-        return self._decoder_sequence_length
-
-    @decoder_sequence_length.setter
-    def decoder_sequence_length(self, value):
-        self._decoder_sequence_length = value
-        if self.decoder_packer is not None:
-            self.decoder_packer.sequence_length = value
-
-    @property
-    def sequence_length(self):
-        """Alias for `decoder_sequence_length`."""
-        return self.decoder_sequence_length
-
-    @sequence_length.setter
-    def sequence_length(self, value):
-        self.decoder_sequence_length = value

keras_hub/src/models/bloom/bloom_preprocessor.py

@@ -1,178 +0,0 @@
-# Copyright 2024 The KerasHub Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-import keras
-
-from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
-from keras_hub.src.models.bloom.bloom_backbone import BloomBackbone
-from keras_hub.src.models.bloom.bloom_tokenizer import BloomTokenizer
-from keras_hub.src.models.preprocessor import Preprocessor
-from keras_hub.src.utils.tensor_utils import preprocessing_function
-
-
-@keras_hub_export("keras_hub.models.BloomPreprocessor")
-class BloomPreprocessor(Preprocessor):
-    """BLOOM preprocessing layer which tokenizes and packs inputs.
-
-    This preprocessing layer will do 2 things:
-
-    - Tokenize the inputs using the `tokenizer`.
-    - Construct a dictionary with keys `"token_ids"`, `"padding_mask"`, that can
-        be passed directly to a `keras_hub.models.BloomBackbone`.
-
-    This layer can be used directly with `tf.data.Dataset.map` to preprocess
-    string data in the `(x, y, sample_weight)` format used by
-    `keras.Model.fit`.
-
-    The call method of this layer accepts three arguments, `x`, `y`, and
-    `sample_weight`. `x` can be a python string or tensor representing a single
-    segment, a list of python strings representing a batch of single segments,
-    or a list of tensors representing multiple segments to be packed together.
-    `y` and `sample_weight` are both optional, can have any format, and will be
-    passed through unaltered.
-
-    Args:
-        tokenizer: A `keras_hub.models.BloomTokenizer` instance.
-        sequence_length: The length of the packed inputs.
-        add_start_token: If `True`, the preprocessor will prepend the tokenizer
-            start token to each input sequence.
-        add_end_token: If `True`, the preprocessor will append the tokenizer
-            end token to each input sequence.
-
-    Call arguments:
-        x: A string, `tf.Tensor` or list of python strings.
-        y: Any label data. Will be passed through unaltered.
-        sample_weight: Any label weight data. Will be passed through unaltered.
-        sequence_length: Pass to override the configured `sequence_length` of
-            the layer.
-
-    Examples:
-
-    Directly calling the layer on data.
-    ```python
-    preprocessor = keras_hub.models.BloomPreprocessor.from_preset(
-        "bloom_560m_multi"
-    )
-    # Tokenize and pack a single sentence.
-    preprocessor("The quick brown fox jumped.")
-
-    # Tokenize a batch of single sentences.
-    preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
-
-    # Custom vocabulary.
-    features = ["a quick fox.", "a fox quick."]
-    vocab = {"<pad>": 0, "<s>":1, "</s>":2, "a": 3, "Ġquick": 4, "Ġfox": 5}
-    merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"]
-    merges += ["Ġ f", "o x", "Ġf ox"]
-    tokenizer = keras_hub.models.BloomTokenizer(
-        vocabulary=vocab,
-        merges=merges,
-    )
-    preprocessor = keras_hub.models.BloomPreprocessor(tokenizer=tokenizer)
-    preprocessor("The quick brown fox jumped.")
-    ```
-
-    Mapping with `tf.data.Dataset`.
-    ```python
-    preprocessor = keras_hub.models.BloomPreprocessor.from_preset(
-        "bloom_560m_multi"
-    )
-
-    text = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
-    label = tf.constant([1, 1])
-
-    # Map labeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices((text, label))
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map unlabeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices(text)
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-    ```
-    """
-
-    backbone_cls = BloomBackbone
-    tokenizer_cls = BloomTokenizer
-
-    def __init__(
-        self,
-        tokenizer,
-        sequence_length=1024,
-        add_start_token=True,
-        add_end_token=True,
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-        self.tokenizer = tokenizer
-        self.packer = None
-        self.sequence_length = sequence_length
-        self.add_start_token = add_start_token
-        self.add_end_token = add_end_token
-
-    def build(self, input_shape):
-        # Defer packer creation to `build()` so that we can be sure tokenizer
-        # assets have loaded when restoring a saved model.
-        self.packer = StartEndPacker(
-            start_value=self.tokenizer.start_token_id,
-            end_value=self.tokenizer.end_token_id,
-            pad_value=self.tokenizer.pad_token_id,
-            sequence_length=self.sequence_length,
-            return_padding_mask=True,
-        )
-        self.built = True
-
-    @preprocessing_function
-    def call(
-        self,
-        x,
-        y=None,
-        sample_weight=None,
-        sequence_length=None,
-    ):
-        sequence_length = sequence_length or self.sequence_length
-        token_ids, padding_mask = self.packer(
-            self.tokenizer(x),
-            sequence_length=sequence_length,
-            add_start_value=self.add_start_token,
-            add_end_value=self.add_end_token,
-        )
-        x = {
-            "token_ids": token_ids,
-            "padding_mask": padding_mask,
-        }
-        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.sequence_length,
-                "add_start_token": self.add_start_token,
-                "add_end_token": self.add_end_token,
-            }
-        )
-        return config
-
-    @property
-    def sequence_length(self):
-        """The padded length of model input sequences."""
-        return self._sequence_length
-
-    @sequence_length.setter
-    def sequence_length(self, value):
-        self._sequence_length = value
-        if self.packer is not None:
-            self.packer.sequence_length = value

keras_hub/src/models/electra/electra_preprocessor.py

@@ -1,155 +0,0 @@
-# Copyright 2024 The KerasHub Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import keras
-
-from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.layers.preprocessing.multi_segment_packer import (
-    MultiSegmentPacker,
-)
-from keras_hub.src.models.electra.electra_backbone import ElectraBackbone
-from keras_hub.src.models.electra.electra_tokenizer import ElectraTokenizer
-from keras_hub.src.models.preprocessor import Preprocessor
-from keras_hub.src.utils.tensor_utils import preprocessing_function
-
-
-@keras_hub_export("keras_hub.models.ElectraPreprocessor")
-class ElectraPreprocessor(Preprocessor):
-    """A ELECTRA preprocessing layer which tokenizes and packs inputs.
-
-    This preprocessing layer will do three things:
-
-     1. Tokenize any number of input segments using the `tokenizer`.
-     2. Pack the inputs together using a `keras_hub.layers.MultiSegmentPacker`.
-       with the appropriate `"[CLS]"`, `"[SEP]"` and `"[PAD]"` tokens.
-     3. Construct a dictionary of with keys `"token_ids"` and `"padding_mask"`,
-       that can be passed directly to a ELECTRA model.
-
-    This layer can be used directly with `tf.data.Dataset.map` to preprocess
-    string data in the `(x, y, sample_weight)` format used by
-    `keras.Model.fit`.
-
-    Args:
-        tokenizer: A `keras_hub.models.ElectraTokenizer` instance.
-        sequence_length: The length of the packed inputs.
-        truncate: string. The algorithm to truncate a list of batched segments
-            to fit within `sequence_length`. The value can be either
-            `round_robin` or `waterfall`:
-            - `"round_robin"`: Available space is assigned one token at a
-                time in a round-robin fashion to the inputs that still need
-                some, until the limit is reached.
-            - `"waterfall"`: The allocation of the budget is done using a
-                "waterfall" algorithm that allocates quota in a
-                left-to-right manner and fills up the buckets until we run
-                out of budget. It supports an arbitrary number of segments.
-
-    Call arguments:
-        x: A tensor of single string sequences, or a tuple of multiple
-            tensor sequences to be packed together. Inputs may be batched or
-            unbatched. For single sequences, raw python inputs will be converted
-            to tensors. For multiple sequences, pass tensors directly.
-        y: Any label data. Will be passed through unaltered.
-        sample_weight: Any label weight data. Will be passed through unaltered.
-
-    Examples:
-
-    Directly calling the layer on data.
-    ```python
-    preprocessor = keras_hub.models.ElectraPreprocessor.from_preset(
-        "electra_base_discriminator_en"
-    )
-    preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
-
-    # Custom vocabulary.
-    vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
-    vocab += ["The", "quick", "brown", "fox", "jumped", "."]
-    tokenizer = keras_hub.models.ElectraTokenizer(vocabulary=vocab)
-    preprocessor = keras_hub.models.ElectraPreprocessor(tokenizer)
-    preprocessor("The quick brown fox jumped.")
-    ```
-
-    Mapping with `tf.data.Dataset`.
-    ```python
-    preprocessor = keras_hub.models.ElectraPreprocessor.from_preset(
-        "electra_base_discriminator_en"
-    )
-
-    first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
-    second = tf.constant(["The fox tripped.", "Oh look, a whale."])
-    label = tf.constant([1, 1])
-    # Map labeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices((first, label))
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-
-    # Map unlabeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices(first)
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map labeled sentence pairs.
-    ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-    # Map unlabeled sentence pairs.
-    ds = tf.data.Dataset.from_tensor_slices((first, second))
-
-    # Watch out for tf.data's default unpacking of tuples here!
-    # Best to invoke the `preprocessor` directly in this case.
-    ds = ds.map(
-        lambda first, second: preprocessor(x=(first, second)),
-        num_parallel_calls=tf.data.AUTOTUNE,
-    )
-    ```
-    """
-
-    backbone_cls = ElectraBackbone
-    tokenizer_cls = ElectraTokenizer
-
-    def __init__(
-        self,
-        tokenizer,
-        sequence_length=512,
-        truncate="round_robin",
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-        self.tokenizer = tokenizer
-        self.packer = MultiSegmentPacker(
-            start_value=self.tokenizer.cls_token_id,
-            end_value=self.tokenizer.sep_token_id,
-            pad_value=self.tokenizer.pad_token_id,
-            truncate=truncate,
-            sequence_length=sequence_length,
-        )
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.packer.sequence_length,
-                "truncate": self.packer.truncate,
-            }
-        )
-        return config
-
-    @preprocessing_function
-    def call(self, x, y=None, sample_weight=None):
-        x = x if isinstance(x, tuple) else (x,)
-        x = tuple(self.tokenizer(segment) for segment in x)
-        token_ids, segment_ids = self.packer(x)
-        x = {
-            "token_ids": token_ids,
-            "segment_ids": segment_ids,
-            "padding_mask": token_ids != self.tokenizer.pad_token_id,
-        }
-        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)