keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl → 0.16.0.dev2024092017__py3-none-any.whl

keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py

@@ -12,19 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import keras
-from absl import logging
-
 from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
-from keras_hub.src.utils.keras_utils import (
-    convert_inputs_to_list_of_tensor_segments,
-)
-from keras_hub.src.utils.tensor_utils import strip_to_ragged
+from keras_hub.src.models.causal_lm_preprocessor import CausalLMPreprocessor
+from keras_hub.src.models.gpt2.gpt2_backbone import GPT2Backbone
+from keras_hub.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
 
 
 @keras_hub_export("keras_hub.models.GPT2CausalLMPreprocessor")
-class GPT2CausalLMPreprocessor(GPT2Preprocessor):
+class GPT2CausalLMPreprocessor(CausalLMPreprocessor):
     """GPT2 Causal LM preprocessor.
 
     This preprocessing layer is meant for use with
@@ -91,83 +86,5 @@ class GPT2CausalLMPreprocessor(GPT2Preprocessor):
     ```
     """
 
-    def call(
-        self,
-        x,
-        y=None,
-        sample_weight=None,
-        sequence_length=None,
-    ):
-        if y is not None or sample_weight is not None:
-            logging.warning(
-                "`GPT2CausalLMPreprocessor` generates `y` and `sample_weight` "
-                "based on your input data, but your data already contains `y` "
-                "or `sample_weight`. Your `y` and `sample_weight` will be "
-                "ignored."
-            )
-        sequence_length = sequence_length or self.sequence_length
-
-        x = convert_inputs_to_list_of_tensor_segments(x)[0]
-        x = self.tokenizer(x)
-        # Pad with one extra token to account for the truncation below.
-        token_ids, padding_mask = self.packer(
-            x,
-            sequence_length=sequence_length + 1,
-            add_start_value=self.add_start_token,
-            add_end_value=self.add_end_token,
-        )
-        # The last token does not have a next token, so we truncate it out.
-        x = {
-            "token_ids": token_ids[..., :-1],
-            "padding_mask": padding_mask[..., :-1],
-        }
-        # Target `y` will be the next token.
-        y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:]
-        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
-
-    def generate_preprocess(
-        self,
-        x,
-        sequence_length=None,
-    ):
-        """Convert strings to integer token input for generation.
-
-        Similar to calling the layer for training, this method takes in strings
-        or tensor strings, tokenizes and packs the input, and computes a padding
-        mask masking all inputs not filled in with a padded value.
-
-        Unlike calling the layer for training, this method does not compute
-        labels and will never append a `tokenizer.end_token_id` to the end of
-        the sequence (as generation is expected to continue at the end of the
-        inputted prompt).
-        """
-        if not self.built:
-            self.build(None)
-
-        x = convert_inputs_to_list_of_tensor_segments(x)[0]
-        x = self.tokenizer(x)
-        token_ids, padding_mask = self.packer(
-            x, sequence_length=sequence_length, add_end_value=False
-        )
-        return {
-            "token_ids": token_ids,
-            "padding_mask": padding_mask,
-        }
-
-    def generate_postprocess(
-        self,
-        x,
-    ):
-        """Convert integer token output to strings for generation.
-
-        This method reverses `generate_preprocess()`, by first removing all
-        padding and start/end tokens, and then converting the integer sequence
-        back to a string.
-        """
-        if not self.built:
-            self.build(None)
-
-        token_ids, padding_mask = x["token_ids"], x["padding_mask"]
-        ids_to_strip = (self.tokenizer.end_token_id,)
-        token_ids = strip_to_ragged(token_ids, padding_mask, ids_to_strip)
-        return self.tokenizer.detokenize(token_ids)
+    backbone_cls = GPT2Backbone
+    tokenizer_cls = GPT2Tokenizer

keras_hub/src/models/gpt2/gpt2_preprocessor.py

@@ -17,96 +17,23 @@ import keras
 
 from keras_hub.src.api_export import keras_hub_export
 from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_hub.src.models.gpt2.gpt2_backbone import GPT2Backbone
 from keras_hub.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
 from keras_hub.src.models.preprocessor import Preprocessor
-from keras_hub.src.utils.keras_utils import (
-    convert_inputs_to_list_of_tensor_segments,
-)
+from keras_hub.src.utils.tensor_utils import preprocessing_function
 
 
 @keras_hub_export("keras_hub.models.GPT2Preprocessor")
 class GPT2Preprocessor(Preprocessor):
-    """GPT2 preprocessing layer which tokenizes and packs inputs.
+    """Legacy preprocessing layer for GPT2.
 
-    This preprocessing layer will do 2 things:
-
-    - Tokenize the inputs using the `tokenizer`.
-    - Construct a dictionary with keys `"token_ids"`, `"padding_mask"`, that can
-        be passed directly to a `keras_hub.models.GPT2Backbone`.
-
-    This layer can be used directly with `tf.data.Dataset.map` to preprocess
-    string data in the `(x, y, sample_weight)` format used by
-    `keras.Model.fit`.
-
-    The call method of this layer accepts three arguments, `x`, `y`, and
-    `sample_weight`. `x` can be a python string or tensor representing a single
-    segment, a list of python strings representing a batch of single segments,
-    or a list of tensors representing multiple segments to be packed together.
-    `y` and `sample_weight` are both optional, can have any format, and will be
-    passed through unaltered.
-
-    `GPT2Preprocessor` forces the input to have only one segment, as GPT2 is
-    mainly used for generation tasks. For tasks having multi-segment inputs
-    like "glue/mnli", please use a model designed for classification purposes
-    such as BERT or RoBERTa.
-
-    Args:
-        tokenizer: A `keras_hub.models.GPT2Tokenizer` instance.
-        sequence_length: The length of the packed inputs.
-        add_start_token: If `True`, the preprocessor will prepend the tokenizer
-            start token to each input sequence.
-        add_end_token: If `True`, the preprocessor will append the tokenizer
-            end token to each input sequence.
-
-    Call arguments:
-        x: A string, `tf.Tensor` or list of python strings.
-        y: Any label data. Will be passed through unaltered.
-        sample_weight: Any label weight data. Will be passed through unaltered.
-        sequence_length: Pass to override the configured `sequence_length` of
-            the layer.
-
-    Examples:
-
-    Directly calling the layer on data.
-    ```python
-    preprocessor = keras_hub.models.GPT2Preprocessor.from_preset("gpt2_base_en")
-
-    # Tokenize and pack a single sentence.
-    preprocessor("The quick brown fox jumped.")
-
-    # Tokenize a batch of single sentences.
-    preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
-
-    # Custom vocabulary.
-    features = ["a quick fox.", "a fox quick."]
-    vocab = {"<|endoftext|>": 0, "a": 4, "Ġquick": 5, "Ġfox": 6}
-    merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"]
-    merges += ["Ġ f", "o x", "Ġf ox"]
-    tokenizer = keras_hub.models.GPT2Tokenizer(
-        vocabulary=vocab,
-        merges=merges,
-    )
-    preprocessor = keras_hub.models.GPT2Preprocessor(tokenizer=tokenizer)
-    preprocessor("The quick brown fox jumped.")
-    ```
-
-    Mapping with `tf.data.Dataset`.
-    ```python
-    preprocessor = keras_hub.models.GPT2Preprocessor.from_preset("gpt2_base_en")
-
-    text = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
-    label = tf.constant([1, 1])
-
-    # Map labeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices((text, label))
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-
-    # Map unlabeled single sentences.
-    ds = tf.data.Dataset.from_tensor_slices(text)
-    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
-    ```
+    This layer should not be used in new code! All preprocessing layers pair
+    directly with a task. E.g. `BertClassifier` and
+    `BertClassifierPreprocessor`. Either use `GPT2CausalLMPreprocessor` or
+    wrap `GPT2Tokenizer` into a custom preprocessing layer or function.
     """
 
+    backbone_cls = GPT2Backbone
     tokenizer_cls = GPT2Tokenizer
 
     def __init__(
@@ -117,6 +44,8 @@ class GPT2Preprocessor(Preprocessor):
         add_end_token=True,
         **kwargs,
     ):
+        # TODO: this class has some usage, but barely any, and is no longer
+        # documented. We should consider dropping it.
         super().__init__(**kwargs)
         self.tokenizer = tokenizer
         self.packer = None
@@ -136,6 +65,7 @@ class GPT2Preprocessor(Preprocessor):
         )
         self.built = True
 
+    @preprocessing_function
     def call(
         self,
         x,
@@ -143,17 +73,9 @@ class GPT2Preprocessor(Preprocessor):
         sample_weight=None,
         sequence_length=None,
     ):
-        x = convert_inputs_to_list_of_tensor_segments(x)
-        if len(x) != 1:
-            raise ValueError(
-                "GPT2 requires each input feature to contain only "
-                f"one segment, but received {len(x)}. If you are using GPT2 "
-                "for a multi-segment classification task, please refer to "
-                "classification models like BERT or RoBERTa."
-            )
         sequence_length = sequence_length or self.sequence_length
         token_ids, padding_mask = self.packer(
-            self.tokenizer(x[0]),
+            self.tokenizer(x),
             sequence_length=sequence_length,
             add_start_value=self.add_start_token,
             add_end_value=self.add_end_token,

keras_hub/src/models/gpt2/gpt2_tokenizer.py

@@ -14,10 +14,16 @@
 
 
 from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.gpt2.gpt2_backbone import GPT2Backbone
 from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
-@keras_hub_export("keras_hub.models.GPT2Tokenizer")
+@keras_hub_export(
+    [
+        "keras_hub.tokenizers.GPT2Tokenizer",
+        "keras_hub.models.GPT2Tokenizer",
+    ]
+)
 class GPT2Tokenizer(BytePairTokenizer):
     """A GPT-2 tokenizer using Byte-Pair Encoding subword segmentation.
 
@@ -27,8 +33,6 @@ class GPT2Tokenizer(BytePairTokenizer):
     models and provides a `from_preset()` method to automatically download
     a matching vocabulary for a GPT-2 preset.
 
-    This tokenizer does not provide truncation or padding of inputs.
-
     If input is a batch of strings (rank > 0), the layer will output a
     `tf.RaggedTensor` where the last dimension of the output is ragged.
 
@@ -65,6 +69,8 @@ class GPT2Tokenizer(BytePairTokenizer):
     ```
     """
 
+    backbone_cls = GPT2Backbone
+
     def __init__(
         self,
         vocabulary=None,
@@ -72,39 +78,11 @@ class GPT2Tokenizer(BytePairTokenizer):
         **kwargs,
     ):
         # GPT2 uses the same start as end token, i.e., "<|endoftext|>".
-        self.end_token = self.start_token = "<|endoftext|>"
-
+        self._add_special_token("<|endoftext|>", "end_token")
+        self._add_special_token("<|endoftext|>", "start_token")
+        self.pad_token_id = 0
         super().__init__(
             vocabulary=vocabulary,
             merges=merges,
-            unsplittable_tokens=[self.end_token],
             **kwargs,
         )
-
-    def set_vocabulary_and_merges(self, vocabulary, merges):
-        super().set_vocabulary_and_merges(vocabulary, merges)
-
-        if vocabulary is not None:
-            # Check for necessary special tokens.
-            if self.end_token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{self.end_token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{self.end_token}'` in "
-                    "your `vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-            self.end_token_id = self.token_to_id(self.end_token)
-            self.start_token_id = self.end_token_id
-            self.pad_token_id = 0
-        else:
-            self.end_token_id = None
-            self.start_token_id = None
-            self.pad_token_id = None
-
-    def get_config(self):
-        config = super().get_config()
-        # In the constructor, we pass the list of special tokens to the
-        # `unsplittable_tokens` arg of the superclass' constructor. Hence, we
-        # delete it from the config here.
-        del config["unsplittable_tokens"]
-        return config

keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py

@@ -12,21 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import keras
-from absl import logging
-
 from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.models.gpt_neo_x.gpt_neo_x_preprocessor import (
-    GPTNeoXPreprocessor,
-)
-from keras_hub.src.utils.keras_utils import (
-    convert_inputs_to_list_of_tensor_segments,
-)
-from keras_hub.src.utils.tensor_utils import strip_to_ragged
+from keras_hub.src.models.causal_lm_preprocessor import CausalLMPreprocessor
+from keras_hub.src.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
+from keras_hub.src.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
 
 
 @keras_hub_export("keras_hub.models.GPTNeoXCausalLMPreprocessor")
-class GPTNeoXCausalLMPreprocessor(GPTNeoXPreprocessor):
+class GPTNeoXCausalLMPreprocessor(CausalLMPreprocessor):
     """GPT-NeoX Causal LM preprocessor.
 
     This preprocessing layer is meant for use with
@@ -59,83 +52,5 @@ class GPTNeoXCausalLMPreprocessor(GPTNeoXPreprocessor):
 
     """
 
-    def call(
-        self,
-        x,
-        y=None,
-        sample_weight=None,
-        sequence_length=None,
-    ):
-        if y is not None or sample_weight is not None:
-            logging.warning(
-                "`GPTNeoXCausalLMPreprocessor` generates `y` and `sample_weight` "
-                "based on your input data, but your data already contains `y` "
-                "or `sample_weight`. Your `y` and `sample_weight` will be "
-                "ignored."
-            )
-        sequence_length = sequence_length or self.sequence_length
-
-        x = convert_inputs_to_list_of_tensor_segments(x)[0]
-        x = self.tokenizer(x)
-        # Pad with one extra token to account for the truncation below.
-        token_ids, padding_mask = self.packer(
-            x,
-            sequence_length=sequence_length + 1,
-            add_start_value=self.add_start_token,
-            add_end_value=self.add_end_token,
-        )
-        # The last token does not have a next token, so we truncate it out.
-        x = {
-            "token_ids": token_ids[..., :-1],
-            "padding_mask": padding_mask[..., :-1],
-        }
-        # Target `y` will be the next token.
-        y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:]
-        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
-
-    def generate_preprocess(
-        self,
-        x,
-        sequence_length=None,
-    ):
-        """Convert strings to integer token input for generation.
-
-        Similar to calling the layer for training, this method takes in strings
-        or tensor strings, tokenizes and packs the input, and computes a padding
-        mask masking all inputs not filled in with a padded value.
-
-        Unlike calling the layer for training, this method does not compute
-        labels and will never append a `tokenizer.end_token_id` to the end of
-        the sequence (as generation is expected to continue at the end of the
-        inputted prompt).
-        """
-        if not self.built:
-            self.build(None)
-
-        x = convert_inputs_to_list_of_tensor_segments(x)[0]
-        x = self.tokenizer(x)
-        token_ids, padding_mask = self.packer(
-            x, sequence_length=sequence_length, add_end_value=False
-        )
-        return {
-            "token_ids": token_ids,
-            "padding_mask": padding_mask,
-        }
-
-    def generate_postprocess(
-        self,
-        x,
-    ):
-        """Convert integer token output to strings for generation.
-
-        This method reverses `generate_preprocess()`, by first removing all
-        padding and start/end tokens, and then converting the integer sequence
-        back to a string.
-        """
-        if not self.built:
-            self.build(None)
-
-        token_ids, padding_mask = x["token_ids"], x["padding_mask"]
-        ids_to_strip = (self.tokenizer.end_token_id,)
-        token_ids = strip_to_ragged(token_ids, padding_mask, ids_to_strip)
-        return self.tokenizer.detokenize(token_ids)
+    backbone_cls = GPTNeoXBackbone
+    tokenizer_cls = GPTNeoXTokenizer

keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py

@@ -13,10 +13,16 @@
 # limitations under the License.
 
 from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
 from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
-@keras_hub_export("keras_hub.models.GPTNeoXTokenizer")
+@keras_hub_export(
+    [
+        "keras_hub.tokenizers.GPTNeoXTokenizer",
+        "keras_hub.models.GPTNeoXTokenizer",
+    ]
+)
 class GPTNeoXTokenizer(BytePairTokenizer):
     """A GPTNeoX tokenizer using Byte-Pair Encoding subword segmentation.
 
@@ -26,8 +32,6 @@ class GPTNeoXTokenizer(BytePairTokenizer):
     models and provides a `from_preset()` method to automatically download
     a matching vocabulary for a GPTNeoX preset.
 
-    This tokenizer does not provide truncation or padding of inputs.
-
     If input is a batch of strings (rank > 0), the layer will output a
     `tf.RaggedTensor` where the last dimension of the output is ragged.
 
@@ -43,6 +47,8 @@ class GPTNeoXTokenizer(BytePairTokenizer):
             merge entities separated by a space.
     """
 
+    backbone_cls = GPTNeoXBackbone
+
     def __init__(
         self,
         vocabulary=None,
@@ -50,39 +56,11 @@ class GPTNeoXTokenizer(BytePairTokenizer):
         **kwargs,
     ):
         # GPTNeoX uses the same start as end token, i.e., "<|endoftext|>".
-        self.end_token = self.start_token = "<|endoftext|>"
-
+        self._add_special_token("<|endoftext|>", "end_token")
+        self._add_special_token("<|endoftext|>", "start_token")
+        self.pad_token_id = 0
         super().__init__(
             vocabulary=vocabulary,
             merges=merges,
-            unsplittable_tokens=[self.end_token],
             **kwargs,
         )
-
-    def set_vocabulary_and_merges(self, vocabulary, merges):
-        super().set_vocabulary_and_merges(vocabulary, merges)
-
-        if vocabulary is not None:
-            # Check for necessary special tokens.
-            if self.end_token not in self.get_vocabulary():
-                raise ValueError(
-                    f"Cannot find token `'{self.end_token}'` in the provided "
-                    f"`vocabulary`. Please provide `'{self.end_token}'` in "
-                    "your `vocabulary` or use a pretrained `vocabulary` name."
-                )
-
-            self.end_token_id = self.token_to_id(self.end_token)
-            self.start_token_id = self.end_token_id
-            self.pad_token_id = 0
-        else:
-            self.end_token_id = None
-            self.start_token_id = None
-            self.pad_token_id = None
-
-    def get_config(self):
-        config = super().get_config()
-        # In the constructor, we pass the list of special tokens to the
-        # `unsplittable_tokens` arg of the superclass' constructor. Hence, we
-        # delete it from the config here.
-        del config["unsplittable_tokens"]
-        return config

keras_hub/src/models/image_classifier.py

@@ -33,11 +33,6 @@ class ImageClassifier(Task):
     used to load a pre-trained config and weights.
     """
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        # Default compilation.
-        self.compile()
-
     def compile(
         self,
         optimizer="auto",

keras_hub/src/models/image_classifier_preprocessor.py

@@ -0,0 +1,83 @@
+# Copyright 2024 The KerasHub Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import keras
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.preprocessor import Preprocessor
+from keras_hub.src.utils.tensor_utils import preprocessing_function
+
+
+@keras_hub_export("keras_hub.models.ImageClassifierPreprocessor")
+class ImageClassifierPreprocessor(Preprocessor):
+    """Base class for image classification preprocessing layers.
+
+    `ImageClassifierPreprocessor` tasks wraps a
+    `keras_hub.layers.ImageConverter` to create a preprocessing layer for
+    image classification tasks. It is intended to be paired with a
+    `keras_hub.models.ImageClassifier` task.
+
+    All `ImageClassifierPreprocessor` take inputs three inputs, `x`, `y`, and
+    `sample_weight`. `x`, the first input, should always be included. It can
+    be a image or batch of images. See examples below. `y` and `sample_weight`
+    are optional inputs that will be passed through unaltered. Usually, `y` will
+    be the classification label, and `sample_weight` will not be provided.
+
+    The layer will output either `x`, an `(x, y)` tuple if labels were provided,
+    or an `(x, y, sample_weight)` tuple if labels and sample weight were
+    provided. `x` will be the input images after all model preprocessing has
+    been applied.
+
+    All `ImageClassifierPreprocessor` tasks include a `from_preset()`
+    constructor which can be used to load a pre-trained config and vocabularies.
+    You can call the `from_preset()` constructor directly on this base class, in
+    which case the correct class for your model will be automatically
+    instantiated.
+
+    Examples.
+    ```python
+    preprocessor = keras_hub.models.ImageClassifierPreprocessor.from_preset(
+        "resnet_50",
+    )
+
+    # Resize a single image for resnet 50.
+    x = np.ones((512, 512, 3))
+    x = preprocessor(x)
+
+    # Resize a labeled image.
+    x, y = np.ones((512, 512, 3)), 1
+    x, y = preprocessor(x, y)
+
+    # Resize a batch of labeled images.
+    x, y = [np.ones((512, 512, 3)), np.zeros((512, 512, 3))], [1, 0]
+    x, y = preprocessor(x, y)
+
+    # Use a `tf.data.Dataset`.
+    ds = tf.data.Dataset.from_tensor_slices((x, y)).batch(2)
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+    ```
+    """
+
+    def __init__(
+        self,
+        image_converter=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.image_converter = image_converter
+
+    @preprocessing_function
+    def call(self, x, y=None, sample_weight=None):
+        if self.image_converter:
+            x = self.image_converter(x)
+        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)

keras_hub/src/models/llama/__init__.py

@@ -14,7 +14,6 @@
 
 from keras_hub.src.models.llama.llama_backbone import LlamaBackbone
 from keras_hub.src.models.llama.llama_presets import backbone_presets
-from keras_hub.src.models.llama.llama_tokenizer import LlamaTokenizer
 from keras_hub.src.utils.preset_utils import register_presets
 
-register_presets(backbone_presets, (LlamaBackbone, LlamaTokenizer))
+register_presets(backbone_presets, LlamaBackbone)