keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl → 0.16.0.dev2024092017__py3-none-any.whl

keras_hub/src/models/bert/{bert_preprocessor.py → bert_text_classifier_preprocessor.py}

@@ -12,21 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import keras
-
 from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.layers.preprocessing.multi_segment_packer import (
-    MultiSegmentPacker,
-)
+from keras_hub.src.models.bert.bert_backbone import BertBackbone
 from keras_hub.src.models.bert.bert_tokenizer import BertTokenizer
-from keras_hub.src.models.preprocessor import Preprocessor
-from keras_hub.src.utils.keras_utils import (
-    convert_inputs_to_list_of_tensor_segments,
+from keras_hub.src.models.text_classifier_preprocessor import (
+    TextClassifierPreprocessor,
 )
 
 
-@keras_hub_export("keras_hub.models.BertPreprocessor")
-class BertPreprocessor(Preprocessor):
+@keras_hub_export(
+    [
+        "keras_hub.models.BertTextClassifierPreprocessor",
+        "keras_hub.models.BertPreprocessor",
+    ]
+)
+class BertTextClassifierPreprocessor(TextClassifierPreprocessor):
     """A BERT preprocessing layer which tokenizes and packs inputs.
 
     This preprocessing layer will do three things:
@@ -67,7 +67,7 @@ class BertPreprocessor(Preprocessor):
 
     Directly calling the layer on data.
     ```python
-    preprocessor = keras_hub.models.BertPreprocessor.from_preset(
+    preprocessor = keras_hub.models.TextClassifierPreprocessor.from_preset(
         "bert_base_en_uncased"
     )
 
@@ -87,13 +87,13 @@ class BertPreprocessor(Preprocessor):
     vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
     vocab += ["The", "quick", "brown", "fox", "jumped", "."]
     tokenizer = keras_hub.models.BertTokenizer(vocabulary=vocab)
-    preprocessor = keras_hub.models.BertPreprocessor(tokenizer)
+    preprocessor = keras_hub.models.BertTextClassifierPreprocessor(tokenizer)
     preprocessor("The quick brown fox jumped.")
     ```
 
     Mapping with `tf.data.Dataset`.
     ```python
-    preprocessor = keras_hub.models.BertPreprocessor.from_preset(
+    preprocessor = keras_hub.models.TextClassifierPreprocessor.from_preset(
         "bert_base_en_uncased"
     )
 
@@ -124,61 +124,5 @@ class BertPreprocessor(Preprocessor):
     ```
     """
 
+    backbone_cls = BertBackbone
     tokenizer_cls = BertTokenizer
-
-    def __init__(
-        self,
-        tokenizer,
-        sequence_length=512,
-        truncate="round_robin",
-        **kwargs,
-    ):
-        super().__init__(**kwargs)
-        self.tokenizer = tokenizer
-        self.packer = None
-        self.sequence_length = sequence_length
-        self.truncate = truncate
-
-    def build(self, input_shape):
-        # Defer packer creation to `build()` so that we can be sure tokenizer
-        # assets have loaded when restoring a saved model.
-        self.packer = MultiSegmentPacker(
-            start_value=self.tokenizer.cls_token_id,
-            end_value=self.tokenizer.sep_token_id,
-            pad_value=self.tokenizer.pad_token_id,
-            truncate=self.truncate,
-            sequence_length=self.sequence_length,
-        )
-        self.built = True
-
-    def call(self, x, y=None, sample_weight=None):
-        x = convert_inputs_to_list_of_tensor_segments(x)
-        x = [self.tokenizer(segment) for segment in x]
-        token_ids, segment_ids = self.packer(x)
-        x = {
-            "token_ids": token_ids,
-            "segment_ids": segment_ids,
-            "padding_mask": token_ids != self.tokenizer.pad_token_id,
-        }
-        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
-
-    def get_config(self):
-        config = super().get_config()
-        config.update(
-            {
-                "sequence_length": self.sequence_length,
-                "truncate": self.truncate,
-            }
-        )
-        return config
-
-    @property
-    def sequence_length(self):
-        """The padded length of model input sequences."""
-        return self._sequence_length
-
-    @sequence_length.setter
-    def sequence_length(self, value):
-        self._sequence_length = value
-        if self.packer is not None:
-            self.packer.sequence_length = value

keras_hub/src/models/bert/bert_tokenizer.py

@@ -13,10 +13,16 @@
 # limitations under the License.
 
 from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.bert.bert_backbone import BertBackbone
 from keras_hub.src.tokenizers.word_piece_tokenizer import WordPieceTokenizer
 
 
-@keras_hub_export("keras_hub.models.BertTokenizer")
+@keras_hub_export(
+    [
+        "keras_hub.tokenizers.BertTokenizer",
+        "keras_hub.models.BertTokenizer",
+    ]
+)
 class BertTokenizer(WordPieceTokenizer):
     """A BERT tokenizer using WordPiece subword segmentation.
 
@@ -26,9 +32,6 @@ class BertTokenizer(WordPieceTokenizer):
     models and provides a `from_preset()` method to automatically download
     a matching vocabulary for a BERT preset.
 
-    This tokenizer does not provide truncation or padding of inputs. It can be
-    combined with a `keras_hub.models.BertPreprocessor` layer for input packing.
-
     If input is a batch of strings (rank > 0), the layer will output a
     `tf.RaggedTensor` where the last dimension of the output is ragged.
 
@@ -68,45 +71,24 @@ class BertTokenizer(WordPieceTokenizer):
     ```
     """
 
+    backbone_cls = BertBackbone
+
     def __init__(
         self,
         vocabulary=None,
         lowercase=False,
-        special_tokens_in_strings=False,
         **kwargs,
     ):
-        self.cls_token = "[CLS]"
-        self.sep_token = "[SEP]"
-        self.pad_token = "[PAD]"
-        self.mask_token = "[MASK]"
+        self._add_special_token("[CLS]", "cls_token")
+        self._add_special_token("[SEP]", "sep_token")
+        self._add_special_token("[PAD]", "pad_token")
+        self._add_special_token("[MASK]", "mask_token")
+        # Also add `tokenizer.start_token` and `tokenizer.end_token` for
+        # compatibility with other tokenizers.
+        self._add_special_token("[CLS]", "start_token")
+        self._add_special_token("[SEP]", "end_token")
         super().__init__(
             vocabulary=vocabulary,
             lowercase=lowercase,
-            special_tokens=[
-                self.cls_token,
-                self.sep_token,
-                self.pad_token,
-                self.mask_token,
-            ],
-            special_tokens_in_strings=special_tokens_in_strings,
             **kwargs,
         )
-
-    def set_vocabulary(self, vocabulary):
-        super().set_vocabulary(vocabulary)
-
-        if vocabulary is not None:
-            self.cls_token_id = self.token_to_id(self.cls_token)
-            self.sep_token_id = self.token_to_id(self.sep_token)
-            self.pad_token_id = self.token_to_id(self.pad_token)
-            self.mask_token_id = self.token_to_id(self.mask_token)
-        else:
-            self.cls_token_id = None
-            self.sep_token_id = None
-            self.pad_token_id = None
-            self.mask_token_id = None
-
-    def get_config(self):
-        config = super().get_config()
-        del config["special_tokens"]  # Not configurable; set in __init__.
-        return config

keras_hub/src/models/bloom/__init__.py

@@ -14,7 +14,6 @@
 
 from keras_hub.src.models.bloom.bloom_backbone import BloomBackbone
 from keras_hub.src.models.bloom.bloom_presets import backbone_presets
-from keras_hub.src.models.bloom.bloom_tokenizer import BloomTokenizer
 from keras_hub.src.utils.preset_utils import register_presets
 
-register_presets(backbone_presets, (BloomBackbone, BloomTokenizer))
+register_presets(backbone_presets, BloomBackbone)

keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py

@@ -12,19 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import keras
-from absl import logging
 
 from keras_hub.src.api_export import keras_hub_export
-from keras_hub.src.models.bloom.bloom_preprocessor import BloomPreprocessor
-from keras_hub.src.utils.keras_utils import (
-    convert_inputs_to_list_of_tensor_segments,
-)
-from keras_hub.src.utils.tensor_utils import strip_to_ragged
+from keras_hub.src.models.bloom.bloom_backbone import BloomBackbone
+from keras_hub.src.models.bloom.bloom_tokenizer import BloomTokenizer
+from keras_hub.src.models.causal_lm_preprocessor import CausalLMPreprocessor
 
 
 @keras_hub_export("keras_hub.models.BloomCausalLMPreprocessor")
-class BloomCausalLMPreprocessor(BloomPreprocessor):
+class BloomCausalLMPreprocessor(CausalLMPreprocessor):
     """BLOOM Causal LM preprocessor.
 
     This preprocessing layer is meant for use with
@@ -91,86 +87,5 @@ class BloomCausalLMPreprocessor(BloomPreprocessor):
     ```
     """
 
-    def call(
-        self,
-        x,
-        y=None,
-        sample_weight=None,
-        sequence_length=None,
-    ):
-        if y is not None or sample_weight is not None:
-            logging.warning(
-                "`BloomCausalLMPreprocessor` generates `y` and `sample_weight` "
-                "based on your input data, but your data already contains `y` "
-                "or `sample_weight`. Your `y` and `sample_weight` will be "
-                "ignored."
-            )
-        sequence_length = sequence_length or self.sequence_length
-
-        x = convert_inputs_to_list_of_tensor_segments(x)[0]
-        x = self.tokenizer(x)
-        # Pad with one extra token to account for the truncation below.
-        token_ids, padding_mask = self.packer(
-            x,
-            sequence_length=sequence_length + 1,
-            add_start_value=self.add_start_token,
-            add_end_value=self.add_end_token,
-        )
-        # The last token does not have a next token, so we truncate it out.
-        x = {
-            "token_ids": token_ids[..., :-1],
-            "padding_mask": padding_mask[..., :-1],
-        }
-        # Target `y` will be the next token.
-        y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:]
-        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
-
-    def generate_preprocess(
-        self,
-        x,
-        sequence_length=None,
-    ):
-        """Convert strings to integer token input for generation.
-
-        Similar to calling the layer for training, this method takes in strings
-        or tensor strings, tokenizes and packs the input, and computes a padding
-        mask masking all inputs not filled in with a padded value.
-
-        Unlike calling the layer for training, this method does not compute
-        labels and will never append a `tokenizer.end_token_id` to the end of
-        the sequence (as generation is expected to continue at the end of the
-        inputted prompt).
-        """
-        if not self.built:
-            self.build(None)
-
-        x = convert_inputs_to_list_of_tensor_segments(x)[0]
-        x = self.tokenizer(x)
-        token_ids, padding_mask = self.packer(
-            x, sequence_length=sequence_length, add_end_value=False
-        )
-        return {
-            "token_ids": token_ids,
-            "padding_mask": padding_mask,
-        }
-
-    def generate_postprocess(
-        self,
-        x,
-    ):
-        """Convert integer token output to strings for generation.
-
-        This method reverses `generate_preprocess()`, by first removing all
-        padding and start/end tokens, and then converting the integer sequence
-        back to a string.
-        """
-        if not self.built:
-            self.build(None)
-
-        token_ids, padding_mask = x["token_ids"], x["padding_mask"]
-        ids_to_strip = (
-            self.tokenizer.start_token_id,
-            self.tokenizer.end_token_id,
-        )
-        token_ids = strip_to_ragged(token_ids, padding_mask, ids_to_strip)
-        return self.tokenizer.detokenize(token_ids)
+    backbone_cls = BloomBackbone
+    tokenizer_cls = BloomTokenizer

keras_hub/src/models/bloom/bloom_tokenizer.py

@@ -14,10 +14,16 @@
 
 
 from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.models.bloom.bloom_backbone import BloomBackbone
 from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
 
 
-@keras_hub_export("keras_hub.models.BloomTokenizer")
+@keras_hub_export(
+    [
+        "keras_hub.tokenizers.BloomTokenizer",
+        "keras_hub.models.BloomTokenizer",
+    ]
+)
 class BloomTokenizer(BytePairTokenizer):
     """A BLOOM tokenizer using Byte-Pair Encoding subword segmentation.
 
@@ -27,8 +33,6 @@ class BloomTokenizer(BytePairTokenizer):
     models and provides a `from_preset()` method to automatically download
     a matching vocabulary for a BLOOM preset.
 
-    This tokenizer does not provide truncation or padding of inputs.
-
     If input is a batch of strings (rank > 0), the layer will output a
     `tf.RaggedTensor` where the last dimension of the output is ragged.
 
@@ -65,52 +69,19 @@ class BloomTokenizer(BytePairTokenizer):
     ```
     """
 
+    backbone_cls = BloomBackbone
+
     def __init__(
         self,
         vocabulary=None,
         merges=None,
         **kwargs,
     ):
-        self.start_token = "<s>"
-        self.end_token = "</s>"
-        self.pad_token = "<pad>"
-
+        self._add_special_token("<s>", "start_token")
+        self._add_special_token("</s>", "end_token")
+        self._add_special_token("<pad>", "pad_token")
         super().__init__(
             vocabulary=vocabulary,
             merges=merges,
-            unsplittable_tokens=[
-                self.start_token,
-                self.end_token,
-                self.pad_token,
-            ],
             **kwargs,
         )
-
-    def set_vocabulary_and_merges(self, vocabulary, merges):
-        super().set_vocabulary_and_merges(vocabulary, merges)
-
-        if vocabulary is not None:
-            # Check for necessary special tokens.
-            for token in [self.start_token, self.end_token, self.pad_token]:
-                if token not in self.get_vocabulary():
-                    raise ValueError(
-                        f"Cannot find token `'{token}'` in the provided "
-                        f"`vocabulary`. Please provide `'{token}'` in "
-                        "your `vocabulary` or use a pretrained `vocabulary` name."
-                    )
-
-            self.start_token_id = self.token_to_id(self.start_token)
-            self.end_token_id = self.token_to_id(self.end_token)
-            self.pad_token_id = self.token_to_id(self.pad_token)
-        else:
-            self.start_token_id = None
-            self.end_token_id = None
-            self.pad_token_id = None
-
-    def get_config(self):
-        config = super().get_config()
-        # In the constructor, we pass the list of special tokens to the
-        # `unsplittable_tokens` arg of the superclass' constructor. Hence, we
-        # delete it from the config here.
-        del config["unsplittable_tokens"]
-        return config

keras_hub/src/models/causal_lm.py

@@ -22,7 +22,6 @@ from keras import tree
 from keras_hub.src.api_export import keras_hub_export
 from keras_hub.src.models.task import Task
 from keras_hub.src.samplers.serialization import get as get_sampler
-from keras_hub.src.utils.tensor_utils import tensor_to_list
 
 try:
     import tensorflow as tf
@@ -73,8 +72,6 @@ class CausalLM(Task):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        # Default compilation.
-        self.compile()
 
     def compile(
         self,
@@ -234,21 +231,18 @@ class CausalLM(Task):
         necessary, and returns a iterable "dataset like" object (either an
         actual `tf.data.Dataset` or a list with a single batch element).
         """
-        input_is_scalar = False
+        if tf and isinstance(inputs, tf.data.Dataset):
+            return inputs.as_numpy_iterator(), False
 
-        if isinstance(inputs, tf.data.Dataset):
-            return inputs, input_is_scalar
+        if self.preprocessor is None:
+            return [inputs], False
 
         def normalize(x):
-            x_is_scalar = False
-            if isinstance(x, str) or isinstance(x, list):
-                x = tf.convert_to_tensor(x)
-
-            if isinstance(x, tf.Tensor) and x.shape.rank == 0:
-                x_is_scalar = True
-                x = x[tf.newaxis]
-
-            return x, x_is_scalar
+            if isinstance(x, str):
+                return [x], True
+            if tf and isinstance(x, tf.Tensor) and x.shape.rank == 0:
+                return x[tf.newaxis], True
+            return x, False
 
         if isinstance(inputs, dict):
             for key in inputs:
@@ -256,8 +250,6 @@ class CausalLM(Task):
         else:
             inputs, input_is_scalar = normalize(inputs)
 
-        # We avoid converting to a dataset purely for speed, for a single batch
-        # of input, creating a dataset would add significant overhead.
         return [inputs], input_is_scalar
 
     def _normalize_generate_outputs(
@@ -280,10 +272,6 @@ class CausalLM(Task):
                     for e in batch:
                         outputs.append(e)
                 return outputs[0] if input_is_scalar else outputs
-            if isinstance(x[0], tf.Tensor) and x[0].dtype == tf.string:
-                outputs = tf.concat(x, axis=0)
-                outputs = tf.squeeze(outputs, 0) if input_is_scalar else outputs
-                return tensor_to_list(outputs)
             outputs = ops.concatenate(x, axis=0)
             outputs = ops.squeeze(outputs, 0) if input_is_scalar else outputs
             return ops.convert_to_numpy(outputs)
@@ -368,15 +356,8 @@ class CausalLM(Task):
         inputs, input_is_scalar = self._normalize_generate_inputs(inputs)
 
         if self.preprocessor is not None:
-            if isinstance(inputs, tf.data.Dataset):
-                inputs = inputs.map(preprocess, tf.data.AUTOTUNE)
-                inputs = inputs.prefetch(tf.data.AUTOTUNE)
-            else:
-                # Fast path for non-dataset, single-batch input.
-                inputs = [preprocess(x) for x in inputs]
-
+            inputs = [preprocess(x) for x in inputs]
         outputs = [generate(x) for x in inputs]
-
         if self.preprocessor is not None:
             outputs = [postprocess(x) for x in outputs]
 

keras_hub/src/models/causal_lm_preprocessor.py

@@ -0,0 +1,195 @@
+# Copyright 2024 The KerasHub Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import keras
+
+from keras_hub.src.api_export import keras_hub_export
+from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
+from keras_hub.src.models.preprocessor import Preprocessor
+from keras_hub.src.utils.tensor_utils import preprocessing_function
+from keras_hub.src.utils.tensor_utils import strip_to_ragged
+
+
+@keras_hub_export("keras_hub.models.CausalLMPreprocessor")
+class CausalLMPreprocessor(Preprocessor):
+    """Base class for causal language modeling preprocessing layers.
+
+    `CausalLMPreprocessor` tasks wrap a `keras_hub.tokenizer.Tokenizer` to
+    create a preprocessing layer for causal language modeling tasks. It is
+    intended to be paired with a `keras.models.CausalLM` task.
+
+    All `CausalLMPreprocessor` take inputs a single input. This can be a single
+    string or a batch of strings. See examples below. These inputs
+    will be tokenized and padded/truncated to a fixed sequence length.
+
+    This layer will always output a `(x, y, sample_weight)` tuple, where `x`
+    is a dictionary with the tokenized inputs, `y` contains the tokens from `x`
+    offset by 1, and `sample_weight` marks where `y` contains padded
+    values. The exact contents of `x` will vary depending on the model being
+    used.
+
+    a `CausalLMPreprocessor` contains two extra methods, `generate_preprocess`
+    and `generate_postprocess` for use with generation. See examples below.
+
+    All `CausalLMPreprocessor` tasks include a `from_preset()` constructor
+    which can be used to load a pre-trained config and vocabularies. You can
+    call the `from_preset()` constructor directly on this base class, in which
+    case the correct class for you model will be automatically instantiated.
+
+    Examples.
+    ```python
+    preprocessor = keras_hub.models.CausalLMPreprocessor.from_preset(
+        "bert_base_en_uncased",
+        sequence_length=256, # Optional.
+    )
+
+    # Tokenize, mask and pack a single sentence.
+    x = "The quick brown fox jumped."
+    x, y, sample_weight = preprocessor(x)
+
+    # Tokenize and pad/truncate a batch of labeled sentences.
+    x = ["The quick brown fox jumped.", "Call me Ishmael."]
+    x, y, sample_weight = preprocessor(x)
+
+    # With a `tf.data.Dataset`.
+    ds = tf.data.Dataset.from_tensor_slices(x)
+    ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
+
+    # Generate preprocess and postprocess.
+    x = preprocessor.generate_preprocess(x)  # Tokenized numeric inputs.
+    x = preprocessor.generate_postprocess(x)  # Detokenized string outputs.
+    ```
+    """
+
+    def __init__(
+        self,
+        tokenizer,
+        sequence_length=1024,
+        add_start_token=True,
+        add_end_token=True,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.tokenizer = tokenizer
+        self.packer = None
+        self.sequence_length = sequence_length
+        self.add_start_token = add_start_token
+        self.add_end_token = add_end_token
+
+    def build(self, input_shape):
+        # Defer packer creation to `build()` so that we can be sure tokenizer
+        # assets have loaded when restoring a saved model.
+        self.packer = StartEndPacker(
+            start_value=self.tokenizer.start_token_id,
+            end_value=self.tokenizer.end_token_id,
+            pad_value=self.tokenizer.pad_token_id,
+            sequence_length=self.sequence_length,
+            return_padding_mask=True,
+        )
+        self.built = True
+
+    @preprocessing_function
+    def call(
+        self,
+        x,
+        y=None,
+        sample_weight=None,
+        sequence_length=None,
+    ):
+        sequence_length = sequence_length or self.sequence_length
+        x = self.tokenizer(x)
+        # Pad with one extra token to account for the truncation below.
+        token_ids, padding_mask = self.packer(
+            x,
+            sequence_length=sequence_length + 1,
+            add_start_value=self.add_start_token,
+            add_end_value=self.add_end_token,
+        )
+        # The last token does not have a next token, so we truncate it out.
+        x = {
+            "token_ids": token_ids[..., :-1],
+            "padding_mask": padding_mask[..., :-1],
+        }
+        # Target `y` will be the next token.
+        y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:]
+        return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
+
+    @preprocessing_function
+    def generate_preprocess(
+        self,
+        x,
+        sequence_length=None,
+    ):
+        """Convert strings to integer token input for generation.
+
+        Similar to calling the layer for training, this method takes in strings
+        or tensor strings, tokenizes and packs the input, and computes a padding
+        mask masking all inputs not filled in with a padded value.
+
+        Unlike calling the layer for training, this method does not compute
+        labels and will never append a `tokenizer.end_token_id` to the end of
+        the sequence (as generation is expected to continue at the end of the
+        inputted prompt).
+        """
+        if not self.built:
+            self.build(None)
+
+        x = self.tokenizer(x)
+        token_ids, padding_mask = self.packer(
+            x, sequence_length=sequence_length, add_end_value=False
+        )
+        return {
+            "token_ids": token_ids,
+            "padding_mask": padding_mask,
+        }
+
+    @preprocessing_function
+    def generate_postprocess(
+        self,
+        x,
+    ):
+        """Convert integer token output to strings for generation.
+
+        This method reverses `generate_preprocess()`, by first removing all
+        padding and start/end tokens, and then converting the integer sequence
+        back to a string.
+        """
+        if not self.built:
+            self.build(None)
+
+        token_ids, padding_mask = x["token_ids"], x["padding_mask"]
+        ids_to_strip = self.tokenizer.special_token_ids
+        token_ids = strip_to_ragged(token_ids, padding_mask, ids_to_strip)
+        return self.tokenizer.detokenize(token_ids)
+
+    def get_config(self):
+        config = super().get_config()
+        config.update(
+            {
+                "sequence_length": self.sequence_length,
+                "add_start_token": self.add_start_token,
+                "add_end_token": self.add_end_token,
+            }
+        )
+        return config
+
+    @property
+    def sequence_length(self):
+        """The padded length of model input sequences."""
+        return self._sequence_length
+
+    @sequence_length.setter
+    def sequence_length(self, value):
+        self._sequence_length = value
+        if self.packer is not None:
+            self.packer.sequence_length = value