keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keras_hub/__init__.py +52 -0
- keras_hub/api/__init__.py +27 -0
- keras_hub/api/layers/__init__.py +47 -0
- keras_hub/api/metrics/__init__.py +24 -0
- keras_hub/api/models/__init__.py +249 -0
- keras_hub/api/samplers/__init__.py +29 -0
- keras_hub/api/tokenizers/__init__.py +35 -0
- keras_hub/src/__init__.py +13 -0
- keras_hub/src/api_export.py +53 -0
- keras_hub/src/layers/__init__.py +13 -0
- keras_hub/src/layers/modeling/__init__.py +13 -0
- keras_hub/src/layers/modeling/alibi_bias.py +143 -0
- keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
- keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
- keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
- keras_hub/src/layers/modeling/position_embedding.py +123 -0
- keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
- keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
- keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
- keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
- keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
- keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
- keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
- keras_hub/src/layers/preprocessing/__init__.py +13 -0
- keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
- keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
- keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
- keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
- keras_hub/src/layers/preprocessing/random_swap.py +267 -0
- keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
- keras_hub/src/metrics/__init__.py +13 -0
- keras_hub/src/metrics/bleu.py +394 -0
- keras_hub/src/metrics/edit_distance.py +197 -0
- keras_hub/src/metrics/perplexity.py +181 -0
- keras_hub/src/metrics/rouge_base.py +204 -0
- keras_hub/src/metrics/rouge_l.py +97 -0
- keras_hub/src/metrics/rouge_n.py +125 -0
- keras_hub/src/models/__init__.py +13 -0
- keras_hub/src/models/albert/__init__.py +20 -0
- keras_hub/src/models/albert/albert_backbone.py +267 -0
- keras_hub/src/models/albert/albert_classifier.py +202 -0
- keras_hub/src/models/albert/albert_masked_lm.py +129 -0
- keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
- keras_hub/src/models/albert/albert_preprocessor.py +206 -0
- keras_hub/src/models/albert/albert_presets.py +70 -0
- keras_hub/src/models/albert/albert_tokenizer.py +119 -0
- keras_hub/src/models/backbone.py +311 -0
- keras_hub/src/models/bart/__init__.py +20 -0
- keras_hub/src/models/bart/bart_backbone.py +261 -0
- keras_hub/src/models/bart/bart_preprocessor.py +276 -0
- keras_hub/src/models/bart/bart_presets.py +74 -0
- keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
- keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
- keras_hub/src/models/bart/bart_tokenizer.py +124 -0
- keras_hub/src/models/bert/__init__.py +23 -0
- keras_hub/src/models/bert/bert_backbone.py +227 -0
- keras_hub/src/models/bert/bert_classifier.py +183 -0
- keras_hub/src/models/bert/bert_masked_lm.py +131 -0
- keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
- keras_hub/src/models/bert/bert_preprocessor.py +184 -0
- keras_hub/src/models/bert/bert_presets.py +147 -0
- keras_hub/src/models/bert/bert_tokenizer.py +112 -0
- keras_hub/src/models/bloom/__init__.py +20 -0
- keras_hub/src/models/bloom/bloom_attention.py +186 -0
- keras_hub/src/models/bloom/bloom_backbone.py +173 -0
- keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
- keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
- keras_hub/src/models/bloom/bloom_decoder.py +206 -0
- keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
- keras_hub/src/models/bloom/bloom_presets.py +121 -0
- keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
- keras_hub/src/models/causal_lm.py +383 -0
- keras_hub/src/models/classifier.py +109 -0
- keras_hub/src/models/csp_darknet/__init__.py +13 -0
- keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
- keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
- keras_hub/src/models/deberta_v3/__init__.py +24 -0
- keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
- keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
- keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
- keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
- keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
- keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
- keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
- keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
- keras_hub/src/models/densenet/__init__.py +13 -0
- keras_hub/src/models/densenet/densenet_backbone.py +210 -0
- keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
- keras_hub/src/models/distil_bert/__init__.py +26 -0
- keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
- keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
- keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
- keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
- keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
- keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
- keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
- keras_hub/src/models/electra/__init__.py +20 -0
- keras_hub/src/models/electra/electra_backbone.py +247 -0
- keras_hub/src/models/electra/electra_preprocessor.py +154 -0
- keras_hub/src/models/electra/electra_presets.py +95 -0
- keras_hub/src/models/electra/electra_tokenizer.py +104 -0
- keras_hub/src/models/f_net/__init__.py +20 -0
- keras_hub/src/models/f_net/f_net_backbone.py +236 -0
- keras_hub/src/models/f_net/f_net_classifier.py +154 -0
- keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
- keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
- keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
- keras_hub/src/models/f_net/f_net_presets.py +43 -0
- keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
- keras_hub/src/models/falcon/__init__.py +20 -0
- keras_hub/src/models/falcon/falcon_attention.py +156 -0
- keras_hub/src/models/falcon/falcon_backbone.py +164 -0
- keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
- keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
- keras_hub/src/models/falcon/falcon_presets.py +30 -0
- keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
- keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
- keras_hub/src/models/feature_pyramid_backbone.py +73 -0
- keras_hub/src/models/gemma/__init__.py +20 -0
- keras_hub/src/models/gemma/gemma_attention.py +250 -0
- keras_hub/src/models/gemma/gemma_backbone.py +316 -0
- keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
- keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
- keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
- keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
- keras_hub/src/models/gemma/gemma_presets.py +248 -0
- keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
- keras_hub/src/models/gemma/rms_normalization.py +40 -0
- keras_hub/src/models/gpt2/__init__.py +20 -0
- keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
- keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
- keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
- keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
- keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
- keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
- keras_hub/src/models/image_classifier.py +90 -0
- keras_hub/src/models/llama/__init__.py +20 -0
- keras_hub/src/models/llama/llama_attention.py +225 -0
- keras_hub/src/models/llama/llama_backbone.py +188 -0
- keras_hub/src/models/llama/llama_causal_lm.py +327 -0
- keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
- keras_hub/src/models/llama/llama_decoder.py +246 -0
- keras_hub/src/models/llama/llama_layernorm.py +48 -0
- keras_hub/src/models/llama/llama_preprocessor.py +189 -0
- keras_hub/src/models/llama/llama_presets.py +80 -0
- keras_hub/src/models/llama/llama_tokenizer.py +84 -0
- keras_hub/src/models/llama3/__init__.py +20 -0
- keras_hub/src/models/llama3/llama3_backbone.py +84 -0
- keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
- keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
- keras_hub/src/models/llama3/llama3_presets.py +69 -0
- keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
- keras_hub/src/models/masked_lm.py +101 -0
- keras_hub/src/models/mistral/__init__.py +20 -0
- keras_hub/src/models/mistral/mistral_attention.py +238 -0
- keras_hub/src/models/mistral/mistral_backbone.py +203 -0
- keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
- keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
- keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
- keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
- keras_hub/src/models/mistral/mistral_presets.py +48 -0
- keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
- keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
- keras_hub/src/models/mix_transformer/__init__.py +13 -0
- keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
- keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
- keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
- keras_hub/src/models/opt/__init__.py +20 -0
- keras_hub/src/models/opt/opt_backbone.py +173 -0
- keras_hub/src/models/opt/opt_causal_lm.py +301 -0
- keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
- keras_hub/src/models/opt/opt_preprocessor.py +188 -0
- keras_hub/src/models/opt/opt_presets.py +72 -0
- keras_hub/src/models/opt/opt_tokenizer.py +116 -0
- keras_hub/src/models/pali_gemma/__init__.py +23 -0
- keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
- keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
- keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
- keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
- keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
- keras_hub/src/models/phi3/__init__.py +20 -0
- keras_hub/src/models/phi3/phi3_attention.py +260 -0
- keras_hub/src/models/phi3/phi3_backbone.py +224 -0
- keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
- keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/phi3/phi3_decoder.py +260 -0
- keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
- keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
- keras_hub/src/models/phi3/phi3_presets.py +50 -0
- keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
- keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
- keras_hub/src/models/preprocessor.py +207 -0
- keras_hub/src/models/resnet/__init__.py +13 -0
- keras_hub/src/models/resnet/resnet_backbone.py +612 -0
- keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
- keras_hub/src/models/roberta/__init__.py +20 -0
- keras_hub/src/models/roberta/roberta_backbone.py +184 -0
- keras_hub/src/models/roberta/roberta_classifier.py +209 -0
- keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
- keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
- keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
- keras_hub/src/models/roberta/roberta_presets.py +43 -0
- keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
- keras_hub/src/models/seq_2_seq_lm.py +54 -0
- keras_hub/src/models/t5/__init__.py +20 -0
- keras_hub/src/models/t5/t5_backbone.py +261 -0
- keras_hub/src/models/t5/t5_layer_norm.py +35 -0
- keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
- keras_hub/src/models/t5/t5_presets.py +95 -0
- keras_hub/src/models/t5/t5_tokenizer.py +100 -0
- keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
- keras_hub/src/models/task.py +419 -0
- keras_hub/src/models/vgg/__init__.py +13 -0
- keras_hub/src/models/vgg/vgg_backbone.py +158 -0
- keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
- keras_hub/src/models/vit_det/__init__.py +13 -0
- keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
- keras_hub/src/models/vit_det/vit_layers.py +565 -0
- keras_hub/src/models/whisper/__init__.py +20 -0
- keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
- keras_hub/src/models/whisper/whisper_backbone.py +305 -0
- keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
- keras_hub/src/models/whisper/whisper_decoder.py +141 -0
- keras_hub/src/models/whisper/whisper_encoder.py +106 -0
- keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
- keras_hub/src/models/whisper/whisper_presets.py +148 -0
- keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
- keras_hub/src/models/xlm_roberta/__init__.py +26 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
- keras_hub/src/models/xlnet/__init__.py +13 -0
- keras_hub/src/models/xlnet/relative_attention.py +459 -0
- keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
- keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
- keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
- keras_hub/src/samplers/__init__.py +13 -0
- keras_hub/src/samplers/beam_sampler.py +207 -0
- keras_hub/src/samplers/contrastive_sampler.py +231 -0
- keras_hub/src/samplers/greedy_sampler.py +50 -0
- keras_hub/src/samplers/random_sampler.py +77 -0
- keras_hub/src/samplers/sampler.py +237 -0
- keras_hub/src/samplers/serialization.py +97 -0
- keras_hub/src/samplers/top_k_sampler.py +92 -0
- keras_hub/src/samplers/top_p_sampler.py +113 -0
- keras_hub/src/tests/__init__.py +13 -0
- keras_hub/src/tests/test_case.py +608 -0
- keras_hub/src/tokenizers/__init__.py +13 -0
- keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
- keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
- keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
- keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
- keras_hub/src/tokenizers/tokenizer.py +235 -0
- keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
- keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
- keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
- keras_hub/src/utils/__init__.py +13 -0
- keras_hub/src/utils/keras_utils.py +130 -0
- keras_hub/src/utils/pipeline_model.py +293 -0
- keras_hub/src/utils/preset_utils.py +621 -0
- keras_hub/src/utils/python_utils.py +21 -0
- keras_hub/src/utils/tensor_utils.py +206 -0
- keras_hub/src/utils/timm/__init__.py +13 -0
- keras_hub/src/utils/timm/convert.py +37 -0
- keras_hub/src/utils/timm/convert_resnet.py +171 -0
- keras_hub/src/utils/transformers/__init__.py +13 -0
- keras_hub/src/utils/transformers/convert.py +101 -0
- keras_hub/src/utils/transformers/convert_bert.py +173 -0
- keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
- keras_hub/src/utils/transformers/convert_gemma.py +187 -0
- keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
- keras_hub/src/utils/transformers/convert_llama3.py +136 -0
- keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
- keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
- keras_hub/src/version_utils.py +23 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,154 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
import keras
|
17
|
+
|
18
|
+
from keras_hub.src.api_export import keras_hub_export
|
19
|
+
from keras_hub.src.models.classifier import Classifier
|
20
|
+
from keras_hub.src.models.f_net.f_net_backbone import FNetBackbone
|
21
|
+
from keras_hub.src.models.f_net.f_net_backbone import f_net_kernel_initializer
|
22
|
+
from keras_hub.src.models.f_net.f_net_preprocessor import FNetPreprocessor
|
23
|
+
|
24
|
+
|
25
|
+
@keras_hub_export("keras_hub.models.FNetClassifier")
|
26
|
+
class FNetClassifier(Classifier):
|
27
|
+
"""An end-to-end f_net model for classification tasks.
|
28
|
+
|
29
|
+
This model attaches a classification head to a
|
30
|
+
`keras_hub.model.FNetBackbone` instance, mapping from the backbone outputs
|
31
|
+
to logits suitable for a classification task. For usage of this model with
|
32
|
+
pre-trained weights, use the `from_preset()` constructor.
|
33
|
+
|
34
|
+
This model can optionally be configured with a `preprocessor` layer, in
|
35
|
+
which case it will automatically apply preprocessing to raw inputs during
|
36
|
+
`fit()`, `predict()`, and `evaluate()`. This is done by default when
|
37
|
+
creating the model with `from_preset()`.
|
38
|
+
|
39
|
+
Disclaimer: Pre-trained models are provided on an "as is" basis, without
|
40
|
+
warranties or conditions of any kind.
|
41
|
+
|
42
|
+
Args:
|
43
|
+
backbone: A `keras_hub.models.FNetBackbone` instance.
|
44
|
+
num_classes: int. Number of classes to predict.
|
45
|
+
preprocessor: A `keras_hub.models.FNetPreprocessor` or `None`. If
|
46
|
+
`None`, this model will not apply preprocessing, and inputs should
|
47
|
+
be preprocessed before calling the model.
|
48
|
+
activation: Optional `str` or callable. The
|
49
|
+
activation function to use on the model outputs. Set
|
50
|
+
`activation="softmax"` to return output probabilities.
|
51
|
+
Defaults to `None`.
|
52
|
+
hidden_dim: int. The size of the pooler layer.
|
53
|
+
dropout: float. The dropout probability value, applied after the dense
|
54
|
+
layer.
|
55
|
+
|
56
|
+
Examples:
|
57
|
+
|
58
|
+
Raw string data.
|
59
|
+
```python
|
60
|
+
features = ["The quick brown fox jumped.", "I forgot my homework."]
|
61
|
+
labels = [0, 3]
|
62
|
+
|
63
|
+
# Pretrained classifier.
|
64
|
+
classifier = keras_hub.models.FNetClassifier.from_preset(
|
65
|
+
"f_net_base_en",
|
66
|
+
num_classes=4,
|
67
|
+
)
|
68
|
+
classifier.fit(x=features, y=labels, batch_size=2)
|
69
|
+
classifier.predict(x=features, batch_size=2)
|
70
|
+
|
71
|
+
# Re-compile (e.g., with a new learning rate).
|
72
|
+
classifier.compile(
|
73
|
+
loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
|
74
|
+
optimizer=keras.optimizers.Adam(5e-5),
|
75
|
+
jit_compile=True,
|
76
|
+
)
|
77
|
+
# Access backbone programmatically (e.g., to change `trainable`).
|
78
|
+
classifier.backbone.trainable = False
|
79
|
+
# Fit again.
|
80
|
+
classifier.fit(x=features, y=labels, batch_size=2)
|
81
|
+
```
|
82
|
+
|
83
|
+
Preprocessed integer data.
|
84
|
+
```python
|
85
|
+
features = {
|
86
|
+
"token_ids": np.ones(shape=(2, 12), dtype="int32"),
|
87
|
+
"segment_ids": np.array([[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0]] * 2),
|
88
|
+
}
|
89
|
+
labels = [0, 3]
|
90
|
+
|
91
|
+
# Pretrained classifier without preprocessing.
|
92
|
+
classifier = keras_hub.models.FNetClassifier.from_preset(
|
93
|
+
"f_net_base_en",
|
94
|
+
num_classes=4,
|
95
|
+
preprocessor=None,
|
96
|
+
)
|
97
|
+
classifier.fit(x=features, y=labels, batch_size=2)
|
98
|
+
```
|
99
|
+
"""
|
100
|
+
|
101
|
+
backbone_cls = FNetBackbone
|
102
|
+
preprocessor_cls = FNetPreprocessor
|
103
|
+
|
104
|
+
def __init__(
|
105
|
+
self,
|
106
|
+
backbone,
|
107
|
+
num_classes,
|
108
|
+
preprocessor=None,
|
109
|
+
activation=None,
|
110
|
+
dropout=0.1,
|
111
|
+
**kwargs,
|
112
|
+
):
|
113
|
+
# === Layers ===
|
114
|
+
self.backbone = backbone
|
115
|
+
self.preprocessor = preprocessor
|
116
|
+
self.output_dropout = keras.layers.Dropout(
|
117
|
+
dropout,
|
118
|
+
dtype=backbone.dtype_policy,
|
119
|
+
name="output_dropout",
|
120
|
+
)
|
121
|
+
self.output_dense = keras.layers.Dense(
|
122
|
+
num_classes,
|
123
|
+
kernel_initializer=f_net_kernel_initializer(),
|
124
|
+
activation=activation,
|
125
|
+
dtype=backbone.dtype_policy,
|
126
|
+
name="logits",
|
127
|
+
)
|
128
|
+
|
129
|
+
# === Functional Model ===
|
130
|
+
inputs = backbone.input
|
131
|
+
pooled = backbone(inputs)["pooled_output"]
|
132
|
+
pooled = self.output_dropout(pooled)
|
133
|
+
outputs = self.output_dense(pooled)
|
134
|
+
super().__init__(
|
135
|
+
inputs=inputs,
|
136
|
+
outputs=outputs,
|
137
|
+
**kwargs,
|
138
|
+
)
|
139
|
+
|
140
|
+
# === Config ===
|
141
|
+
self.num_classes = num_classes
|
142
|
+
self.activation = keras.activations.get(activation)
|
143
|
+
self.dropout = dropout
|
144
|
+
|
145
|
+
def get_config(self):
|
146
|
+
config = super().get_config()
|
147
|
+
config.update(
|
148
|
+
{
|
149
|
+
"num_classes": self.num_classes,
|
150
|
+
"dropout": self.dropout,
|
151
|
+
"activation": keras.activations.serialize(self.activation),
|
152
|
+
}
|
153
|
+
)
|
154
|
+
return config
|
@@ -0,0 +1,132 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
import keras
|
17
|
+
|
18
|
+
from keras_hub.src.api_export import keras_hub_export
|
19
|
+
from keras_hub.src.layers.modeling.masked_lm_head import MaskedLMHead
|
20
|
+
from keras_hub.src.models.f_net.f_net_backbone import FNetBackbone
|
21
|
+
from keras_hub.src.models.f_net.f_net_backbone import f_net_kernel_initializer
|
22
|
+
from keras_hub.src.models.f_net.f_net_masked_lm_preprocessor import (
|
23
|
+
FNetMaskedLMPreprocessor,
|
24
|
+
)
|
25
|
+
from keras_hub.src.models.masked_lm import MaskedLM
|
26
|
+
|
27
|
+
|
28
|
+
@keras_hub_export("keras_hub.models.FNetMaskedLM")
|
29
|
+
class FNetMaskedLM(MaskedLM):
|
30
|
+
"""An end-to-end FNet model for the masked language modeling task.
|
31
|
+
|
32
|
+
This model will train FNet on a masked language modeling task.
|
33
|
+
The model will predict labels for a number of masked tokens in the
|
34
|
+
input data. For usage of this model with pre-trained weights, see the
|
35
|
+
`from_preset()` constructor.
|
36
|
+
|
37
|
+
This model can optionally be configured with a `preprocessor` layer, in
|
38
|
+
which case inputs can be raw string features during `fit()`, `predict()`,
|
39
|
+
and `evaluate()`. Inputs will be tokenized and dynamically masked during
|
40
|
+
training and evaluation. This is done by default when creating the model
|
41
|
+
with `from_preset()`.
|
42
|
+
|
43
|
+
Disclaimer: Pre-trained models are provided on an "as is" basis, without
|
44
|
+
warranties or conditions of any kind.
|
45
|
+
|
46
|
+
Args:
|
47
|
+
backbone: A `keras_hub.models.FNetBackbone` instance.
|
48
|
+
preprocessor: A `keras_hub.models.FNetMaskedLMPreprocessor` or
|
49
|
+
`None`. If `None`, this model will not apply preprocessing, and
|
50
|
+
inputs should be preprocessed before calling the model.
|
51
|
+
|
52
|
+
Examples:
|
53
|
+
|
54
|
+
Raw string data.
|
55
|
+
```python
|
56
|
+
|
57
|
+
features = ["The quick brown fox jumped.", "I forgot my homework."]
|
58
|
+
|
59
|
+
# Pretrained language model.
|
60
|
+
masked_lm = keras_hub.models.FNetMaskedLM.from_preset(
|
61
|
+
"f_net_base_en",
|
62
|
+
)
|
63
|
+
masked_lm.fit(x=features, batch_size=2)
|
64
|
+
|
65
|
+
# Re-compile (e.g., with a new learning rate).
|
66
|
+
masked_lm.compile(
|
67
|
+
loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
|
68
|
+
optimizer=keras.optimizers.Adam(5e-5),
|
69
|
+
jit_compile=True,
|
70
|
+
)
|
71
|
+
# Access backbone programmatically (e.g., to change `trainable`).
|
72
|
+
masked_lm.backbone.trainable = False
|
73
|
+
# Fit again.
|
74
|
+
masked_lm.fit(x=features, batch_size=2)
|
75
|
+
```
|
76
|
+
|
77
|
+
Preprocessed integer data.
|
78
|
+
```python
|
79
|
+
# Create a preprocessed dataset where 0 is the mask token.
|
80
|
+
features = {
|
81
|
+
"token_ids": np.array([[1, 2, 0, 4, 0, 6, 7, 8]] * 2),
|
82
|
+
"segment_ids": np.array([[0, 0, 0, 1, 1, 1, 0, 0]] * 2),
|
83
|
+
"mask_positions": np.array([[2, 4]] * 2)
|
84
|
+
}
|
85
|
+
# Labels are the original masked values.
|
86
|
+
labels = [[3, 5]] * 2
|
87
|
+
|
88
|
+
masked_lm = keras_hub.models.FNetMaskedLM.from_preset(
|
89
|
+
"f_net_base_en",
|
90
|
+
preprocessor=None,
|
91
|
+
)
|
92
|
+
masked_lm.fit(x=features, y=labels, batch_size=2)
|
93
|
+
```
|
94
|
+
"""
|
95
|
+
|
96
|
+
backbone_cls = FNetBackbone
|
97
|
+
preprocessor_cls = FNetMaskedLMPreprocessor
|
98
|
+
|
99
|
+
def __init__(
|
100
|
+
self,
|
101
|
+
backbone,
|
102
|
+
preprocessor=None,
|
103
|
+
**kwargs,
|
104
|
+
):
|
105
|
+
# === Layers ===
|
106
|
+
self.backbone = backbone
|
107
|
+
self.preprocessor = preprocessor
|
108
|
+
self.masked_lm_head = MaskedLMHead(
|
109
|
+
vocabulary_size=backbone.vocabulary_size,
|
110
|
+
token_embedding=backbone.token_embedding,
|
111
|
+
intermediate_activation="gelu",
|
112
|
+
kernel_initializer=f_net_kernel_initializer(),
|
113
|
+
dtype=backbone.dtype_policy,
|
114
|
+
name="mlm_head",
|
115
|
+
)
|
116
|
+
|
117
|
+
# === Functional Model ===
|
118
|
+
inputs = {
|
119
|
+
**backbone.input,
|
120
|
+
"mask_positions": keras.Input(
|
121
|
+
shape=(None,), dtype="int32", name="mask_positions"
|
122
|
+
),
|
123
|
+
}
|
124
|
+
backbone_outputs = backbone(backbone.input)
|
125
|
+
outputs = self.masked_lm_head(
|
126
|
+
backbone_outputs["sequence_output"], inputs["mask_positions"]
|
127
|
+
)
|
128
|
+
super().__init__(
|
129
|
+
inputs=inputs,
|
130
|
+
outputs=outputs,
|
131
|
+
**kwargs,
|
132
|
+
)
|
@@ -0,0 +1,196 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
import keras
|
16
|
+
from absl import logging
|
17
|
+
|
18
|
+
from keras_hub.src.api_export import keras_hub_export
|
19
|
+
from keras_hub.src.layers.preprocessing.masked_lm_mask_generator import (
|
20
|
+
MaskedLMMaskGenerator,
|
21
|
+
)
|
22
|
+
from keras_hub.src.models.f_net.f_net_preprocessor import FNetPreprocessor
|
23
|
+
|
24
|
+
|
25
|
+
@keras_hub_export("keras_hub.models.FNetMaskedLMPreprocessor")
|
26
|
+
class FNetMaskedLMPreprocessor(FNetPreprocessor):
|
27
|
+
"""FNet preprocessing for the masked language modeling task.
|
28
|
+
|
29
|
+
This preprocessing layer will prepare inputs for a masked language modeling
|
30
|
+
task. It is primarily intended for use with the
|
31
|
+
`keras_hub.models.FNetMaskedLM` task model. Preprocessing will occur in
|
32
|
+
multiple steps.
|
33
|
+
|
34
|
+
1. Tokenize any number of input segments using the `tokenizer`.
|
35
|
+
2. Pack the inputs together with the appropriate `"<s>"`, `"</s>"` and
|
36
|
+
`"<pad>"` tokens, i.e., adding a single `"<s>"` at the start of the
|
37
|
+
entire sequence, `"</s></s>"` between each segment,
|
38
|
+
and a `"</s>"` at the end of the entire sequence.
|
39
|
+
3. Randomly select non-special tokens to mask, controlled by
|
40
|
+
`mask_selection_rate`.
|
41
|
+
4. Construct a `(x, y, sample_weight)` tuple suitable for training with a
|
42
|
+
`keras_hub.models.FNetMaskedLM` task model.
|
43
|
+
|
44
|
+
Args:
|
45
|
+
tokenizer: A `keras_hub.models.FNetTokenizer` instance.
|
46
|
+
sequence_length: The length of the packed inputs.
|
47
|
+
mask_selection_rate: The probability an input token will be dynamically
|
48
|
+
masked.
|
49
|
+
mask_selection_length: The maximum number of masked tokens supported
|
50
|
+
by the layer.
|
51
|
+
mask_token_rate: float. `mask_token_rate` must be
|
52
|
+
between 0 and 1 which indicates how often the mask_token is
|
53
|
+
substituted for tokens selected for masking. Defaults to `0.8`.
|
54
|
+
random_token_rate: float. `random_token_rate` must be
|
55
|
+
between 0 and 1 which indicates how often a random token is
|
56
|
+
substituted for tokens selected for masking.
|
57
|
+
Note: mask_token_rate + random_token_rate <= 1, and for
|
58
|
+
(1 - mask_token_rate - random_token_rate), the token will not be
|
59
|
+
changed. Defaults to `0.1`.
|
60
|
+
truncate: string. The algorithm to truncate a list of batched segments
|
61
|
+
to fit within `sequence_length`. The value can be either
|
62
|
+
`round_robin` or `waterfall`:
|
63
|
+
- `"round_robin"`: Available space is assigned one token at a
|
64
|
+
time in a round-robin fashion to the inputs that still need
|
65
|
+
some, until the limit is reached.
|
66
|
+
- `"waterfall"`: The allocation of the budget is done using a
|
67
|
+
"waterfall" algorithm that allocates quota in a
|
68
|
+
left-to-right manner and fills up the buckets until we run
|
69
|
+
out of budget. It supports an arbitrary number of segments.
|
70
|
+
|
71
|
+
Examples:
|
72
|
+
|
73
|
+
Directly calling the layer on data.
|
74
|
+
```python
|
75
|
+
# Load the preprocessor from a preset.
|
76
|
+
preprocessor = keras_hub.models.FNetMaskedLMPreprocessor.from_preset(
|
77
|
+
"f_net_base_en"
|
78
|
+
)
|
79
|
+
|
80
|
+
# Tokenize and mask a single sentence.
|
81
|
+
preprocessor("The quick brown fox jumped.")
|
82
|
+
|
83
|
+
# Tokenize and mask a batch of single sentences.
|
84
|
+
preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
|
85
|
+
|
86
|
+
# Tokenize and mask sentence pairs.
|
87
|
+
# In this case, always convert input to tensors before calling the layer.
|
88
|
+
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
89
|
+
second = tf.constant(["The fox tripped.", "Oh look, a whale."])
|
90
|
+
preprocessor((first, second))
|
91
|
+
```
|
92
|
+
|
93
|
+
Mapping with `tf.data.Dataset`.
|
94
|
+
```python
|
95
|
+
preprocessor = keras_hub.models.FNetMaskedLMPreprocessor.from_preset(
|
96
|
+
"f_net_base_en"
|
97
|
+
)
|
98
|
+
|
99
|
+
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
100
|
+
second = tf.constant(["The fox tripped.", "Oh look, a whale."])
|
101
|
+
|
102
|
+
# Map single sentences.
|
103
|
+
ds = tf.data.Dataset.from_tensor_slices(first)
|
104
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
105
|
+
|
106
|
+
# Alternatively, you can create a preprocessor from your own vocabulary.
|
107
|
+
vocab_data = tf.data.Dataset.from_tensor_slices(
|
108
|
+
["the quick brown fox", "the earth is round"]
|
109
|
+
)
|
110
|
+
|
111
|
+
# Map sentence pairs.
|
112
|
+
ds = tf.data.Dataset.from_tensor_slices((first, second))
|
113
|
+
# Watch out for tf.data's default unpacking of tuples here!
|
114
|
+
# Best to invoke the `preprocessor` directly in this case.
|
115
|
+
ds = ds.map(
|
116
|
+
lambda first, second: preprocessor(x=(first, second)),
|
117
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
118
|
+
)
|
119
|
+
```
|
120
|
+
"""
|
121
|
+
|
122
|
+
def __init__(
|
123
|
+
self,
|
124
|
+
tokenizer,
|
125
|
+
sequence_length=512,
|
126
|
+
truncate="round_robin",
|
127
|
+
mask_selection_rate=0.15,
|
128
|
+
mask_selection_length=96,
|
129
|
+
mask_token_rate=0.8,
|
130
|
+
random_token_rate=0.1,
|
131
|
+
**kwargs,
|
132
|
+
):
|
133
|
+
super().__init__(
|
134
|
+
tokenizer,
|
135
|
+
sequence_length=sequence_length,
|
136
|
+
truncate=truncate,
|
137
|
+
**kwargs,
|
138
|
+
)
|
139
|
+
self.mask_selection_rate = mask_selection_rate
|
140
|
+
self.mask_selection_length = mask_selection_length
|
141
|
+
self.mask_token_rate = mask_token_rate
|
142
|
+
self.random_token_rate = random_token_rate
|
143
|
+
self.masker = None
|
144
|
+
|
145
|
+
def build(self, input_shape):
|
146
|
+
super().build(input_shape)
|
147
|
+
# Defer masker creation to `build()` so that we can be sure tokenizer
|
148
|
+
# assets have loaded when restoring a saved model.
|
149
|
+
self.masker = MaskedLMMaskGenerator(
|
150
|
+
mask_selection_rate=self.mask_selection_rate,
|
151
|
+
mask_selection_length=self.mask_selection_length,
|
152
|
+
mask_token_rate=self.mask_token_rate,
|
153
|
+
random_token_rate=self.random_token_rate,
|
154
|
+
vocabulary_size=self.tokenizer.vocabulary_size(),
|
155
|
+
mask_token_id=self.tokenizer.mask_token_id,
|
156
|
+
unselectable_token_ids=[
|
157
|
+
self.tokenizer.cls_token_id,
|
158
|
+
self.tokenizer.sep_token_id,
|
159
|
+
self.tokenizer.pad_token_id,
|
160
|
+
],
|
161
|
+
)
|
162
|
+
|
163
|
+
def get_config(self):
|
164
|
+
config = super().get_config()
|
165
|
+
config.update(
|
166
|
+
{
|
167
|
+
"mask_selection_rate": self.mask_selection_rate,
|
168
|
+
"mask_selection_length": self.mask_selection_length,
|
169
|
+
"mask_token_rate": self.mask_token_rate,
|
170
|
+
"random_token_rate": self.random_token_rate,
|
171
|
+
}
|
172
|
+
)
|
173
|
+
return config
|
174
|
+
|
175
|
+
def call(self, x, y=None, sample_weight=None):
|
176
|
+
if y is not None or sample_weight is not None:
|
177
|
+
logging.warning(
|
178
|
+
f"{self.__class__.__name__} generates `y` and `sample_weight` "
|
179
|
+
"based on your input data, but your data already contains `y` "
|
180
|
+
"or `sample_weight`. Your `y` and `sample_weight` will be "
|
181
|
+
"ignored."
|
182
|
+
)
|
183
|
+
x = super().call(x)
|
184
|
+
token_ids, segment_ids = (
|
185
|
+
x["token_ids"],
|
186
|
+
x["segment_ids"],
|
187
|
+
)
|
188
|
+
masker_outputs = self.masker(token_ids)
|
189
|
+
x = {
|
190
|
+
"token_ids": masker_outputs["token_ids"],
|
191
|
+
"segment_ids": segment_ids,
|
192
|
+
"mask_positions": masker_outputs["mask_positions"],
|
193
|
+
}
|
194
|
+
y = masker_outputs["mask_ids"]
|
195
|
+
sample_weight = masker_outputs["mask_weights"]
|
196
|
+
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
|
@@ -0,0 +1,177 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
import keras
|
17
|
+
|
18
|
+
from keras_hub.src.api_export import keras_hub_export
|
19
|
+
from keras_hub.src.layers.preprocessing.multi_segment_packer import (
|
20
|
+
MultiSegmentPacker,
|
21
|
+
)
|
22
|
+
from keras_hub.src.models.f_net.f_net_tokenizer import FNetTokenizer
|
23
|
+
from keras_hub.src.models.preprocessor import Preprocessor
|
24
|
+
from keras_hub.src.utils.keras_utils import (
|
25
|
+
convert_inputs_to_list_of_tensor_segments,
|
26
|
+
)
|
27
|
+
|
28
|
+
|
29
|
+
@keras_hub_export("keras_hub.models.FNetPreprocessor")
|
30
|
+
class FNetPreprocessor(Preprocessor):
|
31
|
+
"""An FNet preprocessing layer which tokenizes and packs inputs.
|
32
|
+
|
33
|
+
This preprocessing layer will do three things:
|
34
|
+
|
35
|
+
1. Tokenize any number of input segments using the `tokenizer`.
|
36
|
+
2. Pack the inputs together using a `keras_hub.layers.MultiSegmentPacker`.
|
37
|
+
with the appropriate `"[CLS]"`, `"[SEP]"` and `"<pad>"` tokens.
|
38
|
+
3. Construct a dictionary with keys `"token_ids"`, and `"segment_ids"` that
|
39
|
+
can be passed directly to `keras_hub.models.FNetBackbone`.
|
40
|
+
|
41
|
+
This layer can be used directly with `tf.data.Dataset.map` to preprocess
|
42
|
+
string data in the `(x, y, sample_weight)` format used by
|
43
|
+
`keras.Model.fit`.
|
44
|
+
|
45
|
+
Args:
|
46
|
+
tokenizer: A `keras_hub.models.FNetTokenizer` instance.
|
47
|
+
sequence_length: The length of the packed inputs.
|
48
|
+
truncate: string. The algorithm to truncate a list of batched segments
|
49
|
+
to fit within `sequence_length`. The value can be either
|
50
|
+
`round_robin` or `waterfall`:
|
51
|
+
- `"round_robin"`: Available space is assigned one token at a
|
52
|
+
time in a round-robin fashion to the inputs that still need
|
53
|
+
some, until the limit is reached.
|
54
|
+
- `"waterfall"`: The allocation of the budget is done using a
|
55
|
+
"waterfall" algorithm that allocates quota in a
|
56
|
+
left-to-right manner and fills up the buckets until we run
|
57
|
+
out of budget. It supports an arbitrary number of segments.
|
58
|
+
|
59
|
+
Call arguments:
|
60
|
+
x: A tensor of single string sequences, or a tuple of multiple
|
61
|
+
tensor sequences to be packed together. Inputs may be batched or
|
62
|
+
unbatched. For single sequences, raw python inputs will be converted
|
63
|
+
to tensors. For multiple sequences, pass tensors directly.
|
64
|
+
y: Any label data. Will be passed through unaltered.
|
65
|
+
sample_weight: Any label weight data. Will be passed through unaltered.
|
66
|
+
|
67
|
+
Examples:
|
68
|
+
|
69
|
+
Directly calling the from_preset().
|
70
|
+
```python
|
71
|
+
preprocessor = keras_hub.models.FNetPreprocessor.from_preset(
|
72
|
+
"f_net_base_en"
|
73
|
+
)
|
74
|
+
|
75
|
+
# Tokenize and pack a single sentence.
|
76
|
+
preprocessor("The quick brown fox jumped.")
|
77
|
+
|
78
|
+
# Tokenize and a batch of single sentences.
|
79
|
+
preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
|
80
|
+
|
81
|
+
# Preprocess a batch of sentence pairs.
|
82
|
+
# When handling multiple sequences, always convert to tensors first!
|
83
|
+
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
84
|
+
second = tf.constant(["The fox tripped.", "Oh look, a whale."])
|
85
|
+
preprocessor((first, second))
|
86
|
+
```
|
87
|
+
|
88
|
+
Mapping with `tf.data.Dataset`.
|
89
|
+
```python
|
90
|
+
preprocessor = keras_hub.models.FNetPreprocessor.from_preset(
|
91
|
+
"f_net_base_en"
|
92
|
+
)
|
93
|
+
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
94
|
+
second = tf.constant(["The fox tripped.", "Oh look, a whale."])
|
95
|
+
label = tf.constant([1, 1])
|
96
|
+
|
97
|
+
# Map labeled single sentences.
|
98
|
+
ds = tf.data.Dataset.from_tensor_slices((first, label))
|
99
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
100
|
+
|
101
|
+
# Map unlabeled single sentences.
|
102
|
+
ds = tf.data.Dataset.from_tensor_slices(first)
|
103
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
104
|
+
|
105
|
+
# Map labeled sentence pairs.
|
106
|
+
ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
|
107
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
108
|
+
|
109
|
+
# Map unlabeled sentence pairs.
|
110
|
+
ds = tf.data.Dataset.from_tensor_slices((first, second))
|
111
|
+
|
112
|
+
# Watch out for tf.data's default unpacking of tuples here!
|
113
|
+
# Best to invoke the `preprocessor` directly in this case.
|
114
|
+
ds = ds.map(
|
115
|
+
lambda first, second: preprocessor(x=(first, second)),
|
116
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
117
|
+
)
|
118
|
+
```
|
119
|
+
"""
|
120
|
+
|
121
|
+
tokenizer_cls = FNetTokenizer
|
122
|
+
|
123
|
+
def __init__(
|
124
|
+
self,
|
125
|
+
tokenizer,
|
126
|
+
sequence_length=512,
|
127
|
+
truncate="round_robin",
|
128
|
+
**kwargs,
|
129
|
+
):
|
130
|
+
super().__init__(**kwargs)
|
131
|
+
self.tokenizer = tokenizer
|
132
|
+
self.packer = None
|
133
|
+
self.truncate = truncate
|
134
|
+
self.sequence_length = sequence_length
|
135
|
+
|
136
|
+
def build(self, input_shape):
|
137
|
+
# Defer packer creation to `build()` so that we can be sure tokenizer
|
138
|
+
# assets have loaded when restoring a saved model.
|
139
|
+
self.packer = MultiSegmentPacker(
|
140
|
+
start_value=self.tokenizer.cls_token_id,
|
141
|
+
end_value=self.tokenizer.sep_token_id,
|
142
|
+
pad_value=self.tokenizer.pad_token_id,
|
143
|
+
truncate=self.truncate,
|
144
|
+
sequence_length=self.sequence_length,
|
145
|
+
)
|
146
|
+
self.built = True
|
147
|
+
|
148
|
+
def get_config(self):
|
149
|
+
config = super().get_config()
|
150
|
+
config.update(
|
151
|
+
{
|
152
|
+
"sequence_length": self.sequence_length,
|
153
|
+
"truncate": self.truncate,
|
154
|
+
}
|
155
|
+
)
|
156
|
+
return config
|
157
|
+
|
158
|
+
def call(self, x, y=None, sample_weight=None):
|
159
|
+
x = convert_inputs_to_list_of_tensor_segments(x)
|
160
|
+
x = [self.tokenizer(segment) for segment in x]
|
161
|
+
token_ids, segment_ids = self.packer(x)
|
162
|
+
x = {
|
163
|
+
"token_ids": token_ids,
|
164
|
+
"segment_ids": segment_ids,
|
165
|
+
}
|
166
|
+
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
|
167
|
+
|
168
|
+
@property
|
169
|
+
def sequence_length(self):
|
170
|
+
"""The padded length of model input sequences."""
|
171
|
+
return self._sequence_length
|
172
|
+
|
173
|
+
@sequence_length.setter
|
174
|
+
def sequence_length(self, value):
|
175
|
+
self._sequence_length = value
|
176
|
+
if self.packer is not None:
|
177
|
+
self.packer.sequence_length = value
|