keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keras_hub/__init__.py +52 -0
- keras_hub/api/__init__.py +27 -0
- keras_hub/api/layers/__init__.py +47 -0
- keras_hub/api/metrics/__init__.py +24 -0
- keras_hub/api/models/__init__.py +249 -0
- keras_hub/api/samplers/__init__.py +29 -0
- keras_hub/api/tokenizers/__init__.py +35 -0
- keras_hub/src/__init__.py +13 -0
- keras_hub/src/api_export.py +53 -0
- keras_hub/src/layers/__init__.py +13 -0
- keras_hub/src/layers/modeling/__init__.py +13 -0
- keras_hub/src/layers/modeling/alibi_bias.py +143 -0
- keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
- keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
- keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
- keras_hub/src/layers/modeling/position_embedding.py +123 -0
- keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
- keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
- keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
- keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
- keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
- keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
- keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
- keras_hub/src/layers/preprocessing/__init__.py +13 -0
- keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
- keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
- keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
- keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
- keras_hub/src/layers/preprocessing/random_swap.py +267 -0
- keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
- keras_hub/src/metrics/__init__.py +13 -0
- keras_hub/src/metrics/bleu.py +394 -0
- keras_hub/src/metrics/edit_distance.py +197 -0
- keras_hub/src/metrics/perplexity.py +181 -0
- keras_hub/src/metrics/rouge_base.py +204 -0
- keras_hub/src/metrics/rouge_l.py +97 -0
- keras_hub/src/metrics/rouge_n.py +125 -0
- keras_hub/src/models/__init__.py +13 -0
- keras_hub/src/models/albert/__init__.py +20 -0
- keras_hub/src/models/albert/albert_backbone.py +267 -0
- keras_hub/src/models/albert/albert_classifier.py +202 -0
- keras_hub/src/models/albert/albert_masked_lm.py +129 -0
- keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
- keras_hub/src/models/albert/albert_preprocessor.py +206 -0
- keras_hub/src/models/albert/albert_presets.py +70 -0
- keras_hub/src/models/albert/albert_tokenizer.py +119 -0
- keras_hub/src/models/backbone.py +311 -0
- keras_hub/src/models/bart/__init__.py +20 -0
- keras_hub/src/models/bart/bart_backbone.py +261 -0
- keras_hub/src/models/bart/bart_preprocessor.py +276 -0
- keras_hub/src/models/bart/bart_presets.py +74 -0
- keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
- keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
- keras_hub/src/models/bart/bart_tokenizer.py +124 -0
- keras_hub/src/models/bert/__init__.py +23 -0
- keras_hub/src/models/bert/bert_backbone.py +227 -0
- keras_hub/src/models/bert/bert_classifier.py +183 -0
- keras_hub/src/models/bert/bert_masked_lm.py +131 -0
- keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
- keras_hub/src/models/bert/bert_preprocessor.py +184 -0
- keras_hub/src/models/bert/bert_presets.py +147 -0
- keras_hub/src/models/bert/bert_tokenizer.py +112 -0
- keras_hub/src/models/bloom/__init__.py +20 -0
- keras_hub/src/models/bloom/bloom_attention.py +186 -0
- keras_hub/src/models/bloom/bloom_backbone.py +173 -0
- keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
- keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
- keras_hub/src/models/bloom/bloom_decoder.py +206 -0
- keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
- keras_hub/src/models/bloom/bloom_presets.py +121 -0
- keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
- keras_hub/src/models/causal_lm.py +383 -0
- keras_hub/src/models/classifier.py +109 -0
- keras_hub/src/models/csp_darknet/__init__.py +13 -0
- keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
- keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
- keras_hub/src/models/deberta_v3/__init__.py +24 -0
- keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
- keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
- keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
- keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
- keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
- keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
- keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
- keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
- keras_hub/src/models/densenet/__init__.py +13 -0
- keras_hub/src/models/densenet/densenet_backbone.py +210 -0
- keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
- keras_hub/src/models/distil_bert/__init__.py +26 -0
- keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
- keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
- keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
- keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
- keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
- keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
- keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
- keras_hub/src/models/electra/__init__.py +20 -0
- keras_hub/src/models/electra/electra_backbone.py +247 -0
- keras_hub/src/models/electra/electra_preprocessor.py +154 -0
- keras_hub/src/models/electra/electra_presets.py +95 -0
- keras_hub/src/models/electra/electra_tokenizer.py +104 -0
- keras_hub/src/models/f_net/__init__.py +20 -0
- keras_hub/src/models/f_net/f_net_backbone.py +236 -0
- keras_hub/src/models/f_net/f_net_classifier.py +154 -0
- keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
- keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
- keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
- keras_hub/src/models/f_net/f_net_presets.py +43 -0
- keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
- keras_hub/src/models/falcon/__init__.py +20 -0
- keras_hub/src/models/falcon/falcon_attention.py +156 -0
- keras_hub/src/models/falcon/falcon_backbone.py +164 -0
- keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
- keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
- keras_hub/src/models/falcon/falcon_presets.py +30 -0
- keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
- keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
- keras_hub/src/models/feature_pyramid_backbone.py +73 -0
- keras_hub/src/models/gemma/__init__.py +20 -0
- keras_hub/src/models/gemma/gemma_attention.py +250 -0
- keras_hub/src/models/gemma/gemma_backbone.py +316 -0
- keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
- keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
- keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
- keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
- keras_hub/src/models/gemma/gemma_presets.py +248 -0
- keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
- keras_hub/src/models/gemma/rms_normalization.py +40 -0
- keras_hub/src/models/gpt2/__init__.py +20 -0
- keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
- keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
- keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
- keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
- keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
- keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
- keras_hub/src/models/image_classifier.py +90 -0
- keras_hub/src/models/llama/__init__.py +20 -0
- keras_hub/src/models/llama/llama_attention.py +225 -0
- keras_hub/src/models/llama/llama_backbone.py +188 -0
- keras_hub/src/models/llama/llama_causal_lm.py +327 -0
- keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
- keras_hub/src/models/llama/llama_decoder.py +246 -0
- keras_hub/src/models/llama/llama_layernorm.py +48 -0
- keras_hub/src/models/llama/llama_preprocessor.py +189 -0
- keras_hub/src/models/llama/llama_presets.py +80 -0
- keras_hub/src/models/llama/llama_tokenizer.py +84 -0
- keras_hub/src/models/llama3/__init__.py +20 -0
- keras_hub/src/models/llama3/llama3_backbone.py +84 -0
- keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
- keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
- keras_hub/src/models/llama3/llama3_presets.py +69 -0
- keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
- keras_hub/src/models/masked_lm.py +101 -0
- keras_hub/src/models/mistral/__init__.py +20 -0
- keras_hub/src/models/mistral/mistral_attention.py +238 -0
- keras_hub/src/models/mistral/mistral_backbone.py +203 -0
- keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
- keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
- keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
- keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
- keras_hub/src/models/mistral/mistral_presets.py +48 -0
- keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
- keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
- keras_hub/src/models/mix_transformer/__init__.py +13 -0
- keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
- keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
- keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
- keras_hub/src/models/opt/__init__.py +20 -0
- keras_hub/src/models/opt/opt_backbone.py +173 -0
- keras_hub/src/models/opt/opt_causal_lm.py +301 -0
- keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
- keras_hub/src/models/opt/opt_preprocessor.py +188 -0
- keras_hub/src/models/opt/opt_presets.py +72 -0
- keras_hub/src/models/opt/opt_tokenizer.py +116 -0
- keras_hub/src/models/pali_gemma/__init__.py +23 -0
- keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
- keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
- keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
- keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
- keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
- keras_hub/src/models/phi3/__init__.py +20 -0
- keras_hub/src/models/phi3/phi3_attention.py +260 -0
- keras_hub/src/models/phi3/phi3_backbone.py +224 -0
- keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
- keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/phi3/phi3_decoder.py +260 -0
- keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
- keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
- keras_hub/src/models/phi3/phi3_presets.py +50 -0
- keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
- keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
- keras_hub/src/models/preprocessor.py +207 -0
- keras_hub/src/models/resnet/__init__.py +13 -0
- keras_hub/src/models/resnet/resnet_backbone.py +612 -0
- keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
- keras_hub/src/models/roberta/__init__.py +20 -0
- keras_hub/src/models/roberta/roberta_backbone.py +184 -0
- keras_hub/src/models/roberta/roberta_classifier.py +209 -0
- keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
- keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
- keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
- keras_hub/src/models/roberta/roberta_presets.py +43 -0
- keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
- keras_hub/src/models/seq_2_seq_lm.py +54 -0
- keras_hub/src/models/t5/__init__.py +20 -0
- keras_hub/src/models/t5/t5_backbone.py +261 -0
- keras_hub/src/models/t5/t5_layer_norm.py +35 -0
- keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
- keras_hub/src/models/t5/t5_presets.py +95 -0
- keras_hub/src/models/t5/t5_tokenizer.py +100 -0
- keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
- keras_hub/src/models/task.py +419 -0
- keras_hub/src/models/vgg/__init__.py +13 -0
- keras_hub/src/models/vgg/vgg_backbone.py +158 -0
- keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
- keras_hub/src/models/vit_det/__init__.py +13 -0
- keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
- keras_hub/src/models/vit_det/vit_layers.py +565 -0
- keras_hub/src/models/whisper/__init__.py +20 -0
- keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
- keras_hub/src/models/whisper/whisper_backbone.py +305 -0
- keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
- keras_hub/src/models/whisper/whisper_decoder.py +141 -0
- keras_hub/src/models/whisper/whisper_encoder.py +106 -0
- keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
- keras_hub/src/models/whisper/whisper_presets.py +148 -0
- keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
- keras_hub/src/models/xlm_roberta/__init__.py +26 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
- keras_hub/src/models/xlnet/__init__.py +13 -0
- keras_hub/src/models/xlnet/relative_attention.py +459 -0
- keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
- keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
- keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
- keras_hub/src/samplers/__init__.py +13 -0
- keras_hub/src/samplers/beam_sampler.py +207 -0
- keras_hub/src/samplers/contrastive_sampler.py +231 -0
- keras_hub/src/samplers/greedy_sampler.py +50 -0
- keras_hub/src/samplers/random_sampler.py +77 -0
- keras_hub/src/samplers/sampler.py +237 -0
- keras_hub/src/samplers/serialization.py +97 -0
- keras_hub/src/samplers/top_k_sampler.py +92 -0
- keras_hub/src/samplers/top_p_sampler.py +113 -0
- keras_hub/src/tests/__init__.py +13 -0
- keras_hub/src/tests/test_case.py +608 -0
- keras_hub/src/tokenizers/__init__.py +13 -0
- keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
- keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
- keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
- keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
- keras_hub/src/tokenizers/tokenizer.py +235 -0
- keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
- keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
- keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
- keras_hub/src/utils/__init__.py +13 -0
- keras_hub/src/utils/keras_utils.py +130 -0
- keras_hub/src/utils/pipeline_model.py +293 -0
- keras_hub/src/utils/preset_utils.py +621 -0
- keras_hub/src/utils/python_utils.py +21 -0
- keras_hub/src/utils/tensor_utils.py +206 -0
- keras_hub/src/utils/timm/__init__.py +13 -0
- keras_hub/src/utils/timm/convert.py +37 -0
- keras_hub/src/utils/timm/convert_resnet.py +171 -0
- keras_hub/src/utils/transformers/__init__.py +13 -0
- keras_hub/src/utils/transformers/convert.py +101 -0
- keras_hub/src/utils/transformers/convert_bert.py +173 -0
- keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
- keras_hub/src/utils/transformers/convert_gemma.py +187 -0
- keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
- keras_hub/src/utils/transformers/convert_llama3.py +136 -0
- keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
- keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
- keras_hub/src/version_utils.py +23 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,191 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
import keras
|
16
|
+
from absl import logging
|
17
|
+
|
18
|
+
from keras_hub.src.api_export import keras_hub_export
|
19
|
+
from keras_hub.src.layers.preprocessing.masked_lm_mask_generator import (
|
20
|
+
MaskedLMMaskGenerator,
|
21
|
+
)
|
22
|
+
from keras_hub.src.models.deberta_v3.deberta_v3_preprocessor import (
|
23
|
+
DebertaV3Preprocessor,
|
24
|
+
)
|
25
|
+
|
26
|
+
|
27
|
+
@keras_hub_export("keras_hub.models.DebertaV3MaskedLMPreprocessor")
|
28
|
+
class DebertaV3MaskedLMPreprocessor(DebertaV3Preprocessor):
|
29
|
+
"""DeBERTa preprocessing for the masked language modeling task.
|
30
|
+
|
31
|
+
This preprocessing layer will prepare inputs for a masked language modeling
|
32
|
+
task. It is primarily intended for use with the
|
33
|
+
`keras_hub.models.DebertaV3MaskedLM` task model. Preprocessing will occur in
|
34
|
+
multiple steps.
|
35
|
+
|
36
|
+
- Tokenize any number of input segments using the `tokenizer`.
|
37
|
+
- Pack the inputs together with the appropriate `"<s>"`, `"</s>"` and
|
38
|
+
`"<pad>"` tokens, i.e., adding a single `"<s>"` at the start of the
|
39
|
+
entire sequence, `"</s></s>"` between each segment,
|
40
|
+
and a `"</s>"` at the end of the entire sequence.
|
41
|
+
- Randomly select non-special tokens to mask, controlled by
|
42
|
+
`mask_selection_rate`.
|
43
|
+
- Construct a `(x, y, sample_weight)` tuple suitable for training with a
|
44
|
+
`keras_hub.models.DebertaV3MaskedLM` task model.
|
45
|
+
|
46
|
+
Args:
|
47
|
+
tokenizer: A `keras_hub.models.DebertaV3Tokenizer` instance.
|
48
|
+
sequence_length: The length of the packed inputs.
|
49
|
+
mask_selection_rate: The probability an input token will be dynamically
|
50
|
+
masked.
|
51
|
+
mask_selection_length: The maximum number of masked tokens supported
|
52
|
+
by the layer.
|
53
|
+
mask_token_rate: float. `mask_token_rate` must be
|
54
|
+
between 0 and 1 which indicates how often the mask_token is
|
55
|
+
substituted for tokens selected for masking.
|
56
|
+
Defaults to `0.8`.
|
57
|
+
random_token_rate: float. `random_token_rate` must be
|
58
|
+
between 0 and 1 which indicates how often a random token is
|
59
|
+
substituted for tokens selected for masking.
|
60
|
+
Note: mask_token_rate + random_token_rate <= 1, and for
|
61
|
+
(1 - mask_token_rate - random_token_rate), the token will not be
|
62
|
+
changed. Defaults to `0.1`.
|
63
|
+
truncate: string. The algorithm to truncate a list of batched segments
|
64
|
+
to fit within `sequence_length`. The value can be either
|
65
|
+
`round_robin` or `waterfall`:
|
66
|
+
- `"round_robin"`: Available space is assigned one token at a
|
67
|
+
time in a round-robin fashion to the inputs that still need
|
68
|
+
some, until the limit is reached.
|
69
|
+
- `"waterfall"`: The allocation of the budget is done using a
|
70
|
+
"waterfall" algorithm that allocates quota in a
|
71
|
+
left-to-right manner and fills up the buckets until we run
|
72
|
+
out of budget. It supports an arbitrary number of segments.
|
73
|
+
|
74
|
+
Examples:
|
75
|
+
Directly calling the layer on data.
|
76
|
+
```python
|
77
|
+
preprocessor = keras_hub.models.DebertaV3MaskedLMPreprocessor.from_preset(
|
78
|
+
"deberta_v3_base_en"
|
79
|
+
)
|
80
|
+
|
81
|
+
# Tokenize and mask a single sentence.
|
82
|
+
preprocessor("The quick brown fox jumped.")
|
83
|
+
|
84
|
+
# Tokenize and mask a batch of single sentences.
|
85
|
+
preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
|
86
|
+
|
87
|
+
# Tokenize and mask sentence pairs.
|
88
|
+
# In this case, always convert input to tensors before calling the layer.
|
89
|
+
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
90
|
+
second = tf.constant(["The fox tripped.", "Oh look, a whale."])
|
91
|
+
preprocessor((first, second))
|
92
|
+
```
|
93
|
+
|
94
|
+
Mapping with `tf.data.Dataset`.
|
95
|
+
```python
|
96
|
+
preprocessor = keras_hub.models.DebertaV3MaskedLMPreprocessor.from_preset(
|
97
|
+
"deberta_v3_base_en"
|
98
|
+
)
|
99
|
+
|
100
|
+
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
101
|
+
second = tf.constant(["The fox tripped.", "Oh look, a whale."])
|
102
|
+
|
103
|
+
# Map single sentences.
|
104
|
+
ds = tf.data.Dataset.from_tensor_slices(first)
|
105
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
106
|
+
|
107
|
+
# Map sentence pairs.
|
108
|
+
ds = tf.data.Dataset.from_tensor_slices((first, second))
|
109
|
+
# Watch out for tf.data's default unpacking of tuples here!
|
110
|
+
# Best to invoke the `preprocessor` directly in this case.
|
111
|
+
ds = ds.map(
|
112
|
+
lambda first, second: preprocessor(x=(first, second)),
|
113
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
114
|
+
)
|
115
|
+
```
|
116
|
+
"""
|
117
|
+
|
118
|
+
def __init__(
|
119
|
+
self,
|
120
|
+
tokenizer,
|
121
|
+
sequence_length=512,
|
122
|
+
truncate="round_robin",
|
123
|
+
mask_selection_rate=0.15,
|
124
|
+
mask_selection_length=96,
|
125
|
+
mask_token_rate=0.8,
|
126
|
+
random_token_rate=0.1,
|
127
|
+
**kwargs,
|
128
|
+
):
|
129
|
+
super().__init__(
|
130
|
+
tokenizer,
|
131
|
+
sequence_length=sequence_length,
|
132
|
+
truncate=truncate,
|
133
|
+
**kwargs,
|
134
|
+
)
|
135
|
+
|
136
|
+
self.mask_selection_rate = mask_selection_rate
|
137
|
+
self.mask_selection_length = mask_selection_length
|
138
|
+
self.mask_token_rate = mask_token_rate
|
139
|
+
self.random_token_rate = random_token_rate
|
140
|
+
self.masker = None
|
141
|
+
|
142
|
+
def build(self, input_shape):
|
143
|
+
super().build(input_shape)
|
144
|
+
# Defer masker creation to `build()` so that we can be sure tokenizer
|
145
|
+
# assets have loaded when restoring a saved model.
|
146
|
+
self.masker = MaskedLMMaskGenerator(
|
147
|
+
mask_selection_rate=self.mask_selection_rate,
|
148
|
+
mask_selection_length=self.mask_selection_length,
|
149
|
+
mask_token_rate=self.mask_token_rate,
|
150
|
+
random_token_rate=self.random_token_rate,
|
151
|
+
vocabulary_size=self.tokenizer.vocabulary_size(),
|
152
|
+
mask_token_id=self.tokenizer.mask_token_id,
|
153
|
+
unselectable_token_ids=[
|
154
|
+
self.tokenizer.cls_token_id,
|
155
|
+
self.tokenizer.sep_token_id,
|
156
|
+
self.tokenizer.pad_token_id,
|
157
|
+
],
|
158
|
+
)
|
159
|
+
|
160
|
+
def get_config(self):
|
161
|
+
config = super().get_config()
|
162
|
+
config.update(
|
163
|
+
{
|
164
|
+
"mask_selection_rate": self.mask_selection_rate,
|
165
|
+
"mask_selection_length": self.mask_selection_length,
|
166
|
+
"mask_token_rate": self.mask_token_rate,
|
167
|
+
"random_token_rate": self.random_token_rate,
|
168
|
+
}
|
169
|
+
)
|
170
|
+
return config
|
171
|
+
|
172
|
+
def call(self, x, y=None, sample_weight=None):
|
173
|
+
if y is not None or sample_weight is not None:
|
174
|
+
logging.warning(
|
175
|
+
f"{self.__class__.__name__} generates `y` and `sample_weight` "
|
176
|
+
"based on your input data, but your data already contains `y` "
|
177
|
+
"or `sample_weight`. Your `y` and `sample_weight` will be "
|
178
|
+
"ignored."
|
179
|
+
)
|
180
|
+
|
181
|
+
x = super().call(x)
|
182
|
+
token_ids, padding_mask = x["token_ids"], x["padding_mask"]
|
183
|
+
masker_outputs = self.masker(token_ids)
|
184
|
+
x = {
|
185
|
+
"token_ids": masker_outputs["token_ids"],
|
186
|
+
"padding_mask": padding_mask,
|
187
|
+
"mask_positions": masker_outputs["mask_positions"],
|
188
|
+
}
|
189
|
+
y = masker_outputs["mask_ids"]
|
190
|
+
sample_weight = masker_outputs["mask_weights"]
|
191
|
+
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
|
@@ -0,0 +1,206 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
import keras
|
17
|
+
|
18
|
+
from keras_hub.src.api_export import keras_hub_export
|
19
|
+
from keras_hub.src.layers.preprocessing.multi_segment_packer import (
|
20
|
+
MultiSegmentPacker,
|
21
|
+
)
|
22
|
+
from keras_hub.src.models.deberta_v3.deberta_v3_tokenizer import (
|
23
|
+
DebertaV3Tokenizer,
|
24
|
+
)
|
25
|
+
from keras_hub.src.models.preprocessor import Preprocessor
|
26
|
+
from keras_hub.src.utils.keras_utils import (
|
27
|
+
convert_inputs_to_list_of_tensor_segments,
|
28
|
+
)
|
29
|
+
|
30
|
+
|
31
|
+
@keras_hub_export("keras_hub.models.DebertaV3Preprocessor")
|
32
|
+
class DebertaV3Preprocessor(Preprocessor):
|
33
|
+
"""A DeBERTa preprocessing layer which tokenizes and packs inputs.
|
34
|
+
|
35
|
+
This preprocessing layer will do three things:
|
36
|
+
|
37
|
+
- Tokenize any number of input segments using the `tokenizer`.
|
38
|
+
- Pack the inputs together using a `keras_hub.layers.MultiSegmentPacker`.
|
39
|
+
with the appropriate `"[CLS]"`, `"[SEP]"` and `"[PAD]"` tokens.
|
40
|
+
- Construct a dictionary with keys `"token_ids"` and `"padding_mask"`, that
|
41
|
+
can be passed directly to a DeBERTa model.
|
42
|
+
|
43
|
+
This layer can be used directly with `tf.data.Dataset.map` to preprocess
|
44
|
+
string data in the `(x, y, sample_weight)` format used by
|
45
|
+
`keras.Model.fit`.
|
46
|
+
|
47
|
+
The call method of this layer accepts three arguments, `x`, `y`, and
|
48
|
+
`sample_weight`. `x` can be a python string or tensor representing a single
|
49
|
+
segment, a list of python strings representing a batch of single segments,
|
50
|
+
or a list of tensors representing multiple segments to be packed together.
|
51
|
+
`y` and `sample_weight` are both optional, can have any format, and will be
|
52
|
+
passed through unaltered.
|
53
|
+
|
54
|
+
Special care should be taken when using `tf.data` to map over an unlabeled
|
55
|
+
tuple of string segments. `tf.data.Dataset.map` will unpack this tuple
|
56
|
+
directly into the call arguments of this layer, rather than forward all
|
57
|
+
argument to `x`. To handle this case, it is recommended to explicitly call
|
58
|
+
the layer, e.g. `ds.map(lambda seg1, seg2: preprocessor(x=(seg1, seg2)))`.
|
59
|
+
|
60
|
+
Args:
|
61
|
+
tokenizer: A `keras_hub.models.DebertaV3Tokenizer` instance.
|
62
|
+
sequence_length: The length of the packed inputs.
|
63
|
+
truncate: string. The algorithm to truncate a list of batched segments
|
64
|
+
to fit within `sequence_length`. The value can be either
|
65
|
+
`round_robin` or `waterfall`:
|
66
|
+
- `"round_robin"`: Available space is assigned one token at a
|
67
|
+
time in a round-robin fashion to the inputs that still need
|
68
|
+
some, until the limit is reached.
|
69
|
+
- `"waterfall"`: The allocation of the budget is done using a
|
70
|
+
"waterfall" algorithm that allocates quota in a
|
71
|
+
left-to-right manner and fills up the buckets until we run
|
72
|
+
out of budget. It supports an arbitrary number of segments.
|
73
|
+
|
74
|
+
Examples:
|
75
|
+
Directly calling the layer on data.
|
76
|
+
```python
|
77
|
+
preprocessor = keras_hub.models.DebertaV3Preprocessor.from_preset(
|
78
|
+
"deberta_v3_base_en"
|
79
|
+
)
|
80
|
+
|
81
|
+
# Tokenize and pack a single sentence.
|
82
|
+
preprocessor("The quick brown fox jumped.")
|
83
|
+
|
84
|
+
# Tokenize a batch of single sentences.
|
85
|
+
preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
|
86
|
+
|
87
|
+
# Preprocess a batch of sentence pairs.
|
88
|
+
# When handling multiple sequences, always convert to tensors first!
|
89
|
+
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
90
|
+
second = tf.constant(["The fox tripped.", "Oh look, a whale."])
|
91
|
+
preprocessor((first, second))
|
92
|
+
|
93
|
+
# Custom vocabulary.
|
94
|
+
bytes_io = io.BytesIO()
|
95
|
+
ds = tf.data.Dataset.from_tensor_slices(["The quick brown fox jumped."])
|
96
|
+
sentencepiece.SentencePieceTrainer.train(
|
97
|
+
sentence_iterator=ds.as_numpy_iterator(),
|
98
|
+
model_writer=bytes_io,
|
99
|
+
vocab_size=9,
|
100
|
+
model_type="WORD",
|
101
|
+
pad_id=0,
|
102
|
+
bos_id=1,
|
103
|
+
eos_id=2,
|
104
|
+
unk_id=3,
|
105
|
+
pad_piece="[PAD]",
|
106
|
+
bos_piece="[CLS]",
|
107
|
+
eos_piece="[SEP]",
|
108
|
+
unk_piece="[UNK]",
|
109
|
+
)
|
110
|
+
tokenizer = keras_hub.models.DebertaV3Tokenizer(
|
111
|
+
proto=bytes_io.getvalue(),
|
112
|
+
)
|
113
|
+
preprocessor = keras_hub.models.DebertaV3Preprocessor(tokenizer)
|
114
|
+
preprocessor("The quick brown fox jumped.")
|
115
|
+
```
|
116
|
+
|
117
|
+
Mapping with `tf.data.Dataset`.
|
118
|
+
```python
|
119
|
+
preprocessor = keras_hub.models.DebertaV3Preprocessor.from_preset(
|
120
|
+
"deberta_v3_base_en"
|
121
|
+
)
|
122
|
+
|
123
|
+
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
124
|
+
second = tf.constant(["The fox tripped.", "Oh look, a whale."])
|
125
|
+
label = tf.constant([1, 1])
|
126
|
+
|
127
|
+
# Map labeled single sentences.
|
128
|
+
ds = tf.data.Dataset.from_tensor_slices((first, label))
|
129
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
130
|
+
|
131
|
+
# Map unlabeled single sentences.
|
132
|
+
ds = tf.data.Dataset.from_tensor_slices(first)
|
133
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
134
|
+
|
135
|
+
# Map labeled sentence pairs.
|
136
|
+
ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
|
137
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
138
|
+
|
139
|
+
# Map unlabeled sentence pairs.
|
140
|
+
ds = tf.data.Dataset.from_tensor_slices((first, second))
|
141
|
+
# Watch out for tf.data's default unpacking of tuples here!
|
142
|
+
# Best to invoke the `preprocessor` directly in this case.
|
143
|
+
ds = ds.map(
|
144
|
+
lambda first, second: preprocessor(x=(first, second)),
|
145
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
146
|
+
)
|
147
|
+
```
|
148
|
+
"""
|
149
|
+
|
150
|
+
tokenizer_cls = DebertaV3Tokenizer
|
151
|
+
|
152
|
+
def __init__(
|
153
|
+
self,
|
154
|
+
tokenizer,
|
155
|
+
sequence_length=512,
|
156
|
+
truncate="round_robin",
|
157
|
+
**kwargs,
|
158
|
+
):
|
159
|
+
super().__init__(**kwargs)
|
160
|
+
self.tokenizer = tokenizer
|
161
|
+
self.packer = None
|
162
|
+
self.truncate = truncate
|
163
|
+
self.sequence_length = sequence_length
|
164
|
+
|
165
|
+
def build(self, input_shape):
|
166
|
+
# Defer packer creation to `build()` so that we can be sure tokenizer
|
167
|
+
# assets have loaded when restoring a saved model.
|
168
|
+
self.packer = MultiSegmentPacker(
|
169
|
+
start_value=self.tokenizer.cls_token_id,
|
170
|
+
end_value=self.tokenizer.sep_token_id,
|
171
|
+
pad_value=self.tokenizer.pad_token_id,
|
172
|
+
truncate=self.truncate,
|
173
|
+
sequence_length=self.sequence_length,
|
174
|
+
)
|
175
|
+
self.built = True
|
176
|
+
|
177
|
+
def get_config(self):
|
178
|
+
config = super().get_config()
|
179
|
+
config.update(
|
180
|
+
{
|
181
|
+
"sequence_length": self.sequence_length,
|
182
|
+
"truncate": self.truncate,
|
183
|
+
}
|
184
|
+
)
|
185
|
+
return config
|
186
|
+
|
187
|
+
def call(self, x, y=None, sample_weight=None):
|
188
|
+
x = convert_inputs_to_list_of_tensor_segments(x)
|
189
|
+
x = [self.tokenizer(segment) for segment in x]
|
190
|
+
token_ids, _ = self.packer(x)
|
191
|
+
x = {
|
192
|
+
"token_ids": token_ids,
|
193
|
+
"padding_mask": token_ids != self.tokenizer.pad_token_id,
|
194
|
+
}
|
195
|
+
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
|
196
|
+
|
197
|
+
@property
|
198
|
+
def sequence_length(self):
|
199
|
+
"""The padded length of model input sequences."""
|
200
|
+
return self._sequence_length
|
201
|
+
|
202
|
+
@sequence_length.setter
|
203
|
+
def sequence_length(self, value):
|
204
|
+
self._sequence_length = value
|
205
|
+
if self.packer is not None:
|
206
|
+
self.packer.sequence_length = value
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
"""DeBERTa model preset configurations."""
|
15
|
+
|
16
|
+
backbone_presets = {
|
17
|
+
"deberta_v3_extra_small_en": {
|
18
|
+
"metadata": {
|
19
|
+
"description": (
|
20
|
+
"12-layer DeBERTaV3 model where case is maintained. "
|
21
|
+
"Trained on English Wikipedia, BookCorpus and OpenWebText."
|
22
|
+
),
|
23
|
+
"params": 70682112,
|
24
|
+
"official_name": "DeBERTaV3",
|
25
|
+
"path": "deberta_v3",
|
26
|
+
"model_card": "https://huggingface.co/microsoft/deberta-v3-xsmall",
|
27
|
+
},
|
28
|
+
"kaggle_handle": "kaggle://keras/deberta_v3/keras/deberta_v3_extra_small_en/2",
|
29
|
+
},
|
30
|
+
"deberta_v3_small_en": {
|
31
|
+
"metadata": {
|
32
|
+
"description": (
|
33
|
+
"6-layer DeBERTaV3 model where case is maintained. "
|
34
|
+
"Trained on English Wikipedia, BookCorpus and OpenWebText."
|
35
|
+
),
|
36
|
+
"params": 141304320,
|
37
|
+
"official_name": "DeBERTaV3",
|
38
|
+
"path": "deberta_v3",
|
39
|
+
"model_card": "https://huggingface.co/microsoft/deberta-v3-small",
|
40
|
+
},
|
41
|
+
"kaggle_handle": "kaggle://keras/deberta_v3/keras/deberta_v3_small_en/2",
|
42
|
+
},
|
43
|
+
"deberta_v3_base_en": {
|
44
|
+
"metadata": {
|
45
|
+
"description": (
|
46
|
+
"12-layer DeBERTaV3 model where case is maintained. "
|
47
|
+
"Trained on English Wikipedia, BookCorpus and OpenWebText."
|
48
|
+
),
|
49
|
+
"params": 183831552,
|
50
|
+
"official_name": "DeBERTaV3",
|
51
|
+
"path": "deberta_v3",
|
52
|
+
"model_card": "https://huggingface.co/microsoft/deberta-v3-base",
|
53
|
+
},
|
54
|
+
"kaggle_handle": "kaggle://keras/deberta_v3/keras/deberta_v3_base_en/2",
|
55
|
+
},
|
56
|
+
"deberta_v3_large_en": {
|
57
|
+
"metadata": {
|
58
|
+
"description": (
|
59
|
+
"24-layer DeBERTaV3 model where case is maintained. "
|
60
|
+
"Trained on English Wikipedia, BookCorpus and OpenWebText."
|
61
|
+
),
|
62
|
+
"params": 434012160,
|
63
|
+
"official_name": "DeBERTaV3",
|
64
|
+
"path": "deberta_v3",
|
65
|
+
"model_card": "https://huggingface.co/microsoft/deberta-v3-large",
|
66
|
+
},
|
67
|
+
"kaggle_handle": "kaggle://keras/deberta_v3/keras/deberta_v3_large_en/2",
|
68
|
+
},
|
69
|
+
"deberta_v3_base_multi": {
|
70
|
+
"metadata": {
|
71
|
+
"description": (
|
72
|
+
"12-layer DeBERTaV3 model where case is maintained. "
|
73
|
+
"Trained on the 2.5TB multilingual CC100 dataset."
|
74
|
+
),
|
75
|
+
"params": 278218752,
|
76
|
+
"official_name": "DeBERTaV3",
|
77
|
+
"path": "deberta_v3",
|
78
|
+
"model_card": "https://huggingface.co/microsoft/mdeberta-v3-base",
|
79
|
+
},
|
80
|
+
"kaggle_handle": "kaggle://keras/deberta_v3/keras/deberta_v3_base_multi/2",
|
81
|
+
},
|
82
|
+
}
|
@@ -0,0 +1,155 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
from keras_hub.src.api_export import keras_hub_export
|
17
|
+
from keras_hub.src.tokenizers.sentence_piece_tokenizer import (
|
18
|
+
SentencePieceTokenizer,
|
19
|
+
)
|
20
|
+
|
21
|
+
try:
|
22
|
+
import tensorflow as tf
|
23
|
+
except ImportError:
|
24
|
+
tf = None
|
25
|
+
|
26
|
+
|
27
|
+
@keras_hub_export("keras_hub.models.DebertaV3Tokenizer")
|
28
|
+
class DebertaV3Tokenizer(SentencePieceTokenizer):
|
29
|
+
"""DeBERTa tokenizer layer based on SentencePiece.
|
30
|
+
|
31
|
+
This tokenizer class will tokenize raw strings into integer sequences and
|
32
|
+
is based on `keras_hub.tokenizers.SentencePieceTokenizer`. Unlike the
|
33
|
+
underlying tokenizer, it will check for all special tokens needed by
|
34
|
+
DeBERTa models and provides a `from_preset()` method to automatically
|
35
|
+
download a matching vocabulary for a DeBERTa preset.
|
36
|
+
|
37
|
+
This tokenizer does not provide truncation or padding of inputs. It can be
|
38
|
+
combined with a `keras_hub.models.DebertaV3Preprocessor` layer for input
|
39
|
+
packing.
|
40
|
+
|
41
|
+
If input is a batch of strings (rank > 0), the layer will output a
|
42
|
+
`tf.RaggedTensor` where the last dimension of the output is ragged.
|
43
|
+
|
44
|
+
If input is a scalar string (rank == 0), the layer will output a dense
|
45
|
+
`tf.Tensor` with static shape `[None]`.
|
46
|
+
|
47
|
+
Note: The mask token (`"[MASK]"`) is handled differently in this tokenizer.
|
48
|
+
If the token is not present in the provided SentencePiece vocabulary, the
|
49
|
+
token will be appended to the vocabulary. For example, if the vocabulary
|
50
|
+
size is 100, the mask token will be assigned the ID 100.
|
51
|
+
|
52
|
+
Args:
|
53
|
+
proto: Either a `string` path to a SentencePiece proto file, or a
|
54
|
+
`bytes` object with a serialized SentencePiece proto. See the
|
55
|
+
[SentencePiece repository](https://github.com/google/sentencepiece)
|
56
|
+
for more details on the format.
|
57
|
+
|
58
|
+
Examples:
|
59
|
+
|
60
|
+
```python
|
61
|
+
# Unbatched input.
|
62
|
+
tokenizer = keras_hub.models.DebertaV3Tokenizer.from_preset(
|
63
|
+
"deberta_v3_base_en",
|
64
|
+
)
|
65
|
+
tokenizer("The quick brown fox jumped.")
|
66
|
+
|
67
|
+
# Batched inputs.
|
68
|
+
tokenizer(["the quick brown fox", "the earth is round"])
|
69
|
+
|
70
|
+
# Detokenization.
|
71
|
+
tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
|
72
|
+
|
73
|
+
# Custom vocabulary.
|
74
|
+
bytes_io = io.BytesIO()
|
75
|
+
ds = tf.data.Dataset.from_tensor_slices(["The quick brown fox jumped."])
|
76
|
+
sentencepiece.SentencePieceTrainer.train(
|
77
|
+
sentence_iterator=ds.as_numpy_iterator(),
|
78
|
+
model_writer=bytes_io,
|
79
|
+
vocab_size=9,
|
80
|
+
model_type="WORD",
|
81
|
+
pad_id=0,
|
82
|
+
bos_id=1,
|
83
|
+
eos_id=2,
|
84
|
+
unk_id=3,
|
85
|
+
pad_piece="[PAD]",
|
86
|
+
bos_piece="[CLS]",
|
87
|
+
eos_piece="[SEP]",
|
88
|
+
unk_piece="[UNK]",
|
89
|
+
)
|
90
|
+
tokenizer = keras_hub.models.DebertaV3Tokenizer(
|
91
|
+
proto=bytes_io.getvalue(),
|
92
|
+
)
|
93
|
+
tokenizer("The quick brown fox jumped.")
|
94
|
+
```
|
95
|
+
"""
|
96
|
+
|
97
|
+
def __init__(self, proto, **kwargs):
|
98
|
+
self.cls_token = "[CLS]"
|
99
|
+
self.sep_token = "[SEP]"
|
100
|
+
self.pad_token = "[PAD]"
|
101
|
+
self.mask_token = "[MASK]"
|
102
|
+
|
103
|
+
super().__init__(proto=proto, **kwargs)
|
104
|
+
|
105
|
+
def set_proto(self, proto):
|
106
|
+
super().set_proto(proto)
|
107
|
+
if proto is not None:
|
108
|
+
for token in [self.cls_token, self.pad_token, self.sep_token]:
|
109
|
+
if token not in super().get_vocabulary():
|
110
|
+
raise ValueError(
|
111
|
+
f"Cannot find token `'{token}'` in the provided "
|
112
|
+
f"`vocabulary`. Please provide `'{token}'` in your "
|
113
|
+
"`vocabulary` or use a pretrained `vocabulary` name."
|
114
|
+
)
|
115
|
+
|
116
|
+
self.cls_token_id = self.token_to_id(self.cls_token)
|
117
|
+
self.sep_token_id = self.token_to_id(self.sep_token)
|
118
|
+
self.pad_token_id = self.token_to_id(self.pad_token)
|
119
|
+
# If the mask token is not in the vocabulary, add it to the end of the
|
120
|
+
# vocabulary.
|
121
|
+
if self.mask_token in super().get_vocabulary():
|
122
|
+
self.mask_token_id = super().token_to_id(self.mask_token)
|
123
|
+
else:
|
124
|
+
self.mask_token_id = super().vocabulary_size()
|
125
|
+
else:
|
126
|
+
self.cls_token_id = None
|
127
|
+
self.sep_token_id = None
|
128
|
+
self.pad_token_id = None
|
129
|
+
self.mask_token_id = None
|
130
|
+
|
131
|
+
def vocabulary_size(self):
|
132
|
+
sentence_piece_size = super().vocabulary_size()
|
133
|
+
if sentence_piece_size == self.mask_token_id:
|
134
|
+
return sentence_piece_size + 1
|
135
|
+
return sentence_piece_size
|
136
|
+
|
137
|
+
def get_vocabulary(self):
|
138
|
+
sentence_piece_vocabulary = super().get_vocabulary()
|
139
|
+
if self.mask_token_id < super().vocabulary_size():
|
140
|
+
return sentence_piece_vocabulary
|
141
|
+
return sentence_piece_vocabulary + ["[MASK]"]
|
142
|
+
|
143
|
+
def id_to_token(self, id):
|
144
|
+
if id == self.mask_token_id:
|
145
|
+
return "[MASK]"
|
146
|
+
return super().id_to_token(id)
|
147
|
+
|
148
|
+
def token_to_id(self, token):
|
149
|
+
if token == "[MASK]":
|
150
|
+
return self.mask_token_id
|
151
|
+
return super().token_to_id(token)
|
152
|
+
|
153
|
+
def detokenize(self, ids):
|
154
|
+
ids = tf.ragged.boolean_mask(ids, tf.not_equal(ids, self.mask_token_id))
|
155
|
+
return super().detokenize(ids)
|