keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keras_hub/__init__.py +52 -0
- keras_hub/api/__init__.py +27 -0
- keras_hub/api/layers/__init__.py +47 -0
- keras_hub/api/metrics/__init__.py +24 -0
- keras_hub/api/models/__init__.py +249 -0
- keras_hub/api/samplers/__init__.py +29 -0
- keras_hub/api/tokenizers/__init__.py +35 -0
- keras_hub/src/__init__.py +13 -0
- keras_hub/src/api_export.py +53 -0
- keras_hub/src/layers/__init__.py +13 -0
- keras_hub/src/layers/modeling/__init__.py +13 -0
- keras_hub/src/layers/modeling/alibi_bias.py +143 -0
- keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
- keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
- keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
- keras_hub/src/layers/modeling/position_embedding.py +123 -0
- keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
- keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
- keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
- keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
- keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
- keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
- keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
- keras_hub/src/layers/preprocessing/__init__.py +13 -0
- keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
- keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
- keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
- keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
- keras_hub/src/layers/preprocessing/random_swap.py +267 -0
- keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
- keras_hub/src/metrics/__init__.py +13 -0
- keras_hub/src/metrics/bleu.py +394 -0
- keras_hub/src/metrics/edit_distance.py +197 -0
- keras_hub/src/metrics/perplexity.py +181 -0
- keras_hub/src/metrics/rouge_base.py +204 -0
- keras_hub/src/metrics/rouge_l.py +97 -0
- keras_hub/src/metrics/rouge_n.py +125 -0
- keras_hub/src/models/__init__.py +13 -0
- keras_hub/src/models/albert/__init__.py +20 -0
- keras_hub/src/models/albert/albert_backbone.py +267 -0
- keras_hub/src/models/albert/albert_classifier.py +202 -0
- keras_hub/src/models/albert/albert_masked_lm.py +129 -0
- keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
- keras_hub/src/models/albert/albert_preprocessor.py +206 -0
- keras_hub/src/models/albert/albert_presets.py +70 -0
- keras_hub/src/models/albert/albert_tokenizer.py +119 -0
- keras_hub/src/models/backbone.py +311 -0
- keras_hub/src/models/bart/__init__.py +20 -0
- keras_hub/src/models/bart/bart_backbone.py +261 -0
- keras_hub/src/models/bart/bart_preprocessor.py +276 -0
- keras_hub/src/models/bart/bart_presets.py +74 -0
- keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
- keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
- keras_hub/src/models/bart/bart_tokenizer.py +124 -0
- keras_hub/src/models/bert/__init__.py +23 -0
- keras_hub/src/models/bert/bert_backbone.py +227 -0
- keras_hub/src/models/bert/bert_classifier.py +183 -0
- keras_hub/src/models/bert/bert_masked_lm.py +131 -0
- keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
- keras_hub/src/models/bert/bert_preprocessor.py +184 -0
- keras_hub/src/models/bert/bert_presets.py +147 -0
- keras_hub/src/models/bert/bert_tokenizer.py +112 -0
- keras_hub/src/models/bloom/__init__.py +20 -0
- keras_hub/src/models/bloom/bloom_attention.py +186 -0
- keras_hub/src/models/bloom/bloom_backbone.py +173 -0
- keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
- keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
- keras_hub/src/models/bloom/bloom_decoder.py +206 -0
- keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
- keras_hub/src/models/bloom/bloom_presets.py +121 -0
- keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
- keras_hub/src/models/causal_lm.py +383 -0
- keras_hub/src/models/classifier.py +109 -0
- keras_hub/src/models/csp_darknet/__init__.py +13 -0
- keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
- keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
- keras_hub/src/models/deberta_v3/__init__.py +24 -0
- keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
- keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
- keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
- keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
- keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
- keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
- keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
- keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
- keras_hub/src/models/densenet/__init__.py +13 -0
- keras_hub/src/models/densenet/densenet_backbone.py +210 -0
- keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
- keras_hub/src/models/distil_bert/__init__.py +26 -0
- keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
- keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
- keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
- keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
- keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
- keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
- keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
- keras_hub/src/models/electra/__init__.py +20 -0
- keras_hub/src/models/electra/electra_backbone.py +247 -0
- keras_hub/src/models/electra/electra_preprocessor.py +154 -0
- keras_hub/src/models/electra/electra_presets.py +95 -0
- keras_hub/src/models/electra/electra_tokenizer.py +104 -0
- keras_hub/src/models/f_net/__init__.py +20 -0
- keras_hub/src/models/f_net/f_net_backbone.py +236 -0
- keras_hub/src/models/f_net/f_net_classifier.py +154 -0
- keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
- keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
- keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
- keras_hub/src/models/f_net/f_net_presets.py +43 -0
- keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
- keras_hub/src/models/falcon/__init__.py +20 -0
- keras_hub/src/models/falcon/falcon_attention.py +156 -0
- keras_hub/src/models/falcon/falcon_backbone.py +164 -0
- keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
- keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
- keras_hub/src/models/falcon/falcon_presets.py +30 -0
- keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
- keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
- keras_hub/src/models/feature_pyramid_backbone.py +73 -0
- keras_hub/src/models/gemma/__init__.py +20 -0
- keras_hub/src/models/gemma/gemma_attention.py +250 -0
- keras_hub/src/models/gemma/gemma_backbone.py +316 -0
- keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
- keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
- keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
- keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
- keras_hub/src/models/gemma/gemma_presets.py +248 -0
- keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
- keras_hub/src/models/gemma/rms_normalization.py +40 -0
- keras_hub/src/models/gpt2/__init__.py +20 -0
- keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
- keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
- keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
- keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
- keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
- keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
- keras_hub/src/models/image_classifier.py +90 -0
- keras_hub/src/models/llama/__init__.py +20 -0
- keras_hub/src/models/llama/llama_attention.py +225 -0
- keras_hub/src/models/llama/llama_backbone.py +188 -0
- keras_hub/src/models/llama/llama_causal_lm.py +327 -0
- keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
- keras_hub/src/models/llama/llama_decoder.py +246 -0
- keras_hub/src/models/llama/llama_layernorm.py +48 -0
- keras_hub/src/models/llama/llama_preprocessor.py +189 -0
- keras_hub/src/models/llama/llama_presets.py +80 -0
- keras_hub/src/models/llama/llama_tokenizer.py +84 -0
- keras_hub/src/models/llama3/__init__.py +20 -0
- keras_hub/src/models/llama3/llama3_backbone.py +84 -0
- keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
- keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
- keras_hub/src/models/llama3/llama3_presets.py +69 -0
- keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
- keras_hub/src/models/masked_lm.py +101 -0
- keras_hub/src/models/mistral/__init__.py +20 -0
- keras_hub/src/models/mistral/mistral_attention.py +238 -0
- keras_hub/src/models/mistral/mistral_backbone.py +203 -0
- keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
- keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
- keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
- keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
- keras_hub/src/models/mistral/mistral_presets.py +48 -0
- keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
- keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
- keras_hub/src/models/mix_transformer/__init__.py +13 -0
- keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
- keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
- keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
- keras_hub/src/models/opt/__init__.py +20 -0
- keras_hub/src/models/opt/opt_backbone.py +173 -0
- keras_hub/src/models/opt/opt_causal_lm.py +301 -0
- keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
- keras_hub/src/models/opt/opt_preprocessor.py +188 -0
- keras_hub/src/models/opt/opt_presets.py +72 -0
- keras_hub/src/models/opt/opt_tokenizer.py +116 -0
- keras_hub/src/models/pali_gemma/__init__.py +23 -0
- keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
- keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
- keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
- keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
- keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
- keras_hub/src/models/phi3/__init__.py +20 -0
- keras_hub/src/models/phi3/phi3_attention.py +260 -0
- keras_hub/src/models/phi3/phi3_backbone.py +224 -0
- keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
- keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/phi3/phi3_decoder.py +260 -0
- keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
- keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
- keras_hub/src/models/phi3/phi3_presets.py +50 -0
- keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
- keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
- keras_hub/src/models/preprocessor.py +207 -0
- keras_hub/src/models/resnet/__init__.py +13 -0
- keras_hub/src/models/resnet/resnet_backbone.py +612 -0
- keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
- keras_hub/src/models/roberta/__init__.py +20 -0
- keras_hub/src/models/roberta/roberta_backbone.py +184 -0
- keras_hub/src/models/roberta/roberta_classifier.py +209 -0
- keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
- keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
- keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
- keras_hub/src/models/roberta/roberta_presets.py +43 -0
- keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
- keras_hub/src/models/seq_2_seq_lm.py +54 -0
- keras_hub/src/models/t5/__init__.py +20 -0
- keras_hub/src/models/t5/t5_backbone.py +261 -0
- keras_hub/src/models/t5/t5_layer_norm.py +35 -0
- keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
- keras_hub/src/models/t5/t5_presets.py +95 -0
- keras_hub/src/models/t5/t5_tokenizer.py +100 -0
- keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
- keras_hub/src/models/task.py +419 -0
- keras_hub/src/models/vgg/__init__.py +13 -0
- keras_hub/src/models/vgg/vgg_backbone.py +158 -0
- keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
- keras_hub/src/models/vit_det/__init__.py +13 -0
- keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
- keras_hub/src/models/vit_det/vit_layers.py +565 -0
- keras_hub/src/models/whisper/__init__.py +20 -0
- keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
- keras_hub/src/models/whisper/whisper_backbone.py +305 -0
- keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
- keras_hub/src/models/whisper/whisper_decoder.py +141 -0
- keras_hub/src/models/whisper/whisper_encoder.py +106 -0
- keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
- keras_hub/src/models/whisper/whisper_presets.py +148 -0
- keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
- keras_hub/src/models/xlm_roberta/__init__.py +26 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
- keras_hub/src/models/xlnet/__init__.py +13 -0
- keras_hub/src/models/xlnet/relative_attention.py +459 -0
- keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
- keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
- keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
- keras_hub/src/samplers/__init__.py +13 -0
- keras_hub/src/samplers/beam_sampler.py +207 -0
- keras_hub/src/samplers/contrastive_sampler.py +231 -0
- keras_hub/src/samplers/greedy_sampler.py +50 -0
- keras_hub/src/samplers/random_sampler.py +77 -0
- keras_hub/src/samplers/sampler.py +237 -0
- keras_hub/src/samplers/serialization.py +97 -0
- keras_hub/src/samplers/top_k_sampler.py +92 -0
- keras_hub/src/samplers/top_p_sampler.py +113 -0
- keras_hub/src/tests/__init__.py +13 -0
- keras_hub/src/tests/test_case.py +608 -0
- keras_hub/src/tokenizers/__init__.py +13 -0
- keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
- keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
- keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
- keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
- keras_hub/src/tokenizers/tokenizer.py +235 -0
- keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
- keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
- keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
- keras_hub/src/utils/__init__.py +13 -0
- keras_hub/src/utils/keras_utils.py +130 -0
- keras_hub/src/utils/pipeline_model.py +293 -0
- keras_hub/src/utils/preset_utils.py +621 -0
- keras_hub/src/utils/python_utils.py +21 -0
- keras_hub/src/utils/tensor_utils.py +206 -0
- keras_hub/src/utils/timm/__init__.py +13 -0
- keras_hub/src/utils/timm/convert.py +37 -0
- keras_hub/src/utils/timm/convert_resnet.py +171 -0
- keras_hub/src/utils/transformers/__init__.py +13 -0
- keras_hub/src/utils/transformers/convert.py +101 -0
- keras_hub/src/utils/transformers/convert_bert.py +173 -0
- keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
- keras_hub/src/utils/transformers/convert_gemma.py +187 -0
- keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
- keras_hub/src/utils/transformers/convert_llama3.py +136 -0
- keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
- keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
- keras_hub/src/version_utils.py +23 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,110 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
from keras_hub.src.api_export import keras_hub_export
|
17
|
+
from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
|
18
|
+
|
19
|
+
|
20
|
+
@keras_hub_export("keras_hub.models.FalconTokenizer")
|
21
|
+
class FalconTokenizer(BytePairTokenizer):
|
22
|
+
"""Falcon tokenizer based on BytePairTokenizer.
|
23
|
+
|
24
|
+
This tokenizer class will tokenize raw strings into integer sequences and
|
25
|
+
is based on `keras_hub.tokenizers.BytePairTokenizer`. Unlike the
|
26
|
+
underlying tokenizer, it will check for all special tokens needed by Falcon
|
27
|
+
models and provides a `from_preset()` method to automatically download
|
28
|
+
a matching vocabulary for a Falcon preset.
|
29
|
+
|
30
|
+
This tokenizer does not provide truncation or padding of inputs.
|
31
|
+
|
32
|
+
If input is a batch of strings (rank > 0), the layer will output a
|
33
|
+
`tf.RaggedTensor` where the last dimension of the output is ragged.
|
34
|
+
|
35
|
+
If input is a scalar string (rank == 0), the layer will output a dense
|
36
|
+
`tf.Tensor` with static shape `[None]`.
|
37
|
+
|
38
|
+
Args:
|
39
|
+
vocabulary: string or dict, maps token to integer ids. If it is a
|
40
|
+
string, it should be the file path to a json file.
|
41
|
+
merges: string or list, contains the merge rule. If it is a string,
|
42
|
+
it should be the file path to merge rules. The merge rule file
|
43
|
+
should have one merge rule per line. Every merge rule contains
|
44
|
+
merge entities separated by a space.
|
45
|
+
|
46
|
+
Examples:
|
47
|
+
|
48
|
+
```python
|
49
|
+
# Unbatched input.
|
50
|
+
tokenizer = keras_hub.models.FalconTokenizer.from_preset("falcon_refinedweb_1b_en")
|
51
|
+
tokenizer("The quick brown fox jumped.")
|
52
|
+
|
53
|
+
# Batched input.
|
54
|
+
tokenizer(["The quick brown fox jumped.", "The fox slept."])
|
55
|
+
|
56
|
+
# Detokenization.
|
57
|
+
tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
|
58
|
+
|
59
|
+
# Custom vocabulary.
|
60
|
+
vocab = {"<|endoftext|>": 0, "a": 4, "Ġquick": 5, "Ġfox": 6}
|
61
|
+
merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"]
|
62
|
+
merges += ["Ġ f", "o x", "Ġf ox"]
|
63
|
+
tokenizer = keras_hub.models.FalconTokenizer(vocabulary=vocab, merges=merges)
|
64
|
+
tokenizer("a quick fox.")
|
65
|
+
```
|
66
|
+
"""
|
67
|
+
|
68
|
+
def __init__(
|
69
|
+
self,
|
70
|
+
vocabulary=None,
|
71
|
+
merges=None,
|
72
|
+
**kwargs,
|
73
|
+
):
|
74
|
+
# Falcon uses the same start as end token, i.e., "<|endoftext|>".
|
75
|
+
self.end_token = self.start_token = "<|endoftext|>"
|
76
|
+
|
77
|
+
super().__init__(
|
78
|
+
vocabulary=vocabulary,
|
79
|
+
merges=merges,
|
80
|
+
unsplittable_tokens=[self.end_token],
|
81
|
+
**kwargs,
|
82
|
+
)
|
83
|
+
|
84
|
+
def set_vocabulary_and_merges(self, vocabulary, merges):
|
85
|
+
super().set_vocabulary_and_merges(vocabulary, merges)
|
86
|
+
|
87
|
+
if vocabulary is not None:
|
88
|
+
# Check for necessary special tokens.
|
89
|
+
if self.end_token not in self.get_vocabulary():
|
90
|
+
raise ValueError(
|
91
|
+
f"Cannot find token `'{self.end_token}'` in the provided "
|
92
|
+
f"`vocabulary`. Please provide `'{self.end_token}'` in "
|
93
|
+
"your `vocabulary` or use a pretrained `vocabulary` name."
|
94
|
+
)
|
95
|
+
|
96
|
+
self.end_token_id = self.token_to_id(self.end_token)
|
97
|
+
self.start_token_id = self.end_token_id
|
98
|
+
self.pad_token_id = 0
|
99
|
+
else:
|
100
|
+
self.end_token_id = None
|
101
|
+
self.start_token_id = None
|
102
|
+
self.pad_token_id = None
|
103
|
+
|
104
|
+
def get_config(self):
|
105
|
+
config = super().get_config()
|
106
|
+
# In the constructor, we pass the list of special tokens to the
|
107
|
+
# `unsplittable_tokens` arg of the superclass' constructor. Hence, we
|
108
|
+
# delete it from the config here.
|
109
|
+
del config["unsplittable_tokens"]
|
110
|
+
return config
|
@@ -0,0 +1,255 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
import math
|
15
|
+
|
16
|
+
import keras
|
17
|
+
from keras import ops
|
18
|
+
|
19
|
+
from keras_hub.src.layers.modeling.transformer_layer_utils import (
|
20
|
+
compute_causal_mask,
|
21
|
+
)
|
22
|
+
from keras_hub.src.layers.modeling.transformer_layer_utils import (
|
23
|
+
merge_padding_and_attention_mask,
|
24
|
+
)
|
25
|
+
from keras_hub.src.models.falcon.falcon_attention import FalconAttention
|
26
|
+
|
27
|
+
|
28
|
+
class FalconTransformerDecoder(keras.layers.Layer):
|
29
|
+
def __init__(
|
30
|
+
self,
|
31
|
+
num_attention_heads,
|
32
|
+
intermediate_dim,
|
33
|
+
layer_norm_epsilon=1e-5,
|
34
|
+
attention_dropout_rate=0,
|
35
|
+
feedforward_dropout_rate=0,
|
36
|
+
**kwargs,
|
37
|
+
):
|
38
|
+
super().__init__(**kwargs)
|
39
|
+
self.num_attention_heads = num_attention_heads
|
40
|
+
self.intermediate_dim = intermediate_dim
|
41
|
+
self.layer_norm_epsilon = layer_norm_epsilon
|
42
|
+
self.attention_dropout_rate = attention_dropout_rate
|
43
|
+
self.feedforward_dropout_rate = feedforward_dropout_rate
|
44
|
+
|
45
|
+
def build(self, decoder_sequence_shape):
|
46
|
+
self.hidden_dim = decoder_sequence_shape[-1]
|
47
|
+
self.input_layernorm = keras.layers.LayerNormalization(
|
48
|
+
epsilon=self.layer_norm_epsilon,
|
49
|
+
dtype=self.dtype_policy,
|
50
|
+
name="input_layernorm",
|
51
|
+
)
|
52
|
+
self.input_layernorm.build(decoder_sequence_shape)
|
53
|
+
|
54
|
+
# Attention layers.
|
55
|
+
self.key_dim = self.hidden_dim // self.num_attention_heads
|
56
|
+
self.attention_layer = FalconAttention(
|
57
|
+
num_heads=self.num_attention_heads,
|
58
|
+
attention_dropout_rate=self.attention_dropout_rate,
|
59
|
+
dtype=self.dtype_policy,
|
60
|
+
name="attention",
|
61
|
+
)
|
62
|
+
self.attention_layer.build(
|
63
|
+
decoder_sequence_shape,
|
64
|
+
)
|
65
|
+
|
66
|
+
self.attention_dropout = keras.layers.Dropout(
|
67
|
+
rate=self.attention_dropout_rate,
|
68
|
+
dtype=self.dtype_policy,
|
69
|
+
name="attention_dropout",
|
70
|
+
)
|
71
|
+
|
72
|
+
self.post_attention_layernorm = keras.layers.LayerNormalization(
|
73
|
+
epsilon=self.layer_norm_epsilon,
|
74
|
+
dtype=self.dtype_policy,
|
75
|
+
name="post_attention_layernorm",
|
76
|
+
)
|
77
|
+
self.post_attention_layernorm.build(decoder_sequence_shape)
|
78
|
+
|
79
|
+
# Feedforward layers.
|
80
|
+
# TODO: use_bias should be an argument to the transformer to support
|
81
|
+
# other sizes of models, e.g. 7B, that don't use bias.
|
82
|
+
self.dense_h_to_4h = keras.layers.Dense(
|
83
|
+
self.intermediate_dim,
|
84
|
+
activation=keras.activations.gelu,
|
85
|
+
use_bias=True,
|
86
|
+
dtype=self.dtype_policy,
|
87
|
+
name="dense_h_to_4h",
|
88
|
+
)
|
89
|
+
self.dense_h_to_4h.build(decoder_sequence_shape)
|
90
|
+
|
91
|
+
self.dense_4h_to_h = keras.layers.Dense(
|
92
|
+
self.hidden_dim,
|
93
|
+
use_bias=True,
|
94
|
+
dtype=self.dtype_policy,
|
95
|
+
name="dense_4h_to_h",
|
96
|
+
)
|
97
|
+
self.dense_4h_to_h.build(
|
98
|
+
(
|
99
|
+
decoder_sequence_shape[0],
|
100
|
+
decoder_sequence_shape[1],
|
101
|
+
self.intermediate_dim,
|
102
|
+
)
|
103
|
+
)
|
104
|
+
|
105
|
+
self.feedforward_dropout = keras.layers.Dropout(
|
106
|
+
rate=self.feedforward_dropout_rate,
|
107
|
+
dtype=self.dtype_policy,
|
108
|
+
name="feedforward_dropout",
|
109
|
+
)
|
110
|
+
|
111
|
+
self.built = True
|
112
|
+
|
113
|
+
def call(
|
114
|
+
self,
|
115
|
+
inputs,
|
116
|
+
decoder_padding_mask=None,
|
117
|
+
decoder_attention_mask=None,
|
118
|
+
attention_cache=None,
|
119
|
+
attention_cache_update_index=None,
|
120
|
+
training=None,
|
121
|
+
):
|
122
|
+
attention_mask = self._compute_attention_mask(
|
123
|
+
decoder_sequence=inputs,
|
124
|
+
decoder_padding_mask=decoder_padding_mask,
|
125
|
+
decoder_attention_mask=decoder_attention_mask,
|
126
|
+
attention_cache=attention_cache,
|
127
|
+
attention_cache_update_index=attention_cache_update_index,
|
128
|
+
)
|
129
|
+
|
130
|
+
residual = inputs
|
131
|
+
|
132
|
+
x = self.input_layernorm(inputs)
|
133
|
+
|
134
|
+
mask = decoder_padding_mask
|
135
|
+
if mask is None:
|
136
|
+
batch_size, seq_length = ops.shape(inputs)[:2]
|
137
|
+
mask = ops.ones((batch_size, seq_length), dtype="int32")
|
138
|
+
alibi = self._build_alibi_tensor(self.num_attention_heads, mask)
|
139
|
+
|
140
|
+
# Attention block.
|
141
|
+
attention_output = self.attention_layer(
|
142
|
+
inputs=x,
|
143
|
+
alibi=alibi,
|
144
|
+
attention_mask=attention_mask,
|
145
|
+
cache=attention_cache,
|
146
|
+
cache_update_index=attention_cache_update_index,
|
147
|
+
)
|
148
|
+
|
149
|
+
if attention_cache is None:
|
150
|
+
x = attention_output
|
151
|
+
else:
|
152
|
+
x, attention_cache = attention_output
|
153
|
+
|
154
|
+
x = self.attention_dropout(x, training=training)
|
155
|
+
|
156
|
+
x = x + residual
|
157
|
+
residual = x
|
158
|
+
|
159
|
+
x = self.post_attention_layernorm(x)
|
160
|
+
|
161
|
+
x = self.dense_h_to_4h(x)
|
162
|
+
x = self.dense_4h_to_h(x)
|
163
|
+
|
164
|
+
x = self.feedforward_dropout(x, training=training)
|
165
|
+
|
166
|
+
x = x + residual
|
167
|
+
|
168
|
+
if attention_cache is not None:
|
169
|
+
return x, attention_cache
|
170
|
+
else:
|
171
|
+
return x
|
172
|
+
|
173
|
+
def get_config(self):
|
174
|
+
config = super().get_config()
|
175
|
+
config.update(
|
176
|
+
{
|
177
|
+
"num_attention_heads": self.num_attention_heads,
|
178
|
+
"intermediate_dim": self.intermediate_dim,
|
179
|
+
"layer_norm_epsilon": self.layer_norm_epsilon,
|
180
|
+
"attention_dropout_rate": self.attention_dropout_rate,
|
181
|
+
"feedforward_dropout_rate": self.feedforward_dropout_rate,
|
182
|
+
}
|
183
|
+
)
|
184
|
+
return config
|
185
|
+
|
186
|
+
def compute_output_shape(self, decoder_sequence_shape):
|
187
|
+
return decoder_sequence_shape
|
188
|
+
|
189
|
+
def _compute_attention_mask(
|
190
|
+
self,
|
191
|
+
decoder_sequence,
|
192
|
+
decoder_padding_mask,
|
193
|
+
decoder_attention_mask,
|
194
|
+
attention_cache=None,
|
195
|
+
attention_cache_update_index=None,
|
196
|
+
):
|
197
|
+
decoder_mask = merge_padding_and_attention_mask(
|
198
|
+
decoder_sequence, decoder_padding_mask, decoder_attention_mask
|
199
|
+
)
|
200
|
+
batch_size = ops.shape(decoder_sequence)[0]
|
201
|
+
input_length = output_length = ops.shape(decoder_sequence)[1]
|
202
|
+
# We need to handle a rectangular causal mask when doing cached
|
203
|
+
# decoding. For generative inference, `decoder_sequence` will
|
204
|
+
# generally be length 1, and `cache` will be the full generation length.
|
205
|
+
if attention_cache is not None:
|
206
|
+
input_length = ops.shape(attention_cache)[2]
|
207
|
+
|
208
|
+
causal_mask = compute_causal_mask(
|
209
|
+
batch_size,
|
210
|
+
input_length,
|
211
|
+
output_length,
|
212
|
+
(
|
213
|
+
0
|
214
|
+
if attention_cache_update_index is None
|
215
|
+
else attention_cache_update_index
|
216
|
+
),
|
217
|
+
)
|
218
|
+
return (
|
219
|
+
ops.minimum(decoder_mask, causal_mask)
|
220
|
+
if decoder_mask is not None
|
221
|
+
else causal_mask
|
222
|
+
)
|
223
|
+
|
224
|
+
def _build_alibi_tensor(self, num_heads, mask):
|
225
|
+
slopes = ops.convert_to_tensor(
|
226
|
+
self._get_slopes(num_heads),
|
227
|
+
dtype=self.compute_dtype,
|
228
|
+
) # num_heads
|
229
|
+
mask = ops.cast(mask, dtype="int32")
|
230
|
+
# TODO: cumsum always outputs int64 in Keras 2 so the casting of cumsum
|
231
|
+
# result to int32 can be removed when keras 2 support is removed.
|
232
|
+
cumsum_mask = ops.cast(ops.cumsum(mask, axis=-1) - 1, "int32")
|
233
|
+
arange_tensor = (cumsum_mask * mask)[:, None, :]
|
234
|
+
alibi = slopes[..., None] * ops.cast(arange_tensor, self.compute_dtype)
|
235
|
+
alibi = ops.expand_dims(
|
236
|
+
alibi, 0
|
237
|
+
) # [None, batch_size, num_heads, seq_length]
|
238
|
+
return ops.transpose(alibi, [1, 2, 0, 3])
|
239
|
+
|
240
|
+
def _get_slopes(self, num_heads):
|
241
|
+
def get_slopes_power_of_2(n):
|
242
|
+
start = 2 ** (-(2 ** -(math.log2(n) - 3)))
|
243
|
+
ratio = start
|
244
|
+
return [start * ratio**i for i in range(n)]
|
245
|
+
|
246
|
+
if math.log2(num_heads).is_integer():
|
247
|
+
return get_slopes_power_of_2(num_heads)
|
248
|
+
else:
|
249
|
+
closest_power_of_2 = 2 ** math.floor(math.log2(num_heads))
|
250
|
+
return (
|
251
|
+
get_slopes_power_of_2(closest_power_of_2)
|
252
|
+
+ self._get_slopes(2 * closest_power_of_2)[0::2][
|
253
|
+
: num_heads - closest_power_of_2
|
254
|
+
]
|
255
|
+
)
|
@@ -0,0 +1,73 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
import keras
|
15
|
+
|
16
|
+
from keras_hub.src.api_export import keras_hub_export
|
17
|
+
from keras_hub.src.models.backbone import Backbone
|
18
|
+
|
19
|
+
|
20
|
+
@keras_hub_export("keras_hub.models.FeaturePyramidBackbone")
|
21
|
+
class FeaturePyramidBackbone(Backbone):
|
22
|
+
"""A backbone with feature pyramid outputs.
|
23
|
+
|
24
|
+
`FeaturePyramidBackbone` extends `Backbone` with a single `pyramid_outputs`
|
25
|
+
property for accessing the feature pyramid outputs of the model. Subclassers
|
26
|
+
should set the `pyramid_outputs` property during the model constructor.
|
27
|
+
|
28
|
+
Example:
|
29
|
+
|
30
|
+
```python
|
31
|
+
input_data = np.random.uniform(0, 255, size=(2, 224, 224, 3))
|
32
|
+
|
33
|
+
# Convert to feature pyramid output format using ResNet.
|
34
|
+
backbone = ResNetBackbone.from_preset("resnet50")
|
35
|
+
model = keras.Model(
|
36
|
+
inputs=backbone.inputs, outputs=backbone.pyramid_outputs
|
37
|
+
)
|
38
|
+
model(input_data) # A dict containing the keys ["P2", "P3", "P4", "P5"]
|
39
|
+
```
|
40
|
+
"""
|
41
|
+
|
42
|
+
@property
|
43
|
+
def pyramid_outputs(self):
|
44
|
+
"""A dict for feature pyramid outputs.
|
45
|
+
|
46
|
+
The key is a string represents the name of the feature output and the
|
47
|
+
value is a `keras.KerasTensor`. A typical feature pyramid has multiple
|
48
|
+
levels corresponding to scales such as `["P2", "P3", "P4", "P5"]`. Scale
|
49
|
+
`Pn` represents a feature map `2^n` times smaller in width and height
|
50
|
+
than the inputs.
|
51
|
+
"""
|
52
|
+
return getattr(self, "_pyramid_outputs", {})
|
53
|
+
|
54
|
+
@pyramid_outputs.setter
|
55
|
+
def pyramid_outputs(self, value):
|
56
|
+
if not isinstance(value, dict):
|
57
|
+
raise TypeError(
|
58
|
+
"`pyramid_outputs` must be a dictionary. "
|
59
|
+
f"Received: value={value} of type {type(value)}"
|
60
|
+
)
|
61
|
+
for k, v in value.items():
|
62
|
+
if not isinstance(k, str):
|
63
|
+
raise TypeError(
|
64
|
+
"The key of `pyramid_outputs` must be a string. "
|
65
|
+
f"Received: key={k} of type {type(k)}"
|
66
|
+
)
|
67
|
+
if not isinstance(v, keras.KerasTensor):
|
68
|
+
raise TypeError(
|
69
|
+
"The value of `pyramid_outputs` must be a "
|
70
|
+
"`keras.KerasTensor`. "
|
71
|
+
f"Received: value={v} of type {type(v)}"
|
72
|
+
)
|
73
|
+
self._pyramid_outputs = value
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
from keras_hub.src.models.gemma.gemma_backbone import GemmaBackbone
|
16
|
+
from keras_hub.src.models.gemma.gemma_presets import backbone_presets
|
17
|
+
from keras_hub.src.models.gemma.gemma_tokenizer import GemmaTokenizer
|
18
|
+
from keras_hub.src.utils.preset_utils import register_presets
|
19
|
+
|
20
|
+
register_presets(backbone_presets, (GemmaBackbone, GemmaTokenizer))
|