keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keras_hub/__init__.py +52 -0
- keras_hub/api/__init__.py +27 -0
- keras_hub/api/layers/__init__.py +47 -0
- keras_hub/api/metrics/__init__.py +24 -0
- keras_hub/api/models/__init__.py +249 -0
- keras_hub/api/samplers/__init__.py +29 -0
- keras_hub/api/tokenizers/__init__.py +35 -0
- keras_hub/src/__init__.py +13 -0
- keras_hub/src/api_export.py +53 -0
- keras_hub/src/layers/__init__.py +13 -0
- keras_hub/src/layers/modeling/__init__.py +13 -0
- keras_hub/src/layers/modeling/alibi_bias.py +143 -0
- keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
- keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
- keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
- keras_hub/src/layers/modeling/position_embedding.py +123 -0
- keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
- keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
- keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
- keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
- keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
- keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
- keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
- keras_hub/src/layers/preprocessing/__init__.py +13 -0
- keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
- keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
- keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
- keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
- keras_hub/src/layers/preprocessing/random_swap.py +267 -0
- keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
- keras_hub/src/metrics/__init__.py +13 -0
- keras_hub/src/metrics/bleu.py +394 -0
- keras_hub/src/metrics/edit_distance.py +197 -0
- keras_hub/src/metrics/perplexity.py +181 -0
- keras_hub/src/metrics/rouge_base.py +204 -0
- keras_hub/src/metrics/rouge_l.py +97 -0
- keras_hub/src/metrics/rouge_n.py +125 -0
- keras_hub/src/models/__init__.py +13 -0
- keras_hub/src/models/albert/__init__.py +20 -0
- keras_hub/src/models/albert/albert_backbone.py +267 -0
- keras_hub/src/models/albert/albert_classifier.py +202 -0
- keras_hub/src/models/albert/albert_masked_lm.py +129 -0
- keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
- keras_hub/src/models/albert/albert_preprocessor.py +206 -0
- keras_hub/src/models/albert/albert_presets.py +70 -0
- keras_hub/src/models/albert/albert_tokenizer.py +119 -0
- keras_hub/src/models/backbone.py +311 -0
- keras_hub/src/models/bart/__init__.py +20 -0
- keras_hub/src/models/bart/bart_backbone.py +261 -0
- keras_hub/src/models/bart/bart_preprocessor.py +276 -0
- keras_hub/src/models/bart/bart_presets.py +74 -0
- keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
- keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
- keras_hub/src/models/bart/bart_tokenizer.py +124 -0
- keras_hub/src/models/bert/__init__.py +23 -0
- keras_hub/src/models/bert/bert_backbone.py +227 -0
- keras_hub/src/models/bert/bert_classifier.py +183 -0
- keras_hub/src/models/bert/bert_masked_lm.py +131 -0
- keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
- keras_hub/src/models/bert/bert_preprocessor.py +184 -0
- keras_hub/src/models/bert/bert_presets.py +147 -0
- keras_hub/src/models/bert/bert_tokenizer.py +112 -0
- keras_hub/src/models/bloom/__init__.py +20 -0
- keras_hub/src/models/bloom/bloom_attention.py +186 -0
- keras_hub/src/models/bloom/bloom_backbone.py +173 -0
- keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
- keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
- keras_hub/src/models/bloom/bloom_decoder.py +206 -0
- keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
- keras_hub/src/models/bloom/bloom_presets.py +121 -0
- keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
- keras_hub/src/models/causal_lm.py +383 -0
- keras_hub/src/models/classifier.py +109 -0
- keras_hub/src/models/csp_darknet/__init__.py +13 -0
- keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
- keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
- keras_hub/src/models/deberta_v3/__init__.py +24 -0
- keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
- keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
- keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
- keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
- keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
- keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
- keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
- keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
- keras_hub/src/models/densenet/__init__.py +13 -0
- keras_hub/src/models/densenet/densenet_backbone.py +210 -0
- keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
- keras_hub/src/models/distil_bert/__init__.py +26 -0
- keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
- keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
- keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
- keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
- keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
- keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
- keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
- keras_hub/src/models/electra/__init__.py +20 -0
- keras_hub/src/models/electra/electra_backbone.py +247 -0
- keras_hub/src/models/electra/electra_preprocessor.py +154 -0
- keras_hub/src/models/electra/electra_presets.py +95 -0
- keras_hub/src/models/electra/electra_tokenizer.py +104 -0
- keras_hub/src/models/f_net/__init__.py +20 -0
- keras_hub/src/models/f_net/f_net_backbone.py +236 -0
- keras_hub/src/models/f_net/f_net_classifier.py +154 -0
- keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
- keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
- keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
- keras_hub/src/models/f_net/f_net_presets.py +43 -0
- keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
- keras_hub/src/models/falcon/__init__.py +20 -0
- keras_hub/src/models/falcon/falcon_attention.py +156 -0
- keras_hub/src/models/falcon/falcon_backbone.py +164 -0
- keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
- keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
- keras_hub/src/models/falcon/falcon_presets.py +30 -0
- keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
- keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
- keras_hub/src/models/feature_pyramid_backbone.py +73 -0
- keras_hub/src/models/gemma/__init__.py +20 -0
- keras_hub/src/models/gemma/gemma_attention.py +250 -0
- keras_hub/src/models/gemma/gemma_backbone.py +316 -0
- keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
- keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
- keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
- keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
- keras_hub/src/models/gemma/gemma_presets.py +248 -0
- keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
- keras_hub/src/models/gemma/rms_normalization.py +40 -0
- keras_hub/src/models/gpt2/__init__.py +20 -0
- keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
- keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
- keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
- keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
- keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
- keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
- keras_hub/src/models/image_classifier.py +90 -0
- keras_hub/src/models/llama/__init__.py +20 -0
- keras_hub/src/models/llama/llama_attention.py +225 -0
- keras_hub/src/models/llama/llama_backbone.py +188 -0
- keras_hub/src/models/llama/llama_causal_lm.py +327 -0
- keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
- keras_hub/src/models/llama/llama_decoder.py +246 -0
- keras_hub/src/models/llama/llama_layernorm.py +48 -0
- keras_hub/src/models/llama/llama_preprocessor.py +189 -0
- keras_hub/src/models/llama/llama_presets.py +80 -0
- keras_hub/src/models/llama/llama_tokenizer.py +84 -0
- keras_hub/src/models/llama3/__init__.py +20 -0
- keras_hub/src/models/llama3/llama3_backbone.py +84 -0
- keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
- keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
- keras_hub/src/models/llama3/llama3_presets.py +69 -0
- keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
- keras_hub/src/models/masked_lm.py +101 -0
- keras_hub/src/models/mistral/__init__.py +20 -0
- keras_hub/src/models/mistral/mistral_attention.py +238 -0
- keras_hub/src/models/mistral/mistral_backbone.py +203 -0
- keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
- keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
- keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
- keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
- keras_hub/src/models/mistral/mistral_presets.py +48 -0
- keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
- keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
- keras_hub/src/models/mix_transformer/__init__.py +13 -0
- keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
- keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
- keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
- keras_hub/src/models/opt/__init__.py +20 -0
- keras_hub/src/models/opt/opt_backbone.py +173 -0
- keras_hub/src/models/opt/opt_causal_lm.py +301 -0
- keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
- keras_hub/src/models/opt/opt_preprocessor.py +188 -0
- keras_hub/src/models/opt/opt_presets.py +72 -0
- keras_hub/src/models/opt/opt_tokenizer.py +116 -0
- keras_hub/src/models/pali_gemma/__init__.py +23 -0
- keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
- keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
- keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
- keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
- keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
- keras_hub/src/models/phi3/__init__.py +20 -0
- keras_hub/src/models/phi3/phi3_attention.py +260 -0
- keras_hub/src/models/phi3/phi3_backbone.py +224 -0
- keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
- keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/phi3/phi3_decoder.py +260 -0
- keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
- keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
- keras_hub/src/models/phi3/phi3_presets.py +50 -0
- keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
- keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
- keras_hub/src/models/preprocessor.py +207 -0
- keras_hub/src/models/resnet/__init__.py +13 -0
- keras_hub/src/models/resnet/resnet_backbone.py +612 -0
- keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
- keras_hub/src/models/roberta/__init__.py +20 -0
- keras_hub/src/models/roberta/roberta_backbone.py +184 -0
- keras_hub/src/models/roberta/roberta_classifier.py +209 -0
- keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
- keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
- keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
- keras_hub/src/models/roberta/roberta_presets.py +43 -0
- keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
- keras_hub/src/models/seq_2_seq_lm.py +54 -0
- keras_hub/src/models/t5/__init__.py +20 -0
- keras_hub/src/models/t5/t5_backbone.py +261 -0
- keras_hub/src/models/t5/t5_layer_norm.py +35 -0
- keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
- keras_hub/src/models/t5/t5_presets.py +95 -0
- keras_hub/src/models/t5/t5_tokenizer.py +100 -0
- keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
- keras_hub/src/models/task.py +419 -0
- keras_hub/src/models/vgg/__init__.py +13 -0
- keras_hub/src/models/vgg/vgg_backbone.py +158 -0
- keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
- keras_hub/src/models/vit_det/__init__.py +13 -0
- keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
- keras_hub/src/models/vit_det/vit_layers.py +565 -0
- keras_hub/src/models/whisper/__init__.py +20 -0
- keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
- keras_hub/src/models/whisper/whisper_backbone.py +305 -0
- keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
- keras_hub/src/models/whisper/whisper_decoder.py +141 -0
- keras_hub/src/models/whisper/whisper_encoder.py +106 -0
- keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
- keras_hub/src/models/whisper/whisper_presets.py +148 -0
- keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
- keras_hub/src/models/xlm_roberta/__init__.py +26 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
- keras_hub/src/models/xlnet/__init__.py +13 -0
- keras_hub/src/models/xlnet/relative_attention.py +459 -0
- keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
- keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
- keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
- keras_hub/src/samplers/__init__.py +13 -0
- keras_hub/src/samplers/beam_sampler.py +207 -0
- keras_hub/src/samplers/contrastive_sampler.py +231 -0
- keras_hub/src/samplers/greedy_sampler.py +50 -0
- keras_hub/src/samplers/random_sampler.py +77 -0
- keras_hub/src/samplers/sampler.py +237 -0
- keras_hub/src/samplers/serialization.py +97 -0
- keras_hub/src/samplers/top_k_sampler.py +92 -0
- keras_hub/src/samplers/top_p_sampler.py +113 -0
- keras_hub/src/tests/__init__.py +13 -0
- keras_hub/src/tests/test_case.py +608 -0
- keras_hub/src/tokenizers/__init__.py +13 -0
- keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
- keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
- keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
- keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
- keras_hub/src/tokenizers/tokenizer.py +235 -0
- keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
- keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
- keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
- keras_hub/src/utils/__init__.py +13 -0
- keras_hub/src/utils/keras_utils.py +130 -0
- keras_hub/src/utils/pipeline_model.py +293 -0
- keras_hub/src/utils/preset_utils.py +621 -0
- keras_hub/src/utils/python_utils.py +21 -0
- keras_hub/src/utils/tensor_utils.py +206 -0
- keras_hub/src/utils/timm/__init__.py +13 -0
- keras_hub/src/utils/timm/convert.py +37 -0
- keras_hub/src/utils/timm/convert_resnet.py +171 -0
- keras_hub/src/utils/transformers/__init__.py +13 -0
- keras_hub/src/utils/transformers/convert.py +101 -0
- keras_hub/src/utils/transformers/convert_bert.py +173 -0
- keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
- keras_hub/src/utils/transformers/convert_gemma.py +187 -0
- keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
- keras_hub/src/utils/transformers/convert_llama3.py +136 -0
- keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
- keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
- keras_hub/src/version_utils.py +23 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,246 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
import keras
|
15
|
+
from keras import ops
|
16
|
+
|
17
|
+
from keras_hub.src.layers.modeling.transformer_layer_utils import (
|
18
|
+
compute_causal_mask,
|
19
|
+
)
|
20
|
+
from keras_hub.src.layers.modeling.transformer_layer_utils import (
|
21
|
+
merge_padding_and_attention_mask,
|
22
|
+
)
|
23
|
+
from keras_hub.src.models.llama.llama_attention import LlamaAttention
|
24
|
+
from keras_hub.src.models.llama.llama_layernorm import LlamaLayerNorm
|
25
|
+
from keras_hub.src.utils.keras_utils import clone_initializer
|
26
|
+
|
27
|
+
|
28
|
+
class LlamaTransformerDecoder(keras.layers.Layer):
|
29
|
+
"""A Transformer decoder layer for the Llama backbone."""
|
30
|
+
|
31
|
+
def __init__(
|
32
|
+
self,
|
33
|
+
intermediate_dim,
|
34
|
+
num_query_heads,
|
35
|
+
num_key_value_heads,
|
36
|
+
rope_max_wavelength=10000,
|
37
|
+
rope_scaling_factor=1.0,
|
38
|
+
activation="silu",
|
39
|
+
layer_norm_epsilon=1e-5,
|
40
|
+
kernel_initializer="glorot_uniform",
|
41
|
+
dropout=0,
|
42
|
+
**kwargs,
|
43
|
+
):
|
44
|
+
super().__init__(**kwargs)
|
45
|
+
self.intermediate_dim = intermediate_dim
|
46
|
+
self.num_query_heads = num_query_heads
|
47
|
+
self.num_key_value_heads = num_key_value_heads
|
48
|
+
|
49
|
+
self.rope_max_wavelength = rope_max_wavelength
|
50
|
+
self.rope_scaling_factor = rope_scaling_factor
|
51
|
+
|
52
|
+
self.dropout = dropout
|
53
|
+
|
54
|
+
self.activation = keras.activations.get(activation)
|
55
|
+
self.layer_norm_epsilon = layer_norm_epsilon
|
56
|
+
self.kernel_initializer = keras.initializers.get(kernel_initializer)
|
57
|
+
|
58
|
+
self.supports_masking = True
|
59
|
+
|
60
|
+
def build(self, decoder_sequence_shape):
|
61
|
+
self._decoder_sequence_shape = decoder_sequence_shape
|
62
|
+
self.hidden_dim = decoder_sequence_shape[-1]
|
63
|
+
|
64
|
+
# Self attention layer.
|
65
|
+
self._self_attention_layer = LlamaAttention(
|
66
|
+
num_query_heads=self.num_query_heads,
|
67
|
+
num_key_value_heads=self.num_key_value_heads,
|
68
|
+
rope_max_wavelength=self.rope_max_wavelength,
|
69
|
+
rope_scaling_factor=self.rope_scaling_factor,
|
70
|
+
kernel_initializer=clone_initializer(self.kernel_initializer),
|
71
|
+
dropout=self.dropout,
|
72
|
+
dtype=self.dtype_policy,
|
73
|
+
name="self_attention",
|
74
|
+
)
|
75
|
+
self._self_attention_layer.build(decoder_sequence_shape)
|
76
|
+
|
77
|
+
self._self_attention_layernorm = LlamaLayerNorm(
|
78
|
+
epsilon=self.layer_norm_epsilon,
|
79
|
+
dtype=self.dtype_policy,
|
80
|
+
name="self_attention_layernorm",
|
81
|
+
)
|
82
|
+
self._self_attention_layernorm.build(decoder_sequence_shape)
|
83
|
+
self._self_attention_dropout = keras.layers.Dropout(
|
84
|
+
rate=self.dropout,
|
85
|
+
dtype=self.dtype_policy,
|
86
|
+
name="self_attention_dropout",
|
87
|
+
)
|
88
|
+
|
89
|
+
# Feedforward layers.
|
90
|
+
self._feedforward_intermediate_dense = keras.layers.Dense(
|
91
|
+
self.intermediate_dim,
|
92
|
+
kernel_initializer=clone_initializer(self.kernel_initializer),
|
93
|
+
use_bias=False,
|
94
|
+
dtype=self.dtype_policy,
|
95
|
+
name="feedforward_intermediate_dense",
|
96
|
+
)
|
97
|
+
self._feedforward_intermediate_dense.build(decoder_sequence_shape)
|
98
|
+
|
99
|
+
self._feedforward_gate_dense = keras.layers.Dense(
|
100
|
+
self.intermediate_dim,
|
101
|
+
kernel_initializer=clone_initializer(self.kernel_initializer),
|
102
|
+
use_bias=False,
|
103
|
+
dtype=self.dtype_policy,
|
104
|
+
name="feedforward_gate_dense",
|
105
|
+
)
|
106
|
+
self._feedforward_gate_dense.build(decoder_sequence_shape)
|
107
|
+
|
108
|
+
self._feedforward_output_dense = keras.layers.Dense(
|
109
|
+
self.hidden_dim,
|
110
|
+
kernel_initializer=clone_initializer(self.kernel_initializer),
|
111
|
+
use_bias=False,
|
112
|
+
dtype=self.dtype_policy,
|
113
|
+
name="feedforward_output_dense",
|
114
|
+
)
|
115
|
+
|
116
|
+
self._feedforward_output_dense.build(
|
117
|
+
self._feedforward_gate_dense.compute_output_shape(
|
118
|
+
decoder_sequence_shape
|
119
|
+
)
|
120
|
+
)
|
121
|
+
|
122
|
+
self._feedforward_layernorm = LlamaLayerNorm(
|
123
|
+
epsilon=self.layer_norm_epsilon,
|
124
|
+
dtype=self.dtype_policy,
|
125
|
+
name="feedforward_layernorm",
|
126
|
+
)
|
127
|
+
self._feedforward_layernorm.build(decoder_sequence_shape)
|
128
|
+
|
129
|
+
self.built = True
|
130
|
+
|
131
|
+
def call(
|
132
|
+
self,
|
133
|
+
decoder_sequence,
|
134
|
+
decoder_padding_mask=None,
|
135
|
+
decoder_attention_mask=None,
|
136
|
+
self_attention_cache=None,
|
137
|
+
self_attention_cache_update_index=None,
|
138
|
+
training=None,
|
139
|
+
):
|
140
|
+
self_attention_mask = self._compute_self_attention_mask(
|
141
|
+
decoder_sequence=decoder_sequence,
|
142
|
+
decoder_padding_mask=decoder_padding_mask,
|
143
|
+
decoder_attention_mask=decoder_attention_mask,
|
144
|
+
self_attention_cache=self_attention_cache,
|
145
|
+
self_attention_cache_update_index=self_attention_cache_update_index,
|
146
|
+
)
|
147
|
+
residual = decoder_sequence
|
148
|
+
|
149
|
+
x = self._self_attention_layernorm(decoder_sequence)
|
150
|
+
|
151
|
+
# Self attention block.
|
152
|
+
x = self._self_attention_layer(
|
153
|
+
hidden_states=x,
|
154
|
+
attention_mask=self_attention_mask,
|
155
|
+
cache=self_attention_cache,
|
156
|
+
cache_update_index=self_attention_cache_update_index,
|
157
|
+
)
|
158
|
+
|
159
|
+
if self_attention_cache is not None:
|
160
|
+
x, self_attention_cache = x
|
161
|
+
|
162
|
+
x = self._self_attention_dropout(x, training=training)
|
163
|
+
|
164
|
+
x = x + residual
|
165
|
+
residual = x
|
166
|
+
|
167
|
+
x = self._feedforward_layernorm(x)
|
168
|
+
gate_output = self._feedforward_gate_dense(x)
|
169
|
+
|
170
|
+
# Note that we run the activation function in full 32-bit
|
171
|
+
# precision since this is what `torch.nn.functional.silu`
|
172
|
+
# does. Internally, `torch.nn.functional.silu` converts the
|
173
|
+
# inputs to float32, computes SiLU, and converts the outputs
|
174
|
+
# back to compute dtype.
|
175
|
+
# CPU Kernel: https://github.com/pytorch/pytorch/blob/35c493f2cf9b623bfdc7e6b34dc1cb39690a7919/aten/src/ATen/native/cpu/Activation.cpp#L1221-L1235 # noqa: E501
|
176
|
+
# CUDA Kernel: https://github.com/pytorch/pytorch/blob/35c493f2cf9b623bfdc7e6b34dc1cb39690a7919/aten/src/ATen/native/cuda/ActivationSiluKernel.cu # noqa: E501
|
177
|
+
gate_output = ops.cast(gate_output, "float32")
|
178
|
+
gate_output = self.activation(gate_output)
|
179
|
+
gate_output = ops.cast(gate_output, self.compute_dtype)
|
180
|
+
|
181
|
+
x = self._feedforward_intermediate_dense(x)
|
182
|
+
|
183
|
+
x = self._feedforward_output_dense(ops.multiply(x, gate_output))
|
184
|
+
|
185
|
+
decoder_output = x + residual
|
186
|
+
|
187
|
+
if self_attention_cache is not None:
|
188
|
+
return decoder_output, self_attention_cache
|
189
|
+
return decoder_output
|
190
|
+
|
191
|
+
def _compute_self_attention_mask(
|
192
|
+
self,
|
193
|
+
decoder_sequence,
|
194
|
+
decoder_padding_mask,
|
195
|
+
decoder_attention_mask,
|
196
|
+
self_attention_cache,
|
197
|
+
self_attention_cache_update_index,
|
198
|
+
):
|
199
|
+
decoder_mask = merge_padding_and_attention_mask(
|
200
|
+
decoder_sequence, decoder_padding_mask, decoder_attention_mask
|
201
|
+
)
|
202
|
+
batch_size = ops.shape(decoder_sequence)[0]
|
203
|
+
input_length = output_length = ops.shape(decoder_sequence)[1]
|
204
|
+
# We need to handle a rectangular causal mask when doing cached
|
205
|
+
# decoding. For generative inference, `decoder_sequence` will
|
206
|
+
# generally be length 1, and `cache` will be the full generation length.
|
207
|
+
if self_attention_cache is not None:
|
208
|
+
input_length = ops.shape(self_attention_cache)[2]
|
209
|
+
|
210
|
+
cache_update_index = (
|
211
|
+
0
|
212
|
+
if self_attention_cache_update_index is None
|
213
|
+
else self_attention_cache_update_index
|
214
|
+
)
|
215
|
+
|
216
|
+
causal_mask = compute_causal_mask(
|
217
|
+
batch_size, input_length, output_length, cache_update_index
|
218
|
+
)
|
219
|
+
|
220
|
+
return (
|
221
|
+
ops.minimum(decoder_mask, causal_mask)
|
222
|
+
if decoder_mask is not None
|
223
|
+
else causal_mask
|
224
|
+
)
|
225
|
+
|
226
|
+
def compute_output_shape(self, decoder_sequence_shape):
|
227
|
+
return decoder_sequence_shape
|
228
|
+
|
229
|
+
def get_config(self):
|
230
|
+
config = super().get_config()
|
231
|
+
config.update(
|
232
|
+
{
|
233
|
+
"intermediate_dim": self.intermediate_dim,
|
234
|
+
"num_query_heads": self.num_query_heads,
|
235
|
+
"rope_max_wavelength": self.rope_max_wavelength,
|
236
|
+
"rope_scaling_factor": self.rope_scaling_factor,
|
237
|
+
"num_key_value_heads": self.num_key_value_heads,
|
238
|
+
"activation": keras.activations.serialize(self.activation),
|
239
|
+
"layer_norm_epsilon": self.layer_norm_epsilon,
|
240
|
+
"kernel_initializer": keras.initializers.serialize(
|
241
|
+
self.kernel_initializer
|
242
|
+
),
|
243
|
+
"dropout": self.dropout,
|
244
|
+
}
|
245
|
+
)
|
246
|
+
return config
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
import keras
|
15
|
+
from keras import ops
|
16
|
+
|
17
|
+
|
18
|
+
# TODO: Deprecate this in favor of
|
19
|
+
# `keras.layers.LayerNormalization(rms_scaling=True)` once Keras 2 support is
|
20
|
+
# removed.
|
21
|
+
class LlamaLayerNorm(keras.layers.Layer):
|
22
|
+
"""A normalization layer for Llama that implements RMS normalization."""
|
23
|
+
|
24
|
+
def __init__(self, epsilon=1e-6, **kwargs):
|
25
|
+
super().__init__(**kwargs)
|
26
|
+
self.epsilon = epsilon
|
27
|
+
|
28
|
+
def build(self, input_shape):
|
29
|
+
dim = input_shape[-1]
|
30
|
+
self.scale = self.add_weight(
|
31
|
+
name="scale",
|
32
|
+
trainable=True,
|
33
|
+
shape=(dim,),
|
34
|
+
initializer="ones",
|
35
|
+
dtype=self.variable_dtype,
|
36
|
+
)
|
37
|
+
self.built = True
|
38
|
+
|
39
|
+
def call(self, x):
|
40
|
+
x = ops.cast(x, "float32")
|
41
|
+
var = ops.mean(ops.power(x, 2), axis=-1, keepdims=True)
|
42
|
+
x = x * ops.rsqrt(var + self.epsilon)
|
43
|
+
return ops.cast(x * self.scale, self.compute_dtype)
|
44
|
+
|
45
|
+
def get_config(self):
|
46
|
+
config = super().get_config()
|
47
|
+
config.update({"epsilon": self.epsilon})
|
48
|
+
return config
|
@@ -0,0 +1,189 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
import keras
|
15
|
+
|
16
|
+
from keras_hub.src.api_export import keras_hub_export
|
17
|
+
from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
|
18
|
+
from keras_hub.src.models.llama.llama_tokenizer import LlamaTokenizer
|
19
|
+
from keras_hub.src.models.preprocessor import Preprocessor
|
20
|
+
from keras_hub.src.utils.keras_utils import (
|
21
|
+
convert_inputs_to_list_of_tensor_segments,
|
22
|
+
)
|
23
|
+
|
24
|
+
|
25
|
+
@keras_hub_export("keras_hub.models.LlamaPreprocessor")
|
26
|
+
class LlamaPreprocessor(Preprocessor):
|
27
|
+
"""A Llama preprocessing layer which tokenizes and packs inputs.
|
28
|
+
|
29
|
+
This preprocessing layer will do three things:
|
30
|
+
|
31
|
+
1. Tokenize any number of input segments using the `tokenizer`.
|
32
|
+
2. Pack the inputs together using a `keras_hub.layers.StartEndPacker`.
|
33
|
+
with the appropriate tokens.
|
34
|
+
3. Construct a dictionary with keys `"token_ids"`, and `"padding_mask"`
|
35
|
+
that can be passed directly to `keras_hub.models.LlamaBackbone`.
|
36
|
+
|
37
|
+
This layer can be used directly with `tf.data.Dataset.map` to preprocess
|
38
|
+
string data in the `(x, y, sample_weight)` format used by
|
39
|
+
`keras.Model.fit`.
|
40
|
+
|
41
|
+
Args:
|
42
|
+
tokenizer: A `keras_hub.models.LlamaTokenizer` instance.
|
43
|
+
sequence_length: The length of the packed inputs.
|
44
|
+
add_start_token: If `True`, the preprocessor will prepend the tokenizer
|
45
|
+
start token to each input sequence. Default is `True`.
|
46
|
+
add_end_token: If `True`, the preprocessor will append the tokenizer
|
47
|
+
end token to each input sequence. Default is `False`.
|
48
|
+
|
49
|
+
Call arguments:
|
50
|
+
x: A tensor of single string sequences, or a tuple of multiple
|
51
|
+
tensor sequences to be packed together. Inputs may be batched or
|
52
|
+
unbatched. For single sequences, raw python inputs will be converted
|
53
|
+
to tensors. For multiple sequences, pass tensors directly.
|
54
|
+
y: Any label data. Will be passed through unaltered.
|
55
|
+
sample_weight: Any label weight data. Will be passed through unaltered.
|
56
|
+
sequence_length: Pass to override the configured `sequence_length` of
|
57
|
+
the layer.
|
58
|
+
|
59
|
+
Examples:
|
60
|
+
|
61
|
+
Directly calling the from_preset().
|
62
|
+
```python
|
63
|
+
preprocessor = keras_hub.models.LlamaPreprocessor.from_preset(
|
64
|
+
"llama_base_en"
|
65
|
+
)
|
66
|
+
|
67
|
+
# Tokenize and pack a single sentence.
|
68
|
+
preprocessor("The quick brown fox jumped.")
|
69
|
+
|
70
|
+
# Tokenize and a batch of single sentences.
|
71
|
+
preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
|
72
|
+
|
73
|
+
# Preprocess a batch of sentence pairs.
|
74
|
+
# When handling multiple sequences, always convert to tensors first!
|
75
|
+
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
76
|
+
second = tf.constant(["The fox tripped.", "Oh look, a whale."])
|
77
|
+
preprocessor((first, second))
|
78
|
+
```
|
79
|
+
|
80
|
+
Mapping with `tf.data.Dataset`.
|
81
|
+
```python
|
82
|
+
preprocessor = keras_hub.models.LlamaPreprocessor.from_preset(
|
83
|
+
"llama_base_en"
|
84
|
+
)
|
85
|
+
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
86
|
+
second = tf.constant(["The fox tripped.", "Oh look, a whale."])
|
87
|
+
label = tf.constant([1, 1])
|
88
|
+
|
89
|
+
# Map labeled single sentences.
|
90
|
+
ds = tf.data.Dataset.from_tensor_slices((first, label))
|
91
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
92
|
+
|
93
|
+
# Map unlabeled single sentences.
|
94
|
+
ds = tf.data.Dataset.from_tensor_slices(first)
|
95
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
96
|
+
|
97
|
+
# Map labeled sentence pairs.
|
98
|
+
ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
|
99
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
100
|
+
|
101
|
+
# Map unlabeled sentence pairs.
|
102
|
+
ds = tf.data.Dataset.from_tensor_slices((first, second))
|
103
|
+
|
104
|
+
# Watch out for tf.data's default unpacking of tuples here!
|
105
|
+
# Best to invoke the `preprocessor` directly in this case.
|
106
|
+
ds = ds.map(
|
107
|
+
lambda first, second: preprocessor(x=(first, second)),
|
108
|
+
num_parallel_calls=tf.data.AUTOTUNE,
|
109
|
+
)
|
110
|
+
```
|
111
|
+
"""
|
112
|
+
|
113
|
+
tokenizer_cls = LlamaTokenizer
|
114
|
+
|
115
|
+
def __init__(
|
116
|
+
self,
|
117
|
+
tokenizer,
|
118
|
+
sequence_length=1024,
|
119
|
+
add_start_token=True,
|
120
|
+
add_end_token=False,
|
121
|
+
**kwargs,
|
122
|
+
):
|
123
|
+
super().__init__(**kwargs)
|
124
|
+
self.tokenizer = tokenizer
|
125
|
+
self.packer = None
|
126
|
+
self.add_start_token = add_start_token
|
127
|
+
self.add_end_token = add_end_token
|
128
|
+
self.sequence_length = sequence_length
|
129
|
+
|
130
|
+
def build(self, input_shape):
|
131
|
+
# Defer packer creation to `build()` so that we can be sure tokenizer
|
132
|
+
# assets have loaded when restoring a saved model.
|
133
|
+
self.packer = StartEndPacker(
|
134
|
+
start_value=self.tokenizer.start_token_id,
|
135
|
+
end_value=self.tokenizer.end_token_id,
|
136
|
+
sequence_length=self.sequence_length,
|
137
|
+
return_padding_mask=True,
|
138
|
+
)
|
139
|
+
self.built = True
|
140
|
+
|
141
|
+
def get_config(self):
|
142
|
+
config = super().get_config()
|
143
|
+
config.update(
|
144
|
+
{
|
145
|
+
"sequence_length": self.sequence_length,
|
146
|
+
"add_start_token": self.add_start_token,
|
147
|
+
"add_end_token": self.add_end_token,
|
148
|
+
}
|
149
|
+
)
|
150
|
+
return config
|
151
|
+
|
152
|
+
def call(
|
153
|
+
self,
|
154
|
+
x,
|
155
|
+
y=None,
|
156
|
+
sample_weight=None,
|
157
|
+
sequence_length=None,
|
158
|
+
):
|
159
|
+
x = convert_inputs_to_list_of_tensor_segments(x)
|
160
|
+
if len(x) != 1:
|
161
|
+
raise ValueError(
|
162
|
+
"Llama requires each input feature to contain only "
|
163
|
+
f"one segment, but received {len(x)}. If you are using Llama"
|
164
|
+
" for a multi-segment classification task, please refer to "
|
165
|
+
"classification models like BERT or RoBERTa."
|
166
|
+
)
|
167
|
+
sequence_length = sequence_length or self.sequence_length
|
168
|
+
token_ids, padding_mask = self.packer(
|
169
|
+
self.tokenizer(x[0]),
|
170
|
+
sequence_length=sequence_length,
|
171
|
+
add_start_value=self.add_start_token,
|
172
|
+
add_end_value=self.add_end_token,
|
173
|
+
)
|
174
|
+
x = {
|
175
|
+
"token_ids": token_ids,
|
176
|
+
"padding_mask": padding_mask,
|
177
|
+
}
|
178
|
+
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
|
179
|
+
|
180
|
+
@property
|
181
|
+
def sequence_length(self):
|
182
|
+
"""The padded length of model input sequences."""
|
183
|
+
return self._sequence_length
|
184
|
+
|
185
|
+
@sequence_length.setter
|
186
|
+
def sequence_length(self, value):
|
187
|
+
self._sequence_length = value
|
188
|
+
if self.packer is not None:
|
189
|
+
self.packer.sequence_length = value
|
@@ -0,0 +1,80 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
"""Llama model preset configurations."""
|
15
|
+
|
16
|
+
# Metadata for loading pretrained model weights.
|
17
|
+
backbone_presets = {
|
18
|
+
"llama2_7b_en": {
|
19
|
+
"metadata": {
|
20
|
+
"description": "7 billion parameter, 32-layer, base LLaMA 2 model.",
|
21
|
+
"params": 6738415616,
|
22
|
+
"official_name": "LLaMA 2",
|
23
|
+
"path": "llama2",
|
24
|
+
"model_card": "https://github.com/meta-llama/llama",
|
25
|
+
},
|
26
|
+
"kaggle_handle": "kaggle://keras/llama2/keras/llama2_7b_en/1",
|
27
|
+
},
|
28
|
+
"llama2_7b_en_int8": {
|
29
|
+
"metadata": {
|
30
|
+
"description": (
|
31
|
+
"7 billion parameter, 32-layer, base LLaMA 2 model with "
|
32
|
+
"activation and weights quantized to int8."
|
33
|
+
),
|
34
|
+
"params": 6739839488,
|
35
|
+
"official_name": "LLaMA 2",
|
36
|
+
"path": "llama2",
|
37
|
+
"model_card": "https://github.com/meta-llama/llama",
|
38
|
+
},
|
39
|
+
"kaggle_handle": "kaggle://keras/llama2/keras/llama2_7b_en_int8/1",
|
40
|
+
},
|
41
|
+
"llama2_instruct_7b_en": {
|
42
|
+
"metadata": {
|
43
|
+
"description": (
|
44
|
+
"7 billion parameter, 32-layer, instruction tuned LLaMA 2 "
|
45
|
+
"model."
|
46
|
+
),
|
47
|
+
"params": 6738415616,
|
48
|
+
"official_name": "LLaMA 2",
|
49
|
+
"path": "llama2",
|
50
|
+
"model_card": "https://github.com/meta-llama/llama",
|
51
|
+
},
|
52
|
+
"kaggle_handle": "kaggle://keras/llama2/keras/llama2_instruct_7b_en/1",
|
53
|
+
},
|
54
|
+
"llama2_instruct_7b_en_int8": {
|
55
|
+
"metadata": {
|
56
|
+
"description": (
|
57
|
+
"7 billion parameter, 32-layer, instruction tuned LLaMA 2 "
|
58
|
+
"model with activation and weights quantized to int8."
|
59
|
+
),
|
60
|
+
"params": 6739839488,
|
61
|
+
"official_name": "LLaMA 2",
|
62
|
+
"path": "llama2",
|
63
|
+
"model_card": "https://github.com/meta-llama/llama",
|
64
|
+
},
|
65
|
+
"kaggle_handle": "kaggle://keras/llama2/keras/llama2_instruct_7b_en_int8/1",
|
66
|
+
},
|
67
|
+
"vicuna_1.5_7b_en": {
|
68
|
+
"metadata": {
|
69
|
+
"description": (
|
70
|
+
"7 billion parameter, 32-layer, instruction tuned Vicuna v1.5 "
|
71
|
+
"model."
|
72
|
+
),
|
73
|
+
"params": 6738415616,
|
74
|
+
"official_name": "Vicuna",
|
75
|
+
"path": "vicuna",
|
76
|
+
"model_card": "https://github.com/lm-sys/FastChat",
|
77
|
+
},
|
78
|
+
"kaggle_handle": "kaggle://keras/vicuna/keras/vicuna_1.5_7b_en/1",
|
79
|
+
},
|
80
|
+
}
|
@@ -0,0 +1,84 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
from keras_hub.src.api_export import keras_hub_export
|
16
|
+
from keras_hub.src.tokenizers.sentence_piece_tokenizer import (
|
17
|
+
SentencePieceTokenizer,
|
18
|
+
)
|
19
|
+
|
20
|
+
|
21
|
+
@keras_hub_export("keras_hub.models.LlamaTokenizer")
|
22
|
+
class LlamaTokenizer(SentencePieceTokenizer):
|
23
|
+
"""Llama tokenizer layer based on SentencePiece.
|
24
|
+
|
25
|
+
This tokenizer class will tokenize raw strings into integer sequences and
|
26
|
+
is based on `keras_hub.tokenizers.SentencePieceTokenizer`. Unlike the
|
27
|
+
underlying tokenizer, it will check for all special tokens needed by
|
28
|
+
Llama models and provides a `from_preset()` method to automatically
|
29
|
+
download a matching vocabulary for a Llama preset.
|
30
|
+
|
31
|
+
This tokenizer does not provide truncation or padding of inputs. It can be
|
32
|
+
combined with a `keras_hub.models.LlamaPreprocessor` layer for input
|
33
|
+
packing.
|
34
|
+
|
35
|
+
If input is a batch of strings (rank > 0), the layer will output a
|
36
|
+
`tf.RaggedTensor` where the last dimension of the output is ragged.
|
37
|
+
|
38
|
+
If input is a scalar string (rank == 0), the layer will output a dense
|
39
|
+
`tf.Tensor` with static shape `[None]`.
|
40
|
+
|
41
|
+
Args:
|
42
|
+
proto: Either a `string` path to a SentencePiece proto file, or a
|
43
|
+
`bytes` object with a serialized SentencePiece proto. See the
|
44
|
+
[SentencePiece repository](https://github.com/google/sentencepiece)
|
45
|
+
for more details on the format.
|
46
|
+
|
47
|
+
Examples:
|
48
|
+
```python
|
49
|
+
# Unbatched input.
|
50
|
+
tokenizer = keras_hub.models.LlamaTokenizer.from_preset(
|
51
|
+
"llama_7b_en",
|
52
|
+
)
|
53
|
+
tokenizer("The quick brown fox jumped.")
|
54
|
+
|
55
|
+
# Batched input.
|
56
|
+
tokenizer(["The quick brown fox jumped.", "The fox slept."])
|
57
|
+
|
58
|
+
# Detokenization.
|
59
|
+
tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
|
60
|
+
```
|
61
|
+
"""
|
62
|
+
|
63
|
+
def __init__(self, proto, **kwargs):
|
64
|
+
self.start_token = "<s>"
|
65
|
+
self.end_token = "</s>"
|
66
|
+
super().__init__(proto=proto, **kwargs)
|
67
|
+
|
68
|
+
def set_proto(self, proto):
|
69
|
+
super().set_proto(proto)
|
70
|
+
if proto is not None:
|
71
|
+
for token in [self.start_token, self.end_token]:
|
72
|
+
if token not in self.get_vocabulary():
|
73
|
+
raise ValueError(
|
74
|
+
f"Cannot find token `'{token}'` in the provided "
|
75
|
+
f"`vocabulary`. Please provide `'{token}'` in your "
|
76
|
+
"`vocabulary` or use a pretrained `vocabulary` name."
|
77
|
+
)
|
78
|
+
self.start_token_id = self.token_to_id(self.start_token)
|
79
|
+
self.end_token_id = self.token_to_id(self.end_token)
|
80
|
+
self.pad_token_id = 0
|
81
|
+
else:
|
82
|
+
self.start_token_id = None
|
83
|
+
self.end_token_id = None
|
84
|
+
self.pad_token_id = None
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
from keras_hub.src.models.llama3.llama3_backbone import Llama3Backbone
|
16
|
+
from keras_hub.src.models.llama3.llama3_presets import backbone_presets
|
17
|
+
from keras_hub.src.models.llama3.llama3_tokenizer import Llama3Tokenizer
|
18
|
+
from keras_hub.src.utils.preset_utils import register_presets
|
19
|
+
|
20
|
+
register_presets(backbone_presets, (Llama3Backbone, Llama3Tokenizer))
|