keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keras_hub/__init__.py +52 -0
- keras_hub/api/__init__.py +27 -0
- keras_hub/api/layers/__init__.py +47 -0
- keras_hub/api/metrics/__init__.py +24 -0
- keras_hub/api/models/__init__.py +249 -0
- keras_hub/api/samplers/__init__.py +29 -0
- keras_hub/api/tokenizers/__init__.py +35 -0
- keras_hub/src/__init__.py +13 -0
- keras_hub/src/api_export.py +53 -0
- keras_hub/src/layers/__init__.py +13 -0
- keras_hub/src/layers/modeling/__init__.py +13 -0
- keras_hub/src/layers/modeling/alibi_bias.py +143 -0
- keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
- keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
- keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
- keras_hub/src/layers/modeling/position_embedding.py +123 -0
- keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
- keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
- keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
- keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
- keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
- keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
- keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
- keras_hub/src/layers/preprocessing/__init__.py +13 -0
- keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
- keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
- keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
- keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
- keras_hub/src/layers/preprocessing/random_swap.py +267 -0
- keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
- keras_hub/src/metrics/__init__.py +13 -0
- keras_hub/src/metrics/bleu.py +394 -0
- keras_hub/src/metrics/edit_distance.py +197 -0
- keras_hub/src/metrics/perplexity.py +181 -0
- keras_hub/src/metrics/rouge_base.py +204 -0
- keras_hub/src/metrics/rouge_l.py +97 -0
- keras_hub/src/metrics/rouge_n.py +125 -0
- keras_hub/src/models/__init__.py +13 -0
- keras_hub/src/models/albert/__init__.py +20 -0
- keras_hub/src/models/albert/albert_backbone.py +267 -0
- keras_hub/src/models/albert/albert_classifier.py +202 -0
- keras_hub/src/models/albert/albert_masked_lm.py +129 -0
- keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
- keras_hub/src/models/albert/albert_preprocessor.py +206 -0
- keras_hub/src/models/albert/albert_presets.py +70 -0
- keras_hub/src/models/albert/albert_tokenizer.py +119 -0
- keras_hub/src/models/backbone.py +311 -0
- keras_hub/src/models/bart/__init__.py +20 -0
- keras_hub/src/models/bart/bart_backbone.py +261 -0
- keras_hub/src/models/bart/bart_preprocessor.py +276 -0
- keras_hub/src/models/bart/bart_presets.py +74 -0
- keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
- keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
- keras_hub/src/models/bart/bart_tokenizer.py +124 -0
- keras_hub/src/models/bert/__init__.py +23 -0
- keras_hub/src/models/bert/bert_backbone.py +227 -0
- keras_hub/src/models/bert/bert_classifier.py +183 -0
- keras_hub/src/models/bert/bert_masked_lm.py +131 -0
- keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
- keras_hub/src/models/bert/bert_preprocessor.py +184 -0
- keras_hub/src/models/bert/bert_presets.py +147 -0
- keras_hub/src/models/bert/bert_tokenizer.py +112 -0
- keras_hub/src/models/bloom/__init__.py +20 -0
- keras_hub/src/models/bloom/bloom_attention.py +186 -0
- keras_hub/src/models/bloom/bloom_backbone.py +173 -0
- keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
- keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
- keras_hub/src/models/bloom/bloom_decoder.py +206 -0
- keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
- keras_hub/src/models/bloom/bloom_presets.py +121 -0
- keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
- keras_hub/src/models/causal_lm.py +383 -0
- keras_hub/src/models/classifier.py +109 -0
- keras_hub/src/models/csp_darknet/__init__.py +13 -0
- keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
- keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
- keras_hub/src/models/deberta_v3/__init__.py +24 -0
- keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
- keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
- keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
- keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
- keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
- keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
- keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
- keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
- keras_hub/src/models/densenet/__init__.py +13 -0
- keras_hub/src/models/densenet/densenet_backbone.py +210 -0
- keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
- keras_hub/src/models/distil_bert/__init__.py +26 -0
- keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
- keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
- keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
- keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
- keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
- keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
- keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
- keras_hub/src/models/electra/__init__.py +20 -0
- keras_hub/src/models/electra/electra_backbone.py +247 -0
- keras_hub/src/models/electra/electra_preprocessor.py +154 -0
- keras_hub/src/models/electra/electra_presets.py +95 -0
- keras_hub/src/models/electra/electra_tokenizer.py +104 -0
- keras_hub/src/models/f_net/__init__.py +20 -0
- keras_hub/src/models/f_net/f_net_backbone.py +236 -0
- keras_hub/src/models/f_net/f_net_classifier.py +154 -0
- keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
- keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
- keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
- keras_hub/src/models/f_net/f_net_presets.py +43 -0
- keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
- keras_hub/src/models/falcon/__init__.py +20 -0
- keras_hub/src/models/falcon/falcon_attention.py +156 -0
- keras_hub/src/models/falcon/falcon_backbone.py +164 -0
- keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
- keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
- keras_hub/src/models/falcon/falcon_presets.py +30 -0
- keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
- keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
- keras_hub/src/models/feature_pyramid_backbone.py +73 -0
- keras_hub/src/models/gemma/__init__.py +20 -0
- keras_hub/src/models/gemma/gemma_attention.py +250 -0
- keras_hub/src/models/gemma/gemma_backbone.py +316 -0
- keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
- keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
- keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
- keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
- keras_hub/src/models/gemma/gemma_presets.py +248 -0
- keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
- keras_hub/src/models/gemma/rms_normalization.py +40 -0
- keras_hub/src/models/gpt2/__init__.py +20 -0
- keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
- keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
- keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
- keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
- keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
- keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
- keras_hub/src/models/image_classifier.py +90 -0
- keras_hub/src/models/llama/__init__.py +20 -0
- keras_hub/src/models/llama/llama_attention.py +225 -0
- keras_hub/src/models/llama/llama_backbone.py +188 -0
- keras_hub/src/models/llama/llama_causal_lm.py +327 -0
- keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
- keras_hub/src/models/llama/llama_decoder.py +246 -0
- keras_hub/src/models/llama/llama_layernorm.py +48 -0
- keras_hub/src/models/llama/llama_preprocessor.py +189 -0
- keras_hub/src/models/llama/llama_presets.py +80 -0
- keras_hub/src/models/llama/llama_tokenizer.py +84 -0
- keras_hub/src/models/llama3/__init__.py +20 -0
- keras_hub/src/models/llama3/llama3_backbone.py +84 -0
- keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
- keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
- keras_hub/src/models/llama3/llama3_presets.py +69 -0
- keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
- keras_hub/src/models/masked_lm.py +101 -0
- keras_hub/src/models/mistral/__init__.py +20 -0
- keras_hub/src/models/mistral/mistral_attention.py +238 -0
- keras_hub/src/models/mistral/mistral_backbone.py +203 -0
- keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
- keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
- keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
- keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
- keras_hub/src/models/mistral/mistral_presets.py +48 -0
- keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
- keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
- keras_hub/src/models/mix_transformer/__init__.py +13 -0
- keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
- keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
- keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
- keras_hub/src/models/opt/__init__.py +20 -0
- keras_hub/src/models/opt/opt_backbone.py +173 -0
- keras_hub/src/models/opt/opt_causal_lm.py +301 -0
- keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
- keras_hub/src/models/opt/opt_preprocessor.py +188 -0
- keras_hub/src/models/opt/opt_presets.py +72 -0
- keras_hub/src/models/opt/opt_tokenizer.py +116 -0
- keras_hub/src/models/pali_gemma/__init__.py +23 -0
- keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
- keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
- keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
- keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
- keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
- keras_hub/src/models/phi3/__init__.py +20 -0
- keras_hub/src/models/phi3/phi3_attention.py +260 -0
- keras_hub/src/models/phi3/phi3_backbone.py +224 -0
- keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
- keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
- keras_hub/src/models/phi3/phi3_decoder.py +260 -0
- keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
- keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
- keras_hub/src/models/phi3/phi3_presets.py +50 -0
- keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
- keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
- keras_hub/src/models/preprocessor.py +207 -0
- keras_hub/src/models/resnet/__init__.py +13 -0
- keras_hub/src/models/resnet/resnet_backbone.py +612 -0
- keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
- keras_hub/src/models/roberta/__init__.py +20 -0
- keras_hub/src/models/roberta/roberta_backbone.py +184 -0
- keras_hub/src/models/roberta/roberta_classifier.py +209 -0
- keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
- keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
- keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
- keras_hub/src/models/roberta/roberta_presets.py +43 -0
- keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
- keras_hub/src/models/seq_2_seq_lm.py +54 -0
- keras_hub/src/models/t5/__init__.py +20 -0
- keras_hub/src/models/t5/t5_backbone.py +261 -0
- keras_hub/src/models/t5/t5_layer_norm.py +35 -0
- keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
- keras_hub/src/models/t5/t5_presets.py +95 -0
- keras_hub/src/models/t5/t5_tokenizer.py +100 -0
- keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
- keras_hub/src/models/task.py +419 -0
- keras_hub/src/models/vgg/__init__.py +13 -0
- keras_hub/src/models/vgg/vgg_backbone.py +158 -0
- keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
- keras_hub/src/models/vit_det/__init__.py +13 -0
- keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
- keras_hub/src/models/vit_det/vit_layers.py +565 -0
- keras_hub/src/models/whisper/__init__.py +20 -0
- keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
- keras_hub/src/models/whisper/whisper_backbone.py +305 -0
- keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
- keras_hub/src/models/whisper/whisper_decoder.py +141 -0
- keras_hub/src/models/whisper/whisper_encoder.py +106 -0
- keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
- keras_hub/src/models/whisper/whisper_presets.py +148 -0
- keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
- keras_hub/src/models/xlm_roberta/__init__.py +26 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
- keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
- keras_hub/src/models/xlnet/__init__.py +13 -0
- keras_hub/src/models/xlnet/relative_attention.py +459 -0
- keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
- keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
- keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
- keras_hub/src/samplers/__init__.py +13 -0
- keras_hub/src/samplers/beam_sampler.py +207 -0
- keras_hub/src/samplers/contrastive_sampler.py +231 -0
- keras_hub/src/samplers/greedy_sampler.py +50 -0
- keras_hub/src/samplers/random_sampler.py +77 -0
- keras_hub/src/samplers/sampler.py +237 -0
- keras_hub/src/samplers/serialization.py +97 -0
- keras_hub/src/samplers/top_k_sampler.py +92 -0
- keras_hub/src/samplers/top_p_sampler.py +113 -0
- keras_hub/src/tests/__init__.py +13 -0
- keras_hub/src/tests/test_case.py +608 -0
- keras_hub/src/tokenizers/__init__.py +13 -0
- keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
- keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
- keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
- keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
- keras_hub/src/tokenizers/tokenizer.py +235 -0
- keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
- keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
- keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
- keras_hub/src/utils/__init__.py +13 -0
- keras_hub/src/utils/keras_utils.py +130 -0
- keras_hub/src/utils/pipeline_model.py +293 -0
- keras_hub/src/utils/preset_utils.py +621 -0
- keras_hub/src/utils/python_utils.py +21 -0
- keras_hub/src/utils/tensor_utils.py +206 -0
- keras_hub/src/utils/timm/__init__.py +13 -0
- keras_hub/src/utils/timm/convert.py +37 -0
- keras_hub/src/utils/timm/convert_resnet.py +171 -0
- keras_hub/src/utils/transformers/__init__.py +13 -0
- keras_hub/src/utils/transformers/convert.py +101 -0
- keras_hub/src/utils/transformers/convert_bert.py +173 -0
- keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
- keras_hub/src/utils/transformers/convert_gemma.py +187 -0
- keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
- keras_hub/src/utils/transformers/convert_llama3.py +136 -0
- keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
- keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
- keras_hub/src/version_utils.py +23 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,241 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
import keras
|
15
|
+
from keras import ops
|
16
|
+
|
17
|
+
from keras_hub.src.layers.modeling.transformer_layer_utils import (
|
18
|
+
compute_causal_mask,
|
19
|
+
)
|
20
|
+
from keras_hub.src.layers.modeling.transformer_layer_utils import (
|
21
|
+
merge_padding_and_attention_mask,
|
22
|
+
)
|
23
|
+
from keras_hub.src.models.gemma.gemma_attention import CachedGemmaAttention
|
24
|
+
from keras_hub.src.models.gemma.rms_normalization import RMSNormalization
|
25
|
+
|
26
|
+
|
27
|
+
class GemmaDecoderBlock(keras.layers.Layer):
|
28
|
+
def __init__(
|
29
|
+
self,
|
30
|
+
hidden_dim,
|
31
|
+
intermediate_dim,
|
32
|
+
head_dim,
|
33
|
+
num_query_heads,
|
34
|
+
num_key_value_heads,
|
35
|
+
query_head_dim_normalize=True,
|
36
|
+
use_post_ffw_norm=False,
|
37
|
+
use_post_attention_norm=False,
|
38
|
+
logit_soft_cap=None,
|
39
|
+
use_sliding_window_attention=False,
|
40
|
+
sliding_window_size=4096,
|
41
|
+
layer_norm_epsilon=1e-6,
|
42
|
+
dropout=0,
|
43
|
+
**kwargs,
|
44
|
+
):
|
45
|
+
super().__init__(**kwargs)
|
46
|
+
|
47
|
+
self.intermediate_dim = intermediate_dim
|
48
|
+
self.hidden_dim = hidden_dim
|
49
|
+
self.num_query_heads = num_query_heads
|
50
|
+
self.num_key_value_heads = num_key_value_heads
|
51
|
+
self.head_dim = head_dim
|
52
|
+
self.layer_norm_epsilon = layer_norm_epsilon
|
53
|
+
self.dropout = dropout
|
54
|
+
self.query_head_dim_normalize = query_head_dim_normalize
|
55
|
+
self.use_post_ffw_norm = use_post_ffw_norm
|
56
|
+
self.use_post_attention_norm = use_post_attention_norm
|
57
|
+
self.logit_soft_cap = logit_soft_cap
|
58
|
+
self.use_sliding_window_attention = use_sliding_window_attention
|
59
|
+
self.sliding_window_size = sliding_window_size
|
60
|
+
|
61
|
+
self.pre_attention_norm = RMSNormalization(
|
62
|
+
epsilon=self.layer_norm_epsilon,
|
63
|
+
dtype=self.dtype_policy,
|
64
|
+
name="pre_attention_norm",
|
65
|
+
)
|
66
|
+
|
67
|
+
if use_post_attention_norm:
|
68
|
+
self.post_attention_norm = RMSNormalization(
|
69
|
+
epsilon=self.layer_norm_epsilon,
|
70
|
+
dtype=self.dtype_policy,
|
71
|
+
name="pre_attention_norm",
|
72
|
+
)
|
73
|
+
|
74
|
+
self.attention = CachedGemmaAttention(
|
75
|
+
head_dim=head_dim,
|
76
|
+
num_query_heads=num_query_heads,
|
77
|
+
num_key_value_heads=num_key_value_heads,
|
78
|
+
logit_soft_cap=logit_soft_cap,
|
79
|
+
use_sliding_window_attention=use_sliding_window_attention,
|
80
|
+
sliding_window_size=sliding_window_size,
|
81
|
+
query_head_dim_normalize=True,
|
82
|
+
dropout=dropout,
|
83
|
+
dtype=self.dtype_policy,
|
84
|
+
name="attention",
|
85
|
+
)
|
86
|
+
|
87
|
+
if self.dropout > 0:
|
88
|
+
self.attention_dropout = keras.layers.Dropout(rate=dropout)
|
89
|
+
self.feedforward_dropout = keras.layers.Dropout(rate=dropout)
|
90
|
+
|
91
|
+
self.pre_ffw_norm = RMSNormalization(
|
92
|
+
epsilon=self.layer_norm_epsilon,
|
93
|
+
dtype=self.dtype_policy,
|
94
|
+
name="pre_ffw_norm",
|
95
|
+
)
|
96
|
+
|
97
|
+
if use_post_ffw_norm:
|
98
|
+
self.post_ffw_norm = RMSNormalization(
|
99
|
+
epsilon=self.layer_norm_epsilon,
|
100
|
+
dtype=self.dtype_policy,
|
101
|
+
name="post_ffw_norm",
|
102
|
+
)
|
103
|
+
|
104
|
+
self.gating_ffw = keras.layers.EinsumDense(
|
105
|
+
equation="btd,df->btf",
|
106
|
+
output_shape=(None, self.intermediate_dim // 2),
|
107
|
+
dtype=self.dtype_policy,
|
108
|
+
name="ffw_gating",
|
109
|
+
)
|
110
|
+
|
111
|
+
self.gating_ffw_2 = keras.layers.EinsumDense(
|
112
|
+
equation="btd,df->btf",
|
113
|
+
output_shape=(None, self.intermediate_dim // 2),
|
114
|
+
dtype=self.dtype_policy,
|
115
|
+
name="ffw_gating_2",
|
116
|
+
)
|
117
|
+
|
118
|
+
self.ffw_linear = keras.layers.EinsumDense(
|
119
|
+
equation="btf,fd->btd",
|
120
|
+
output_shape=(None, self.hidden_dim),
|
121
|
+
dtype=self.dtype_policy,
|
122
|
+
name="ffw_linear",
|
123
|
+
)
|
124
|
+
|
125
|
+
def build(self, input_shape):
|
126
|
+
self.pre_attention_norm.build(input_shape)
|
127
|
+
self.attention.build(input_shape)
|
128
|
+
|
129
|
+
if self.use_post_attention_norm:
|
130
|
+
shape = self.attention.compute_output_shape(input_shape)
|
131
|
+
self.post_attention_norm.build(shape)
|
132
|
+
|
133
|
+
shape = input_shape
|
134
|
+
self.pre_ffw_norm.build(shape)
|
135
|
+
self.gating_ffw.build(shape)
|
136
|
+
self.gating_ffw_2.build(shape)
|
137
|
+
|
138
|
+
shape = self.gating_ffw.compute_output_shape(shape)
|
139
|
+
self.ffw_linear.build(shape)
|
140
|
+
|
141
|
+
if self.use_post_ffw_norm:
|
142
|
+
shape = self.ffw_linear.compute_output_shape(shape)
|
143
|
+
self.post_ffw_norm.build(shape)
|
144
|
+
|
145
|
+
self.built = True
|
146
|
+
|
147
|
+
def compute_output_shape(self, input_shape):
|
148
|
+
# Isometric
|
149
|
+
return input_shape
|
150
|
+
|
151
|
+
def _compute_attention_mask(
|
152
|
+
self, x, padding_mask, cache, cache_update_index
|
153
|
+
):
|
154
|
+
decoder_mask = merge_padding_and_attention_mask(
|
155
|
+
inputs=x, padding_mask=padding_mask, attention_mask=None
|
156
|
+
)
|
157
|
+
batch_size = ops.shape(x)[0]
|
158
|
+
input_length = output_length = ops.shape(x)[1]
|
159
|
+
if cache is not None:
|
160
|
+
input_length = ops.shape(cache)[2]
|
161
|
+
|
162
|
+
causal_mask = compute_causal_mask(
|
163
|
+
batch_size=batch_size,
|
164
|
+
input_length=input_length,
|
165
|
+
output_length=output_length,
|
166
|
+
cache_index=cache_update_index,
|
167
|
+
)
|
168
|
+
|
169
|
+
return (
|
170
|
+
ops.minimum(decoder_mask, causal_mask)
|
171
|
+
if decoder_mask is not None
|
172
|
+
else causal_mask
|
173
|
+
)
|
174
|
+
|
175
|
+
def call(
|
176
|
+
self,
|
177
|
+
x,
|
178
|
+
padding_mask=None,
|
179
|
+
cache=None,
|
180
|
+
cache_update_index=0,
|
181
|
+
):
|
182
|
+
normalized_x = self.pre_attention_norm(x)
|
183
|
+
attention_mask = self._compute_attention_mask(
|
184
|
+
normalized_x, padding_mask, cache, cache_update_index
|
185
|
+
)
|
186
|
+
if cache is not None:
|
187
|
+
attention, new_cache = self.attention(
|
188
|
+
normalized_x,
|
189
|
+
attention_mask=attention_mask,
|
190
|
+
cache=cache,
|
191
|
+
cache_update_index=cache_update_index,
|
192
|
+
)
|
193
|
+
else:
|
194
|
+
attention = self.attention(
|
195
|
+
normalized_x,
|
196
|
+
attention_mask=attention_mask,
|
197
|
+
)
|
198
|
+
|
199
|
+
if self.use_post_attention_norm:
|
200
|
+
attention = self.post_attention_norm(attention)
|
201
|
+
|
202
|
+
if self.dropout:
|
203
|
+
attention = self.attention_dropout(attention)
|
204
|
+
|
205
|
+
attention_x = x + attention
|
206
|
+
normalized_x = self.pre_ffw_norm(attention_x)
|
207
|
+
|
208
|
+
x1 = self.gating_ffw(normalized_x)
|
209
|
+
x2 = self.gating_ffw_2(normalized_x)
|
210
|
+
x = keras.activations.gelu(x1, approximate=True) * x2
|
211
|
+
x = self.ffw_linear(x)
|
212
|
+
|
213
|
+
if self.use_post_ffw_norm:
|
214
|
+
x = self.post_ffw_norm(x)
|
215
|
+
|
216
|
+
x = x + attention_x
|
217
|
+
|
218
|
+
if cache is not None:
|
219
|
+
return x, new_cache
|
220
|
+
return x
|
221
|
+
|
222
|
+
def get_config(self):
|
223
|
+
config = super().get_config()
|
224
|
+
config.update(
|
225
|
+
{
|
226
|
+
"hidden_dim": self.hidden_dim,
|
227
|
+
"intermediate_dim": self.intermediate_dim,
|
228
|
+
"head_dim": self.head_dim,
|
229
|
+
"num_query_heads": self.num_query_heads,
|
230
|
+
"num_key_value_heads": self.num_key_value_heads,
|
231
|
+
"layer_norm_epsilon": self.layer_norm_epsilon,
|
232
|
+
"dropout": self.dropout,
|
233
|
+
"use_post_ffw_norm": self.use_post_ffw_norm,
|
234
|
+
"use_post_attention_norm": self.use_post_attention_norm,
|
235
|
+
"logit_soft_cap": self.logit_soft_cap,
|
236
|
+
"use_sliding_window_attention": self.use_sliding_window_attention,
|
237
|
+
"sliding_window_size": self.sliding_window_size,
|
238
|
+
"query_head_dim_normalize": self.query_head_dim_normalize,
|
239
|
+
}
|
240
|
+
)
|
241
|
+
return config
|
@@ -0,0 +1,191 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
import keras
|
17
|
+
|
18
|
+
from keras_hub.src.api_export import keras_hub_export
|
19
|
+
from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
|
20
|
+
from keras_hub.src.models.gemma.gemma_tokenizer import GemmaTokenizer
|
21
|
+
from keras_hub.src.models.preprocessor import Preprocessor
|
22
|
+
from keras_hub.src.utils.keras_utils import (
|
23
|
+
convert_inputs_to_list_of_tensor_segments,
|
24
|
+
)
|
25
|
+
|
26
|
+
|
27
|
+
@keras_hub_export("keras_hub.models.GemmaPreprocessor")
|
28
|
+
class GemmaPreprocessor(Preprocessor):
|
29
|
+
"""Gemma preprocessing layer which tokenizes and packs inputs.
|
30
|
+
|
31
|
+
This preprocessing layer will do 2 things:
|
32
|
+
|
33
|
+
- Tokenize the inputs using the `tokenizer`.
|
34
|
+
- Construct a dictionary with keys `"token_ids"`, `"padding_mask"`, that can
|
35
|
+
be passed directly to a `keras_hub.models.GemmaBackbone`.
|
36
|
+
|
37
|
+
This layer can be used directly with `tf.data.Dataset.map` to preprocess
|
38
|
+
string data in the `(x, y, sample_weight)` format used by
|
39
|
+
`keras.Model.fit`.
|
40
|
+
|
41
|
+
The call method of this layer accepts three arguments, `x`, `y`, and
|
42
|
+
`sample_weight`. `x` can be a python string or tensor representing a single
|
43
|
+
segment, a list of python strings representing a batch of single segments,
|
44
|
+
or a list of tensors representing multiple segments to be packed together.
|
45
|
+
`y` and `sample_weight` are both optional, can have any format, and will be
|
46
|
+
passed through unaltered.
|
47
|
+
|
48
|
+
`GemmaPreprocessor` expects the input to have only one segment, as Gemma is
|
49
|
+
mainly used for generation tasks. For tasks having multi-segment inputs
|
50
|
+
please combine inputs into a single string input before passing to the
|
51
|
+
preprocessor layer.
|
52
|
+
|
53
|
+
Args:
|
54
|
+
tokenizer: A `keras_hub.models.GemmaTokenizer` instance.
|
55
|
+
sequence_length: The length of the packed inputs.
|
56
|
+
add_start_token: If `True`, the preprocessor will prepend the tokenizer
|
57
|
+
start token to each input sequence.
|
58
|
+
add_end_token: If `True`, the preprocessor will append the tokenizer
|
59
|
+
end token to each input sequence.
|
60
|
+
|
61
|
+
Call arguments:
|
62
|
+
x: A string, `tf.Tensor` or list of python strings.
|
63
|
+
y: Any label data. Will be passed through unaltered.
|
64
|
+
sample_weight: Any label weight data. Will be passed through unaltered.
|
65
|
+
sequence_length: Pass to override the configured `sequence_length` of
|
66
|
+
the layer.
|
67
|
+
|
68
|
+
Examples:
|
69
|
+
|
70
|
+
Directly calling the layer on data.
|
71
|
+
```python
|
72
|
+
preprocessor = keras_hub.models.GemmaPreprocessor.from_preset(
|
73
|
+
"gemma_2b_en"
|
74
|
+
)
|
75
|
+
|
76
|
+
# Tokenize and pack a single sentence.
|
77
|
+
preprocessor("The quick brown fox jumped.")
|
78
|
+
|
79
|
+
# Tokenize a batch of sentences.
|
80
|
+
preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
|
81
|
+
|
82
|
+
# Custom vocabulary.
|
83
|
+
bytes_io = io.BytesIO()
|
84
|
+
ds = tf.data.Dataset.from_tensor_slices(["The quick brown fox jumped."])
|
85
|
+
sentencepiece.SentencePieceTrainer.train(
|
86
|
+
sentence_iterator=ds.as_numpy_iterator(),
|
87
|
+
model_writer=bytes_io,
|
88
|
+
vocab_size=8,
|
89
|
+
model_type="WORD",
|
90
|
+
pad_id=0,
|
91
|
+
bos_id=1,
|
92
|
+
eos_id=2,
|
93
|
+
unk_id=3,
|
94
|
+
pad_piece="<pad>",
|
95
|
+
bos_piece="<bos>",
|
96
|
+
eos_piece="<eos>",
|
97
|
+
unk_piece="<unk>",
|
98
|
+
)
|
99
|
+
tokenizer = keras_hub.models.GemmaTokenizer(
|
100
|
+
proto=bytes_io.getvalue(),
|
101
|
+
)
|
102
|
+
preprocessor = keras_hub.models.GemmaPreprocessor(tokenizer=tokenizer)
|
103
|
+
preprocessor("The quick brown fox jumped.")
|
104
|
+
```
|
105
|
+
|
106
|
+
Apply preprocessing to a `tf.data.Dataset`.
|
107
|
+
```python
|
108
|
+
preprocessor = keras_hub.models.GemmaPreprocessor.from_preset(
|
109
|
+
"gemma_2b_en"
|
110
|
+
)
|
111
|
+
|
112
|
+
text = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
113
|
+
label = tf.constant([1, 1])
|
114
|
+
|
115
|
+
# Map labeled single sentences.
|
116
|
+
ds = tf.data.Dataset.from_tensor_slices((text, label))
|
117
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
118
|
+
|
119
|
+
# Map unlabeled single sentences.
|
120
|
+
ds = tf.data.Dataset.from_tensor_slices(text)
|
121
|
+
ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
|
122
|
+
```
|
123
|
+
"""
|
124
|
+
|
125
|
+
tokenizer_cls = GemmaTokenizer
|
126
|
+
|
127
|
+
def __init__(
|
128
|
+
self,
|
129
|
+
tokenizer,
|
130
|
+
sequence_length=1024,
|
131
|
+
add_start_token=True,
|
132
|
+
add_end_token=True,
|
133
|
+
**kwargs,
|
134
|
+
):
|
135
|
+
super().__init__(**kwargs)
|
136
|
+
|
137
|
+
self.tokenizer = tokenizer
|
138
|
+
self.sequence_length = sequence_length
|
139
|
+
self.add_start_token = add_start_token
|
140
|
+
self.add_end_token = add_end_token
|
141
|
+
|
142
|
+
def build(self, input_shape):
|
143
|
+
# Defer packer creation to `build()` so that we can be sure tokenizer
|
144
|
+
# assets have loaded when restoring a saved model.
|
145
|
+
self.packer = StartEndPacker(
|
146
|
+
start_value=self.tokenizer.start_token_id,
|
147
|
+
end_value=self.tokenizer.end_token_id,
|
148
|
+
pad_value=self.tokenizer.pad_token_id,
|
149
|
+
sequence_length=self.sequence_length,
|
150
|
+
return_padding_mask=True,
|
151
|
+
)
|
152
|
+
self.built = True
|
153
|
+
|
154
|
+
def call(
|
155
|
+
self,
|
156
|
+
x,
|
157
|
+
y=None,
|
158
|
+
sample_weight=None,
|
159
|
+
sequence_length=None,
|
160
|
+
):
|
161
|
+
x = convert_inputs_to_list_of_tensor_segments(x)
|
162
|
+
if len(x) != 1:
|
163
|
+
raise ValueError(
|
164
|
+
"GemmaPreprocessor requires each input to contain only "
|
165
|
+
f"one segment, but received {len(x)}. If you are using Gemma "
|
166
|
+
"for a multi-segment classification task, please combine your "
|
167
|
+
"input into a single string."
|
168
|
+
)
|
169
|
+
sequence_length = sequence_length or self.sequence_length
|
170
|
+
token_ids, padding_mask = self.packer(
|
171
|
+
self.tokenizer(x[0]),
|
172
|
+
sequence_length=sequence_length,
|
173
|
+
add_start_value=self.add_start_token,
|
174
|
+
add_end_value=self.add_end_token,
|
175
|
+
)
|
176
|
+
x = {
|
177
|
+
"token_ids": token_ids,
|
178
|
+
"padding_mask": padding_mask,
|
179
|
+
}
|
180
|
+
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
|
181
|
+
|
182
|
+
def get_config(self):
|
183
|
+
config = super().get_config()
|
184
|
+
config.update(
|
185
|
+
{
|
186
|
+
"sequence_length": self.sequence_length,
|
187
|
+
"add_start_token": self.add_start_token,
|
188
|
+
"add_end_token": self.add_end_token,
|
189
|
+
}
|
190
|
+
)
|
191
|
+
return config
|
@@ -0,0 +1,248 @@
|
|
1
|
+
# Copyright 2024 The KerasHub Authors
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
"""Gemma model preset configurations."""
|
15
|
+
|
16
|
+
# Metadata for loading pretrained model weights.
|
17
|
+
backbone_presets = {
|
18
|
+
"gemma_2b_en": {
|
19
|
+
"metadata": {
|
20
|
+
"description": "2 billion parameter, 18-layer, base Gemma model.",
|
21
|
+
"params": 2506172416,
|
22
|
+
"official_name": "Gemma",
|
23
|
+
"path": "gemma",
|
24
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
25
|
+
},
|
26
|
+
"kaggle_handle": "kaggle://keras/gemma/keras/gemma_2b_en/2",
|
27
|
+
},
|
28
|
+
"gemma_instruct_2b_en": {
|
29
|
+
"metadata": {
|
30
|
+
"description": (
|
31
|
+
"2 billion parameter, 18-layer, instruction tuned Gemma model."
|
32
|
+
),
|
33
|
+
"params": 2506172416,
|
34
|
+
"official_name": "Gemma",
|
35
|
+
"path": "gemma",
|
36
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
37
|
+
},
|
38
|
+
"kaggle_handle": "kaggle://keras/gemma/keras/gemma_instruct_2b_en/2",
|
39
|
+
},
|
40
|
+
"gemma_1.1_instruct_2b_en": {
|
41
|
+
"metadata": {
|
42
|
+
"description": (
|
43
|
+
"2 billion parameter, 18-layer, instruction tuned Gemma model. "
|
44
|
+
"The 1.1 update improves model quality."
|
45
|
+
),
|
46
|
+
"params": 2506172416,
|
47
|
+
"official_name": "Gemma",
|
48
|
+
"path": "gemma",
|
49
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
50
|
+
},
|
51
|
+
"kaggle_handle": "kaggle://keras/gemma/keras/gemma_1.1_instruct_2b_en/3",
|
52
|
+
},
|
53
|
+
"code_gemma_1.1_2b_en": {
|
54
|
+
"metadata": {
|
55
|
+
"description": (
|
56
|
+
"2 billion parameter, 18-layer, CodeGemma model. This model "
|
57
|
+
"has been trained on a fill-in-the-middle (FIM) task for code "
|
58
|
+
"completion. The 1.1 update improves model quality."
|
59
|
+
),
|
60
|
+
"params": 2506172416,
|
61
|
+
"official_name": "Gemma",
|
62
|
+
"path": "gemma",
|
63
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
64
|
+
},
|
65
|
+
"kaggle_handle": "kaggle://keras/codegemma/keras/code_gemma_1.1_2b_en/1",
|
66
|
+
},
|
67
|
+
"code_gemma_2b_en": {
|
68
|
+
"metadata": {
|
69
|
+
"description": (
|
70
|
+
"2 billion parameter, 18-layer, CodeGemma model. This model "
|
71
|
+
"has been trained on a fill-in-the-middle (FIM) task for code "
|
72
|
+
"completion."
|
73
|
+
),
|
74
|
+
"params": 2506172416,
|
75
|
+
"official_name": "Gemma",
|
76
|
+
"path": "gemma",
|
77
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
78
|
+
},
|
79
|
+
"kaggle_handle": "kaggle://keras/codegemma/keras/code_gemma_2b_en/1",
|
80
|
+
},
|
81
|
+
"gemma_7b_en": {
|
82
|
+
"metadata": {
|
83
|
+
"description": "7 billion parameter, 28-layer, base Gemma model.",
|
84
|
+
"params": 8537680896,
|
85
|
+
"official_name": "Gemma",
|
86
|
+
"path": "gemma",
|
87
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
88
|
+
},
|
89
|
+
"kaggle_handle": "kaggle://keras/gemma/keras/gemma_7b_en/2",
|
90
|
+
},
|
91
|
+
"gemma_instruct_7b_en": {
|
92
|
+
"metadata": {
|
93
|
+
"description": (
|
94
|
+
"7 billion parameter, 28-layer, instruction tuned Gemma model."
|
95
|
+
),
|
96
|
+
"params": 8537680896,
|
97
|
+
"official_name": "Gemma",
|
98
|
+
"path": "gemma",
|
99
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
100
|
+
},
|
101
|
+
"kaggle_handle": "kaggle://keras/gemma/keras/gemma_instruct_7b_en/2",
|
102
|
+
},
|
103
|
+
"gemma_1.1_instruct_7b_en": {
|
104
|
+
"metadata": {
|
105
|
+
"description": (
|
106
|
+
"7 billion parameter, 28-layer, instruction tuned Gemma model. "
|
107
|
+
"The 1.1 update improves model quality."
|
108
|
+
),
|
109
|
+
"params": 8537680896,
|
110
|
+
"official_name": "Gemma",
|
111
|
+
"path": "gemma",
|
112
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
113
|
+
},
|
114
|
+
"kaggle_handle": "kaggle://keras/gemma/keras/gemma_1.1_instruct_7b_en/3",
|
115
|
+
},
|
116
|
+
"code_gemma_7b_en": {
|
117
|
+
"metadata": {
|
118
|
+
"description": (
|
119
|
+
"7 billion parameter, 28-layer, CodeGemma model. This model "
|
120
|
+
"has been trained on a fill-in-the-middle (FIM) task for code "
|
121
|
+
"completion."
|
122
|
+
),
|
123
|
+
"params": 8537680896,
|
124
|
+
"official_name": "Gemma",
|
125
|
+
"path": "gemma",
|
126
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
127
|
+
},
|
128
|
+
"kaggle_handle": "kaggle://keras/codegemma/keras/code_gemma_7b_en/1",
|
129
|
+
},
|
130
|
+
"code_gemma_instruct_7b_en": {
|
131
|
+
"metadata": {
|
132
|
+
"description": (
|
133
|
+
"7 billion parameter, 28-layer, instruction tuned CodeGemma "
|
134
|
+
"model. This model has been trained for chat use cases related "
|
135
|
+
"to code."
|
136
|
+
),
|
137
|
+
"params": 8537680896,
|
138
|
+
"official_name": "Gemma",
|
139
|
+
"path": "gemma",
|
140
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
141
|
+
},
|
142
|
+
"kaggle_handle": "kaggle://keras/codegemma/keras/code_gemma_instruct_7b_en/1",
|
143
|
+
},
|
144
|
+
"code_gemma_1.1_instruct_7b_en": {
|
145
|
+
"metadata": {
|
146
|
+
"description": (
|
147
|
+
"7 billion parameter, 28-layer, instruction tuned CodeGemma "
|
148
|
+
"model. This model has been trained for chat use cases related "
|
149
|
+
"to code. The 1.1 update improves model quality."
|
150
|
+
),
|
151
|
+
"params": 8537680896,
|
152
|
+
"official_name": "Gemma",
|
153
|
+
"path": "gemma",
|
154
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
155
|
+
},
|
156
|
+
"kaggle_handle": "kaggle://keras/codegemma/keras/code_gemma_1.1_instruct_7b_en/1",
|
157
|
+
},
|
158
|
+
"gemma2_2b_en": {
|
159
|
+
"metadata": {
|
160
|
+
"description": "2 billion parameter, 26-layer, base Gemma model.",
|
161
|
+
"params": 2614341888,
|
162
|
+
"official_name": "Gemma",
|
163
|
+
"path": "gemma",
|
164
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
165
|
+
},
|
166
|
+
"kaggle_handle": "kaggle://keras/gemma2/keras/gemma2_2b_en/1",
|
167
|
+
},
|
168
|
+
"gemma2_instruct_2b_en": {
|
169
|
+
"metadata": {
|
170
|
+
"description": "2 billion parameter, 26-layer, instruction tuned Gemma model.",
|
171
|
+
"params": 2614341888,
|
172
|
+
"official_name": "Gemma",
|
173
|
+
"path": "gemma",
|
174
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
175
|
+
},
|
176
|
+
"kaggle_handle": "kaggle://keras/gemma2/keras/gemma2_instruct_2b_en/1",
|
177
|
+
},
|
178
|
+
"gemma2_9b_en": {
|
179
|
+
"metadata": {
|
180
|
+
"description": "9 billion parameter, 42-layer, base Gemma model.",
|
181
|
+
"params": 9241705984,
|
182
|
+
"official_name": "Gemma",
|
183
|
+
"path": "gemma",
|
184
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
185
|
+
},
|
186
|
+
"kaggle_handle": "kaggle://keras/gemma2/keras/gemma2_9b_en/2",
|
187
|
+
},
|
188
|
+
"gemma2_instruct_9b_en": {
|
189
|
+
"metadata": {
|
190
|
+
"description": "9 billion parameter, 42-layer, instruction tuned Gemma model.",
|
191
|
+
"params": 9241705984,
|
192
|
+
"official_name": "Gemma",
|
193
|
+
"path": "gemma",
|
194
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
195
|
+
},
|
196
|
+
"kaggle_handle": "kaggle://keras/gemma2/keras/gemma2_instruct_9b_en/2",
|
197
|
+
},
|
198
|
+
"gemma2_27b_en": {
|
199
|
+
"metadata": {
|
200
|
+
"description": "27 billion parameter, 42-layer, base Gemma model.",
|
201
|
+
"params": 27227128320,
|
202
|
+
"official_name": "Gemma",
|
203
|
+
"path": "gemma",
|
204
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
205
|
+
},
|
206
|
+
"kaggle_handle": "kaggle://keras/gemma2/keras/gemma2_27b_en/1",
|
207
|
+
},
|
208
|
+
"gemma2_instruct_27b_en": {
|
209
|
+
"metadata": {
|
210
|
+
"description": "27 billion parameter, 42-layer, instruction tuned Gemma model.",
|
211
|
+
"params": 27227128320,
|
212
|
+
"official_name": "Gemma",
|
213
|
+
"path": "gemma",
|
214
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
215
|
+
},
|
216
|
+
"kaggle_handle": "kaggle://keras/gemma2/keras/gemma2_instruct_27b_en/1",
|
217
|
+
},
|
218
|
+
"shieldgemma_2b_en": {
|
219
|
+
"metadata": {
|
220
|
+
"description": "2 billion parameter, 26-layer, ShieldGemma model.",
|
221
|
+
"params": 2614341888,
|
222
|
+
"official_name": "Gemma",
|
223
|
+
"path": "gemma",
|
224
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
225
|
+
},
|
226
|
+
"kaggle_handle": "kaggle://google/shieldgemma/keras/shieldgemma_2b_en/1",
|
227
|
+
},
|
228
|
+
"shieldgemma_9b_en": {
|
229
|
+
"metadata": {
|
230
|
+
"description": "9 billion parameter, 42-layer, ShieldGemma model.",
|
231
|
+
"params": 9241705984,
|
232
|
+
"official_name": "Gemma",
|
233
|
+
"path": "gemma",
|
234
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
235
|
+
},
|
236
|
+
"kaggle_handle": "kaggle://google/shieldgemma/keras/shieldgemma_9b_en/1",
|
237
|
+
},
|
238
|
+
"shieldgemma_27b_en": {
|
239
|
+
"metadata": {
|
240
|
+
"description": "27 billion parameter, 42-layer, ShieldGemma model.",
|
241
|
+
"params": 27227128320,
|
242
|
+
"official_name": "Gemma",
|
243
|
+
"path": "gemma",
|
244
|
+
"model_card": "https://www.kaggle.com/models/google/gemma",
|
245
|
+
},
|
246
|
+
"kaggle_handle": "kaggle://google/shieldgemma/keras/shieldgemma_27b_en/1",
|
247
|
+
},
|
248
|
+
}
|