keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (297) hide show
  1. keras_hub/__init__.py +52 -0
  2. keras_hub/api/__init__.py +27 -0
  3. keras_hub/api/layers/__init__.py +47 -0
  4. keras_hub/api/metrics/__init__.py +24 -0
  5. keras_hub/api/models/__init__.py +249 -0
  6. keras_hub/api/samplers/__init__.py +29 -0
  7. keras_hub/api/tokenizers/__init__.py +35 -0
  8. keras_hub/src/__init__.py +13 -0
  9. keras_hub/src/api_export.py +53 -0
  10. keras_hub/src/layers/__init__.py +13 -0
  11. keras_hub/src/layers/modeling/__init__.py +13 -0
  12. keras_hub/src/layers/modeling/alibi_bias.py +143 -0
  13. keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
  14. keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
  15. keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
  16. keras_hub/src/layers/modeling/position_embedding.py +123 -0
  17. keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
  18. keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
  19. keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
  20. keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
  21. keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
  22. keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
  23. keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
  24. keras_hub/src/layers/preprocessing/__init__.py +13 -0
  25. keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
  26. keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
  27. keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
  28. keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
  29. keras_hub/src/layers/preprocessing/random_swap.py +267 -0
  30. keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
  31. keras_hub/src/metrics/__init__.py +13 -0
  32. keras_hub/src/metrics/bleu.py +394 -0
  33. keras_hub/src/metrics/edit_distance.py +197 -0
  34. keras_hub/src/metrics/perplexity.py +181 -0
  35. keras_hub/src/metrics/rouge_base.py +204 -0
  36. keras_hub/src/metrics/rouge_l.py +97 -0
  37. keras_hub/src/metrics/rouge_n.py +125 -0
  38. keras_hub/src/models/__init__.py +13 -0
  39. keras_hub/src/models/albert/__init__.py +20 -0
  40. keras_hub/src/models/albert/albert_backbone.py +267 -0
  41. keras_hub/src/models/albert/albert_classifier.py +202 -0
  42. keras_hub/src/models/albert/albert_masked_lm.py +129 -0
  43. keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
  44. keras_hub/src/models/albert/albert_preprocessor.py +206 -0
  45. keras_hub/src/models/albert/albert_presets.py +70 -0
  46. keras_hub/src/models/albert/albert_tokenizer.py +119 -0
  47. keras_hub/src/models/backbone.py +311 -0
  48. keras_hub/src/models/bart/__init__.py +20 -0
  49. keras_hub/src/models/bart/bart_backbone.py +261 -0
  50. keras_hub/src/models/bart/bart_preprocessor.py +276 -0
  51. keras_hub/src/models/bart/bart_presets.py +74 -0
  52. keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
  53. keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
  54. keras_hub/src/models/bart/bart_tokenizer.py +124 -0
  55. keras_hub/src/models/bert/__init__.py +23 -0
  56. keras_hub/src/models/bert/bert_backbone.py +227 -0
  57. keras_hub/src/models/bert/bert_classifier.py +183 -0
  58. keras_hub/src/models/bert/bert_masked_lm.py +131 -0
  59. keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
  60. keras_hub/src/models/bert/bert_preprocessor.py +184 -0
  61. keras_hub/src/models/bert/bert_presets.py +147 -0
  62. keras_hub/src/models/bert/bert_tokenizer.py +112 -0
  63. keras_hub/src/models/bloom/__init__.py +20 -0
  64. keras_hub/src/models/bloom/bloom_attention.py +186 -0
  65. keras_hub/src/models/bloom/bloom_backbone.py +173 -0
  66. keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
  67. keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
  68. keras_hub/src/models/bloom/bloom_decoder.py +206 -0
  69. keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
  70. keras_hub/src/models/bloom/bloom_presets.py +121 -0
  71. keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
  72. keras_hub/src/models/causal_lm.py +383 -0
  73. keras_hub/src/models/classifier.py +109 -0
  74. keras_hub/src/models/csp_darknet/__init__.py +13 -0
  75. keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
  76. keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
  77. keras_hub/src/models/deberta_v3/__init__.py +24 -0
  78. keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
  79. keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
  80. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
  81. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
  82. keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
  83. keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
  84. keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
  85. keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
  86. keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
  87. keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
  88. keras_hub/src/models/densenet/__init__.py +13 -0
  89. keras_hub/src/models/densenet/densenet_backbone.py +210 -0
  90. keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
  91. keras_hub/src/models/distil_bert/__init__.py +26 -0
  92. keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
  93. keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
  94. keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
  95. keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
  96. keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
  97. keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
  98. keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
  99. keras_hub/src/models/electra/__init__.py +20 -0
  100. keras_hub/src/models/electra/electra_backbone.py +247 -0
  101. keras_hub/src/models/electra/electra_preprocessor.py +154 -0
  102. keras_hub/src/models/electra/electra_presets.py +95 -0
  103. keras_hub/src/models/electra/electra_tokenizer.py +104 -0
  104. keras_hub/src/models/f_net/__init__.py +20 -0
  105. keras_hub/src/models/f_net/f_net_backbone.py +236 -0
  106. keras_hub/src/models/f_net/f_net_classifier.py +154 -0
  107. keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
  108. keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
  109. keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
  110. keras_hub/src/models/f_net/f_net_presets.py +43 -0
  111. keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
  112. keras_hub/src/models/falcon/__init__.py +20 -0
  113. keras_hub/src/models/falcon/falcon_attention.py +156 -0
  114. keras_hub/src/models/falcon/falcon_backbone.py +164 -0
  115. keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
  116. keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
  117. keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
  118. keras_hub/src/models/falcon/falcon_presets.py +30 -0
  119. keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
  120. keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
  121. keras_hub/src/models/feature_pyramid_backbone.py +73 -0
  122. keras_hub/src/models/gemma/__init__.py +20 -0
  123. keras_hub/src/models/gemma/gemma_attention.py +250 -0
  124. keras_hub/src/models/gemma/gemma_backbone.py +316 -0
  125. keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
  126. keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
  127. keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
  128. keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
  129. keras_hub/src/models/gemma/gemma_presets.py +248 -0
  130. keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
  131. keras_hub/src/models/gemma/rms_normalization.py +40 -0
  132. keras_hub/src/models/gpt2/__init__.py +20 -0
  133. keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
  134. keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
  135. keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
  136. keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
  137. keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
  138. keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
  139. keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
  140. keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
  141. keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
  142. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
  143. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
  144. keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
  145. keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
  146. keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
  147. keras_hub/src/models/image_classifier.py +90 -0
  148. keras_hub/src/models/llama/__init__.py +20 -0
  149. keras_hub/src/models/llama/llama_attention.py +225 -0
  150. keras_hub/src/models/llama/llama_backbone.py +188 -0
  151. keras_hub/src/models/llama/llama_causal_lm.py +327 -0
  152. keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
  153. keras_hub/src/models/llama/llama_decoder.py +246 -0
  154. keras_hub/src/models/llama/llama_layernorm.py +48 -0
  155. keras_hub/src/models/llama/llama_preprocessor.py +189 -0
  156. keras_hub/src/models/llama/llama_presets.py +80 -0
  157. keras_hub/src/models/llama/llama_tokenizer.py +84 -0
  158. keras_hub/src/models/llama3/__init__.py +20 -0
  159. keras_hub/src/models/llama3/llama3_backbone.py +84 -0
  160. keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
  161. keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
  162. keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
  163. keras_hub/src/models/llama3/llama3_presets.py +69 -0
  164. keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
  165. keras_hub/src/models/masked_lm.py +101 -0
  166. keras_hub/src/models/mistral/__init__.py +20 -0
  167. keras_hub/src/models/mistral/mistral_attention.py +238 -0
  168. keras_hub/src/models/mistral/mistral_backbone.py +203 -0
  169. keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
  170. keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
  171. keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
  172. keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
  173. keras_hub/src/models/mistral/mistral_presets.py +48 -0
  174. keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
  175. keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
  176. keras_hub/src/models/mix_transformer/__init__.py +13 -0
  177. keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
  178. keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
  179. keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
  180. keras_hub/src/models/opt/__init__.py +20 -0
  181. keras_hub/src/models/opt/opt_backbone.py +173 -0
  182. keras_hub/src/models/opt/opt_causal_lm.py +301 -0
  183. keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
  184. keras_hub/src/models/opt/opt_preprocessor.py +188 -0
  185. keras_hub/src/models/opt/opt_presets.py +72 -0
  186. keras_hub/src/models/opt/opt_tokenizer.py +116 -0
  187. keras_hub/src/models/pali_gemma/__init__.py +23 -0
  188. keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
  189. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
  190. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
  191. keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
  192. keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
  193. keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
  194. keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
  195. keras_hub/src/models/phi3/__init__.py +20 -0
  196. keras_hub/src/models/phi3/phi3_attention.py +260 -0
  197. keras_hub/src/models/phi3/phi3_backbone.py +224 -0
  198. keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
  199. keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
  200. keras_hub/src/models/phi3/phi3_decoder.py +260 -0
  201. keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
  202. keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
  203. keras_hub/src/models/phi3/phi3_presets.py +50 -0
  204. keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
  205. keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
  206. keras_hub/src/models/preprocessor.py +207 -0
  207. keras_hub/src/models/resnet/__init__.py +13 -0
  208. keras_hub/src/models/resnet/resnet_backbone.py +612 -0
  209. keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
  210. keras_hub/src/models/roberta/__init__.py +20 -0
  211. keras_hub/src/models/roberta/roberta_backbone.py +184 -0
  212. keras_hub/src/models/roberta/roberta_classifier.py +209 -0
  213. keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
  214. keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
  215. keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
  216. keras_hub/src/models/roberta/roberta_presets.py +43 -0
  217. keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
  218. keras_hub/src/models/seq_2_seq_lm.py +54 -0
  219. keras_hub/src/models/t5/__init__.py +20 -0
  220. keras_hub/src/models/t5/t5_backbone.py +261 -0
  221. keras_hub/src/models/t5/t5_layer_norm.py +35 -0
  222. keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
  223. keras_hub/src/models/t5/t5_presets.py +95 -0
  224. keras_hub/src/models/t5/t5_tokenizer.py +100 -0
  225. keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
  226. keras_hub/src/models/task.py +419 -0
  227. keras_hub/src/models/vgg/__init__.py +13 -0
  228. keras_hub/src/models/vgg/vgg_backbone.py +158 -0
  229. keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
  230. keras_hub/src/models/vit_det/__init__.py +13 -0
  231. keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
  232. keras_hub/src/models/vit_det/vit_layers.py +565 -0
  233. keras_hub/src/models/whisper/__init__.py +20 -0
  234. keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
  235. keras_hub/src/models/whisper/whisper_backbone.py +305 -0
  236. keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
  237. keras_hub/src/models/whisper/whisper_decoder.py +141 -0
  238. keras_hub/src/models/whisper/whisper_encoder.py +106 -0
  239. keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
  240. keras_hub/src/models/whisper/whisper_presets.py +148 -0
  241. keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
  242. keras_hub/src/models/xlm_roberta/__init__.py +26 -0
  243. keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
  244. keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
  245. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
  246. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
  247. keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
  248. keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
  249. keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
  250. keras_hub/src/models/xlnet/__init__.py +13 -0
  251. keras_hub/src/models/xlnet/relative_attention.py +459 -0
  252. keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
  253. keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
  254. keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
  255. keras_hub/src/samplers/__init__.py +13 -0
  256. keras_hub/src/samplers/beam_sampler.py +207 -0
  257. keras_hub/src/samplers/contrastive_sampler.py +231 -0
  258. keras_hub/src/samplers/greedy_sampler.py +50 -0
  259. keras_hub/src/samplers/random_sampler.py +77 -0
  260. keras_hub/src/samplers/sampler.py +237 -0
  261. keras_hub/src/samplers/serialization.py +97 -0
  262. keras_hub/src/samplers/top_k_sampler.py +92 -0
  263. keras_hub/src/samplers/top_p_sampler.py +113 -0
  264. keras_hub/src/tests/__init__.py +13 -0
  265. keras_hub/src/tests/test_case.py +608 -0
  266. keras_hub/src/tokenizers/__init__.py +13 -0
  267. keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
  268. keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
  269. keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
  270. keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
  271. keras_hub/src/tokenizers/tokenizer.py +235 -0
  272. keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
  273. keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
  274. keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
  275. keras_hub/src/utils/__init__.py +13 -0
  276. keras_hub/src/utils/keras_utils.py +130 -0
  277. keras_hub/src/utils/pipeline_model.py +293 -0
  278. keras_hub/src/utils/preset_utils.py +621 -0
  279. keras_hub/src/utils/python_utils.py +21 -0
  280. keras_hub/src/utils/tensor_utils.py +206 -0
  281. keras_hub/src/utils/timm/__init__.py +13 -0
  282. keras_hub/src/utils/timm/convert.py +37 -0
  283. keras_hub/src/utils/timm/convert_resnet.py +171 -0
  284. keras_hub/src/utils/transformers/__init__.py +13 -0
  285. keras_hub/src/utils/transformers/convert.py +101 -0
  286. keras_hub/src/utils/transformers/convert_bert.py +173 -0
  287. keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
  288. keras_hub/src/utils/transformers/convert_gemma.py +187 -0
  289. keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
  290. keras_hub/src/utils/transformers/convert_llama3.py +136 -0
  291. keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
  292. keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
  293. keras_hub/src/version_utils.py +23 -0
  294. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
  295. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
  296. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
  297. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,110 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ from keras_hub.src.api_export import keras_hub_export
17
+ from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
18
+
19
+
20
+ @keras_hub_export("keras_hub.models.FalconTokenizer")
21
+ class FalconTokenizer(BytePairTokenizer):
22
+ """Falcon tokenizer based on BytePairTokenizer.
23
+
24
+ This tokenizer class will tokenize raw strings into integer sequences and
25
+ is based on `keras_hub.tokenizers.BytePairTokenizer`. Unlike the
26
+ underlying tokenizer, it will check for all special tokens needed by Falcon
27
+ models and provides a `from_preset()` method to automatically download
28
+ a matching vocabulary for a Falcon preset.
29
+
30
+ This tokenizer does not provide truncation or padding of inputs.
31
+
32
+ If input is a batch of strings (rank > 0), the layer will output a
33
+ `tf.RaggedTensor` where the last dimension of the output is ragged.
34
+
35
+ If input is a scalar string (rank == 0), the layer will output a dense
36
+ `tf.Tensor` with static shape `[None]`.
37
+
38
+ Args:
39
+ vocabulary: string or dict, maps token to integer ids. If it is a
40
+ string, it should be the file path to a json file.
41
+ merges: string or list, contains the merge rule. If it is a string,
42
+ it should be the file path to merge rules. The merge rule file
43
+ should have one merge rule per line. Every merge rule contains
44
+ merge entities separated by a space.
45
+
46
+ Examples:
47
+
48
+ ```python
49
+ # Unbatched input.
50
+ tokenizer = keras_hub.models.FalconTokenizer.from_preset("falcon_refinedweb_1b_en")
51
+ tokenizer("The quick brown fox jumped.")
52
+
53
+ # Batched input.
54
+ tokenizer(["The quick brown fox jumped.", "The fox slept."])
55
+
56
+ # Detokenization.
57
+ tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
58
+
59
+ # Custom vocabulary.
60
+ vocab = {"<|endoftext|>": 0, "a": 4, "Ġquick": 5, "Ġfox": 6}
61
+ merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"]
62
+ merges += ["Ġ f", "o x", "Ġf ox"]
63
+ tokenizer = keras_hub.models.FalconTokenizer(vocabulary=vocab, merges=merges)
64
+ tokenizer("a quick fox.")
65
+ ```
66
+ """
67
+
68
+ def __init__(
69
+ self,
70
+ vocabulary=None,
71
+ merges=None,
72
+ **kwargs,
73
+ ):
74
+ # Falcon uses the same start as end token, i.e., "<|endoftext|>".
75
+ self.end_token = self.start_token = "<|endoftext|>"
76
+
77
+ super().__init__(
78
+ vocabulary=vocabulary,
79
+ merges=merges,
80
+ unsplittable_tokens=[self.end_token],
81
+ **kwargs,
82
+ )
83
+
84
+ def set_vocabulary_and_merges(self, vocabulary, merges):
85
+ super().set_vocabulary_and_merges(vocabulary, merges)
86
+
87
+ if vocabulary is not None:
88
+ # Check for necessary special tokens.
89
+ if self.end_token not in self.get_vocabulary():
90
+ raise ValueError(
91
+ f"Cannot find token `'{self.end_token}'` in the provided "
92
+ f"`vocabulary`. Please provide `'{self.end_token}'` in "
93
+ "your `vocabulary` or use a pretrained `vocabulary` name."
94
+ )
95
+
96
+ self.end_token_id = self.token_to_id(self.end_token)
97
+ self.start_token_id = self.end_token_id
98
+ self.pad_token_id = 0
99
+ else:
100
+ self.end_token_id = None
101
+ self.start_token_id = None
102
+ self.pad_token_id = None
103
+
104
+ def get_config(self):
105
+ config = super().get_config()
106
+ # In the constructor, we pass the list of special tokens to the
107
+ # `unsplittable_tokens` arg of the superclass' constructor. Hence, we
108
+ # delete it from the config here.
109
+ del config["unsplittable_tokens"]
110
+ return config
@@ -0,0 +1,255 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ import math
15
+
16
+ import keras
17
+ from keras import ops
18
+
19
+ from keras_hub.src.layers.modeling.transformer_layer_utils import (
20
+ compute_causal_mask,
21
+ )
22
+ from keras_hub.src.layers.modeling.transformer_layer_utils import (
23
+ merge_padding_and_attention_mask,
24
+ )
25
+ from keras_hub.src.models.falcon.falcon_attention import FalconAttention
26
+
27
+
28
+ class FalconTransformerDecoder(keras.layers.Layer):
29
+ def __init__(
30
+ self,
31
+ num_attention_heads,
32
+ intermediate_dim,
33
+ layer_norm_epsilon=1e-5,
34
+ attention_dropout_rate=0,
35
+ feedforward_dropout_rate=0,
36
+ **kwargs,
37
+ ):
38
+ super().__init__(**kwargs)
39
+ self.num_attention_heads = num_attention_heads
40
+ self.intermediate_dim = intermediate_dim
41
+ self.layer_norm_epsilon = layer_norm_epsilon
42
+ self.attention_dropout_rate = attention_dropout_rate
43
+ self.feedforward_dropout_rate = feedforward_dropout_rate
44
+
45
+ def build(self, decoder_sequence_shape):
46
+ self.hidden_dim = decoder_sequence_shape[-1]
47
+ self.input_layernorm = keras.layers.LayerNormalization(
48
+ epsilon=self.layer_norm_epsilon,
49
+ dtype=self.dtype_policy,
50
+ name="input_layernorm",
51
+ )
52
+ self.input_layernorm.build(decoder_sequence_shape)
53
+
54
+ # Attention layers.
55
+ self.key_dim = self.hidden_dim // self.num_attention_heads
56
+ self.attention_layer = FalconAttention(
57
+ num_heads=self.num_attention_heads,
58
+ attention_dropout_rate=self.attention_dropout_rate,
59
+ dtype=self.dtype_policy,
60
+ name="attention",
61
+ )
62
+ self.attention_layer.build(
63
+ decoder_sequence_shape,
64
+ )
65
+
66
+ self.attention_dropout = keras.layers.Dropout(
67
+ rate=self.attention_dropout_rate,
68
+ dtype=self.dtype_policy,
69
+ name="attention_dropout",
70
+ )
71
+
72
+ self.post_attention_layernorm = keras.layers.LayerNormalization(
73
+ epsilon=self.layer_norm_epsilon,
74
+ dtype=self.dtype_policy,
75
+ name="post_attention_layernorm",
76
+ )
77
+ self.post_attention_layernorm.build(decoder_sequence_shape)
78
+
79
+ # Feedforward layers.
80
+ # TODO: use_bias should be an argument to the transformer to support
81
+ # other sizes of models, e.g. 7B, that don't use bias.
82
+ self.dense_h_to_4h = keras.layers.Dense(
83
+ self.intermediate_dim,
84
+ activation=keras.activations.gelu,
85
+ use_bias=True,
86
+ dtype=self.dtype_policy,
87
+ name="dense_h_to_4h",
88
+ )
89
+ self.dense_h_to_4h.build(decoder_sequence_shape)
90
+
91
+ self.dense_4h_to_h = keras.layers.Dense(
92
+ self.hidden_dim,
93
+ use_bias=True,
94
+ dtype=self.dtype_policy,
95
+ name="dense_4h_to_h",
96
+ )
97
+ self.dense_4h_to_h.build(
98
+ (
99
+ decoder_sequence_shape[0],
100
+ decoder_sequence_shape[1],
101
+ self.intermediate_dim,
102
+ )
103
+ )
104
+
105
+ self.feedforward_dropout = keras.layers.Dropout(
106
+ rate=self.feedforward_dropout_rate,
107
+ dtype=self.dtype_policy,
108
+ name="feedforward_dropout",
109
+ )
110
+
111
+ self.built = True
112
+
113
+ def call(
114
+ self,
115
+ inputs,
116
+ decoder_padding_mask=None,
117
+ decoder_attention_mask=None,
118
+ attention_cache=None,
119
+ attention_cache_update_index=None,
120
+ training=None,
121
+ ):
122
+ attention_mask = self._compute_attention_mask(
123
+ decoder_sequence=inputs,
124
+ decoder_padding_mask=decoder_padding_mask,
125
+ decoder_attention_mask=decoder_attention_mask,
126
+ attention_cache=attention_cache,
127
+ attention_cache_update_index=attention_cache_update_index,
128
+ )
129
+
130
+ residual = inputs
131
+
132
+ x = self.input_layernorm(inputs)
133
+
134
+ mask = decoder_padding_mask
135
+ if mask is None:
136
+ batch_size, seq_length = ops.shape(inputs)[:2]
137
+ mask = ops.ones((batch_size, seq_length), dtype="int32")
138
+ alibi = self._build_alibi_tensor(self.num_attention_heads, mask)
139
+
140
+ # Attention block.
141
+ attention_output = self.attention_layer(
142
+ inputs=x,
143
+ alibi=alibi,
144
+ attention_mask=attention_mask,
145
+ cache=attention_cache,
146
+ cache_update_index=attention_cache_update_index,
147
+ )
148
+
149
+ if attention_cache is None:
150
+ x = attention_output
151
+ else:
152
+ x, attention_cache = attention_output
153
+
154
+ x = self.attention_dropout(x, training=training)
155
+
156
+ x = x + residual
157
+ residual = x
158
+
159
+ x = self.post_attention_layernorm(x)
160
+
161
+ x = self.dense_h_to_4h(x)
162
+ x = self.dense_4h_to_h(x)
163
+
164
+ x = self.feedforward_dropout(x, training=training)
165
+
166
+ x = x + residual
167
+
168
+ if attention_cache is not None:
169
+ return x, attention_cache
170
+ else:
171
+ return x
172
+
173
+ def get_config(self):
174
+ config = super().get_config()
175
+ config.update(
176
+ {
177
+ "num_attention_heads": self.num_attention_heads,
178
+ "intermediate_dim": self.intermediate_dim,
179
+ "layer_norm_epsilon": self.layer_norm_epsilon,
180
+ "attention_dropout_rate": self.attention_dropout_rate,
181
+ "feedforward_dropout_rate": self.feedforward_dropout_rate,
182
+ }
183
+ )
184
+ return config
185
+
186
+ def compute_output_shape(self, decoder_sequence_shape):
187
+ return decoder_sequence_shape
188
+
189
+ def _compute_attention_mask(
190
+ self,
191
+ decoder_sequence,
192
+ decoder_padding_mask,
193
+ decoder_attention_mask,
194
+ attention_cache=None,
195
+ attention_cache_update_index=None,
196
+ ):
197
+ decoder_mask = merge_padding_and_attention_mask(
198
+ decoder_sequence, decoder_padding_mask, decoder_attention_mask
199
+ )
200
+ batch_size = ops.shape(decoder_sequence)[0]
201
+ input_length = output_length = ops.shape(decoder_sequence)[1]
202
+ # We need to handle a rectangular causal mask when doing cached
203
+ # decoding. For generative inference, `decoder_sequence` will
204
+ # generally be length 1, and `cache` will be the full generation length.
205
+ if attention_cache is not None:
206
+ input_length = ops.shape(attention_cache)[2]
207
+
208
+ causal_mask = compute_causal_mask(
209
+ batch_size,
210
+ input_length,
211
+ output_length,
212
+ (
213
+ 0
214
+ if attention_cache_update_index is None
215
+ else attention_cache_update_index
216
+ ),
217
+ )
218
+ return (
219
+ ops.minimum(decoder_mask, causal_mask)
220
+ if decoder_mask is not None
221
+ else causal_mask
222
+ )
223
+
224
+ def _build_alibi_tensor(self, num_heads, mask):
225
+ slopes = ops.convert_to_tensor(
226
+ self._get_slopes(num_heads),
227
+ dtype=self.compute_dtype,
228
+ ) # num_heads
229
+ mask = ops.cast(mask, dtype="int32")
230
+ # TODO: cumsum always outputs int64 in Keras 2 so the casting of cumsum
231
+ # result to int32 can be removed when keras 2 support is removed.
232
+ cumsum_mask = ops.cast(ops.cumsum(mask, axis=-1) - 1, "int32")
233
+ arange_tensor = (cumsum_mask * mask)[:, None, :]
234
+ alibi = slopes[..., None] * ops.cast(arange_tensor, self.compute_dtype)
235
+ alibi = ops.expand_dims(
236
+ alibi, 0
237
+ ) # [None, batch_size, num_heads, seq_length]
238
+ return ops.transpose(alibi, [1, 2, 0, 3])
239
+
240
+ def _get_slopes(self, num_heads):
241
+ def get_slopes_power_of_2(n):
242
+ start = 2 ** (-(2 ** -(math.log2(n) - 3)))
243
+ ratio = start
244
+ return [start * ratio**i for i in range(n)]
245
+
246
+ if math.log2(num_heads).is_integer():
247
+ return get_slopes_power_of_2(num_heads)
248
+ else:
249
+ closest_power_of_2 = 2 ** math.floor(math.log2(num_heads))
250
+ return (
251
+ get_slopes_power_of_2(closest_power_of_2)
252
+ + self._get_slopes(2 * closest_power_of_2)[0::2][
253
+ : num_heads - closest_power_of_2
254
+ ]
255
+ )
@@ -0,0 +1,73 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ import keras
15
+
16
+ from keras_hub.src.api_export import keras_hub_export
17
+ from keras_hub.src.models.backbone import Backbone
18
+
19
+
20
+ @keras_hub_export("keras_hub.models.FeaturePyramidBackbone")
21
+ class FeaturePyramidBackbone(Backbone):
22
+ """A backbone with feature pyramid outputs.
23
+
24
+ `FeaturePyramidBackbone` extends `Backbone` with a single `pyramid_outputs`
25
+ property for accessing the feature pyramid outputs of the model. Subclassers
26
+ should set the `pyramid_outputs` property during the model constructor.
27
+
28
+ Example:
29
+
30
+ ```python
31
+ input_data = np.random.uniform(0, 255, size=(2, 224, 224, 3))
32
+
33
+ # Convert to feature pyramid output format using ResNet.
34
+ backbone = ResNetBackbone.from_preset("resnet50")
35
+ model = keras.Model(
36
+ inputs=backbone.inputs, outputs=backbone.pyramid_outputs
37
+ )
38
+ model(input_data) # A dict containing the keys ["P2", "P3", "P4", "P5"]
39
+ ```
40
+ """
41
+
42
+ @property
43
+ def pyramid_outputs(self):
44
+ """A dict for feature pyramid outputs.
45
+
46
+ The key is a string represents the name of the feature output and the
47
+ value is a `keras.KerasTensor`. A typical feature pyramid has multiple
48
+ levels corresponding to scales such as `["P2", "P3", "P4", "P5"]`. Scale
49
+ `Pn` represents a feature map `2^n` times smaller in width and height
50
+ than the inputs.
51
+ """
52
+ return getattr(self, "_pyramid_outputs", {})
53
+
54
+ @pyramid_outputs.setter
55
+ def pyramid_outputs(self, value):
56
+ if not isinstance(value, dict):
57
+ raise TypeError(
58
+ "`pyramid_outputs` must be a dictionary. "
59
+ f"Received: value={value} of type {type(value)}"
60
+ )
61
+ for k, v in value.items():
62
+ if not isinstance(k, str):
63
+ raise TypeError(
64
+ "The key of `pyramid_outputs` must be a string. "
65
+ f"Received: key={k} of type {type(k)}"
66
+ )
67
+ if not isinstance(v, keras.KerasTensor):
68
+ raise TypeError(
69
+ "The value of `pyramid_outputs` must be a "
70
+ "`keras.KerasTensor`. "
71
+ f"Received: value={v} of type {type(v)}"
72
+ )
73
+ self._pyramid_outputs = value
@@ -0,0 +1,20 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from keras_hub.src.models.gemma.gemma_backbone import GemmaBackbone
16
+ from keras_hub.src.models.gemma.gemma_presets import backbone_presets
17
+ from keras_hub.src.models.gemma.gemma_tokenizer import GemmaTokenizer
18
+ from keras_hub.src.utils.preset_utils import register_presets
19
+
20
+ register_presets(backbone_presets, (GemmaBackbone, GemmaTokenizer))