keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (297) hide show
  1. keras_hub/__init__.py +52 -0
  2. keras_hub/api/__init__.py +27 -0
  3. keras_hub/api/layers/__init__.py +47 -0
  4. keras_hub/api/metrics/__init__.py +24 -0
  5. keras_hub/api/models/__init__.py +249 -0
  6. keras_hub/api/samplers/__init__.py +29 -0
  7. keras_hub/api/tokenizers/__init__.py +35 -0
  8. keras_hub/src/__init__.py +13 -0
  9. keras_hub/src/api_export.py +53 -0
  10. keras_hub/src/layers/__init__.py +13 -0
  11. keras_hub/src/layers/modeling/__init__.py +13 -0
  12. keras_hub/src/layers/modeling/alibi_bias.py +143 -0
  13. keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
  14. keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
  15. keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
  16. keras_hub/src/layers/modeling/position_embedding.py +123 -0
  17. keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
  18. keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
  19. keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
  20. keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
  21. keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
  22. keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
  23. keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
  24. keras_hub/src/layers/preprocessing/__init__.py +13 -0
  25. keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
  26. keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
  27. keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
  28. keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
  29. keras_hub/src/layers/preprocessing/random_swap.py +267 -0
  30. keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
  31. keras_hub/src/metrics/__init__.py +13 -0
  32. keras_hub/src/metrics/bleu.py +394 -0
  33. keras_hub/src/metrics/edit_distance.py +197 -0
  34. keras_hub/src/metrics/perplexity.py +181 -0
  35. keras_hub/src/metrics/rouge_base.py +204 -0
  36. keras_hub/src/metrics/rouge_l.py +97 -0
  37. keras_hub/src/metrics/rouge_n.py +125 -0
  38. keras_hub/src/models/__init__.py +13 -0
  39. keras_hub/src/models/albert/__init__.py +20 -0
  40. keras_hub/src/models/albert/albert_backbone.py +267 -0
  41. keras_hub/src/models/albert/albert_classifier.py +202 -0
  42. keras_hub/src/models/albert/albert_masked_lm.py +129 -0
  43. keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
  44. keras_hub/src/models/albert/albert_preprocessor.py +206 -0
  45. keras_hub/src/models/albert/albert_presets.py +70 -0
  46. keras_hub/src/models/albert/albert_tokenizer.py +119 -0
  47. keras_hub/src/models/backbone.py +311 -0
  48. keras_hub/src/models/bart/__init__.py +20 -0
  49. keras_hub/src/models/bart/bart_backbone.py +261 -0
  50. keras_hub/src/models/bart/bart_preprocessor.py +276 -0
  51. keras_hub/src/models/bart/bart_presets.py +74 -0
  52. keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
  53. keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
  54. keras_hub/src/models/bart/bart_tokenizer.py +124 -0
  55. keras_hub/src/models/bert/__init__.py +23 -0
  56. keras_hub/src/models/bert/bert_backbone.py +227 -0
  57. keras_hub/src/models/bert/bert_classifier.py +183 -0
  58. keras_hub/src/models/bert/bert_masked_lm.py +131 -0
  59. keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
  60. keras_hub/src/models/bert/bert_preprocessor.py +184 -0
  61. keras_hub/src/models/bert/bert_presets.py +147 -0
  62. keras_hub/src/models/bert/bert_tokenizer.py +112 -0
  63. keras_hub/src/models/bloom/__init__.py +20 -0
  64. keras_hub/src/models/bloom/bloom_attention.py +186 -0
  65. keras_hub/src/models/bloom/bloom_backbone.py +173 -0
  66. keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
  67. keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
  68. keras_hub/src/models/bloom/bloom_decoder.py +206 -0
  69. keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
  70. keras_hub/src/models/bloom/bloom_presets.py +121 -0
  71. keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
  72. keras_hub/src/models/causal_lm.py +383 -0
  73. keras_hub/src/models/classifier.py +109 -0
  74. keras_hub/src/models/csp_darknet/__init__.py +13 -0
  75. keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
  76. keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
  77. keras_hub/src/models/deberta_v3/__init__.py +24 -0
  78. keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
  79. keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
  80. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
  81. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
  82. keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
  83. keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
  84. keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
  85. keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
  86. keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
  87. keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
  88. keras_hub/src/models/densenet/__init__.py +13 -0
  89. keras_hub/src/models/densenet/densenet_backbone.py +210 -0
  90. keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
  91. keras_hub/src/models/distil_bert/__init__.py +26 -0
  92. keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
  93. keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
  94. keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
  95. keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
  96. keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
  97. keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
  98. keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
  99. keras_hub/src/models/electra/__init__.py +20 -0
  100. keras_hub/src/models/electra/electra_backbone.py +247 -0
  101. keras_hub/src/models/electra/electra_preprocessor.py +154 -0
  102. keras_hub/src/models/electra/electra_presets.py +95 -0
  103. keras_hub/src/models/electra/electra_tokenizer.py +104 -0
  104. keras_hub/src/models/f_net/__init__.py +20 -0
  105. keras_hub/src/models/f_net/f_net_backbone.py +236 -0
  106. keras_hub/src/models/f_net/f_net_classifier.py +154 -0
  107. keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
  108. keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
  109. keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
  110. keras_hub/src/models/f_net/f_net_presets.py +43 -0
  111. keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
  112. keras_hub/src/models/falcon/__init__.py +20 -0
  113. keras_hub/src/models/falcon/falcon_attention.py +156 -0
  114. keras_hub/src/models/falcon/falcon_backbone.py +164 -0
  115. keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
  116. keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
  117. keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
  118. keras_hub/src/models/falcon/falcon_presets.py +30 -0
  119. keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
  120. keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
  121. keras_hub/src/models/feature_pyramid_backbone.py +73 -0
  122. keras_hub/src/models/gemma/__init__.py +20 -0
  123. keras_hub/src/models/gemma/gemma_attention.py +250 -0
  124. keras_hub/src/models/gemma/gemma_backbone.py +316 -0
  125. keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
  126. keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
  127. keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
  128. keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
  129. keras_hub/src/models/gemma/gemma_presets.py +248 -0
  130. keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
  131. keras_hub/src/models/gemma/rms_normalization.py +40 -0
  132. keras_hub/src/models/gpt2/__init__.py +20 -0
  133. keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
  134. keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
  135. keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
  136. keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
  137. keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
  138. keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
  139. keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
  140. keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
  141. keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
  142. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
  143. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
  144. keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
  145. keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
  146. keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
  147. keras_hub/src/models/image_classifier.py +90 -0
  148. keras_hub/src/models/llama/__init__.py +20 -0
  149. keras_hub/src/models/llama/llama_attention.py +225 -0
  150. keras_hub/src/models/llama/llama_backbone.py +188 -0
  151. keras_hub/src/models/llama/llama_causal_lm.py +327 -0
  152. keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
  153. keras_hub/src/models/llama/llama_decoder.py +246 -0
  154. keras_hub/src/models/llama/llama_layernorm.py +48 -0
  155. keras_hub/src/models/llama/llama_preprocessor.py +189 -0
  156. keras_hub/src/models/llama/llama_presets.py +80 -0
  157. keras_hub/src/models/llama/llama_tokenizer.py +84 -0
  158. keras_hub/src/models/llama3/__init__.py +20 -0
  159. keras_hub/src/models/llama3/llama3_backbone.py +84 -0
  160. keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
  161. keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
  162. keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
  163. keras_hub/src/models/llama3/llama3_presets.py +69 -0
  164. keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
  165. keras_hub/src/models/masked_lm.py +101 -0
  166. keras_hub/src/models/mistral/__init__.py +20 -0
  167. keras_hub/src/models/mistral/mistral_attention.py +238 -0
  168. keras_hub/src/models/mistral/mistral_backbone.py +203 -0
  169. keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
  170. keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
  171. keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
  172. keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
  173. keras_hub/src/models/mistral/mistral_presets.py +48 -0
  174. keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
  175. keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
  176. keras_hub/src/models/mix_transformer/__init__.py +13 -0
  177. keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
  178. keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
  179. keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
  180. keras_hub/src/models/opt/__init__.py +20 -0
  181. keras_hub/src/models/opt/opt_backbone.py +173 -0
  182. keras_hub/src/models/opt/opt_causal_lm.py +301 -0
  183. keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
  184. keras_hub/src/models/opt/opt_preprocessor.py +188 -0
  185. keras_hub/src/models/opt/opt_presets.py +72 -0
  186. keras_hub/src/models/opt/opt_tokenizer.py +116 -0
  187. keras_hub/src/models/pali_gemma/__init__.py +23 -0
  188. keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
  189. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
  190. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
  191. keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
  192. keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
  193. keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
  194. keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
  195. keras_hub/src/models/phi3/__init__.py +20 -0
  196. keras_hub/src/models/phi3/phi3_attention.py +260 -0
  197. keras_hub/src/models/phi3/phi3_backbone.py +224 -0
  198. keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
  199. keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
  200. keras_hub/src/models/phi3/phi3_decoder.py +260 -0
  201. keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
  202. keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
  203. keras_hub/src/models/phi3/phi3_presets.py +50 -0
  204. keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
  205. keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
  206. keras_hub/src/models/preprocessor.py +207 -0
  207. keras_hub/src/models/resnet/__init__.py +13 -0
  208. keras_hub/src/models/resnet/resnet_backbone.py +612 -0
  209. keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
  210. keras_hub/src/models/roberta/__init__.py +20 -0
  211. keras_hub/src/models/roberta/roberta_backbone.py +184 -0
  212. keras_hub/src/models/roberta/roberta_classifier.py +209 -0
  213. keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
  214. keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
  215. keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
  216. keras_hub/src/models/roberta/roberta_presets.py +43 -0
  217. keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
  218. keras_hub/src/models/seq_2_seq_lm.py +54 -0
  219. keras_hub/src/models/t5/__init__.py +20 -0
  220. keras_hub/src/models/t5/t5_backbone.py +261 -0
  221. keras_hub/src/models/t5/t5_layer_norm.py +35 -0
  222. keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
  223. keras_hub/src/models/t5/t5_presets.py +95 -0
  224. keras_hub/src/models/t5/t5_tokenizer.py +100 -0
  225. keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
  226. keras_hub/src/models/task.py +419 -0
  227. keras_hub/src/models/vgg/__init__.py +13 -0
  228. keras_hub/src/models/vgg/vgg_backbone.py +158 -0
  229. keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
  230. keras_hub/src/models/vit_det/__init__.py +13 -0
  231. keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
  232. keras_hub/src/models/vit_det/vit_layers.py +565 -0
  233. keras_hub/src/models/whisper/__init__.py +20 -0
  234. keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
  235. keras_hub/src/models/whisper/whisper_backbone.py +305 -0
  236. keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
  237. keras_hub/src/models/whisper/whisper_decoder.py +141 -0
  238. keras_hub/src/models/whisper/whisper_encoder.py +106 -0
  239. keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
  240. keras_hub/src/models/whisper/whisper_presets.py +148 -0
  241. keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
  242. keras_hub/src/models/xlm_roberta/__init__.py +26 -0
  243. keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
  244. keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
  245. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
  246. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
  247. keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
  248. keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
  249. keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
  250. keras_hub/src/models/xlnet/__init__.py +13 -0
  251. keras_hub/src/models/xlnet/relative_attention.py +459 -0
  252. keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
  253. keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
  254. keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
  255. keras_hub/src/samplers/__init__.py +13 -0
  256. keras_hub/src/samplers/beam_sampler.py +207 -0
  257. keras_hub/src/samplers/contrastive_sampler.py +231 -0
  258. keras_hub/src/samplers/greedy_sampler.py +50 -0
  259. keras_hub/src/samplers/random_sampler.py +77 -0
  260. keras_hub/src/samplers/sampler.py +237 -0
  261. keras_hub/src/samplers/serialization.py +97 -0
  262. keras_hub/src/samplers/top_k_sampler.py +92 -0
  263. keras_hub/src/samplers/top_p_sampler.py +113 -0
  264. keras_hub/src/tests/__init__.py +13 -0
  265. keras_hub/src/tests/test_case.py +608 -0
  266. keras_hub/src/tokenizers/__init__.py +13 -0
  267. keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
  268. keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
  269. keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
  270. keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
  271. keras_hub/src/tokenizers/tokenizer.py +235 -0
  272. keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
  273. keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
  274. keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
  275. keras_hub/src/utils/__init__.py +13 -0
  276. keras_hub/src/utils/keras_utils.py +130 -0
  277. keras_hub/src/utils/pipeline_model.py +293 -0
  278. keras_hub/src/utils/preset_utils.py +621 -0
  279. keras_hub/src/utils/python_utils.py +21 -0
  280. keras_hub/src/utils/tensor_utils.py +206 -0
  281. keras_hub/src/utils/timm/__init__.py +13 -0
  282. keras_hub/src/utils/timm/convert.py +37 -0
  283. keras_hub/src/utils/timm/convert_resnet.py +171 -0
  284. keras_hub/src/utils/transformers/__init__.py +13 -0
  285. keras_hub/src/utils/transformers/convert.py +101 -0
  286. keras_hub/src/utils/transformers/convert_bert.py +173 -0
  287. keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
  288. keras_hub/src/utils/transformers/convert_gemma.py +187 -0
  289. keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
  290. keras_hub/src/utils/transformers/convert_llama3.py +136 -0
  291. keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
  292. keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
  293. keras_hub/src/version_utils.py +23 -0
  294. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
  295. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
  296. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
  297. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,185 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ import keras
17
+
18
+ from keras_hub.src.api_export import keras_hub_export
19
+ from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
20
+ from keras_hub.src.models.bloom.bloom_tokenizer import BloomTokenizer
21
+ from keras_hub.src.models.preprocessor import Preprocessor
22
+ from keras_hub.src.utils.keras_utils import (
23
+ convert_inputs_to_list_of_tensor_segments,
24
+ )
25
+
26
+
27
+ @keras_hub_export("keras_hub.models.BloomPreprocessor")
28
+ class BloomPreprocessor(Preprocessor):
29
+ """BLOOM preprocessing layer which tokenizes and packs inputs.
30
+
31
+ This preprocessing layer will do 2 things:
32
+
33
+ - Tokenize the inputs using the `tokenizer`.
34
+ - Construct a dictionary with keys `"token_ids"`, `"padding_mask"`, that can
35
+ be passed directly to a `keras_hub.models.BloomBackbone`.
36
+
37
+ This layer can be used directly with `tf.data.Dataset.map` to preprocess
38
+ string data in the `(x, y, sample_weight)` format used by
39
+ `keras.Model.fit`.
40
+
41
+ The call method of this layer accepts three arguments, `x`, `y`, and
42
+ `sample_weight`. `x` can be a python string or tensor representing a single
43
+ segment, a list of python strings representing a batch of single segments,
44
+ or a list of tensors representing multiple segments to be packed together.
45
+ `y` and `sample_weight` are both optional, can have any format, and will be
46
+ passed through unaltered.
47
+
48
+ Args:
49
+ tokenizer: A `keras_hub.models.BloomTokenizer` instance.
50
+ sequence_length: The length of the packed inputs.
51
+ add_start_token: If `True`, the preprocessor will prepend the tokenizer
52
+ start token to each input sequence.
53
+ add_end_token: If `True`, the preprocessor will append the tokenizer
54
+ end token to each input sequence.
55
+
56
+ Call arguments:
57
+ x: A string, `tf.Tensor` or list of python strings.
58
+ y: Any label data. Will be passed through unaltered.
59
+ sample_weight: Any label weight data. Will be passed through unaltered.
60
+ sequence_length: Pass to override the configured `sequence_length` of
61
+ the layer.
62
+
63
+ Examples:
64
+
65
+ Directly calling the layer on data.
66
+ ```python
67
+ preprocessor = keras_hub.models.BloomPreprocessor.from_preset(
68
+ "bloom_560m_multi"
69
+ )
70
+ # Tokenize and pack a single sentence.
71
+ preprocessor("The quick brown fox jumped.")
72
+
73
+ # Tokenize a batch of single sentences.
74
+ preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
75
+
76
+ # Custom vocabulary.
77
+ features = ["a quick fox.", "a fox quick."]
78
+ vocab = {"<pad>": 0, "<s>":1, "</s>":2, "a": 3, "Ġquick": 4, "Ġfox": 5}
79
+ merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"]
80
+ merges += ["Ġ f", "o x", "Ġf ox"]
81
+ tokenizer = keras_hub.models.BloomTokenizer(
82
+ vocabulary=vocab,
83
+ merges=merges,
84
+ )
85
+ preprocessor = keras_hub.models.BloomPreprocessor(tokenizer=tokenizer)
86
+ preprocessor("The quick brown fox jumped.")
87
+ ```
88
+
89
+ Mapping with `tf.data.Dataset`.
90
+ ```python
91
+ preprocessor = keras_hub.models.BloomPreprocessor.from_preset(
92
+ "bloom_560m_multi"
93
+ )
94
+
95
+ text = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
96
+ label = tf.constant([1, 1])
97
+
98
+ # Map labeled single sentences.
99
+ ds = tf.data.Dataset.from_tensor_slices((text, label))
100
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
101
+
102
+ # Map unlabeled single sentences.
103
+ ds = tf.data.Dataset.from_tensor_slices(text)
104
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
105
+ ```
106
+ """
107
+
108
+ tokenizer_cls = BloomTokenizer
109
+
110
+ def __init__(
111
+ self,
112
+ tokenizer,
113
+ sequence_length=1024,
114
+ add_start_token=True,
115
+ add_end_token=True,
116
+ **kwargs,
117
+ ):
118
+ super().__init__(**kwargs)
119
+ self.tokenizer = tokenizer
120
+ self.packer = None
121
+ self.sequence_length = sequence_length
122
+ self.add_start_token = add_start_token
123
+ self.add_end_token = add_end_token
124
+
125
+ def build(self, input_shape):
126
+ # Defer packer creation to `build()` so that we can be sure tokenizer
127
+ # assets have loaded when restoring a saved model.
128
+ self.packer = StartEndPacker(
129
+ start_value=self.tokenizer.start_token_id,
130
+ end_value=self.tokenizer.end_token_id,
131
+ pad_value=self.tokenizer.pad_token_id,
132
+ sequence_length=self.sequence_length,
133
+ return_padding_mask=True,
134
+ )
135
+ self.built = True
136
+
137
+ def call(
138
+ self,
139
+ x,
140
+ y=None,
141
+ sample_weight=None,
142
+ sequence_length=None,
143
+ ):
144
+ x = convert_inputs_to_list_of_tensor_segments(x)
145
+ if len(x) != 1:
146
+ raise ValueError(
147
+ "BLOOM requires each input feature to contain only "
148
+ f"one segment, but received {len(x)}. If you are using BLOOM "
149
+ "for a multi-segment classification task, please refer to "
150
+ "classification models like BERT or RoBERTa."
151
+ )
152
+ sequence_length = sequence_length or self.sequence_length
153
+ token_ids, padding_mask = self.packer(
154
+ self.tokenizer(x[0]),
155
+ sequence_length=sequence_length,
156
+ add_start_value=self.add_start_token,
157
+ add_end_value=self.add_end_token,
158
+ )
159
+ x = {
160
+ "token_ids": token_ids,
161
+ "padding_mask": padding_mask,
162
+ }
163
+ return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
164
+
165
+ def get_config(self):
166
+ config = super().get_config()
167
+ config.update(
168
+ {
169
+ "sequence_length": self.sequence_length,
170
+ "add_start_token": self.add_start_token,
171
+ "add_end_token": self.add_end_token,
172
+ }
173
+ )
174
+ return config
175
+
176
+ @property
177
+ def sequence_length(self):
178
+ """The padded length of model input sequences."""
179
+ return self._sequence_length
180
+
181
+ @sequence_length.setter
182
+ def sequence_length(self, value):
183
+ self._sequence_length = value
184
+ if self.packer is not None:
185
+ self.packer.sequence_length = value
@@ -0,0 +1,121 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """BLOOM model preset configurations."""
15
+
16
+ backbone_presets = {
17
+ "bloom_560m_multi": {
18
+ "metadata": {
19
+ "description": (
20
+ "24-layer Bloom model with hidden dimension of 1024. "
21
+ "trained on 45 natural languages and 12 programming languages."
22
+ ),
23
+ "params": 559214592,
24
+ "official_name": "BLOOM",
25
+ "path": "bloom",
26
+ "model_card": "https://huggingface.co/bigscience/bloom-560m",
27
+ },
28
+ "kaggle_handle": "kaggle://keras/bloom/keras/bloom_560m_multi/3",
29
+ },
30
+ "bloom_1.1b_multi": {
31
+ "metadata": {
32
+ "description": (
33
+ "24-layer Bloom model with hidden dimension of 1536. "
34
+ "trained on 45 natural languages and 12 programming languages."
35
+ ),
36
+ "params": 1065314304,
37
+ "official_name": "BLOOM",
38
+ "path": "bloom",
39
+ "model_card": "https://huggingface.co/bigscience/bloom-1b1",
40
+ },
41
+ "kaggle_handle": "kaggle://keras/bloom/keras/bloom_1.1b_multi/1",
42
+ },
43
+ "bloom_1.7b_multi": {
44
+ "metadata": {
45
+ "description": (
46
+ "24-layer Bloom model with hidden dimension of 2048. "
47
+ "trained on 45 natural languages and 12 programming languages."
48
+ ),
49
+ "params": 1722408960,
50
+ "official_name": "BLOOM",
51
+ "path": "bloom",
52
+ "model_card": "https://huggingface.co/bigscience/bloom-1b7",
53
+ },
54
+ "kaggle_handle": "kaggle://keras/bloom/keras/bloom_1.7b_multi/1",
55
+ },
56
+ "bloom_3b_multi": {
57
+ "metadata": {
58
+ "description": (
59
+ "30-layer Bloom model with hidden dimension of 2560. "
60
+ "trained on 45 natural languages and 12 programming languages."
61
+ ),
62
+ "params": 3002557440,
63
+ "official_name": "BLOOM",
64
+ "path": "bloom",
65
+ "model_card": "https://huggingface.co/bigscience/bloom-3b",
66
+ },
67
+ "kaggle_handle": "kaggle://keras/bloom/keras/bloom_3b_multi/1",
68
+ },
69
+ "bloomz_560m_multi": {
70
+ "metadata": {
71
+ "description": (
72
+ "24-layer Bloom model with hidden dimension of 1024. "
73
+ "finetuned on crosslingual task mixture (xP3) dataset."
74
+ ),
75
+ "params": 559214592,
76
+ "official_name": "BLOOMZ",
77
+ "path": "bloom",
78
+ "model_card": "https://huggingface.co/bigscience/bloomz-560m",
79
+ },
80
+ "kaggle_handle": "kaggle://keras/bloom/keras/bloomz_560m_multi/1",
81
+ },
82
+ "bloomz_1.1b_multi": {
83
+ "metadata": {
84
+ "description": (
85
+ "24-layer Bloom model with hidden dimension of 1536. "
86
+ "finetuned on crosslingual task mixture (xP3) dataset."
87
+ ),
88
+ "params": 1065314304,
89
+ "official_name": "BLOOMZ",
90
+ "path": "bloom",
91
+ "model_card": "https://huggingface.co/bigscience/bloomz-1b1",
92
+ },
93
+ "kaggle_handle": "kaggle://keras/bloom/keras/bloomz_1.1b_multi/1",
94
+ },
95
+ "bloomz_1.7b_multi": {
96
+ "metadata": {
97
+ "description": (
98
+ "24-layer Bloom model with hidden dimension of 2048. "
99
+ "finetuned on crosslingual task mixture (xP3) dataset."
100
+ ),
101
+ "params": 1722408960,
102
+ "official_name": "BLOOMZ",
103
+ "path": "bloom",
104
+ "model_card": "https://huggingface.co/bigscience/bloomz-1b7",
105
+ },
106
+ "kaggle_handle": "kaggle://keras/bloom/keras/bloomz_1.7b_multi/1",
107
+ },
108
+ "bloomz_3b_multi": {
109
+ "metadata": {
110
+ "description": (
111
+ "30-layer Bloom model with hidden dimension of 2560. "
112
+ "finetuned on crosslingual task mixture (xP3) dataset."
113
+ ),
114
+ "params": 3002557440,
115
+ "official_name": "BLOOMZ",
116
+ "path": "bloom",
117
+ "model_card": "https://huggingface.co/bigscience/bloomz-3b",
118
+ },
119
+ "kaggle_handle": "kaggle://keras/bloom/keras/bloomz_3b_multi/1",
120
+ },
121
+ }
@@ -0,0 +1,116 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ from keras_hub.src.api_export import keras_hub_export
17
+ from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
18
+
19
+
20
+ @keras_hub_export("keras_hub.models.BloomTokenizer")
21
+ class BloomTokenizer(BytePairTokenizer):
22
+ """A BLOOM tokenizer using Byte-Pair Encoding subword segmentation.
23
+
24
+ This tokenizer class will tokenize raw strings into integer sequences and
25
+ is based on `keras_hub.tokenizers.BytePairTokenizer`. Unlike the
26
+ underlying tokenizer, it will check for all special tokens needed by BLOOM
27
+ models and provides a `from_preset()` method to automatically download
28
+ a matching vocabulary for a BLOOM preset.
29
+
30
+ This tokenizer does not provide truncation or padding of inputs.
31
+
32
+ If input is a batch of strings (rank > 0), the layer will output a
33
+ `tf.RaggedTensor` where the last dimension of the output is ragged.
34
+
35
+ If input is a scalar string (rank == 0), the layer will output a dense
36
+ `tf.Tensor` with static shape `[None]`.
37
+
38
+ Args:
39
+ vocabulary: string or dict, maps token to integer ids. If it is a
40
+ string, it should be the file path to a json file.
41
+ merges: string or list, contains the merge rule. If it is a string,
42
+ it should be the file path to merge rules. The merge rule file
43
+ should have one merge rule per line. Every merge rule contains
44
+ merge entities separated by a space.
45
+
46
+ Examples:
47
+
48
+ ```python
49
+ # Unbatched input.
50
+ tokenizer = keras_hub.models.BloomTokenizer.from_preset("bloom_560m_multi")
51
+ tokenizer("The quick brown fox jumped.")
52
+
53
+ # Batched input.
54
+ tokenizer(["The quick brown fox jumped.", "The fox slept."])
55
+
56
+ # Detokenization.
57
+ tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
58
+
59
+ # Custom vocabulary.
60
+ vocab = {"<s>": 0, "</s>": 1, "<pad>": 2, "a": 3, "Ġquick": 4, "Ġfox": 5}
61
+ merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"]
62
+ merges += ["Ġ f", "o x", "Ġf ox"]
63
+ tokenizer = keras_hub.models.BloomTokenizer(vocabulary=vocab, merges=merges)
64
+ tokenizer("a quick fox.")
65
+ ```
66
+ """
67
+
68
+ def __init__(
69
+ self,
70
+ vocabulary=None,
71
+ merges=None,
72
+ **kwargs,
73
+ ):
74
+ self.start_token = "<s>"
75
+ self.end_token = "</s>"
76
+ self.pad_token = "<pad>"
77
+
78
+ super().__init__(
79
+ vocabulary=vocabulary,
80
+ merges=merges,
81
+ unsplittable_tokens=[
82
+ self.start_token,
83
+ self.end_token,
84
+ self.pad_token,
85
+ ],
86
+ **kwargs,
87
+ )
88
+
89
+ def set_vocabulary_and_merges(self, vocabulary, merges):
90
+ super().set_vocabulary_and_merges(vocabulary, merges)
91
+
92
+ if vocabulary is not None:
93
+ # Check for necessary special tokens.
94
+ for token in [self.start_token, self.end_token, self.pad_token]:
95
+ if token not in self.get_vocabulary():
96
+ raise ValueError(
97
+ f"Cannot find token `'{token}'` in the provided "
98
+ f"`vocabulary`. Please provide `'{token}'` in "
99
+ "your `vocabulary` or use a pretrained `vocabulary` name."
100
+ )
101
+
102
+ self.start_token_id = self.token_to_id(self.start_token)
103
+ self.end_token_id = self.token_to_id(self.end_token)
104
+ self.pad_token_id = self.token_to_id(self.pad_token)
105
+ else:
106
+ self.start_token_id = None
107
+ self.end_token_id = None
108
+ self.pad_token_id = None
109
+
110
+ def get_config(self):
111
+ config = super().get_config()
112
+ # In the constructor, we pass the list of special tokens to the
113
+ # `unsplittable_tokens` arg of the superclass' constructor. Hence, we
114
+ # delete it from the config here.
115
+ del config["unsplittable_tokens"]
116
+ return config