keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (297) hide show
  1. keras_hub/__init__.py +52 -0
  2. keras_hub/api/__init__.py +27 -0
  3. keras_hub/api/layers/__init__.py +47 -0
  4. keras_hub/api/metrics/__init__.py +24 -0
  5. keras_hub/api/models/__init__.py +249 -0
  6. keras_hub/api/samplers/__init__.py +29 -0
  7. keras_hub/api/tokenizers/__init__.py +35 -0
  8. keras_hub/src/__init__.py +13 -0
  9. keras_hub/src/api_export.py +53 -0
  10. keras_hub/src/layers/__init__.py +13 -0
  11. keras_hub/src/layers/modeling/__init__.py +13 -0
  12. keras_hub/src/layers/modeling/alibi_bias.py +143 -0
  13. keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
  14. keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
  15. keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
  16. keras_hub/src/layers/modeling/position_embedding.py +123 -0
  17. keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
  18. keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
  19. keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
  20. keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
  21. keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
  22. keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
  23. keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
  24. keras_hub/src/layers/preprocessing/__init__.py +13 -0
  25. keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
  26. keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
  27. keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
  28. keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
  29. keras_hub/src/layers/preprocessing/random_swap.py +267 -0
  30. keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
  31. keras_hub/src/metrics/__init__.py +13 -0
  32. keras_hub/src/metrics/bleu.py +394 -0
  33. keras_hub/src/metrics/edit_distance.py +197 -0
  34. keras_hub/src/metrics/perplexity.py +181 -0
  35. keras_hub/src/metrics/rouge_base.py +204 -0
  36. keras_hub/src/metrics/rouge_l.py +97 -0
  37. keras_hub/src/metrics/rouge_n.py +125 -0
  38. keras_hub/src/models/__init__.py +13 -0
  39. keras_hub/src/models/albert/__init__.py +20 -0
  40. keras_hub/src/models/albert/albert_backbone.py +267 -0
  41. keras_hub/src/models/albert/albert_classifier.py +202 -0
  42. keras_hub/src/models/albert/albert_masked_lm.py +129 -0
  43. keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
  44. keras_hub/src/models/albert/albert_preprocessor.py +206 -0
  45. keras_hub/src/models/albert/albert_presets.py +70 -0
  46. keras_hub/src/models/albert/albert_tokenizer.py +119 -0
  47. keras_hub/src/models/backbone.py +311 -0
  48. keras_hub/src/models/bart/__init__.py +20 -0
  49. keras_hub/src/models/bart/bart_backbone.py +261 -0
  50. keras_hub/src/models/bart/bart_preprocessor.py +276 -0
  51. keras_hub/src/models/bart/bart_presets.py +74 -0
  52. keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
  53. keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
  54. keras_hub/src/models/bart/bart_tokenizer.py +124 -0
  55. keras_hub/src/models/bert/__init__.py +23 -0
  56. keras_hub/src/models/bert/bert_backbone.py +227 -0
  57. keras_hub/src/models/bert/bert_classifier.py +183 -0
  58. keras_hub/src/models/bert/bert_masked_lm.py +131 -0
  59. keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
  60. keras_hub/src/models/bert/bert_preprocessor.py +184 -0
  61. keras_hub/src/models/bert/bert_presets.py +147 -0
  62. keras_hub/src/models/bert/bert_tokenizer.py +112 -0
  63. keras_hub/src/models/bloom/__init__.py +20 -0
  64. keras_hub/src/models/bloom/bloom_attention.py +186 -0
  65. keras_hub/src/models/bloom/bloom_backbone.py +173 -0
  66. keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
  67. keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
  68. keras_hub/src/models/bloom/bloom_decoder.py +206 -0
  69. keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
  70. keras_hub/src/models/bloom/bloom_presets.py +121 -0
  71. keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
  72. keras_hub/src/models/causal_lm.py +383 -0
  73. keras_hub/src/models/classifier.py +109 -0
  74. keras_hub/src/models/csp_darknet/__init__.py +13 -0
  75. keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
  76. keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
  77. keras_hub/src/models/deberta_v3/__init__.py +24 -0
  78. keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
  79. keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
  80. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
  81. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
  82. keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
  83. keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
  84. keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
  85. keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
  86. keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
  87. keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
  88. keras_hub/src/models/densenet/__init__.py +13 -0
  89. keras_hub/src/models/densenet/densenet_backbone.py +210 -0
  90. keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
  91. keras_hub/src/models/distil_bert/__init__.py +26 -0
  92. keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
  93. keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
  94. keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
  95. keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
  96. keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
  97. keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
  98. keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
  99. keras_hub/src/models/electra/__init__.py +20 -0
  100. keras_hub/src/models/electra/electra_backbone.py +247 -0
  101. keras_hub/src/models/electra/electra_preprocessor.py +154 -0
  102. keras_hub/src/models/electra/electra_presets.py +95 -0
  103. keras_hub/src/models/electra/electra_tokenizer.py +104 -0
  104. keras_hub/src/models/f_net/__init__.py +20 -0
  105. keras_hub/src/models/f_net/f_net_backbone.py +236 -0
  106. keras_hub/src/models/f_net/f_net_classifier.py +154 -0
  107. keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
  108. keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
  109. keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
  110. keras_hub/src/models/f_net/f_net_presets.py +43 -0
  111. keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
  112. keras_hub/src/models/falcon/__init__.py +20 -0
  113. keras_hub/src/models/falcon/falcon_attention.py +156 -0
  114. keras_hub/src/models/falcon/falcon_backbone.py +164 -0
  115. keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
  116. keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
  117. keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
  118. keras_hub/src/models/falcon/falcon_presets.py +30 -0
  119. keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
  120. keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
  121. keras_hub/src/models/feature_pyramid_backbone.py +73 -0
  122. keras_hub/src/models/gemma/__init__.py +20 -0
  123. keras_hub/src/models/gemma/gemma_attention.py +250 -0
  124. keras_hub/src/models/gemma/gemma_backbone.py +316 -0
  125. keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
  126. keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
  127. keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
  128. keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
  129. keras_hub/src/models/gemma/gemma_presets.py +248 -0
  130. keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
  131. keras_hub/src/models/gemma/rms_normalization.py +40 -0
  132. keras_hub/src/models/gpt2/__init__.py +20 -0
  133. keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
  134. keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
  135. keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
  136. keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
  137. keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
  138. keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
  139. keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
  140. keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
  141. keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
  142. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
  143. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
  144. keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
  145. keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
  146. keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
  147. keras_hub/src/models/image_classifier.py +90 -0
  148. keras_hub/src/models/llama/__init__.py +20 -0
  149. keras_hub/src/models/llama/llama_attention.py +225 -0
  150. keras_hub/src/models/llama/llama_backbone.py +188 -0
  151. keras_hub/src/models/llama/llama_causal_lm.py +327 -0
  152. keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
  153. keras_hub/src/models/llama/llama_decoder.py +246 -0
  154. keras_hub/src/models/llama/llama_layernorm.py +48 -0
  155. keras_hub/src/models/llama/llama_preprocessor.py +189 -0
  156. keras_hub/src/models/llama/llama_presets.py +80 -0
  157. keras_hub/src/models/llama/llama_tokenizer.py +84 -0
  158. keras_hub/src/models/llama3/__init__.py +20 -0
  159. keras_hub/src/models/llama3/llama3_backbone.py +84 -0
  160. keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
  161. keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
  162. keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
  163. keras_hub/src/models/llama3/llama3_presets.py +69 -0
  164. keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
  165. keras_hub/src/models/masked_lm.py +101 -0
  166. keras_hub/src/models/mistral/__init__.py +20 -0
  167. keras_hub/src/models/mistral/mistral_attention.py +238 -0
  168. keras_hub/src/models/mistral/mistral_backbone.py +203 -0
  169. keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
  170. keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
  171. keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
  172. keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
  173. keras_hub/src/models/mistral/mistral_presets.py +48 -0
  174. keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
  175. keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
  176. keras_hub/src/models/mix_transformer/__init__.py +13 -0
  177. keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
  178. keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
  179. keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
  180. keras_hub/src/models/opt/__init__.py +20 -0
  181. keras_hub/src/models/opt/opt_backbone.py +173 -0
  182. keras_hub/src/models/opt/opt_causal_lm.py +301 -0
  183. keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
  184. keras_hub/src/models/opt/opt_preprocessor.py +188 -0
  185. keras_hub/src/models/opt/opt_presets.py +72 -0
  186. keras_hub/src/models/opt/opt_tokenizer.py +116 -0
  187. keras_hub/src/models/pali_gemma/__init__.py +23 -0
  188. keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
  189. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
  190. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
  191. keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
  192. keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
  193. keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
  194. keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
  195. keras_hub/src/models/phi3/__init__.py +20 -0
  196. keras_hub/src/models/phi3/phi3_attention.py +260 -0
  197. keras_hub/src/models/phi3/phi3_backbone.py +224 -0
  198. keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
  199. keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
  200. keras_hub/src/models/phi3/phi3_decoder.py +260 -0
  201. keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
  202. keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
  203. keras_hub/src/models/phi3/phi3_presets.py +50 -0
  204. keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
  205. keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
  206. keras_hub/src/models/preprocessor.py +207 -0
  207. keras_hub/src/models/resnet/__init__.py +13 -0
  208. keras_hub/src/models/resnet/resnet_backbone.py +612 -0
  209. keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
  210. keras_hub/src/models/roberta/__init__.py +20 -0
  211. keras_hub/src/models/roberta/roberta_backbone.py +184 -0
  212. keras_hub/src/models/roberta/roberta_classifier.py +209 -0
  213. keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
  214. keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
  215. keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
  216. keras_hub/src/models/roberta/roberta_presets.py +43 -0
  217. keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
  218. keras_hub/src/models/seq_2_seq_lm.py +54 -0
  219. keras_hub/src/models/t5/__init__.py +20 -0
  220. keras_hub/src/models/t5/t5_backbone.py +261 -0
  221. keras_hub/src/models/t5/t5_layer_norm.py +35 -0
  222. keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
  223. keras_hub/src/models/t5/t5_presets.py +95 -0
  224. keras_hub/src/models/t5/t5_tokenizer.py +100 -0
  225. keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
  226. keras_hub/src/models/task.py +419 -0
  227. keras_hub/src/models/vgg/__init__.py +13 -0
  228. keras_hub/src/models/vgg/vgg_backbone.py +158 -0
  229. keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
  230. keras_hub/src/models/vit_det/__init__.py +13 -0
  231. keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
  232. keras_hub/src/models/vit_det/vit_layers.py +565 -0
  233. keras_hub/src/models/whisper/__init__.py +20 -0
  234. keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
  235. keras_hub/src/models/whisper/whisper_backbone.py +305 -0
  236. keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
  237. keras_hub/src/models/whisper/whisper_decoder.py +141 -0
  238. keras_hub/src/models/whisper/whisper_encoder.py +106 -0
  239. keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
  240. keras_hub/src/models/whisper/whisper_presets.py +148 -0
  241. keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
  242. keras_hub/src/models/xlm_roberta/__init__.py +26 -0
  243. keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
  244. keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
  245. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
  246. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
  247. keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
  248. keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
  249. keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
  250. keras_hub/src/models/xlnet/__init__.py +13 -0
  251. keras_hub/src/models/xlnet/relative_attention.py +459 -0
  252. keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
  253. keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
  254. keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
  255. keras_hub/src/samplers/__init__.py +13 -0
  256. keras_hub/src/samplers/beam_sampler.py +207 -0
  257. keras_hub/src/samplers/contrastive_sampler.py +231 -0
  258. keras_hub/src/samplers/greedy_sampler.py +50 -0
  259. keras_hub/src/samplers/random_sampler.py +77 -0
  260. keras_hub/src/samplers/sampler.py +237 -0
  261. keras_hub/src/samplers/serialization.py +97 -0
  262. keras_hub/src/samplers/top_k_sampler.py +92 -0
  263. keras_hub/src/samplers/top_p_sampler.py +113 -0
  264. keras_hub/src/tests/__init__.py +13 -0
  265. keras_hub/src/tests/test_case.py +608 -0
  266. keras_hub/src/tokenizers/__init__.py +13 -0
  267. keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
  268. keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
  269. keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
  270. keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
  271. keras_hub/src/tokenizers/tokenizer.py +235 -0
  272. keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
  273. keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
  274. keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
  275. keras_hub/src/utils/__init__.py +13 -0
  276. keras_hub/src/utils/keras_utils.py +130 -0
  277. keras_hub/src/utils/pipeline_model.py +293 -0
  278. keras_hub/src/utils/preset_utils.py +621 -0
  279. keras_hub/src/utils/python_utils.py +21 -0
  280. keras_hub/src/utils/tensor_utils.py +206 -0
  281. keras_hub/src/utils/timm/__init__.py +13 -0
  282. keras_hub/src/utils/timm/convert.py +37 -0
  283. keras_hub/src/utils/timm/convert_resnet.py +171 -0
  284. keras_hub/src/utils/transformers/__init__.py +13 -0
  285. keras_hub/src/utils/transformers/convert.py +101 -0
  286. keras_hub/src/utils/transformers/convert_bert.py +173 -0
  287. keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
  288. keras_hub/src/utils/transformers/convert_gemma.py +187 -0
  289. keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
  290. keras_hub/src/utils/transformers/convert_llama3.py +136 -0
  291. keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
  292. keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
  293. keras_hub/src/version_utils.py +23 -0
  294. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
  295. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
  296. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
  297. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,198 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import keras
16
+ from absl import logging
17
+
18
+ from keras_hub.src.api_export import keras_hub_export
19
+ from keras_hub.src.layers.preprocessing.masked_lm_mask_generator import (
20
+ MaskedLMMaskGenerator,
21
+ )
22
+ from keras_hub.src.models.roberta.roberta_preprocessor import (
23
+ RobertaPreprocessor,
24
+ )
25
+
26
+
27
+ @keras_hub_export("keras_hub.models.RobertaMaskedLMPreprocessor")
28
+ class RobertaMaskedLMPreprocessor(RobertaPreprocessor):
29
+ """RoBERTa preprocessing for the masked language modeling task.
30
+
31
+ This preprocessing layer will prepare inputs for a masked language modeling
32
+ task. It is primarily intended for use with the
33
+ `keras_hub.models.RobertaMaskedLM` task model. Preprocessing will occur in
34
+ multiple steps.
35
+
36
+ 1. Tokenize any number of input segments using the `tokenizer`.
37
+ 2. Pack the inputs together with the appropriate `"<s>"`, `"</s>"` and
38
+ `"<pad>"` tokens, i.e., adding a single `"<s>"` at the start of the
39
+ entire sequence, `"</s></s>"` between each segment,
40
+ and a `"</s>"` at the end of the entire sequence.
41
+ 3. Randomly select non-special tokens to mask, controlled by
42
+ `mask_selection_rate`.
43
+ 4. Construct a `(x, y, sample_weight)` tuple suitable for training with a
44
+ `keras_hub.models.RobertaMaskedLM` task model.
45
+
46
+ Args:
47
+ tokenizer: A `keras_hub.models.RobertaTokenizer` instance.
48
+ sequence_length: int. The length of the packed inputs.
49
+ truncate: string. The algorithm to truncate a list of batched segments
50
+ to fit within `sequence_length`. The value can be either
51
+ `round_robin` or `waterfall`:
52
+ - `"round_robin"`: Available space is assigned one token at a
53
+ time in a round-robin fashion to the inputs that still need
54
+ some, until the limit is reached.
55
+ - `"waterfall"`: The allocation of the budget is done using a
56
+ "waterfall" algorithm that allocates quota in a
57
+ left-to-right manner and fills up the buckets until we run
58
+ out of budget. It supports an arbitrary number of segments.
59
+ mask_selection_rate: float. The probability an input token will be
60
+ dynamically masked.
61
+ mask_selection_length: int. The maximum number of masked tokens
62
+ in a given sample.
63
+ mask_token_rate: float. The probability the a selected token will be
64
+ replaced with the mask token.
65
+ random_token_rate: float. The probability the a selected token will be
66
+ replaced with a random token from the vocabulary. A selected token
67
+ will be left as is with probability
68
+ `1 - mask_token_rate - random_token_rate`.
69
+
70
+ Call arguments:
71
+ x: A tensor of single string sequences, or a tuple of multiple
72
+ tensor sequences to be packed together. Inputs may be batched or
73
+ unbatched. For single sequences, raw python inputs will be converted
74
+ to tensors. For multiple sequences, pass tensors directly.
75
+ y: Label data. Should always be `None` as the layer generates labels.
76
+ sample_weight: Label weights. Should always be `None` as the layer
77
+ generates label weights.
78
+
79
+ Examples:
80
+
81
+ Directly calling the layer on data.
82
+ ```python
83
+ # Load the preprocessor from a preset.
84
+ preprocessor = keras_hub.models.RobertaMaskedLMPreprocessor.from_preset(
85
+ "roberta_base_en"
86
+ )
87
+
88
+ # Tokenize and mask a single sentence.
89
+ preprocessor("The quick brown fox jumped.")
90
+
91
+ # Tokenize and mask a batch of single sentences.
92
+ preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
93
+
94
+ # Tokenize and mask sentence pairs.
95
+ # In this case, always convert input to tensors before calling the layer.
96
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
97
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
98
+ preprocessor((first, second))
99
+ ```
100
+
101
+ Mapping with `tf.data.Dataset`.
102
+ ```python
103
+ preprocessor = keras_hub.models.RobertaMaskedLMPreprocessor.from_preset(
104
+ "roberta_base_en"
105
+ )
106
+
107
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
108
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
109
+
110
+ # Map single sentences.
111
+ ds = tf.data.Dataset.from_tensor_slices(first)
112
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
113
+
114
+ # Map sentence pairs.
115
+ ds = tf.data.Dataset.from_tensor_slices((first, second))
116
+ # Watch out for tf.data's default unpacking of tuples here!
117
+ # Best to invoke the `preprocessor` directly in this case.
118
+ ds = ds.map(
119
+ lambda first, second: preprocessor(x=(first, second)),
120
+ num_parallel_calls=tf.data.AUTOTUNE,
121
+ )
122
+ ```
123
+ """
124
+
125
+ def __init__(
126
+ self,
127
+ tokenizer,
128
+ sequence_length=512,
129
+ truncate="round_robin",
130
+ mask_selection_rate=0.15,
131
+ mask_selection_length=96,
132
+ mask_token_rate=0.8,
133
+ random_token_rate=0.1,
134
+ **kwargs,
135
+ ):
136
+ super().__init__(
137
+ tokenizer,
138
+ sequence_length=sequence_length,
139
+ truncate=truncate,
140
+ **kwargs,
141
+ )
142
+ self.mask_selection_rate = mask_selection_rate
143
+ self.mask_selection_length = mask_selection_length
144
+ self.mask_token_rate = mask_token_rate
145
+ self.random_token_rate = random_token_rate
146
+ self.masker = None
147
+
148
+ def build(self, input_shape):
149
+ super().build(input_shape)
150
+ # Defer packer creation to `build()` so that we can be sure tokenizer
151
+ # assets have loaded when restoring a saved model.
152
+ self.masker = MaskedLMMaskGenerator(
153
+ mask_selection_rate=self.mask_selection_rate,
154
+ mask_selection_length=self.mask_selection_length,
155
+ mask_token_rate=self.mask_token_rate,
156
+ random_token_rate=self.random_token_rate,
157
+ vocabulary_size=self.tokenizer.vocabulary_size(),
158
+ mask_token_id=self.tokenizer.mask_token_id,
159
+ unselectable_token_ids=[
160
+ self.tokenizer.start_token_id,
161
+ self.tokenizer.end_token_id,
162
+ self.tokenizer.pad_token_id,
163
+ ],
164
+ )
165
+ self.built = True
166
+
167
+ def call(self, x, y=None, sample_weight=None):
168
+ if y is not None or sample_weight is not None:
169
+ logging.warning(
170
+ f"{self.__class__.__name__} generates `y` and `sample_weight` "
171
+ "based on your input data, but your data already contains `y` "
172
+ "or `sample_weight`. Your `y` and `sample_weight` will be "
173
+ "ignored."
174
+ )
175
+
176
+ x = super().call(x)
177
+ token_ids, padding_mask = x["token_ids"], x["padding_mask"]
178
+ masker_outputs = self.masker(token_ids)
179
+ x = {
180
+ "token_ids": masker_outputs["token_ids"],
181
+ "padding_mask": padding_mask,
182
+ "mask_positions": masker_outputs["mask_positions"],
183
+ }
184
+ y = masker_outputs["mask_ids"]
185
+ sample_weight = masker_outputs["mask_weights"]
186
+ return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
187
+
188
+ def get_config(self):
189
+ config = super().get_config()
190
+ config.update(
191
+ {
192
+ "mask_selection_rate": self.mask_selection_rate,
193
+ "mask_selection_length": self.mask_selection_length,
194
+ "mask_token_rate": self.mask_token_rate,
195
+ "random_token_rate": self.random_token_rate,
196
+ }
197
+ )
198
+ return config
@@ -0,0 +1,192 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ import keras
17
+
18
+ from keras_hub.src.api_export import keras_hub_export
19
+ from keras_hub.src.layers.preprocessing.multi_segment_packer import (
20
+ MultiSegmentPacker,
21
+ )
22
+ from keras_hub.src.models.preprocessor import Preprocessor
23
+ from keras_hub.src.models.roberta.roberta_tokenizer import RobertaTokenizer
24
+ from keras_hub.src.utils.keras_utils import (
25
+ convert_inputs_to_list_of_tensor_segments,
26
+ )
27
+
28
+
29
+ @keras_hub_export("keras_hub.models.RobertaPreprocessor")
30
+ class RobertaPreprocessor(Preprocessor):
31
+ """A RoBERTa preprocessing layer which tokenizes and packs inputs.
32
+
33
+ This preprocessing layer will do three things:
34
+
35
+ 1. Tokenize any number of input segments using the `tokenizer`.
36
+ 2. Pack the inputs together with the appropriate `"<s>"`, `"</s>"` and
37
+ `"<pad>"` tokens, i.e., adding a single `"<s>"` at the start of the
38
+ entire sequence, `"</s></s>"` at the end of each segment, save the last
39
+ and a `"</s>"` at the end of the entire sequence.
40
+ 3. Construct a dictionary with keys `"token_ids"`, `"padding_mask"` that
41
+ can be passed directly to a RoBERTa model.
42
+
43
+ This layer can be used directly with `tf.data.Dataset.map` to preprocess
44
+ string data in the `(x, y, sample_weight)` format used by
45
+ `keras.Model.fit`.
46
+
47
+ Args:
48
+ tokenizer: A `keras_hub.models.RobertaTokenizer` instance.
49
+ sequence_length: The length of the packed inputs.
50
+ truncate: string. The algorithm to truncate a list of batched segments
51
+ to fit within `sequence_length`. The value can be either
52
+ `round_robin` or `waterfall`:
53
+ - `"round_robin"`: Available space is assigned one token at a
54
+ time in a round-robin fashion to the inputs that still need
55
+ some, until the limit is reached.
56
+ - `"waterfall"`: The allocation of the budget is done using a
57
+ "waterfall" algorithm that allocates quota in a
58
+ left-to-right manner and fills up the buckets until we run
59
+ out of budget. It supports an arbitrary number of segments.
60
+
61
+ Call arguments:
62
+ x: A tensor of single string sequences, or a tuple of multiple
63
+ tensor sequences to be packed together. Inputs may be batched or
64
+ unbatched. For single sequences, raw python inputs will be converted
65
+ to tensors. For multiple sequences, pass tensors directly.
66
+ y: Any label data. Will be passed through unaltered.
67
+ sample_weight: Any label weight data. Will be passed through unaltered.
68
+
69
+
70
+ Examples:
71
+
72
+ Directly calling the layer on data.
73
+ ```python
74
+ preprocessor = keras_hub.models.RobertaPreprocessor.from_preset(
75
+ "roberta_base_en"
76
+ )
77
+
78
+ # Tokenize and pack a single sentence.
79
+ preprocessor("The quick brown fox jumped.")
80
+
81
+ # Tokenize a batch of single sentences.
82
+ preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
83
+
84
+ # Preprocess a batch of sentence pairs.
85
+ # When handling multiple sequences, always convert to tensors first!
86
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
87
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
88
+ preprocessor((first, second))
89
+
90
+ # Custom vocabulary.
91
+ vocab = {"<s>": 0, "<pad>": 1, "</s>": 2, "<mask>": 3}
92
+ vocab = {**vocab, "a": 4, "Ġquick": 5, "Ġfox": 6}
93
+ merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick", "Ġ f", "o x", "Ġf ox"]
94
+ tokenizer = keras_hub.models.RobertaTokenizer(
95
+ vocabulary=vocab,
96
+ merges=merges
97
+ )
98
+ preprocessor = keras_hub.models.RobertaPreprocessor(tokenizer)
99
+ preprocessor("a quick fox")
100
+ ```
101
+ Mapping with `tf.data.Dataset`.
102
+ ```python
103
+ preprocessor = keras_hub.models.RobertaPreprocessor.from_preset(
104
+ "roberta_base_en"
105
+ )
106
+
107
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
108
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
109
+ label = tf.constant([1, 1])
110
+
111
+ # Map labeled single sentences.
112
+ ds = tf.data.Dataset.from_tensor_slices((first, label))
113
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
114
+
115
+ # Map unlabeled single sentences.
116
+ ds = tf.data.Dataset.from_tensor_slices(first)
117
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
118
+
119
+ # Map labeled sentence pairs.
120
+ ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
121
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
122
+
123
+ # Map unlabeled sentence pairs.
124
+ ds = tf.data.Dataset.from_tensor_slices((first, second))
125
+ # Watch out for tf.data's default unpacking of tuples here!
126
+ # Best to invoke the `preprocessor` directly in this case.
127
+ ds = ds.map(
128
+ lambda first, second: preprocessor(x=(first, second)),
129
+ num_parallel_calls=tf.data.AUTOTUNE,
130
+ )
131
+ ```
132
+ """
133
+
134
+ tokenizer_cls = RobertaTokenizer
135
+
136
+ def __init__(
137
+ self,
138
+ tokenizer,
139
+ sequence_length=512,
140
+ truncate="round_robin",
141
+ **kwargs,
142
+ ):
143
+ super().__init__(**kwargs)
144
+
145
+ self.tokenizer = tokenizer
146
+ self.packer = None
147
+ self.truncate = truncate
148
+ self.sequence_length = sequence_length
149
+
150
+ def build(self, input_shape):
151
+ # Defer packer creation to `build()` so that we can be sure tokenizer
152
+ # assets have loaded when restoring a saved model.
153
+ self.packer = MultiSegmentPacker(
154
+ start_value=self.tokenizer.start_token_id,
155
+ end_value=self.tokenizer.end_token_id,
156
+ sep_value=[self.tokenizer.end_token_id] * 2,
157
+ pad_value=self.tokenizer.pad_token_id,
158
+ truncate=self.truncate,
159
+ sequence_length=self.sequence_length,
160
+ )
161
+ self.built = True
162
+
163
+ def call(self, x, y=None, sample_weight=None):
164
+ x = convert_inputs_to_list_of_tensor_segments(x)
165
+ x = [self.tokenizer(segment) for segment in x]
166
+ token_ids, _ = self.packer(x)
167
+ x = {
168
+ "token_ids": token_ids,
169
+ "padding_mask": token_ids != self.tokenizer.pad_token_id,
170
+ }
171
+ return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
172
+
173
+ def get_config(self):
174
+ config = super().get_config()
175
+ config.update(
176
+ {
177
+ "sequence_length": self.sequence_length,
178
+ "truncate": self.truncate,
179
+ }
180
+ )
181
+ return config
182
+
183
+ @property
184
+ def sequence_length(self):
185
+ """The padded length of model input sequences."""
186
+ return self._sequence_length
187
+
188
+ @sequence_length.setter
189
+ def sequence_length(self, value):
190
+ self._sequence_length = value
191
+ if self.packer is not None:
192
+ self.packer.sequence_length = value
@@ -0,0 +1,43 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """RoBERTa model preset configurations."""
15
+
16
+ backbone_presets = {
17
+ "roberta_base_en": {
18
+ "metadata": {
19
+ "description": (
20
+ "12-layer RoBERTa model where case is maintained."
21
+ "Trained on English Wikipedia, BooksCorpus, CommonCraw, and OpenWebText."
22
+ ),
23
+ "params": 124052736,
24
+ "official_name": "RoBERTa",
25
+ "path": "roberta",
26
+ "model_card": "https://github.com/facebookresearch/fairseq/blob/main/examples/roberta/README.md",
27
+ },
28
+ "kaggle_handle": "kaggle://keras/roberta/keras/roberta_base_en/2",
29
+ },
30
+ "roberta_large_en": {
31
+ "metadata": {
32
+ "description": (
33
+ "24-layer RoBERTa model where case is maintained."
34
+ "Trained on English Wikipedia, BooksCorpus, CommonCraw, and OpenWebText."
35
+ ),
36
+ "params": 354307072,
37
+ "official_name": "RoBERTa",
38
+ "path": "roberta",
39
+ "model_card": "https://github.com/facebookresearch/fairseq/blob/main/examples/roberta/README.md",
40
+ },
41
+ "kaggle_handle": "kaggle://keras/roberta/keras/roberta_large_en/2",
42
+ },
43
+ }
@@ -0,0 +1,132 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ from keras_hub.src.api_export import keras_hub_export
17
+ from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
18
+
19
+
20
+ @keras_hub_export("keras_hub.models.RobertaTokenizer")
21
+ class RobertaTokenizer(BytePairTokenizer):
22
+ """A RoBERTa tokenizer using Byte-Pair Encoding subword segmentation.
23
+
24
+ This tokenizer class will tokenize raw strings into integer sequences and
25
+ is based on `keras_hub.tokenizers.BytePairTokenizer`. Unlike the
26
+ underlying tokenizer, it will check for all special tokens needed by RoBERTa
27
+ models and provides a `from_preset()` method to automatically download
28
+ a matching vocabulary for a RoBERTa preset.
29
+
30
+ This tokenizer does not provide truncation or padding of inputs. It can be
31
+ combined with a `keras_hub.models.RobertaPreprocessor` layer for input
32
+ packing.
33
+
34
+ If input is a batch of strings (rank > 0), the layer will output a
35
+ `tf.RaggedTensor` where the last dimension of the output is ragged.
36
+
37
+ If input is a scalar string (rank == 0), the layer will output a dense
38
+ `tf.Tensor` with static shape `[None]`.
39
+
40
+ Args:
41
+ vocabulary: A dictionary mapping tokens to integer ids, or file path
42
+ to a json file containing the token to id mapping.
43
+ merges: A list of merge rules or a string file path, If passing a file
44
+ path. the file should have one merge rule per line. Every merge
45
+ rule contains merge entities separated by a space.
46
+
47
+ Examples:
48
+ ```python
49
+ # Unbatched input.
50
+ tokenizer = keras_hub.models.RobertaTokenizer.from_preset(
51
+ "roberta_base_en",
52
+ )
53
+ tokenizer("The quick brown fox jumped.")
54
+
55
+ # Batched input.
56
+ tokenizer(["The quick brown fox jumped.", "The fox slept."])
57
+
58
+ # Detokenization.
59
+ tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
60
+
61
+ # Custom vocabulary.
62
+ # Note: 'Ġ' is space
63
+ vocab = {"<s>": 0, "<pad>": 1, "</s>": 2, "<mask>": 3}
64
+ vocab = {**vocab, "a": 4, "Ġquick": 5, "Ġfox": 6}
65
+ merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"]
66
+ merges += ["Ġ f", "o x", "Ġf ox"]
67
+ tokenizer = keras_hub.models.RobertaTokenizer(
68
+ vocabulary=vocab,
69
+ merges=merges
70
+ )
71
+ tokenizer(["a quick fox", "a fox quick"])
72
+ ```
73
+ """
74
+
75
+ def __init__(
76
+ self,
77
+ vocabulary=None,
78
+ merges=None,
79
+ **kwargs,
80
+ ):
81
+ self.start_token = "<s>"
82
+ self.pad_token = "<pad>"
83
+ self.end_token = "</s>"
84
+ self.mask_token = "<mask>"
85
+
86
+ super().__init__(
87
+ vocabulary=vocabulary,
88
+ merges=merges,
89
+ unsplittable_tokens=[
90
+ self.start_token,
91
+ self.pad_token,
92
+ self.end_token,
93
+ self.mask_token,
94
+ ],
95
+ **kwargs,
96
+ )
97
+
98
+ def set_vocabulary_and_merges(self, vocabulary, merges):
99
+ super().set_vocabulary_and_merges(vocabulary, merges)
100
+
101
+ if vocabulary is not None:
102
+ # Check for necessary special tokens.
103
+ for token in [
104
+ self.start_token,
105
+ self.pad_token,
106
+ self.end_token,
107
+ self.mask_token,
108
+ ]:
109
+ if token not in self.vocabulary:
110
+ raise ValueError(
111
+ f"Cannot find token `'{token}'` in the provided "
112
+ f"`vocabulary`. Please provide `'{token}'` in your "
113
+ "`vocabulary` or use a pretrained `vocabulary` name."
114
+ )
115
+
116
+ self.start_token_id = self.token_to_id(self.start_token)
117
+ self.pad_token_id = self.token_to_id(self.pad_token)
118
+ self.end_token_id = self.token_to_id(self.end_token)
119
+ self.mask_token_id = self.token_to_id(self.mask_token)
120
+ else:
121
+ self.start_token_id = None
122
+ self.pad_token_id = None
123
+ self.end_token_id = None
124
+ self.mask_token_id = None
125
+
126
+ def get_config(self):
127
+ config = super().get_config()
128
+ # In the constructor, we pass the list of special tokens to the
129
+ # `unsplittable_tokens` arg of the superclass' constructor. Hence, we
130
+ # delete it from the config here.
131
+ del config["unsplittable_tokens"]
132
+ return config
@@ -0,0 +1,54 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from keras_hub.src.api_export import keras_hub_export
15
+ from keras_hub.src.models.causal_lm import CausalLM
16
+
17
+
18
+ @keras_hub_export("keras_hub.models.Seq2SeqLM")
19
+ class Seq2SeqLM(CausalLM):
20
+ """Base class for sequence to sequence language modeling tasks.
21
+
22
+ `Seq2SeqLM` tasks wrap a `keras_hub.models.Backbone` and
23
+ a `keras_hub.models.Preprocessor` to create a model that can be used for
24
+ generation and generative fine-tuning, when generation is conditioned on
25
+ additional input sequence in a sequence-to-sequence setting.
26
+
27
+ `Seq2SeqLM` tasks provide an additional, high-level `generate()` function
28
+ which can be used to auto-regressively sample an output sequence token by
29
+ token. The `compile()` method of `Seq2SeqLM` classes contains an additional
30
+ `sampler` argument, which can be used to pass a `keras_hub.samplers.Sampler`
31
+ to control how the predicted distribution will be sampled.
32
+
33
+ When calling `fit()`, each input should contain an input and output
34
+ sequence. The model will be trained to predict the output sequence
35
+ token-by-token using a causal mask, similar to a `keras_hub.models.CausalLM`
36
+ task. Unlike the `CausalLM` task, an input sequence must be passed, and
37
+ can be attended to in full by all tokens in the output sequence.
38
+
39
+ All `Seq2SeqLM` tasks include a `from_preset()` constructor which can be
40
+ used to load a pre-trained config and weights.
41
+
42
+ Example:
43
+ ```python
44
+ # Load a Bart backbone with pre-trained weights.
45
+ seq_2_seq_lm = keras_hub.models.Seq2SeqLM.from_preset(
46
+ "bart_base_en",
47
+ )
48
+ seq_2_seq_lm.compile(sampler="top_k")
49
+ # Generate conditioned on the `"The quick brown fox."` as an input sequence.
50
+ seq_2_seq_lm.generate("The quick brown fox.", max_length=30)
51
+ ```
52
+ """
53
+
54
+ # TODO: fill in during https://github.com/keras-team/keras-hub/pull/1425
@@ -0,0 +1,20 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from keras_hub.src.models.t5.t5_backbone import T5Backbone
16
+ from keras_hub.src.models.t5.t5_presets import backbone_presets
17
+ from keras_hub.src.models.t5.t5_tokenizer import T5Tokenizer
18
+ from keras_hub.src.utils.preset_utils import register_presets
19
+
20
+ register_presets(backbone_presets, (T5Backbone, T5Tokenizer))