keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (297) hide show
  1. keras_hub/__init__.py +52 -0
  2. keras_hub/api/__init__.py +27 -0
  3. keras_hub/api/layers/__init__.py +47 -0
  4. keras_hub/api/metrics/__init__.py +24 -0
  5. keras_hub/api/models/__init__.py +249 -0
  6. keras_hub/api/samplers/__init__.py +29 -0
  7. keras_hub/api/tokenizers/__init__.py +35 -0
  8. keras_hub/src/__init__.py +13 -0
  9. keras_hub/src/api_export.py +53 -0
  10. keras_hub/src/layers/__init__.py +13 -0
  11. keras_hub/src/layers/modeling/__init__.py +13 -0
  12. keras_hub/src/layers/modeling/alibi_bias.py +143 -0
  13. keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
  14. keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
  15. keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
  16. keras_hub/src/layers/modeling/position_embedding.py +123 -0
  17. keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
  18. keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
  19. keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
  20. keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
  21. keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
  22. keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
  23. keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
  24. keras_hub/src/layers/preprocessing/__init__.py +13 -0
  25. keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
  26. keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
  27. keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
  28. keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
  29. keras_hub/src/layers/preprocessing/random_swap.py +267 -0
  30. keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
  31. keras_hub/src/metrics/__init__.py +13 -0
  32. keras_hub/src/metrics/bleu.py +394 -0
  33. keras_hub/src/metrics/edit_distance.py +197 -0
  34. keras_hub/src/metrics/perplexity.py +181 -0
  35. keras_hub/src/metrics/rouge_base.py +204 -0
  36. keras_hub/src/metrics/rouge_l.py +97 -0
  37. keras_hub/src/metrics/rouge_n.py +125 -0
  38. keras_hub/src/models/__init__.py +13 -0
  39. keras_hub/src/models/albert/__init__.py +20 -0
  40. keras_hub/src/models/albert/albert_backbone.py +267 -0
  41. keras_hub/src/models/albert/albert_classifier.py +202 -0
  42. keras_hub/src/models/albert/albert_masked_lm.py +129 -0
  43. keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
  44. keras_hub/src/models/albert/albert_preprocessor.py +206 -0
  45. keras_hub/src/models/albert/albert_presets.py +70 -0
  46. keras_hub/src/models/albert/albert_tokenizer.py +119 -0
  47. keras_hub/src/models/backbone.py +311 -0
  48. keras_hub/src/models/bart/__init__.py +20 -0
  49. keras_hub/src/models/bart/bart_backbone.py +261 -0
  50. keras_hub/src/models/bart/bart_preprocessor.py +276 -0
  51. keras_hub/src/models/bart/bart_presets.py +74 -0
  52. keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
  53. keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
  54. keras_hub/src/models/bart/bart_tokenizer.py +124 -0
  55. keras_hub/src/models/bert/__init__.py +23 -0
  56. keras_hub/src/models/bert/bert_backbone.py +227 -0
  57. keras_hub/src/models/bert/bert_classifier.py +183 -0
  58. keras_hub/src/models/bert/bert_masked_lm.py +131 -0
  59. keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
  60. keras_hub/src/models/bert/bert_preprocessor.py +184 -0
  61. keras_hub/src/models/bert/bert_presets.py +147 -0
  62. keras_hub/src/models/bert/bert_tokenizer.py +112 -0
  63. keras_hub/src/models/bloom/__init__.py +20 -0
  64. keras_hub/src/models/bloom/bloom_attention.py +186 -0
  65. keras_hub/src/models/bloom/bloom_backbone.py +173 -0
  66. keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
  67. keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
  68. keras_hub/src/models/bloom/bloom_decoder.py +206 -0
  69. keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
  70. keras_hub/src/models/bloom/bloom_presets.py +121 -0
  71. keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
  72. keras_hub/src/models/causal_lm.py +383 -0
  73. keras_hub/src/models/classifier.py +109 -0
  74. keras_hub/src/models/csp_darknet/__init__.py +13 -0
  75. keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
  76. keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
  77. keras_hub/src/models/deberta_v3/__init__.py +24 -0
  78. keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
  79. keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
  80. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
  81. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
  82. keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
  83. keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
  84. keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
  85. keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
  86. keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
  87. keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
  88. keras_hub/src/models/densenet/__init__.py +13 -0
  89. keras_hub/src/models/densenet/densenet_backbone.py +210 -0
  90. keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
  91. keras_hub/src/models/distil_bert/__init__.py +26 -0
  92. keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
  93. keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
  94. keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
  95. keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
  96. keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
  97. keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
  98. keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
  99. keras_hub/src/models/electra/__init__.py +20 -0
  100. keras_hub/src/models/electra/electra_backbone.py +247 -0
  101. keras_hub/src/models/electra/electra_preprocessor.py +154 -0
  102. keras_hub/src/models/electra/electra_presets.py +95 -0
  103. keras_hub/src/models/electra/electra_tokenizer.py +104 -0
  104. keras_hub/src/models/f_net/__init__.py +20 -0
  105. keras_hub/src/models/f_net/f_net_backbone.py +236 -0
  106. keras_hub/src/models/f_net/f_net_classifier.py +154 -0
  107. keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
  108. keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
  109. keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
  110. keras_hub/src/models/f_net/f_net_presets.py +43 -0
  111. keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
  112. keras_hub/src/models/falcon/__init__.py +20 -0
  113. keras_hub/src/models/falcon/falcon_attention.py +156 -0
  114. keras_hub/src/models/falcon/falcon_backbone.py +164 -0
  115. keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
  116. keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
  117. keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
  118. keras_hub/src/models/falcon/falcon_presets.py +30 -0
  119. keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
  120. keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
  121. keras_hub/src/models/feature_pyramid_backbone.py +73 -0
  122. keras_hub/src/models/gemma/__init__.py +20 -0
  123. keras_hub/src/models/gemma/gemma_attention.py +250 -0
  124. keras_hub/src/models/gemma/gemma_backbone.py +316 -0
  125. keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
  126. keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
  127. keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
  128. keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
  129. keras_hub/src/models/gemma/gemma_presets.py +248 -0
  130. keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
  131. keras_hub/src/models/gemma/rms_normalization.py +40 -0
  132. keras_hub/src/models/gpt2/__init__.py +20 -0
  133. keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
  134. keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
  135. keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
  136. keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
  137. keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
  138. keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
  139. keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
  140. keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
  141. keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
  142. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
  143. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
  144. keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
  145. keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
  146. keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
  147. keras_hub/src/models/image_classifier.py +90 -0
  148. keras_hub/src/models/llama/__init__.py +20 -0
  149. keras_hub/src/models/llama/llama_attention.py +225 -0
  150. keras_hub/src/models/llama/llama_backbone.py +188 -0
  151. keras_hub/src/models/llama/llama_causal_lm.py +327 -0
  152. keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
  153. keras_hub/src/models/llama/llama_decoder.py +246 -0
  154. keras_hub/src/models/llama/llama_layernorm.py +48 -0
  155. keras_hub/src/models/llama/llama_preprocessor.py +189 -0
  156. keras_hub/src/models/llama/llama_presets.py +80 -0
  157. keras_hub/src/models/llama/llama_tokenizer.py +84 -0
  158. keras_hub/src/models/llama3/__init__.py +20 -0
  159. keras_hub/src/models/llama3/llama3_backbone.py +84 -0
  160. keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
  161. keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
  162. keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
  163. keras_hub/src/models/llama3/llama3_presets.py +69 -0
  164. keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
  165. keras_hub/src/models/masked_lm.py +101 -0
  166. keras_hub/src/models/mistral/__init__.py +20 -0
  167. keras_hub/src/models/mistral/mistral_attention.py +238 -0
  168. keras_hub/src/models/mistral/mistral_backbone.py +203 -0
  169. keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
  170. keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
  171. keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
  172. keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
  173. keras_hub/src/models/mistral/mistral_presets.py +48 -0
  174. keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
  175. keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
  176. keras_hub/src/models/mix_transformer/__init__.py +13 -0
  177. keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
  178. keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
  179. keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
  180. keras_hub/src/models/opt/__init__.py +20 -0
  181. keras_hub/src/models/opt/opt_backbone.py +173 -0
  182. keras_hub/src/models/opt/opt_causal_lm.py +301 -0
  183. keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
  184. keras_hub/src/models/opt/opt_preprocessor.py +188 -0
  185. keras_hub/src/models/opt/opt_presets.py +72 -0
  186. keras_hub/src/models/opt/opt_tokenizer.py +116 -0
  187. keras_hub/src/models/pali_gemma/__init__.py +23 -0
  188. keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
  189. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
  190. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
  191. keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
  192. keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
  193. keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
  194. keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
  195. keras_hub/src/models/phi3/__init__.py +20 -0
  196. keras_hub/src/models/phi3/phi3_attention.py +260 -0
  197. keras_hub/src/models/phi3/phi3_backbone.py +224 -0
  198. keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
  199. keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
  200. keras_hub/src/models/phi3/phi3_decoder.py +260 -0
  201. keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
  202. keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
  203. keras_hub/src/models/phi3/phi3_presets.py +50 -0
  204. keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
  205. keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
  206. keras_hub/src/models/preprocessor.py +207 -0
  207. keras_hub/src/models/resnet/__init__.py +13 -0
  208. keras_hub/src/models/resnet/resnet_backbone.py +612 -0
  209. keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
  210. keras_hub/src/models/roberta/__init__.py +20 -0
  211. keras_hub/src/models/roberta/roberta_backbone.py +184 -0
  212. keras_hub/src/models/roberta/roberta_classifier.py +209 -0
  213. keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
  214. keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
  215. keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
  216. keras_hub/src/models/roberta/roberta_presets.py +43 -0
  217. keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
  218. keras_hub/src/models/seq_2_seq_lm.py +54 -0
  219. keras_hub/src/models/t5/__init__.py +20 -0
  220. keras_hub/src/models/t5/t5_backbone.py +261 -0
  221. keras_hub/src/models/t5/t5_layer_norm.py +35 -0
  222. keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
  223. keras_hub/src/models/t5/t5_presets.py +95 -0
  224. keras_hub/src/models/t5/t5_tokenizer.py +100 -0
  225. keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
  226. keras_hub/src/models/task.py +419 -0
  227. keras_hub/src/models/vgg/__init__.py +13 -0
  228. keras_hub/src/models/vgg/vgg_backbone.py +158 -0
  229. keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
  230. keras_hub/src/models/vit_det/__init__.py +13 -0
  231. keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
  232. keras_hub/src/models/vit_det/vit_layers.py +565 -0
  233. keras_hub/src/models/whisper/__init__.py +20 -0
  234. keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
  235. keras_hub/src/models/whisper/whisper_backbone.py +305 -0
  236. keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
  237. keras_hub/src/models/whisper/whisper_decoder.py +141 -0
  238. keras_hub/src/models/whisper/whisper_encoder.py +106 -0
  239. keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
  240. keras_hub/src/models/whisper/whisper_presets.py +148 -0
  241. keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
  242. keras_hub/src/models/xlm_roberta/__init__.py +26 -0
  243. keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
  244. keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
  245. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
  246. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
  247. keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
  248. keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
  249. keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
  250. keras_hub/src/models/xlnet/__init__.py +13 -0
  251. keras_hub/src/models/xlnet/relative_attention.py +459 -0
  252. keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
  253. keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
  254. keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
  255. keras_hub/src/samplers/__init__.py +13 -0
  256. keras_hub/src/samplers/beam_sampler.py +207 -0
  257. keras_hub/src/samplers/contrastive_sampler.py +231 -0
  258. keras_hub/src/samplers/greedy_sampler.py +50 -0
  259. keras_hub/src/samplers/random_sampler.py +77 -0
  260. keras_hub/src/samplers/sampler.py +237 -0
  261. keras_hub/src/samplers/serialization.py +97 -0
  262. keras_hub/src/samplers/top_k_sampler.py +92 -0
  263. keras_hub/src/samplers/top_p_sampler.py +113 -0
  264. keras_hub/src/tests/__init__.py +13 -0
  265. keras_hub/src/tests/test_case.py +608 -0
  266. keras_hub/src/tokenizers/__init__.py +13 -0
  267. keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
  268. keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
  269. keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
  270. keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
  271. keras_hub/src/tokenizers/tokenizer.py +235 -0
  272. keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
  273. keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
  274. keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
  275. keras_hub/src/utils/__init__.py +13 -0
  276. keras_hub/src/utils/keras_utils.py +130 -0
  277. keras_hub/src/utils/pipeline_model.py +293 -0
  278. keras_hub/src/utils/preset_utils.py +621 -0
  279. keras_hub/src/utils/python_utils.py +21 -0
  280. keras_hub/src/utils/tensor_utils.py +206 -0
  281. keras_hub/src/utils/timm/__init__.py +13 -0
  282. keras_hub/src/utils/timm/convert.py +37 -0
  283. keras_hub/src/utils/timm/convert_resnet.py +171 -0
  284. keras_hub/src/utils/transformers/__init__.py +13 -0
  285. keras_hub/src/utils/transformers/convert.py +101 -0
  286. keras_hub/src/utils/transformers/convert_bert.py +173 -0
  287. keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
  288. keras_hub/src/utils/transformers/convert_gemma.py +187 -0
  289. keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
  290. keras_hub/src/utils/transformers/convert_llama3.py +136 -0
  291. keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
  292. keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
  293. keras_hub/src/version_utils.py +23 -0
  294. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
  295. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
  296. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
  297. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,183 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import keras
16
+
17
+ from keras_hub.src.api_export import keras_hub_export
18
+ from keras_hub.src.models.bert.bert_backbone import BertBackbone
19
+ from keras_hub.src.models.bert.bert_backbone import bert_kernel_initializer
20
+ from keras_hub.src.models.bert.bert_preprocessor import BertPreprocessor
21
+ from keras_hub.src.models.classifier import Classifier
22
+
23
+
24
+ @keras_hub_export("keras_hub.models.BertClassifier")
25
+ class BertClassifier(Classifier):
26
+ """An end-to-end BERT model for classification tasks.
27
+
28
+ This model attaches a classification head to a
29
+ `keras_hub.model.BertBackbone` instance, mapping from the backbone outputs
30
+ to logits suitable for a classification task. For usage of this model with
31
+ pre-trained weights, use the `from_preset()` constructor.
32
+
33
+ This model can optionally be configured with a `preprocessor` layer, in
34
+ which case it will automatically apply preprocessing to raw inputs during
35
+ `fit()`, `predict()`, and `evaluate()`. This is done by default when
36
+ creating the model with `from_preset()`.
37
+
38
+ Disclaimer: Pre-trained models are provided on an "as is" basis, without
39
+ warranties or conditions of any kind.
40
+
41
+ Args:
42
+ backbone: A `keras_hub.models.BertBackbone` instance.
43
+ num_classes: int. Number of classes to predict.
44
+ preprocessor: A `keras_hub.models.BertPreprocessor` or `None`. If
45
+ `None`, this model will not apply preprocessing, and inputs should
46
+ be preprocessed before calling the model.
47
+ activation: Optional `str` or callable. The
48
+ activation function to use on the model outputs. Set
49
+ `activation="softmax"` to return output probabilities.
50
+ Defaults to `None`.
51
+ dropout: float. The dropout probability value, applied after the dense
52
+ layer.
53
+
54
+ Examples:
55
+
56
+ Raw string data.
57
+ ```python
58
+ features = ["The quick brown fox jumped.", "I forgot my homework."]
59
+ labels = [0, 3]
60
+
61
+ # Pretrained classifier.
62
+ classifier = keras_hub.models.BertClassifier.from_preset(
63
+ "bert_base_en_uncased",
64
+ num_classes=4,
65
+ )
66
+ classifier.fit(x=features, y=labels, batch_size=2)
67
+ classifier.predict(x=features, batch_size=2)
68
+
69
+ # Re-compile (e.g., with a new learning rate).
70
+ classifier.compile(
71
+ loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
72
+ optimizer=keras.optimizers.Adam(5e-5),
73
+ jit_compile=True,
74
+ )
75
+ # Access backbone programmatically (e.g., to change `trainable`).
76
+ classifier.backbone.trainable = False
77
+ # Fit again.
78
+ classifier.fit(x=features, y=labels, batch_size=2)
79
+ ```
80
+
81
+ Preprocessed integer data.
82
+ ```python
83
+ features = {
84
+ "token_ids": np.ones(shape=(2, 12), dtype="int32"),
85
+ "segment_ids": np.array([[0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0]] * 2),
86
+ "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]] * 2),
87
+ }
88
+ labels = [0, 3]
89
+
90
+ # Pretrained classifier without preprocessing.
91
+ classifier = keras_hub.models.BertClassifier.from_preset(
92
+ "bert_base_en_uncased",
93
+ num_classes=4,
94
+ preprocessor=None,
95
+ )
96
+ classifier.fit(x=features, y=labels, batch_size=2)
97
+ ```
98
+
99
+ Custom backbone and vocabulary.
100
+ ```python
101
+ features = ["The quick brown fox jumped.", "I forgot my homework."]
102
+ labels = [0, 3]
103
+
104
+ vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
105
+ vocab += ["The", "quick", "brown", "fox", "jumped", "."]
106
+ tokenizer = keras_hub.models.BertTokenizer(
107
+ vocabulary=vocab,
108
+ )
109
+ preprocessor = keras_hub.models.BertPreprocessor(
110
+ tokenizer=tokenizer,
111
+ sequence_length=128,
112
+ )
113
+ backbone = keras_hub.models.BertBackbone(
114
+ vocabulary_size=30552,
115
+ num_layers=4,
116
+ num_heads=4,
117
+ hidden_dim=256,
118
+ intermediate_dim=512,
119
+ max_sequence_length=128,
120
+ )
121
+ classifier = keras_hub.models.BertClassifier(
122
+ backbone=backbone,
123
+ preprocessor=preprocessor,
124
+ num_classes=4,
125
+ )
126
+ classifier.fit(x=features, y=labels, batch_size=2)
127
+ ```
128
+ """
129
+
130
+ backbone_cls = BertBackbone
131
+ preprocessor_cls = BertPreprocessor
132
+
133
+ def __init__(
134
+ self,
135
+ backbone,
136
+ num_classes,
137
+ preprocessor=None,
138
+ activation=None,
139
+ dropout=0.1,
140
+ **kwargs,
141
+ ):
142
+ # === Layers ===
143
+ self.backbone = backbone
144
+ self.preprocessor = preprocessor
145
+ self.output_dropout = keras.layers.Dropout(
146
+ dropout,
147
+ dtype=backbone.dtype_policy,
148
+ name="classifier_dropout",
149
+ )
150
+ self.output_dense = keras.layers.Dense(
151
+ num_classes,
152
+ kernel_initializer=bert_kernel_initializer(),
153
+ activation=activation,
154
+ dtype=backbone.dtype_policy,
155
+ name="logits",
156
+ )
157
+
158
+ # === Functional Model ===
159
+ inputs = backbone.input
160
+ pooled = backbone(inputs)["pooled_output"]
161
+ pooled = self.output_dropout(pooled)
162
+ outputs = self.output_dense(pooled)
163
+ super().__init__(
164
+ inputs=inputs,
165
+ outputs=outputs,
166
+ **kwargs,
167
+ )
168
+
169
+ # === Config ===
170
+ self.num_classes = num_classes
171
+ self.activation = keras.activations.get(activation)
172
+ self.dropout = dropout
173
+
174
+ def get_config(self):
175
+ config = super().get_config()
176
+ config.update(
177
+ {
178
+ "num_classes": self.num_classes,
179
+ "activation": keras.activations.serialize(self.activation),
180
+ "dropout": self.dropout,
181
+ }
182
+ )
183
+ return config
@@ -0,0 +1,131 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import keras
16
+
17
+ from keras_hub.src.api_export import keras_hub_export
18
+ from keras_hub.src.layers.modeling.masked_lm_head import MaskedLMHead
19
+ from keras_hub.src.models.bert.bert_backbone import BertBackbone
20
+ from keras_hub.src.models.bert.bert_backbone import bert_kernel_initializer
21
+ from keras_hub.src.models.bert.bert_masked_lm_preprocessor import (
22
+ BertMaskedLMPreprocessor,
23
+ )
24
+ from keras_hub.src.models.masked_lm import MaskedLM
25
+
26
+
27
+ @keras_hub_export("keras_hub.models.BertMaskedLM")
28
+ class BertMaskedLM(MaskedLM):
29
+ """An end-to-end BERT model for the masked language modeling task.
30
+
31
+ This model will train BERT on a masked language modeling task.
32
+ The model will predict labels for a number of masked tokens in the
33
+ input data. For usage of this model with pre-trained weights, see the
34
+ `from_preset()` constructor.
35
+
36
+ This model can optionally be configured with a `preprocessor` layer, in
37
+ which case inputs can be raw string features during `fit()`, `predict()`,
38
+ and `evaluate()`. Inputs will be tokenized and dynamically masked during
39
+ training and evaluation. This is done by default when creating the model
40
+ with `from_preset()`.
41
+
42
+ Disclaimer: Pre-trained models are provided on an "as is" basis, without
43
+ warranties or conditions of any kind.
44
+
45
+ Args:
46
+ backbone: A `keras_hub.models.BertBackbone` instance.
47
+ preprocessor: A `keras_hub.models.BertMaskedLMPreprocessor` or
48
+ `None`. If `None`, this model will not apply preprocessing, and
49
+ inputs should be preprocessed before calling the model.
50
+
51
+ Examples:
52
+
53
+ Raw string data.
54
+ ```python
55
+ features = ["The quick brown fox jumped.", "I forgot my homework."]
56
+
57
+ # Pretrained language model.
58
+ masked_lm = keras_hub.models.BertMaskedLM.from_preset(
59
+ "bert_base_en_uncased",
60
+ )
61
+ masked_lm.fit(x=features, batch_size=2)
62
+
63
+ # Re-compile (e.g., with a new learning rate).
64
+ masked_lm.compile(
65
+ loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
66
+ optimizer=keras.optimizers.Adam(5e-5),
67
+ jit_compile=True,
68
+ )
69
+ # Access backbone programmatically (e.g., to change `trainable`).
70
+ masked_lm.backbone.trainable = False
71
+ # Fit again.
72
+ masked_lm.fit(x=features, batch_size=2)
73
+ ```
74
+
75
+ Preprocessed integer data.
76
+ ```python
77
+ # Create preprocessed batch where 0 is the mask token.
78
+ features = {
79
+ "token_ids": np.array([[1, 2, 0, 4, 0, 6, 7, 8]] * 2),
80
+ "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 1]] * 2),
81
+ "mask_positions": np.array([[2, 4]] * 2),
82
+ "segment_ids": np.array([[0, 0, 0, 0, 0, 0, 0, 0]] * 2)
83
+ }
84
+ # Labels are the original masked values.
85
+ labels = [[3, 5]] * 2
86
+
87
+ masked_lm = keras_hub.models.BertMaskedLM.from_preset(
88
+ "bert_base_en_uncased",
89
+ preprocessor=None,
90
+ )
91
+ masked_lm.fit(x=features, y=labels, batch_size=2)
92
+ ```
93
+ """
94
+
95
+ backbone_cls = BertBackbone
96
+ preprocessor_cls = BertMaskedLMPreprocessor
97
+
98
+ def __init__(
99
+ self,
100
+ backbone,
101
+ preprocessor=None,
102
+ **kwargs,
103
+ ):
104
+ # === Layers ===
105
+ self.backbone = backbone
106
+ self.preprocessor = preprocessor
107
+ self.masked_lm_head = MaskedLMHead(
108
+ vocabulary_size=backbone.vocabulary_size,
109
+ token_embedding=backbone.token_embedding,
110
+ intermediate_activation="gelu",
111
+ kernel_initializer=bert_kernel_initializer(),
112
+ dtype=backbone.dtype_policy,
113
+ name="mlm_head",
114
+ )
115
+
116
+ # === Functional Model ===
117
+ inputs = {
118
+ **backbone.input,
119
+ "mask_positions": keras.Input(
120
+ shape=(None,), dtype="int32", name="mask_positions"
121
+ ),
122
+ }
123
+ backbone_outputs = backbone(backbone.input)
124
+ outputs = self.masked_lm_head(
125
+ backbone_outputs["sequence_output"], inputs["mask_positions"]
126
+ )
127
+ super().__init__(
128
+ inputs=inputs,
129
+ outputs=outputs,
130
+ **kwargs,
131
+ )
@@ -0,0 +1,198 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import keras
16
+ from absl import logging
17
+
18
+ from keras_hub.src.api_export import keras_hub_export
19
+ from keras_hub.src.layers.preprocessing.masked_lm_mask_generator import (
20
+ MaskedLMMaskGenerator,
21
+ )
22
+ from keras_hub.src.models.bert.bert_preprocessor import BertPreprocessor
23
+
24
+
25
+ @keras_hub_export("keras_hub.models.BertMaskedLMPreprocessor")
26
+ class BertMaskedLMPreprocessor(BertPreprocessor):
27
+ """BERT preprocessing for the masked language modeling task.
28
+
29
+ This preprocessing layer will prepare inputs for a masked language modeling
30
+ task. It is primarily intended for use with the
31
+ `keras_hub.models.BertMaskedLM` task model. Preprocessing will occur in
32
+ multiple steps.
33
+
34
+ 1. Tokenize any number of input segments using the `tokenizer`.
35
+ 2. Pack the inputs together with the appropriate `"[CLS]"`, `"[SEP]"` and
36
+ `"[PAD]"` tokens.
37
+ 3. Randomly select non-special tokens to mask, controlled by
38
+ `mask_selection_rate`.
39
+ 4. Construct a `(x, y, sample_weight)` tuple suitable for training with a
40
+ `keras_hub.models.BertMaskedLM` task model.
41
+
42
+ Args:
43
+ tokenizer: A `keras_hub.models.BertTokenizer` instance.
44
+ sequence_length: int. The length of the packed inputs.
45
+ truncate: string. The algorithm to truncate a list of batched segments
46
+ to fit within `sequence_length`. The value can be either
47
+ `round_robin` or `waterfall`:
48
+ - `"round_robin"`: Available space is assigned one token at a
49
+ time in a round-robin fashion to the inputs that still need
50
+ some, until the limit is reached.
51
+ - `"waterfall"`: The allocation of the budget is done using a
52
+ "waterfall" algorithm that allocates quota in a
53
+ left-to-right manner and fills up the buckets until we run
54
+ out of budget. It supports an arbitrary number of segments.
55
+ mask_selection_rate: float. The probability an input token will be
56
+ dynamically masked.
57
+ mask_selection_length: int. The maximum number of masked tokens
58
+ in a given sample.
59
+ mask_token_rate: float. The probability the a selected token will be
60
+ replaced with the mask token.
61
+ random_token_rate: float. The probability the a selected token will be
62
+ replaced with a random token from the vocabulary. A selected token
63
+ will be left as is with probability
64
+ `1 - mask_token_rate - random_token_rate`.
65
+
66
+ Call arguments:
67
+ x: A tensor of single string sequences, or a tuple of multiple
68
+ tensor sequences to be packed together. Inputs may be batched or
69
+ unbatched. For single sequences, raw python inputs will be converted
70
+ to tensors. For multiple sequences, pass tensors directly.
71
+ y: Label data. Should always be `None` as the layer generates labels.
72
+ sample_weight: Label weights. Should always be `None` as the layer
73
+ generates label weights.
74
+
75
+ Examples:
76
+
77
+ Directly calling the layer on data.
78
+ ```python
79
+ preprocessor = keras_hub.models.BertMaskedLMPreprocessor.from_preset(
80
+ "bert_base_en_uncased"
81
+ )
82
+
83
+ # Tokenize and mask a single sentence.
84
+ preprocessor("The quick brown fox jumped.")
85
+
86
+ # Tokenize and mask a batch of single sentences.
87
+ preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
88
+
89
+ # Tokenize and mask sentence pairs.
90
+ # In this case, always convert input to tensors before calling the layer.
91
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
92
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
93
+ preprocessor((first, second))
94
+ ```
95
+
96
+ Mapping with `tf.data.Dataset`.
97
+ ```python
98
+ preprocessor = keras_hub.models.BertMaskedLMPreprocessor.from_preset(
99
+ "bert_base_en_uncased"
100
+ )
101
+
102
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
103
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
104
+
105
+ # Map single sentences.
106
+ ds = tf.data.Dataset.from_tensor_slices(first)
107
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
108
+
109
+ # Map sentence pairs.
110
+ ds = tf.data.Dataset.from_tensor_slices((first, second))
111
+ # Watch out for tf.data's default unpacking of tuples here!
112
+ # Best to invoke the `preprocessor` directly in this case.
113
+ ds = ds.map(
114
+ lambda first, second: preprocessor(x=(first, second)),
115
+ num_parallel_calls=tf.data.AUTOTUNE,
116
+ )
117
+ ```
118
+ """
119
+
120
+ def __init__(
121
+ self,
122
+ tokenizer,
123
+ sequence_length=512,
124
+ truncate="round_robin",
125
+ mask_selection_rate=0.15,
126
+ mask_selection_length=96,
127
+ mask_token_rate=0.8,
128
+ random_token_rate=0.1,
129
+ **kwargs,
130
+ ):
131
+ super().__init__(
132
+ tokenizer,
133
+ sequence_length=sequence_length,
134
+ truncate=truncate,
135
+ **kwargs,
136
+ )
137
+ self.mask_selection_rate = mask_selection_rate
138
+ self.mask_selection_length = mask_selection_length
139
+ self.mask_token_rate = mask_token_rate
140
+ self.random_token_rate = random_token_rate
141
+ self.masker = None
142
+
143
+ def build(self, input_shape):
144
+ super().build(input_shape)
145
+ # Defer masker creation to `build()` so that we can be sure tokenizer
146
+ # assets have loaded when restoring a saved model.
147
+ self.masker = MaskedLMMaskGenerator(
148
+ mask_selection_rate=self.mask_selection_rate,
149
+ mask_selection_length=self.mask_selection_length,
150
+ mask_token_rate=self.mask_token_rate,
151
+ random_token_rate=self.random_token_rate,
152
+ vocabulary_size=self.tokenizer.vocabulary_size(),
153
+ mask_token_id=self.tokenizer.mask_token_id,
154
+ unselectable_token_ids=[
155
+ self.tokenizer.cls_token_id,
156
+ self.tokenizer.sep_token_id,
157
+ self.tokenizer.pad_token_id,
158
+ ],
159
+ )
160
+
161
+ def call(self, x, y=None, sample_weight=None):
162
+ if y is not None or sample_weight is not None:
163
+ logging.warning(
164
+ f"{self.__class__.__name__} generates `y` and `sample_weight` "
165
+ "based on your input data, but your data already contains `y` "
166
+ "or `sample_weight`. Your `y` and `sample_weight` will be "
167
+ "ignored."
168
+ )
169
+
170
+ x = super().call(x)
171
+
172
+ token_ids, padding_mask, segment_ids = (
173
+ x["token_ids"],
174
+ x["padding_mask"],
175
+ x["segment_ids"],
176
+ )
177
+ masker_outputs = self.masker(token_ids)
178
+ x = {
179
+ "token_ids": masker_outputs["token_ids"],
180
+ "padding_mask": padding_mask,
181
+ "segment_ids": segment_ids,
182
+ "mask_positions": masker_outputs["mask_positions"],
183
+ }
184
+ y = masker_outputs["mask_ids"]
185
+ sample_weight = masker_outputs["mask_weights"]
186
+ return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
187
+
188
+ def get_config(self):
189
+ config = super().get_config()
190
+ config.update(
191
+ {
192
+ "mask_selection_rate": self.mask_selection_rate,
193
+ "mask_selection_length": self.mask_selection_length,
194
+ "mask_token_rate": self.mask_token_rate,
195
+ "random_token_rate": self.random_token_rate,
196
+ }
197
+ )
198
+ return config
@@ -0,0 +1,184 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import keras
16
+
17
+ from keras_hub.src.api_export import keras_hub_export
18
+ from keras_hub.src.layers.preprocessing.multi_segment_packer import (
19
+ MultiSegmentPacker,
20
+ )
21
+ from keras_hub.src.models.bert.bert_tokenizer import BertTokenizer
22
+ from keras_hub.src.models.preprocessor import Preprocessor
23
+ from keras_hub.src.utils.keras_utils import (
24
+ convert_inputs_to_list_of_tensor_segments,
25
+ )
26
+
27
+
28
+ @keras_hub_export("keras_hub.models.BertPreprocessor")
29
+ class BertPreprocessor(Preprocessor):
30
+ """A BERT preprocessing layer which tokenizes and packs inputs.
31
+
32
+ This preprocessing layer will do three things:
33
+
34
+ 1. Tokenize any number of input segments using the `tokenizer`.
35
+ 2. Pack the inputs together using a `keras_hub.layers.MultiSegmentPacker`.
36
+ with the appropriate `"[CLS]"`, `"[SEP]"` and `"[PAD]"` tokens.
37
+ 3. Construct a dictionary with keys `"token_ids"`, `"segment_ids"`,
38
+ `"padding_mask"`, that can be passed directly to a BERT model.
39
+
40
+ This layer can be used directly with `tf.data.Dataset.map` to preprocess
41
+ string data in the `(x, y, sample_weight)` format used by
42
+ `keras.Model.fit`.
43
+
44
+ Args:
45
+ tokenizer: A `keras_hub.models.BertTokenizer` instance.
46
+ sequence_length: The length of the packed inputs.
47
+ truncate: string. The algorithm to truncate a list of batched segments
48
+ to fit within `sequence_length`. The value can be either
49
+ `round_robin` or `waterfall`:
50
+ - `"round_robin"`: Available space is assigned one token at a
51
+ time in a round-robin fashion to the inputs that still need
52
+ some, until the limit is reached.
53
+ - `"waterfall"`: The allocation of the budget is done using a
54
+ "waterfall" algorithm that allocates quota in a
55
+ left-to-right manner and fills up the buckets until we run
56
+ out of budget. It supports an arbitrary number of segments.
57
+
58
+ Call arguments:
59
+ x: A tensor of single string sequences, or a tuple of multiple
60
+ tensor sequences to be packed together. Inputs may be batched or
61
+ unbatched. For single sequences, raw python inputs will be converted
62
+ to tensors. For multiple sequences, pass tensors directly.
63
+ y: Any label data. Will be passed through unaltered.
64
+ sample_weight: Any label weight data. Will be passed through unaltered.
65
+
66
+ Examples:
67
+
68
+ Directly calling the layer on data.
69
+ ```python
70
+ preprocessor = keras_hub.models.BertPreprocessor.from_preset(
71
+ "bert_base_en_uncased"
72
+ )
73
+
74
+ # Tokenize and pack a single sentence.
75
+ preprocessor("The quick brown fox jumped.")
76
+
77
+ # Tokenize a batch of single sentences.
78
+ preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
79
+
80
+ # Preprocess a batch of sentence pairs.
81
+ # When handling multiple sequences, always convert to tensors first!
82
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
83
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
84
+ preprocessor((first, second))
85
+
86
+ # Custom vocabulary.
87
+ vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
88
+ vocab += ["The", "quick", "brown", "fox", "jumped", "."]
89
+ tokenizer = keras_hub.models.BertTokenizer(vocabulary=vocab)
90
+ preprocessor = keras_hub.models.BertPreprocessor(tokenizer)
91
+ preprocessor("The quick brown fox jumped.")
92
+ ```
93
+
94
+ Mapping with `tf.data.Dataset`.
95
+ ```python
96
+ preprocessor = keras_hub.models.BertPreprocessor.from_preset(
97
+ "bert_base_en_uncased"
98
+ )
99
+
100
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
101
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
102
+ label = tf.constant([1, 1])
103
+
104
+ # Map labeled single sentences.
105
+ ds = tf.data.Dataset.from_tensor_slices((first, label))
106
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
107
+
108
+ # Map unlabeled single sentences.
109
+ ds = tf.data.Dataset.from_tensor_slices(first)
110
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
111
+
112
+ # Map labeled sentence pairs.
113
+ ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
114
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
115
+
116
+ # Map unlabeled sentence pairs.
117
+ ds = tf.data.Dataset.from_tensor_slices((first, second))
118
+ # Watch out for tf.data's default unpacking of tuples here!
119
+ # Best to invoke the `preprocessor` directly in this case.
120
+ ds = ds.map(
121
+ lambda first, second: preprocessor(x=(first, second)),
122
+ num_parallel_calls=tf.data.AUTOTUNE,
123
+ )
124
+ ```
125
+ """
126
+
127
+ tokenizer_cls = BertTokenizer
128
+
129
+ def __init__(
130
+ self,
131
+ tokenizer,
132
+ sequence_length=512,
133
+ truncate="round_robin",
134
+ **kwargs,
135
+ ):
136
+ super().__init__(**kwargs)
137
+ self.tokenizer = tokenizer
138
+ self.packer = None
139
+ self.sequence_length = sequence_length
140
+ self.truncate = truncate
141
+
142
+ def build(self, input_shape):
143
+ # Defer packer creation to `build()` so that we can be sure tokenizer
144
+ # assets have loaded when restoring a saved model.
145
+ self.packer = MultiSegmentPacker(
146
+ start_value=self.tokenizer.cls_token_id,
147
+ end_value=self.tokenizer.sep_token_id,
148
+ pad_value=self.tokenizer.pad_token_id,
149
+ truncate=self.truncate,
150
+ sequence_length=self.sequence_length,
151
+ )
152
+ self.built = True
153
+
154
+ def call(self, x, y=None, sample_weight=None):
155
+ x = convert_inputs_to_list_of_tensor_segments(x)
156
+ x = [self.tokenizer(segment) for segment in x]
157
+ token_ids, segment_ids = self.packer(x)
158
+ x = {
159
+ "token_ids": token_ids,
160
+ "segment_ids": segment_ids,
161
+ "padding_mask": token_ids != self.tokenizer.pad_token_id,
162
+ }
163
+ return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
164
+
165
+ def get_config(self):
166
+ config = super().get_config()
167
+ config.update(
168
+ {
169
+ "sequence_length": self.sequence_length,
170
+ "truncate": self.truncate,
171
+ }
172
+ )
173
+ return config
174
+
175
+ @property
176
+ def sequence_length(self):
177
+ """The padded length of model input sequences."""
178
+ return self._sequence_length
179
+
180
+ @sequence_length.setter
181
+ def sequence_length(self, value):
182
+ self._sequence_length = value
183
+ if self.packer is not None:
184
+ self.packer.sequence_length = value