keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (297) hide show
  1. keras_hub/__init__.py +52 -0
  2. keras_hub/api/__init__.py +27 -0
  3. keras_hub/api/layers/__init__.py +47 -0
  4. keras_hub/api/metrics/__init__.py +24 -0
  5. keras_hub/api/models/__init__.py +249 -0
  6. keras_hub/api/samplers/__init__.py +29 -0
  7. keras_hub/api/tokenizers/__init__.py +35 -0
  8. keras_hub/src/__init__.py +13 -0
  9. keras_hub/src/api_export.py +53 -0
  10. keras_hub/src/layers/__init__.py +13 -0
  11. keras_hub/src/layers/modeling/__init__.py +13 -0
  12. keras_hub/src/layers/modeling/alibi_bias.py +143 -0
  13. keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
  14. keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
  15. keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
  16. keras_hub/src/layers/modeling/position_embedding.py +123 -0
  17. keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
  18. keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
  19. keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
  20. keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
  21. keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
  22. keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
  23. keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
  24. keras_hub/src/layers/preprocessing/__init__.py +13 -0
  25. keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
  26. keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
  27. keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
  28. keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
  29. keras_hub/src/layers/preprocessing/random_swap.py +267 -0
  30. keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
  31. keras_hub/src/metrics/__init__.py +13 -0
  32. keras_hub/src/metrics/bleu.py +394 -0
  33. keras_hub/src/metrics/edit_distance.py +197 -0
  34. keras_hub/src/metrics/perplexity.py +181 -0
  35. keras_hub/src/metrics/rouge_base.py +204 -0
  36. keras_hub/src/metrics/rouge_l.py +97 -0
  37. keras_hub/src/metrics/rouge_n.py +125 -0
  38. keras_hub/src/models/__init__.py +13 -0
  39. keras_hub/src/models/albert/__init__.py +20 -0
  40. keras_hub/src/models/albert/albert_backbone.py +267 -0
  41. keras_hub/src/models/albert/albert_classifier.py +202 -0
  42. keras_hub/src/models/albert/albert_masked_lm.py +129 -0
  43. keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
  44. keras_hub/src/models/albert/albert_preprocessor.py +206 -0
  45. keras_hub/src/models/albert/albert_presets.py +70 -0
  46. keras_hub/src/models/albert/albert_tokenizer.py +119 -0
  47. keras_hub/src/models/backbone.py +311 -0
  48. keras_hub/src/models/bart/__init__.py +20 -0
  49. keras_hub/src/models/bart/bart_backbone.py +261 -0
  50. keras_hub/src/models/bart/bart_preprocessor.py +276 -0
  51. keras_hub/src/models/bart/bart_presets.py +74 -0
  52. keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
  53. keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
  54. keras_hub/src/models/bart/bart_tokenizer.py +124 -0
  55. keras_hub/src/models/bert/__init__.py +23 -0
  56. keras_hub/src/models/bert/bert_backbone.py +227 -0
  57. keras_hub/src/models/bert/bert_classifier.py +183 -0
  58. keras_hub/src/models/bert/bert_masked_lm.py +131 -0
  59. keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
  60. keras_hub/src/models/bert/bert_preprocessor.py +184 -0
  61. keras_hub/src/models/bert/bert_presets.py +147 -0
  62. keras_hub/src/models/bert/bert_tokenizer.py +112 -0
  63. keras_hub/src/models/bloom/__init__.py +20 -0
  64. keras_hub/src/models/bloom/bloom_attention.py +186 -0
  65. keras_hub/src/models/bloom/bloom_backbone.py +173 -0
  66. keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
  67. keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
  68. keras_hub/src/models/bloom/bloom_decoder.py +206 -0
  69. keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
  70. keras_hub/src/models/bloom/bloom_presets.py +121 -0
  71. keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
  72. keras_hub/src/models/causal_lm.py +383 -0
  73. keras_hub/src/models/classifier.py +109 -0
  74. keras_hub/src/models/csp_darknet/__init__.py +13 -0
  75. keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
  76. keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
  77. keras_hub/src/models/deberta_v3/__init__.py +24 -0
  78. keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
  79. keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
  80. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
  81. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
  82. keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
  83. keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
  84. keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
  85. keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
  86. keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
  87. keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
  88. keras_hub/src/models/densenet/__init__.py +13 -0
  89. keras_hub/src/models/densenet/densenet_backbone.py +210 -0
  90. keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
  91. keras_hub/src/models/distil_bert/__init__.py +26 -0
  92. keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
  93. keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
  94. keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
  95. keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
  96. keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
  97. keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
  98. keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
  99. keras_hub/src/models/electra/__init__.py +20 -0
  100. keras_hub/src/models/electra/electra_backbone.py +247 -0
  101. keras_hub/src/models/electra/electra_preprocessor.py +154 -0
  102. keras_hub/src/models/electra/electra_presets.py +95 -0
  103. keras_hub/src/models/electra/electra_tokenizer.py +104 -0
  104. keras_hub/src/models/f_net/__init__.py +20 -0
  105. keras_hub/src/models/f_net/f_net_backbone.py +236 -0
  106. keras_hub/src/models/f_net/f_net_classifier.py +154 -0
  107. keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
  108. keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
  109. keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
  110. keras_hub/src/models/f_net/f_net_presets.py +43 -0
  111. keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
  112. keras_hub/src/models/falcon/__init__.py +20 -0
  113. keras_hub/src/models/falcon/falcon_attention.py +156 -0
  114. keras_hub/src/models/falcon/falcon_backbone.py +164 -0
  115. keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
  116. keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
  117. keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
  118. keras_hub/src/models/falcon/falcon_presets.py +30 -0
  119. keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
  120. keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
  121. keras_hub/src/models/feature_pyramid_backbone.py +73 -0
  122. keras_hub/src/models/gemma/__init__.py +20 -0
  123. keras_hub/src/models/gemma/gemma_attention.py +250 -0
  124. keras_hub/src/models/gemma/gemma_backbone.py +316 -0
  125. keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
  126. keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
  127. keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
  128. keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
  129. keras_hub/src/models/gemma/gemma_presets.py +248 -0
  130. keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
  131. keras_hub/src/models/gemma/rms_normalization.py +40 -0
  132. keras_hub/src/models/gpt2/__init__.py +20 -0
  133. keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
  134. keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
  135. keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
  136. keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
  137. keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
  138. keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
  139. keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
  140. keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
  141. keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
  142. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
  143. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
  144. keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
  145. keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
  146. keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
  147. keras_hub/src/models/image_classifier.py +90 -0
  148. keras_hub/src/models/llama/__init__.py +20 -0
  149. keras_hub/src/models/llama/llama_attention.py +225 -0
  150. keras_hub/src/models/llama/llama_backbone.py +188 -0
  151. keras_hub/src/models/llama/llama_causal_lm.py +327 -0
  152. keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
  153. keras_hub/src/models/llama/llama_decoder.py +246 -0
  154. keras_hub/src/models/llama/llama_layernorm.py +48 -0
  155. keras_hub/src/models/llama/llama_preprocessor.py +189 -0
  156. keras_hub/src/models/llama/llama_presets.py +80 -0
  157. keras_hub/src/models/llama/llama_tokenizer.py +84 -0
  158. keras_hub/src/models/llama3/__init__.py +20 -0
  159. keras_hub/src/models/llama3/llama3_backbone.py +84 -0
  160. keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
  161. keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
  162. keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
  163. keras_hub/src/models/llama3/llama3_presets.py +69 -0
  164. keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
  165. keras_hub/src/models/masked_lm.py +101 -0
  166. keras_hub/src/models/mistral/__init__.py +20 -0
  167. keras_hub/src/models/mistral/mistral_attention.py +238 -0
  168. keras_hub/src/models/mistral/mistral_backbone.py +203 -0
  169. keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
  170. keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
  171. keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
  172. keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
  173. keras_hub/src/models/mistral/mistral_presets.py +48 -0
  174. keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
  175. keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
  176. keras_hub/src/models/mix_transformer/__init__.py +13 -0
  177. keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
  178. keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
  179. keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
  180. keras_hub/src/models/opt/__init__.py +20 -0
  181. keras_hub/src/models/opt/opt_backbone.py +173 -0
  182. keras_hub/src/models/opt/opt_causal_lm.py +301 -0
  183. keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
  184. keras_hub/src/models/opt/opt_preprocessor.py +188 -0
  185. keras_hub/src/models/opt/opt_presets.py +72 -0
  186. keras_hub/src/models/opt/opt_tokenizer.py +116 -0
  187. keras_hub/src/models/pali_gemma/__init__.py +23 -0
  188. keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
  189. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
  190. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
  191. keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
  192. keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
  193. keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
  194. keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
  195. keras_hub/src/models/phi3/__init__.py +20 -0
  196. keras_hub/src/models/phi3/phi3_attention.py +260 -0
  197. keras_hub/src/models/phi3/phi3_backbone.py +224 -0
  198. keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
  199. keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
  200. keras_hub/src/models/phi3/phi3_decoder.py +260 -0
  201. keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
  202. keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
  203. keras_hub/src/models/phi3/phi3_presets.py +50 -0
  204. keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
  205. keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
  206. keras_hub/src/models/preprocessor.py +207 -0
  207. keras_hub/src/models/resnet/__init__.py +13 -0
  208. keras_hub/src/models/resnet/resnet_backbone.py +612 -0
  209. keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
  210. keras_hub/src/models/roberta/__init__.py +20 -0
  211. keras_hub/src/models/roberta/roberta_backbone.py +184 -0
  212. keras_hub/src/models/roberta/roberta_classifier.py +209 -0
  213. keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
  214. keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
  215. keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
  216. keras_hub/src/models/roberta/roberta_presets.py +43 -0
  217. keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
  218. keras_hub/src/models/seq_2_seq_lm.py +54 -0
  219. keras_hub/src/models/t5/__init__.py +20 -0
  220. keras_hub/src/models/t5/t5_backbone.py +261 -0
  221. keras_hub/src/models/t5/t5_layer_norm.py +35 -0
  222. keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
  223. keras_hub/src/models/t5/t5_presets.py +95 -0
  224. keras_hub/src/models/t5/t5_tokenizer.py +100 -0
  225. keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
  226. keras_hub/src/models/task.py +419 -0
  227. keras_hub/src/models/vgg/__init__.py +13 -0
  228. keras_hub/src/models/vgg/vgg_backbone.py +158 -0
  229. keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
  230. keras_hub/src/models/vit_det/__init__.py +13 -0
  231. keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
  232. keras_hub/src/models/vit_det/vit_layers.py +565 -0
  233. keras_hub/src/models/whisper/__init__.py +20 -0
  234. keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
  235. keras_hub/src/models/whisper/whisper_backbone.py +305 -0
  236. keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
  237. keras_hub/src/models/whisper/whisper_decoder.py +141 -0
  238. keras_hub/src/models/whisper/whisper_encoder.py +106 -0
  239. keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
  240. keras_hub/src/models/whisper/whisper_presets.py +148 -0
  241. keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
  242. keras_hub/src/models/xlm_roberta/__init__.py +26 -0
  243. keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
  244. keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
  245. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
  246. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
  247. keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
  248. keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
  249. keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
  250. keras_hub/src/models/xlnet/__init__.py +13 -0
  251. keras_hub/src/models/xlnet/relative_attention.py +459 -0
  252. keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
  253. keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
  254. keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
  255. keras_hub/src/samplers/__init__.py +13 -0
  256. keras_hub/src/samplers/beam_sampler.py +207 -0
  257. keras_hub/src/samplers/contrastive_sampler.py +231 -0
  258. keras_hub/src/samplers/greedy_sampler.py +50 -0
  259. keras_hub/src/samplers/random_sampler.py +77 -0
  260. keras_hub/src/samplers/sampler.py +237 -0
  261. keras_hub/src/samplers/serialization.py +97 -0
  262. keras_hub/src/samplers/top_k_sampler.py +92 -0
  263. keras_hub/src/samplers/top_p_sampler.py +113 -0
  264. keras_hub/src/tests/__init__.py +13 -0
  265. keras_hub/src/tests/test_case.py +608 -0
  266. keras_hub/src/tokenizers/__init__.py +13 -0
  267. keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
  268. keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
  269. keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
  270. keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
  271. keras_hub/src/tokenizers/tokenizer.py +235 -0
  272. keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
  273. keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
  274. keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
  275. keras_hub/src/utils/__init__.py +13 -0
  276. keras_hub/src/utils/keras_utils.py +130 -0
  277. keras_hub/src/utils/pipeline_model.py +293 -0
  278. keras_hub/src/utils/preset_utils.py +621 -0
  279. keras_hub/src/utils/python_utils.py +21 -0
  280. keras_hub/src/utils/tensor_utils.py +206 -0
  281. keras_hub/src/utils/timm/__init__.py +13 -0
  282. keras_hub/src/utils/timm/convert.py +37 -0
  283. keras_hub/src/utils/timm/convert_resnet.py +171 -0
  284. keras_hub/src/utils/transformers/__init__.py +13 -0
  285. keras_hub/src/utils/transformers/convert.py +101 -0
  286. keras_hub/src/utils/transformers/convert_bert.py +173 -0
  287. keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
  288. keras_hub/src/utils/transformers/convert_gemma.py +187 -0
  289. keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
  290. keras_hub/src/utils/transformers/convert_llama3.py +136 -0
  291. keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
  292. keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
  293. keras_hub/src/version_utils.py +23 -0
  294. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
  295. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
  296. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
  297. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,191 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import keras
16
+ from absl import logging
17
+
18
+ from keras_hub.src.api_export import keras_hub_export
19
+ from keras_hub.src.layers.preprocessing.masked_lm_mask_generator import (
20
+ MaskedLMMaskGenerator,
21
+ )
22
+ from keras_hub.src.models.deberta_v3.deberta_v3_preprocessor import (
23
+ DebertaV3Preprocessor,
24
+ )
25
+
26
+
27
+ @keras_hub_export("keras_hub.models.DebertaV3MaskedLMPreprocessor")
28
+ class DebertaV3MaskedLMPreprocessor(DebertaV3Preprocessor):
29
+ """DeBERTa preprocessing for the masked language modeling task.
30
+
31
+ This preprocessing layer will prepare inputs for a masked language modeling
32
+ task. It is primarily intended for use with the
33
+ `keras_hub.models.DebertaV3MaskedLM` task model. Preprocessing will occur in
34
+ multiple steps.
35
+
36
+ - Tokenize any number of input segments using the `tokenizer`.
37
+ - Pack the inputs together with the appropriate `"<s>"`, `"</s>"` and
38
+ `"<pad>"` tokens, i.e., adding a single `"<s>"` at the start of the
39
+ entire sequence, `"</s></s>"` between each segment,
40
+ and a `"</s>"` at the end of the entire sequence.
41
+ - Randomly select non-special tokens to mask, controlled by
42
+ `mask_selection_rate`.
43
+ - Construct a `(x, y, sample_weight)` tuple suitable for training with a
44
+ `keras_hub.models.DebertaV3MaskedLM` task model.
45
+
46
+ Args:
47
+ tokenizer: A `keras_hub.models.DebertaV3Tokenizer` instance.
48
+ sequence_length: The length of the packed inputs.
49
+ mask_selection_rate: The probability an input token will be dynamically
50
+ masked.
51
+ mask_selection_length: The maximum number of masked tokens supported
52
+ by the layer.
53
+ mask_token_rate: float. `mask_token_rate` must be
54
+ between 0 and 1 which indicates how often the mask_token is
55
+ substituted for tokens selected for masking.
56
+ Defaults to `0.8`.
57
+ random_token_rate: float. `random_token_rate` must be
58
+ between 0 and 1 which indicates how often a random token is
59
+ substituted for tokens selected for masking.
60
+ Note: mask_token_rate + random_token_rate <= 1, and for
61
+ (1 - mask_token_rate - random_token_rate), the token will not be
62
+ changed. Defaults to `0.1`.
63
+ truncate: string. The algorithm to truncate a list of batched segments
64
+ to fit within `sequence_length`. The value can be either
65
+ `round_robin` or `waterfall`:
66
+ - `"round_robin"`: Available space is assigned one token at a
67
+ time in a round-robin fashion to the inputs that still need
68
+ some, until the limit is reached.
69
+ - `"waterfall"`: The allocation of the budget is done using a
70
+ "waterfall" algorithm that allocates quota in a
71
+ left-to-right manner and fills up the buckets until we run
72
+ out of budget. It supports an arbitrary number of segments.
73
+
74
+ Examples:
75
+ Directly calling the layer on data.
76
+ ```python
77
+ preprocessor = keras_hub.models.DebertaV3MaskedLMPreprocessor.from_preset(
78
+ "deberta_v3_base_en"
79
+ )
80
+
81
+ # Tokenize and mask a single sentence.
82
+ preprocessor("The quick brown fox jumped.")
83
+
84
+ # Tokenize and mask a batch of single sentences.
85
+ preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
86
+
87
+ # Tokenize and mask sentence pairs.
88
+ # In this case, always convert input to tensors before calling the layer.
89
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
90
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
91
+ preprocessor((first, second))
92
+ ```
93
+
94
+ Mapping with `tf.data.Dataset`.
95
+ ```python
96
+ preprocessor = keras_hub.models.DebertaV3MaskedLMPreprocessor.from_preset(
97
+ "deberta_v3_base_en"
98
+ )
99
+
100
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
101
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
102
+
103
+ # Map single sentences.
104
+ ds = tf.data.Dataset.from_tensor_slices(first)
105
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
106
+
107
+ # Map sentence pairs.
108
+ ds = tf.data.Dataset.from_tensor_slices((first, second))
109
+ # Watch out for tf.data's default unpacking of tuples here!
110
+ # Best to invoke the `preprocessor` directly in this case.
111
+ ds = ds.map(
112
+ lambda first, second: preprocessor(x=(first, second)),
113
+ num_parallel_calls=tf.data.AUTOTUNE,
114
+ )
115
+ ```
116
+ """
117
+
118
+ def __init__(
119
+ self,
120
+ tokenizer,
121
+ sequence_length=512,
122
+ truncate="round_robin",
123
+ mask_selection_rate=0.15,
124
+ mask_selection_length=96,
125
+ mask_token_rate=0.8,
126
+ random_token_rate=0.1,
127
+ **kwargs,
128
+ ):
129
+ super().__init__(
130
+ tokenizer,
131
+ sequence_length=sequence_length,
132
+ truncate=truncate,
133
+ **kwargs,
134
+ )
135
+
136
+ self.mask_selection_rate = mask_selection_rate
137
+ self.mask_selection_length = mask_selection_length
138
+ self.mask_token_rate = mask_token_rate
139
+ self.random_token_rate = random_token_rate
140
+ self.masker = None
141
+
142
+ def build(self, input_shape):
143
+ super().build(input_shape)
144
+ # Defer masker creation to `build()` so that we can be sure tokenizer
145
+ # assets have loaded when restoring a saved model.
146
+ self.masker = MaskedLMMaskGenerator(
147
+ mask_selection_rate=self.mask_selection_rate,
148
+ mask_selection_length=self.mask_selection_length,
149
+ mask_token_rate=self.mask_token_rate,
150
+ random_token_rate=self.random_token_rate,
151
+ vocabulary_size=self.tokenizer.vocabulary_size(),
152
+ mask_token_id=self.tokenizer.mask_token_id,
153
+ unselectable_token_ids=[
154
+ self.tokenizer.cls_token_id,
155
+ self.tokenizer.sep_token_id,
156
+ self.tokenizer.pad_token_id,
157
+ ],
158
+ )
159
+
160
+ def get_config(self):
161
+ config = super().get_config()
162
+ config.update(
163
+ {
164
+ "mask_selection_rate": self.mask_selection_rate,
165
+ "mask_selection_length": self.mask_selection_length,
166
+ "mask_token_rate": self.mask_token_rate,
167
+ "random_token_rate": self.random_token_rate,
168
+ }
169
+ )
170
+ return config
171
+
172
+ def call(self, x, y=None, sample_weight=None):
173
+ if y is not None or sample_weight is not None:
174
+ logging.warning(
175
+ f"{self.__class__.__name__} generates `y` and `sample_weight` "
176
+ "based on your input data, but your data already contains `y` "
177
+ "or `sample_weight`. Your `y` and `sample_weight` will be "
178
+ "ignored."
179
+ )
180
+
181
+ x = super().call(x)
182
+ token_ids, padding_mask = x["token_ids"], x["padding_mask"]
183
+ masker_outputs = self.masker(token_ids)
184
+ x = {
185
+ "token_ids": masker_outputs["token_ids"],
186
+ "padding_mask": padding_mask,
187
+ "mask_positions": masker_outputs["mask_positions"],
188
+ }
189
+ y = masker_outputs["mask_ids"]
190
+ sample_weight = masker_outputs["mask_weights"]
191
+ return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
@@ -0,0 +1,206 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ import keras
17
+
18
+ from keras_hub.src.api_export import keras_hub_export
19
+ from keras_hub.src.layers.preprocessing.multi_segment_packer import (
20
+ MultiSegmentPacker,
21
+ )
22
+ from keras_hub.src.models.deberta_v3.deberta_v3_tokenizer import (
23
+ DebertaV3Tokenizer,
24
+ )
25
+ from keras_hub.src.models.preprocessor import Preprocessor
26
+ from keras_hub.src.utils.keras_utils import (
27
+ convert_inputs_to_list_of_tensor_segments,
28
+ )
29
+
30
+
31
+ @keras_hub_export("keras_hub.models.DebertaV3Preprocessor")
32
+ class DebertaV3Preprocessor(Preprocessor):
33
+ """A DeBERTa preprocessing layer which tokenizes and packs inputs.
34
+
35
+ This preprocessing layer will do three things:
36
+
37
+ - Tokenize any number of input segments using the `tokenizer`.
38
+ - Pack the inputs together using a `keras_hub.layers.MultiSegmentPacker`.
39
+ with the appropriate `"[CLS]"`, `"[SEP]"` and `"[PAD]"` tokens.
40
+ - Construct a dictionary with keys `"token_ids"` and `"padding_mask"`, that
41
+ can be passed directly to a DeBERTa model.
42
+
43
+ This layer can be used directly with `tf.data.Dataset.map` to preprocess
44
+ string data in the `(x, y, sample_weight)` format used by
45
+ `keras.Model.fit`.
46
+
47
+ The call method of this layer accepts three arguments, `x`, `y`, and
48
+ `sample_weight`. `x` can be a python string or tensor representing a single
49
+ segment, a list of python strings representing a batch of single segments,
50
+ or a list of tensors representing multiple segments to be packed together.
51
+ `y` and `sample_weight` are both optional, can have any format, and will be
52
+ passed through unaltered.
53
+
54
+ Special care should be taken when using `tf.data` to map over an unlabeled
55
+ tuple of string segments. `tf.data.Dataset.map` will unpack this tuple
56
+ directly into the call arguments of this layer, rather than forward all
57
+ argument to `x`. To handle this case, it is recommended to explicitly call
58
+ the layer, e.g. `ds.map(lambda seg1, seg2: preprocessor(x=(seg1, seg2)))`.
59
+
60
+ Args:
61
+ tokenizer: A `keras_hub.models.DebertaV3Tokenizer` instance.
62
+ sequence_length: The length of the packed inputs.
63
+ truncate: string. The algorithm to truncate a list of batched segments
64
+ to fit within `sequence_length`. The value can be either
65
+ `round_robin` or `waterfall`:
66
+ - `"round_robin"`: Available space is assigned one token at a
67
+ time in a round-robin fashion to the inputs that still need
68
+ some, until the limit is reached.
69
+ - `"waterfall"`: The allocation of the budget is done using a
70
+ "waterfall" algorithm that allocates quota in a
71
+ left-to-right manner and fills up the buckets until we run
72
+ out of budget. It supports an arbitrary number of segments.
73
+
74
+ Examples:
75
+ Directly calling the layer on data.
76
+ ```python
77
+ preprocessor = keras_hub.models.DebertaV3Preprocessor.from_preset(
78
+ "deberta_v3_base_en"
79
+ )
80
+
81
+ # Tokenize and pack a single sentence.
82
+ preprocessor("The quick brown fox jumped.")
83
+
84
+ # Tokenize a batch of single sentences.
85
+ preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
86
+
87
+ # Preprocess a batch of sentence pairs.
88
+ # When handling multiple sequences, always convert to tensors first!
89
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
90
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
91
+ preprocessor((first, second))
92
+
93
+ # Custom vocabulary.
94
+ bytes_io = io.BytesIO()
95
+ ds = tf.data.Dataset.from_tensor_slices(["The quick brown fox jumped."])
96
+ sentencepiece.SentencePieceTrainer.train(
97
+ sentence_iterator=ds.as_numpy_iterator(),
98
+ model_writer=bytes_io,
99
+ vocab_size=9,
100
+ model_type="WORD",
101
+ pad_id=0,
102
+ bos_id=1,
103
+ eos_id=2,
104
+ unk_id=3,
105
+ pad_piece="[PAD]",
106
+ bos_piece="[CLS]",
107
+ eos_piece="[SEP]",
108
+ unk_piece="[UNK]",
109
+ )
110
+ tokenizer = keras_hub.models.DebertaV3Tokenizer(
111
+ proto=bytes_io.getvalue(),
112
+ )
113
+ preprocessor = keras_hub.models.DebertaV3Preprocessor(tokenizer)
114
+ preprocessor("The quick brown fox jumped.")
115
+ ```
116
+
117
+ Mapping with `tf.data.Dataset`.
118
+ ```python
119
+ preprocessor = keras_hub.models.DebertaV3Preprocessor.from_preset(
120
+ "deberta_v3_base_en"
121
+ )
122
+
123
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
124
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
125
+ label = tf.constant([1, 1])
126
+
127
+ # Map labeled single sentences.
128
+ ds = tf.data.Dataset.from_tensor_slices((first, label))
129
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
130
+
131
+ # Map unlabeled single sentences.
132
+ ds = tf.data.Dataset.from_tensor_slices(first)
133
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
134
+
135
+ # Map labeled sentence pairs.
136
+ ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
137
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
138
+
139
+ # Map unlabeled sentence pairs.
140
+ ds = tf.data.Dataset.from_tensor_slices((first, second))
141
+ # Watch out for tf.data's default unpacking of tuples here!
142
+ # Best to invoke the `preprocessor` directly in this case.
143
+ ds = ds.map(
144
+ lambda first, second: preprocessor(x=(first, second)),
145
+ num_parallel_calls=tf.data.AUTOTUNE,
146
+ )
147
+ ```
148
+ """
149
+
150
+ tokenizer_cls = DebertaV3Tokenizer
151
+
152
+ def __init__(
153
+ self,
154
+ tokenizer,
155
+ sequence_length=512,
156
+ truncate="round_robin",
157
+ **kwargs,
158
+ ):
159
+ super().__init__(**kwargs)
160
+ self.tokenizer = tokenizer
161
+ self.packer = None
162
+ self.truncate = truncate
163
+ self.sequence_length = sequence_length
164
+
165
+ def build(self, input_shape):
166
+ # Defer packer creation to `build()` so that we can be sure tokenizer
167
+ # assets have loaded when restoring a saved model.
168
+ self.packer = MultiSegmentPacker(
169
+ start_value=self.tokenizer.cls_token_id,
170
+ end_value=self.tokenizer.sep_token_id,
171
+ pad_value=self.tokenizer.pad_token_id,
172
+ truncate=self.truncate,
173
+ sequence_length=self.sequence_length,
174
+ )
175
+ self.built = True
176
+
177
+ def get_config(self):
178
+ config = super().get_config()
179
+ config.update(
180
+ {
181
+ "sequence_length": self.sequence_length,
182
+ "truncate": self.truncate,
183
+ }
184
+ )
185
+ return config
186
+
187
+ def call(self, x, y=None, sample_weight=None):
188
+ x = convert_inputs_to_list_of_tensor_segments(x)
189
+ x = [self.tokenizer(segment) for segment in x]
190
+ token_ids, _ = self.packer(x)
191
+ x = {
192
+ "token_ids": token_ids,
193
+ "padding_mask": token_ids != self.tokenizer.pad_token_id,
194
+ }
195
+ return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
196
+
197
+ @property
198
+ def sequence_length(self):
199
+ """The padded length of model input sequences."""
200
+ return self._sequence_length
201
+
202
+ @sequence_length.setter
203
+ def sequence_length(self, value):
204
+ self._sequence_length = value
205
+ if self.packer is not None:
206
+ self.packer.sequence_length = value
@@ -0,0 +1,82 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """DeBERTa model preset configurations."""
15
+
16
+ backbone_presets = {
17
+ "deberta_v3_extra_small_en": {
18
+ "metadata": {
19
+ "description": (
20
+ "12-layer DeBERTaV3 model where case is maintained. "
21
+ "Trained on English Wikipedia, BookCorpus and OpenWebText."
22
+ ),
23
+ "params": 70682112,
24
+ "official_name": "DeBERTaV3",
25
+ "path": "deberta_v3",
26
+ "model_card": "https://huggingface.co/microsoft/deberta-v3-xsmall",
27
+ },
28
+ "kaggle_handle": "kaggle://keras/deberta_v3/keras/deberta_v3_extra_small_en/2",
29
+ },
30
+ "deberta_v3_small_en": {
31
+ "metadata": {
32
+ "description": (
33
+ "6-layer DeBERTaV3 model where case is maintained. "
34
+ "Trained on English Wikipedia, BookCorpus and OpenWebText."
35
+ ),
36
+ "params": 141304320,
37
+ "official_name": "DeBERTaV3",
38
+ "path": "deberta_v3",
39
+ "model_card": "https://huggingface.co/microsoft/deberta-v3-small",
40
+ },
41
+ "kaggle_handle": "kaggle://keras/deberta_v3/keras/deberta_v3_small_en/2",
42
+ },
43
+ "deberta_v3_base_en": {
44
+ "metadata": {
45
+ "description": (
46
+ "12-layer DeBERTaV3 model where case is maintained. "
47
+ "Trained on English Wikipedia, BookCorpus and OpenWebText."
48
+ ),
49
+ "params": 183831552,
50
+ "official_name": "DeBERTaV3",
51
+ "path": "deberta_v3",
52
+ "model_card": "https://huggingface.co/microsoft/deberta-v3-base",
53
+ },
54
+ "kaggle_handle": "kaggle://keras/deberta_v3/keras/deberta_v3_base_en/2",
55
+ },
56
+ "deberta_v3_large_en": {
57
+ "metadata": {
58
+ "description": (
59
+ "24-layer DeBERTaV3 model where case is maintained. "
60
+ "Trained on English Wikipedia, BookCorpus and OpenWebText."
61
+ ),
62
+ "params": 434012160,
63
+ "official_name": "DeBERTaV3",
64
+ "path": "deberta_v3",
65
+ "model_card": "https://huggingface.co/microsoft/deberta-v3-large",
66
+ },
67
+ "kaggle_handle": "kaggle://keras/deberta_v3/keras/deberta_v3_large_en/2",
68
+ },
69
+ "deberta_v3_base_multi": {
70
+ "metadata": {
71
+ "description": (
72
+ "12-layer DeBERTaV3 model where case is maintained. "
73
+ "Trained on the 2.5TB multilingual CC100 dataset."
74
+ ),
75
+ "params": 278218752,
76
+ "official_name": "DeBERTaV3",
77
+ "path": "deberta_v3",
78
+ "model_card": "https://huggingface.co/microsoft/mdeberta-v3-base",
79
+ },
80
+ "kaggle_handle": "kaggle://keras/deberta_v3/keras/deberta_v3_base_multi/2",
81
+ },
82
+ }
@@ -0,0 +1,155 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ from keras_hub.src.api_export import keras_hub_export
17
+ from keras_hub.src.tokenizers.sentence_piece_tokenizer import (
18
+ SentencePieceTokenizer,
19
+ )
20
+
21
+ try:
22
+ import tensorflow as tf
23
+ except ImportError:
24
+ tf = None
25
+
26
+
27
+ @keras_hub_export("keras_hub.models.DebertaV3Tokenizer")
28
+ class DebertaV3Tokenizer(SentencePieceTokenizer):
29
+ """DeBERTa tokenizer layer based on SentencePiece.
30
+
31
+ This tokenizer class will tokenize raw strings into integer sequences and
32
+ is based on `keras_hub.tokenizers.SentencePieceTokenizer`. Unlike the
33
+ underlying tokenizer, it will check for all special tokens needed by
34
+ DeBERTa models and provides a `from_preset()` method to automatically
35
+ download a matching vocabulary for a DeBERTa preset.
36
+
37
+ This tokenizer does not provide truncation or padding of inputs. It can be
38
+ combined with a `keras_hub.models.DebertaV3Preprocessor` layer for input
39
+ packing.
40
+
41
+ If input is a batch of strings (rank > 0), the layer will output a
42
+ `tf.RaggedTensor` where the last dimension of the output is ragged.
43
+
44
+ If input is a scalar string (rank == 0), the layer will output a dense
45
+ `tf.Tensor` with static shape `[None]`.
46
+
47
+ Note: The mask token (`"[MASK]"`) is handled differently in this tokenizer.
48
+ If the token is not present in the provided SentencePiece vocabulary, the
49
+ token will be appended to the vocabulary. For example, if the vocabulary
50
+ size is 100, the mask token will be assigned the ID 100.
51
+
52
+ Args:
53
+ proto: Either a `string` path to a SentencePiece proto file, or a
54
+ `bytes` object with a serialized SentencePiece proto. See the
55
+ [SentencePiece repository](https://github.com/google/sentencepiece)
56
+ for more details on the format.
57
+
58
+ Examples:
59
+
60
+ ```python
61
+ # Unbatched input.
62
+ tokenizer = keras_hub.models.DebertaV3Tokenizer.from_preset(
63
+ "deberta_v3_base_en",
64
+ )
65
+ tokenizer("The quick brown fox jumped.")
66
+
67
+ # Batched inputs.
68
+ tokenizer(["the quick brown fox", "the earth is round"])
69
+
70
+ # Detokenization.
71
+ tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
72
+
73
+ # Custom vocabulary.
74
+ bytes_io = io.BytesIO()
75
+ ds = tf.data.Dataset.from_tensor_slices(["The quick brown fox jumped."])
76
+ sentencepiece.SentencePieceTrainer.train(
77
+ sentence_iterator=ds.as_numpy_iterator(),
78
+ model_writer=bytes_io,
79
+ vocab_size=9,
80
+ model_type="WORD",
81
+ pad_id=0,
82
+ bos_id=1,
83
+ eos_id=2,
84
+ unk_id=3,
85
+ pad_piece="[PAD]",
86
+ bos_piece="[CLS]",
87
+ eos_piece="[SEP]",
88
+ unk_piece="[UNK]",
89
+ )
90
+ tokenizer = keras_hub.models.DebertaV3Tokenizer(
91
+ proto=bytes_io.getvalue(),
92
+ )
93
+ tokenizer("The quick brown fox jumped.")
94
+ ```
95
+ """
96
+
97
+ def __init__(self, proto, **kwargs):
98
+ self.cls_token = "[CLS]"
99
+ self.sep_token = "[SEP]"
100
+ self.pad_token = "[PAD]"
101
+ self.mask_token = "[MASK]"
102
+
103
+ super().__init__(proto=proto, **kwargs)
104
+
105
+ def set_proto(self, proto):
106
+ super().set_proto(proto)
107
+ if proto is not None:
108
+ for token in [self.cls_token, self.pad_token, self.sep_token]:
109
+ if token not in super().get_vocabulary():
110
+ raise ValueError(
111
+ f"Cannot find token `'{token}'` in the provided "
112
+ f"`vocabulary`. Please provide `'{token}'` in your "
113
+ "`vocabulary` or use a pretrained `vocabulary` name."
114
+ )
115
+
116
+ self.cls_token_id = self.token_to_id(self.cls_token)
117
+ self.sep_token_id = self.token_to_id(self.sep_token)
118
+ self.pad_token_id = self.token_to_id(self.pad_token)
119
+ # If the mask token is not in the vocabulary, add it to the end of the
120
+ # vocabulary.
121
+ if self.mask_token in super().get_vocabulary():
122
+ self.mask_token_id = super().token_to_id(self.mask_token)
123
+ else:
124
+ self.mask_token_id = super().vocabulary_size()
125
+ else:
126
+ self.cls_token_id = None
127
+ self.sep_token_id = None
128
+ self.pad_token_id = None
129
+ self.mask_token_id = None
130
+
131
+ def vocabulary_size(self):
132
+ sentence_piece_size = super().vocabulary_size()
133
+ if sentence_piece_size == self.mask_token_id:
134
+ return sentence_piece_size + 1
135
+ return sentence_piece_size
136
+
137
+ def get_vocabulary(self):
138
+ sentence_piece_vocabulary = super().get_vocabulary()
139
+ if self.mask_token_id < super().vocabulary_size():
140
+ return sentence_piece_vocabulary
141
+ return sentence_piece_vocabulary + ["[MASK]"]
142
+
143
+ def id_to_token(self, id):
144
+ if id == self.mask_token_id:
145
+ return "[MASK]"
146
+ return super().id_to_token(id)
147
+
148
+ def token_to_id(self, token):
149
+ if token == "[MASK]":
150
+ return self.mask_token_id
151
+ return super().token_to_id(token)
152
+
153
+ def detokenize(self, ids):
154
+ ids = tf.ragged.boolean_mask(ids, tf.not_equal(ids, self.mask_token_id))
155
+ return super().detokenize(ids)