keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (297) hide show
  1. keras_hub/__init__.py +52 -0
  2. keras_hub/api/__init__.py +27 -0
  3. keras_hub/api/layers/__init__.py +47 -0
  4. keras_hub/api/metrics/__init__.py +24 -0
  5. keras_hub/api/models/__init__.py +249 -0
  6. keras_hub/api/samplers/__init__.py +29 -0
  7. keras_hub/api/tokenizers/__init__.py +35 -0
  8. keras_hub/src/__init__.py +13 -0
  9. keras_hub/src/api_export.py +53 -0
  10. keras_hub/src/layers/__init__.py +13 -0
  11. keras_hub/src/layers/modeling/__init__.py +13 -0
  12. keras_hub/src/layers/modeling/alibi_bias.py +143 -0
  13. keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
  14. keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
  15. keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
  16. keras_hub/src/layers/modeling/position_embedding.py +123 -0
  17. keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
  18. keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
  19. keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
  20. keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
  21. keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
  22. keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
  23. keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
  24. keras_hub/src/layers/preprocessing/__init__.py +13 -0
  25. keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
  26. keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
  27. keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
  28. keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
  29. keras_hub/src/layers/preprocessing/random_swap.py +267 -0
  30. keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
  31. keras_hub/src/metrics/__init__.py +13 -0
  32. keras_hub/src/metrics/bleu.py +394 -0
  33. keras_hub/src/metrics/edit_distance.py +197 -0
  34. keras_hub/src/metrics/perplexity.py +181 -0
  35. keras_hub/src/metrics/rouge_base.py +204 -0
  36. keras_hub/src/metrics/rouge_l.py +97 -0
  37. keras_hub/src/metrics/rouge_n.py +125 -0
  38. keras_hub/src/models/__init__.py +13 -0
  39. keras_hub/src/models/albert/__init__.py +20 -0
  40. keras_hub/src/models/albert/albert_backbone.py +267 -0
  41. keras_hub/src/models/albert/albert_classifier.py +202 -0
  42. keras_hub/src/models/albert/albert_masked_lm.py +129 -0
  43. keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
  44. keras_hub/src/models/albert/albert_preprocessor.py +206 -0
  45. keras_hub/src/models/albert/albert_presets.py +70 -0
  46. keras_hub/src/models/albert/albert_tokenizer.py +119 -0
  47. keras_hub/src/models/backbone.py +311 -0
  48. keras_hub/src/models/bart/__init__.py +20 -0
  49. keras_hub/src/models/bart/bart_backbone.py +261 -0
  50. keras_hub/src/models/bart/bart_preprocessor.py +276 -0
  51. keras_hub/src/models/bart/bart_presets.py +74 -0
  52. keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
  53. keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
  54. keras_hub/src/models/bart/bart_tokenizer.py +124 -0
  55. keras_hub/src/models/bert/__init__.py +23 -0
  56. keras_hub/src/models/bert/bert_backbone.py +227 -0
  57. keras_hub/src/models/bert/bert_classifier.py +183 -0
  58. keras_hub/src/models/bert/bert_masked_lm.py +131 -0
  59. keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
  60. keras_hub/src/models/bert/bert_preprocessor.py +184 -0
  61. keras_hub/src/models/bert/bert_presets.py +147 -0
  62. keras_hub/src/models/bert/bert_tokenizer.py +112 -0
  63. keras_hub/src/models/bloom/__init__.py +20 -0
  64. keras_hub/src/models/bloom/bloom_attention.py +186 -0
  65. keras_hub/src/models/bloom/bloom_backbone.py +173 -0
  66. keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
  67. keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
  68. keras_hub/src/models/bloom/bloom_decoder.py +206 -0
  69. keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
  70. keras_hub/src/models/bloom/bloom_presets.py +121 -0
  71. keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
  72. keras_hub/src/models/causal_lm.py +383 -0
  73. keras_hub/src/models/classifier.py +109 -0
  74. keras_hub/src/models/csp_darknet/__init__.py +13 -0
  75. keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
  76. keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
  77. keras_hub/src/models/deberta_v3/__init__.py +24 -0
  78. keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
  79. keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
  80. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
  81. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
  82. keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
  83. keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
  84. keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
  85. keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
  86. keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
  87. keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
  88. keras_hub/src/models/densenet/__init__.py +13 -0
  89. keras_hub/src/models/densenet/densenet_backbone.py +210 -0
  90. keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
  91. keras_hub/src/models/distil_bert/__init__.py +26 -0
  92. keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
  93. keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
  94. keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
  95. keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
  96. keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
  97. keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
  98. keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
  99. keras_hub/src/models/electra/__init__.py +20 -0
  100. keras_hub/src/models/electra/electra_backbone.py +247 -0
  101. keras_hub/src/models/electra/electra_preprocessor.py +154 -0
  102. keras_hub/src/models/electra/electra_presets.py +95 -0
  103. keras_hub/src/models/electra/electra_tokenizer.py +104 -0
  104. keras_hub/src/models/f_net/__init__.py +20 -0
  105. keras_hub/src/models/f_net/f_net_backbone.py +236 -0
  106. keras_hub/src/models/f_net/f_net_classifier.py +154 -0
  107. keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
  108. keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
  109. keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
  110. keras_hub/src/models/f_net/f_net_presets.py +43 -0
  111. keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
  112. keras_hub/src/models/falcon/__init__.py +20 -0
  113. keras_hub/src/models/falcon/falcon_attention.py +156 -0
  114. keras_hub/src/models/falcon/falcon_backbone.py +164 -0
  115. keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
  116. keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
  117. keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
  118. keras_hub/src/models/falcon/falcon_presets.py +30 -0
  119. keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
  120. keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
  121. keras_hub/src/models/feature_pyramid_backbone.py +73 -0
  122. keras_hub/src/models/gemma/__init__.py +20 -0
  123. keras_hub/src/models/gemma/gemma_attention.py +250 -0
  124. keras_hub/src/models/gemma/gemma_backbone.py +316 -0
  125. keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
  126. keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
  127. keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
  128. keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
  129. keras_hub/src/models/gemma/gemma_presets.py +248 -0
  130. keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
  131. keras_hub/src/models/gemma/rms_normalization.py +40 -0
  132. keras_hub/src/models/gpt2/__init__.py +20 -0
  133. keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
  134. keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
  135. keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
  136. keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
  137. keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
  138. keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
  139. keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
  140. keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
  141. keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
  142. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
  143. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
  144. keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
  145. keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
  146. keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
  147. keras_hub/src/models/image_classifier.py +90 -0
  148. keras_hub/src/models/llama/__init__.py +20 -0
  149. keras_hub/src/models/llama/llama_attention.py +225 -0
  150. keras_hub/src/models/llama/llama_backbone.py +188 -0
  151. keras_hub/src/models/llama/llama_causal_lm.py +327 -0
  152. keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
  153. keras_hub/src/models/llama/llama_decoder.py +246 -0
  154. keras_hub/src/models/llama/llama_layernorm.py +48 -0
  155. keras_hub/src/models/llama/llama_preprocessor.py +189 -0
  156. keras_hub/src/models/llama/llama_presets.py +80 -0
  157. keras_hub/src/models/llama/llama_tokenizer.py +84 -0
  158. keras_hub/src/models/llama3/__init__.py +20 -0
  159. keras_hub/src/models/llama3/llama3_backbone.py +84 -0
  160. keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
  161. keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
  162. keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
  163. keras_hub/src/models/llama3/llama3_presets.py +69 -0
  164. keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
  165. keras_hub/src/models/masked_lm.py +101 -0
  166. keras_hub/src/models/mistral/__init__.py +20 -0
  167. keras_hub/src/models/mistral/mistral_attention.py +238 -0
  168. keras_hub/src/models/mistral/mistral_backbone.py +203 -0
  169. keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
  170. keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
  171. keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
  172. keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
  173. keras_hub/src/models/mistral/mistral_presets.py +48 -0
  174. keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
  175. keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
  176. keras_hub/src/models/mix_transformer/__init__.py +13 -0
  177. keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
  178. keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
  179. keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
  180. keras_hub/src/models/opt/__init__.py +20 -0
  181. keras_hub/src/models/opt/opt_backbone.py +173 -0
  182. keras_hub/src/models/opt/opt_causal_lm.py +301 -0
  183. keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
  184. keras_hub/src/models/opt/opt_preprocessor.py +188 -0
  185. keras_hub/src/models/opt/opt_presets.py +72 -0
  186. keras_hub/src/models/opt/opt_tokenizer.py +116 -0
  187. keras_hub/src/models/pali_gemma/__init__.py +23 -0
  188. keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
  189. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
  190. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
  191. keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
  192. keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
  193. keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
  194. keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
  195. keras_hub/src/models/phi3/__init__.py +20 -0
  196. keras_hub/src/models/phi3/phi3_attention.py +260 -0
  197. keras_hub/src/models/phi3/phi3_backbone.py +224 -0
  198. keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
  199. keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
  200. keras_hub/src/models/phi3/phi3_decoder.py +260 -0
  201. keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
  202. keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
  203. keras_hub/src/models/phi3/phi3_presets.py +50 -0
  204. keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
  205. keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
  206. keras_hub/src/models/preprocessor.py +207 -0
  207. keras_hub/src/models/resnet/__init__.py +13 -0
  208. keras_hub/src/models/resnet/resnet_backbone.py +612 -0
  209. keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
  210. keras_hub/src/models/roberta/__init__.py +20 -0
  211. keras_hub/src/models/roberta/roberta_backbone.py +184 -0
  212. keras_hub/src/models/roberta/roberta_classifier.py +209 -0
  213. keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
  214. keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
  215. keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
  216. keras_hub/src/models/roberta/roberta_presets.py +43 -0
  217. keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
  218. keras_hub/src/models/seq_2_seq_lm.py +54 -0
  219. keras_hub/src/models/t5/__init__.py +20 -0
  220. keras_hub/src/models/t5/t5_backbone.py +261 -0
  221. keras_hub/src/models/t5/t5_layer_norm.py +35 -0
  222. keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
  223. keras_hub/src/models/t5/t5_presets.py +95 -0
  224. keras_hub/src/models/t5/t5_tokenizer.py +100 -0
  225. keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
  226. keras_hub/src/models/task.py +419 -0
  227. keras_hub/src/models/vgg/__init__.py +13 -0
  228. keras_hub/src/models/vgg/vgg_backbone.py +158 -0
  229. keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
  230. keras_hub/src/models/vit_det/__init__.py +13 -0
  231. keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
  232. keras_hub/src/models/vit_det/vit_layers.py +565 -0
  233. keras_hub/src/models/whisper/__init__.py +20 -0
  234. keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
  235. keras_hub/src/models/whisper/whisper_backbone.py +305 -0
  236. keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
  237. keras_hub/src/models/whisper/whisper_decoder.py +141 -0
  238. keras_hub/src/models/whisper/whisper_encoder.py +106 -0
  239. keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
  240. keras_hub/src/models/whisper/whisper_presets.py +148 -0
  241. keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
  242. keras_hub/src/models/xlm_roberta/__init__.py +26 -0
  243. keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
  244. keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
  245. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
  246. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
  247. keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
  248. keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
  249. keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
  250. keras_hub/src/models/xlnet/__init__.py +13 -0
  251. keras_hub/src/models/xlnet/relative_attention.py +459 -0
  252. keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
  253. keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
  254. keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
  255. keras_hub/src/samplers/__init__.py +13 -0
  256. keras_hub/src/samplers/beam_sampler.py +207 -0
  257. keras_hub/src/samplers/contrastive_sampler.py +231 -0
  258. keras_hub/src/samplers/greedy_sampler.py +50 -0
  259. keras_hub/src/samplers/random_sampler.py +77 -0
  260. keras_hub/src/samplers/sampler.py +237 -0
  261. keras_hub/src/samplers/serialization.py +97 -0
  262. keras_hub/src/samplers/top_k_sampler.py +92 -0
  263. keras_hub/src/samplers/top_p_sampler.py +113 -0
  264. keras_hub/src/tests/__init__.py +13 -0
  265. keras_hub/src/tests/test_case.py +608 -0
  266. keras_hub/src/tokenizers/__init__.py +13 -0
  267. keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
  268. keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
  269. keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
  270. keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
  271. keras_hub/src/tokenizers/tokenizer.py +235 -0
  272. keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
  273. keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
  274. keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
  275. keras_hub/src/utils/__init__.py +13 -0
  276. keras_hub/src/utils/keras_utils.py +130 -0
  277. keras_hub/src/utils/pipeline_model.py +293 -0
  278. keras_hub/src/utils/preset_utils.py +621 -0
  279. keras_hub/src/utils/python_utils.py +21 -0
  280. keras_hub/src/utils/tensor_utils.py +206 -0
  281. keras_hub/src/utils/timm/__init__.py +13 -0
  282. keras_hub/src/utils/timm/convert.py +37 -0
  283. keras_hub/src/utils/timm/convert_resnet.py +171 -0
  284. keras_hub/src/utils/transformers/__init__.py +13 -0
  285. keras_hub/src/utils/transformers/convert.py +101 -0
  286. keras_hub/src/utils/transformers/convert_bert.py +173 -0
  287. keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
  288. keras_hub/src/utils/transformers/convert_gemma.py +187 -0
  289. keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
  290. keras_hub/src/utils/transformers/convert_llama3.py +136 -0
  291. keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
  292. keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
  293. keras_hub/src/version_utils.py +23 -0
  294. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
  295. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
  296. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
  297. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
@@ -0,0 +1,194 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import keras
16
+ from absl import logging
17
+
18
+ from keras_hub.src.api_export import keras_hub_export
19
+ from keras_hub.src.layers.preprocessing.masked_lm_mask_generator import (
20
+ MaskedLMMaskGenerator,
21
+ )
22
+ from keras_hub.src.models.albert.albert_preprocessor import AlbertPreprocessor
23
+
24
+
25
+ @keras_hub_export("keras_hub.models.AlbertMaskedLMPreprocessor")
26
+ class AlbertMaskedLMPreprocessor(AlbertPreprocessor):
27
+ """ALBERT preprocessing for the masked language modeling task.
28
+
29
+ This preprocessing layer will prepare inputs for a masked language modeling
30
+ task. It is primarily intended for use with the
31
+ `keras_hub.models.AlbertMaskedLM` task model. Preprocessing will occur in
32
+ multiple steps.
33
+
34
+ - Tokenize any number of input segments using the `tokenizer`.
35
+ - Pack the inputs together with the appropriate `"<s>"`, `"</s>"` and
36
+ `"<pad>"` tokens, i.e., adding a single `"<s>"` at the start of the
37
+ entire sequence, `"</s></s>"` between each segment,
38
+ and a `"</s>"` at the end of the entire sequence.
39
+ - Randomly select non-special tokens to mask, controlled by
40
+ `mask_selection_rate`.
41
+ - Construct a `(x, y, sample_weight)` tuple suitable for training with a
42
+ `keras_hub.models.AlbertMaskedLM` task model.
43
+
44
+ Args:
45
+ tokenizer: A `keras_hub.models.AlbertTokenizer` instance.
46
+ sequence_length: The length of the packed inputs.
47
+ mask_selection_rate: The probability an input token will be dynamically
48
+ masked.
49
+ mask_selection_length: The maximum number of masked tokens supported
50
+ by the layer.
51
+ mask_token_rate: float. `mask_token_rate` must be
52
+ between 0 and 1 which indicates how often the mask_token is
53
+ substituted for tokens selected for masking.
54
+ Defaults to `0.8`.
55
+ random_token_rate: float. `random_token_rate` must be
56
+ between 0 and 1 which indicates how often a random token is
57
+ substituted for tokens selected for masking. Default is 0.1.
58
+ Note: mask_token_rate + random_token_rate <= 1, and for
59
+ (1 - mask_token_rate - random_token_rate), the token will not be
60
+ changed. Defaults to `0.1`.
61
+ truncate: string. The algorithm to truncate a list of batched segments
62
+ to fit within `sequence_length`. The value can be either
63
+ `round_robin` or `waterfall`:
64
+ - `"round_robin"`: Available space is assigned one token at a
65
+ time in a round-robin fashion to the inputs that still need
66
+ some, until the limit is reached.
67
+ - `"waterfall"`: The allocation of the budget is done using a
68
+ "waterfall" algorithm that allocates quota in a
69
+ left-to-right manner and fills up the buckets until we run
70
+ out of budget. It supports an arbitrary number of segments.
71
+
72
+ Examples:
73
+
74
+ Directly calling the layer on data.
75
+ ```python
76
+ preprocessor = keras_hub.models.AlbertMaskedLMPreprocessor.from_preset(
77
+ "albert_base_en_uncased"
78
+ )
79
+
80
+ # Tokenize and mask a single sentence.
81
+ preprocessor("The quick brown fox jumped.")
82
+
83
+ # Tokenize and mask a batch of single sentences.
84
+ preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
85
+
86
+ # Tokenize and mask sentence pairs.
87
+ # In this case, always convert input to tensors before calling the layer.
88
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
89
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
90
+ preprocessor((first, second))
91
+ ```
92
+
93
+ Mapping with `tf.data.Dataset`.
94
+ ```python
95
+ preprocessor = keras_hub.models.AlbertMaskedLMPreprocessor.from_preset(
96
+ "albert_base_en_uncased"
97
+ )
98
+
99
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
100
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
101
+
102
+ # Map single sentences.
103
+ ds = tf.data.Dataset.from_tensor_slices(first)
104
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
105
+
106
+ # Map sentence pairs.
107
+ ds = tf.data.Dataset.from_tensor_slices((first, second))
108
+ # Watch out for tf.data's default unpacking of tuples here!
109
+ # Best to invoke the `preprocessor` directly in this case.
110
+ ds = ds.map(
111
+ lambda first, second: preprocessor(x=(first, second)),
112
+ num_parallel_calls=tf.data.AUTOTUNE,
113
+ )
114
+ ```
115
+ """
116
+
117
+ def __init__(
118
+ self,
119
+ tokenizer,
120
+ sequence_length=512,
121
+ truncate="round_robin",
122
+ mask_selection_rate=0.15,
123
+ mask_selection_length=96,
124
+ mask_token_rate=0.8,
125
+ random_token_rate=0.1,
126
+ **kwargs,
127
+ ):
128
+ super().__init__(
129
+ tokenizer,
130
+ sequence_length=sequence_length,
131
+ truncate=truncate,
132
+ **kwargs,
133
+ )
134
+ self.mask_selection_rate = mask_selection_rate
135
+ self.mask_selection_length = mask_selection_length
136
+ self.mask_token_rate = mask_token_rate
137
+ self.random_token_rate = random_token_rate
138
+ self.masker = None
139
+
140
+ def build(self, input_shape):
141
+ super().build(input_shape)
142
+ # Defer masker creation to `build()` so that we can be sure tokenizer
143
+ # assets have loaded when restoring a saved model.
144
+ self.masker = MaskedLMMaskGenerator(
145
+ mask_selection_rate=self.mask_selection_rate,
146
+ mask_selection_length=self.mask_selection_length,
147
+ mask_token_rate=self.mask_token_rate,
148
+ random_token_rate=self.random_token_rate,
149
+ vocabulary_size=self.tokenizer.vocabulary_size(),
150
+ mask_token_id=self.tokenizer.mask_token_id,
151
+ unselectable_token_ids=[
152
+ self.tokenizer.cls_token_id,
153
+ self.tokenizer.sep_token_id,
154
+ self.tokenizer.pad_token_id,
155
+ ],
156
+ )
157
+
158
+ def get_config(self):
159
+ config = super().get_config()
160
+ config.update(
161
+ {
162
+ "mask_selection_rate": self.mask_selection_rate,
163
+ "mask_selection_length": self.mask_selection_length,
164
+ "mask_token_rate": self.mask_token_rate,
165
+ "random_token_rate": self.random_token_rate,
166
+ }
167
+ )
168
+ return config
169
+
170
+ def call(self, x, y=None, sample_weight=None):
171
+ if y is not None or sample_weight is not None:
172
+ logging.warning(
173
+ f"{self.__class__.__name__} generates `y` and `sample_weight` "
174
+ "based on your input data, but your data already contains `y` "
175
+ "or `sample_weight`. Your `y` and `sample_weight` will be "
176
+ "ignored."
177
+ )
178
+
179
+ x = super().call(x)
180
+ token_ids, segment_ids, padding_mask = (
181
+ x["token_ids"],
182
+ x["segment_ids"],
183
+ x["padding_mask"],
184
+ )
185
+ masker_outputs = self.masker(token_ids)
186
+ x = {
187
+ "token_ids": masker_outputs["token_ids"],
188
+ "segment_ids": segment_ids,
189
+ "padding_mask": padding_mask,
190
+ "mask_positions": masker_outputs["mask_positions"],
191
+ }
192
+ y = masker_outputs["mask_ids"]
193
+ sample_weight = masker_outputs["mask_weights"]
194
+ return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
@@ -0,0 +1,206 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import keras
16
+
17
+ from keras_hub.src.api_export import keras_hub_export
18
+ from keras_hub.src.layers.preprocessing.multi_segment_packer import (
19
+ MultiSegmentPacker,
20
+ )
21
+ from keras_hub.src.models.albert.albert_tokenizer import AlbertTokenizer
22
+ from keras_hub.src.models.preprocessor import Preprocessor
23
+ from keras_hub.src.utils.keras_utils import (
24
+ convert_inputs_to_list_of_tensor_segments,
25
+ )
26
+
27
+
28
+ @keras_hub_export("keras_hub.models.AlbertPreprocessor")
29
+ class AlbertPreprocessor(Preprocessor):
30
+ """An ALBERT preprocessing layer which tokenizes and packs inputs.
31
+
32
+ This preprocessing layer will do three things:
33
+
34
+ - Tokenize any number of input segments using the `tokenizer`.
35
+ - Pack the inputs together using a `keras_hub.layers.MultiSegmentPacker`.
36
+ with the appropriate `"[CLS]"`, `"[SEP]"` and `"<pad>"` tokens.
37
+ - Construct a dictionary with keys `"token_ids"`, `"segment_ids"` and
38
+ `"padding_mask"`, that can be passed directly to
39
+ `keras_hub.models.AlbertBackbone`.
40
+
41
+ This layer can be used directly with `tf.data.Dataset.map` to preprocess
42
+ string data in the `(x, y, sample_weight)` format used by
43
+ `keras.Model.fit`.
44
+
45
+ The call method of this layer accepts three arguments, `x`, `y`, and
46
+ `sample_weight`. `x` can be a python string or tensor representing a single
47
+ segment, a list of python strings representing a batch of single segments,
48
+ or a list of tensors representing multiple segments to be packed together.
49
+ `y` and `sample_weight` are both optional, can have any format, and will be
50
+ passed through unaltered.
51
+
52
+ Special care should be taken when using `tf.data` to map over an unlabeled
53
+ tuple of string segments. `tf.data.Dataset.map` will unpack this tuple
54
+ directly into the call arguments of this layer, rather than forward all
55
+ argument to `x`. To handle this case, it is recommended to explicitly call
56
+ the layer, e.g. `ds.map(lambda seg1, seg2: preprocessor(x=(seg1, seg2)))`.
57
+
58
+ Args:
59
+ tokenizer: A `keras_hub.models.AlbertTokenizer` instance.
60
+ sequence_length: The length of the packed inputs.
61
+ truncate: string. The algorithm to truncate a list of batched segments
62
+ to fit within `sequence_length`. The value can be either
63
+ `round_robin` or `waterfall`:
64
+ - `"round_robin"`: Available space is assigned one token at a
65
+ time in a round-robin fashion to the inputs that still need
66
+ some, until the limit is reached.
67
+ - `"waterfall"`: The allocation of the budget is done using a
68
+ "waterfall" algorithm that allocates quota in a
69
+ left-to-right manner and fills up the buckets until we run
70
+ out of budget. It supports an arbitrary number of segments.
71
+
72
+ Examples:
73
+ Directly calling the layer on data.
74
+ ```python
75
+ preprocessor = keras_hub.models.AlbertPreprocessor.from_preset(
76
+ "albert_base_en_uncased"
77
+ )
78
+
79
+ # Tokenize and pack a single sentence.
80
+ preprocessor("The quick brown fox jumped.")
81
+
82
+ # Tokenize a batch of single sentences.
83
+ preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
84
+
85
+ # Preprocess a batch of sentence pairs.
86
+ # When handling multiple sequences, always convert to tensors first!
87
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
88
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
89
+ preprocessor((first, second))
90
+
91
+ # Custom vocabulary.
92
+ bytes_io = io.BytesIO()
93
+ ds = tf.data.Dataset.from_tensor_slices(["The quick brown fox jumped."])
94
+ sentencepiece.SentencePieceTrainer.train(
95
+ sentence_iterator=ds.as_numpy_iterator(),
96
+ model_writer=bytes_io,
97
+ vocab_size=10,
98
+ model_type="WORD",
99
+ pad_id=0,
100
+ unk_id=1,
101
+ bos_id=2,
102
+ eos_id=3,
103
+ pad_piece="<pad>",
104
+ unk_piece="<unk>",
105
+ bos_piece="[CLS]",
106
+ eos_piece="[SEP]",
107
+ user_defined_symbols="[MASK]",
108
+ )
109
+ tokenizer = keras_hub.models.AlbertTokenizer(
110
+ proto=bytes_io.getvalue(),
111
+ )
112
+ preprocessor = keras_hub.models.AlbertPreprocessor(tokenizer)
113
+ preprocessor("The quick brown fox jumped.")
114
+ ```
115
+
116
+ Mapping with `tf.data.Dataset`.
117
+ ```python
118
+ preprocessor = keras_hub.models.AlbertPreprocessor.from_preset(
119
+ "albert_base_en_uncased"
120
+ )
121
+
122
+ first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
123
+ second = tf.constant(["The fox tripped.", "Oh look, a whale."])
124
+ label = tf.constant([1, 1])
125
+
126
+ # Map labeled single sentences.
127
+ ds = tf.data.Dataset.from_tensor_slices((first, label))
128
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
129
+
130
+ # Map unlabeled single sentences.
131
+ ds = tf.data.Dataset.from_tensor_slices(first)
132
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
133
+
134
+ # Map labeled sentence pairs.
135
+ ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
136
+ ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
137
+
138
+ # Map unlabeled sentence pairs.
139
+ ds = tf.data.Dataset.from_tensor_slices((first, second))
140
+ # Watch out for tf.data's default unpacking of tuples here!
141
+ # Best to invoke the `preprocessor` directly in this case.
142
+ ds = ds.map(
143
+ lambda first, second: preprocessor(x=(first, second)),
144
+ num_parallel_calls=tf.data.AUTOTUNE,
145
+ )
146
+ ```
147
+ """
148
+
149
+ tokenizer_cls = AlbertTokenizer
150
+
151
+ def __init__(
152
+ self,
153
+ tokenizer,
154
+ sequence_length=512,
155
+ truncate="round_robin",
156
+ **kwargs,
157
+ ):
158
+ super().__init__(**kwargs)
159
+ self.tokenizer = tokenizer
160
+ self.packer = None
161
+ self.truncate = truncate
162
+ self.sequence_length = sequence_length
163
+
164
+ def build(self, input_shape):
165
+ # Defer packer creation to `build()` so that we can be sure tokenizer
166
+ # assets have loaded when restoring a saved model.
167
+ self.packer = MultiSegmentPacker(
168
+ start_value=self.tokenizer.cls_token_id,
169
+ end_value=self.tokenizer.sep_token_id,
170
+ pad_value=self.tokenizer.pad_token_id,
171
+ truncate=self.truncate,
172
+ sequence_length=self.sequence_length,
173
+ )
174
+ self.built = True
175
+
176
+ def get_config(self):
177
+ config = super().get_config()
178
+ config.update(
179
+ {
180
+ "sequence_length": self.sequence_length,
181
+ "truncate": self.truncate,
182
+ }
183
+ )
184
+ return config
185
+
186
+ def call(self, x, y=None, sample_weight=None):
187
+ x = convert_inputs_to_list_of_tensor_segments(x)
188
+ x = [self.tokenizer(segment) for segment in x]
189
+ token_ids, segment_ids = self.packer(x)
190
+ x = {
191
+ "token_ids": token_ids,
192
+ "segment_ids": segment_ids,
193
+ "padding_mask": token_ids != self.tokenizer.pad_token_id,
194
+ }
195
+ return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
196
+
197
+ @property
198
+ def sequence_length(self):
199
+ """The padded length of model input sequences."""
200
+ return self._sequence_length
201
+
202
+ @sequence_length.setter
203
+ def sequence_length(self, value):
204
+ self._sequence_length = value
205
+ if self.packer is not None:
206
+ self.packer.sequence_length = value
@@ -0,0 +1,70 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """ALBERT model preset configurations."""
15
+
16
+
17
+ backbone_presets = {
18
+ "albert_base_en_uncased": {
19
+ "metadata": {
20
+ "description": (
21
+ "12-layer ALBERT model where all input is lowercased. "
22
+ "Trained on English Wikipedia + BooksCorpus."
23
+ ),
24
+ "params": 11683584,
25
+ "official_name": "ALBERT",
26
+ "path": "albert",
27
+ "model_card": "https://github.com/google-research/albert/blob/master/README.md",
28
+ },
29
+ "kaggle_handle": "kaggle://keras/albert/keras/albert_base_en_uncased/2",
30
+ },
31
+ "albert_large_en_uncased": {
32
+ "metadata": {
33
+ "description": (
34
+ "24-layer ALBERT model where all input is lowercased. "
35
+ "Trained on English Wikipedia + BooksCorpus."
36
+ ),
37
+ "params": 17683968,
38
+ "official_name": "ALBERT",
39
+ "path": "albert",
40
+ "model_card": "https://github.com/google-research/albert/blob/master/README.md",
41
+ },
42
+ "kaggle_handle": "kaggle://keras/albert/keras/albert_large_en_uncased/2",
43
+ },
44
+ "albert_extra_large_en_uncased": {
45
+ "metadata": {
46
+ "description": (
47
+ "24-layer ALBERT model where all input is lowercased. "
48
+ "Trained on English Wikipedia + BooksCorpus."
49
+ ),
50
+ "params": 58724864,
51
+ "official_name": "ALBERT",
52
+ "path": "albert",
53
+ "model_card": "https://github.com/google-research/albert/blob/master/README.md",
54
+ },
55
+ "kaggle_handle": "kaggle://keras/albert/keras/albert_extra_large_en_uncased/2",
56
+ },
57
+ "albert_extra_extra_large_en_uncased": {
58
+ "metadata": {
59
+ "description": (
60
+ "12-layer ALBERT model where all input is lowercased. "
61
+ "Trained on English Wikipedia + BooksCorpus."
62
+ ),
63
+ "params": 222595584,
64
+ "official_name": "ALBERT",
65
+ "path": "albert",
66
+ "model_card": "https://github.com/google-research/albert/blob/master/README.md",
67
+ },
68
+ "kaggle_handle": "kaggle://keras/albert/keras/albert_extra_extra_large_en_uncased/2",
69
+ },
70
+ }
@@ -0,0 +1,119 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from keras_hub.src.api_export import keras_hub_export
16
+ from keras_hub.src.tokenizers.sentence_piece_tokenizer import (
17
+ SentencePieceTokenizer,
18
+ )
19
+
20
+
21
+ @keras_hub_export("keras_hub.models.AlbertTokenizer")
22
+ class AlbertTokenizer(SentencePieceTokenizer):
23
+ """ALBERT tokenizer layer based on SentencePiece.
24
+
25
+ This tokenizer class will tokenize raw strings into integer sequences and
26
+ is based on `keras_hub.tokenizers.SentencePieceTokenizer`. Unlike the
27
+ underlying tokenizer, it will check for all special tokens needed by
28
+ ALBERT models and provides a `from_preset()` method to automatically
29
+ download a matching vocabulary for a ALBERT preset.
30
+
31
+ This tokenizer does not provide truncation or padding of inputs. It can be
32
+ combined with a `keras_hub.models.AlbertPreprocessor` layer for input
33
+ packing.
34
+
35
+ If input is a batch of strings (rank > 0), the layer will output a
36
+ `tf.RaggedTensor` where the last dimension of the output is ragged.
37
+
38
+ If input is a scalar string (rank == 0), the layer will output a dense
39
+ `tf.Tensor` with static shape `[None]`.
40
+
41
+ Args:
42
+ proto: Either a `string` path to a SentencePiece proto file, or a
43
+ `bytes` object with a serialized SentencePiece proto. See the
44
+ [SentencePiece repository](https://github.com/google/sentencepiece)
45
+ for more details on the format.
46
+
47
+ Examples:
48
+
49
+ ```python
50
+ # Unbatched input.
51
+ tokenizer = keras_hub.models.AlbertTokenizer.from_preset(
52
+ "albert_base_en_uncased",
53
+ )
54
+ tokenizer("The quick brown fox jumped.")
55
+
56
+ # Batched input.
57
+ tokenizer(["The quick brown fox jumped.", "The fox slept."])
58
+
59
+ # Detokenization.
60
+ tokenizer.detokenize(tokenizer("The quick brown fox jumped."))
61
+
62
+ # Custom vocabulary.
63
+ bytes_io = io.BytesIO()
64
+ ds = tf.data.Dataset.from_tensor_slices(["The quick brown fox jumped."])
65
+ sentencepiece.SentencePieceTrainer.train(
66
+ sentence_iterator=ds.as_numpy_iterator(),
67
+ model_writer=bytes_io,
68
+ vocab_size=10,
69
+ model_type="WORD",
70
+ pad_id=0,
71
+ unk_id=1,
72
+ bos_id=2,
73
+ eos_id=3,
74
+ pad_piece="<pad>",
75
+ unk_piece="<unk>",
76
+ bos_piece="[CLS]",
77
+ eos_piece="[SEP]",
78
+ user_defined_symbols="[MASK]",
79
+ )
80
+ tokenizer = keras_hub.models.AlbertTokenizer(
81
+ proto=bytes_io.getvalue(),
82
+ )
83
+ tokenizer("The quick brown fox jumped.")
84
+ ```
85
+ """
86
+
87
+ def __init__(self, proto, **kwargs):
88
+ self.cls_token = "[CLS]"
89
+ self.sep_token = "[SEP]"
90
+ self.pad_token = "<pad>"
91
+ self.mask_token = "[MASK]"
92
+
93
+ super().__init__(proto=proto, **kwargs)
94
+
95
+ def set_proto(self, proto):
96
+ super().set_proto(proto)
97
+ if proto is not None:
98
+ for token in [
99
+ self.cls_token,
100
+ self.sep_token,
101
+ self.pad_token,
102
+ self.mask_token,
103
+ ]:
104
+ if token not in self.get_vocabulary():
105
+ raise ValueError(
106
+ f"Cannot find token `'{token}'` in the provided "
107
+ f"`vocabulary`. Please provide `'{token}'` in your "
108
+ "`vocabulary` or use a pretrained `vocabulary` name."
109
+ )
110
+
111
+ self.cls_token_id = self.token_to_id(self.cls_token)
112
+ self.sep_token_id = self.token_to_id(self.sep_token)
113
+ self.pad_token_id = self.token_to_id(self.pad_token)
114
+ self.mask_token_id = self.token_to_id(self.mask_token)
115
+ else:
116
+ self.cls_token_id = None
117
+ self.sep_token_id = None
118
+ self.pad_token_id = None
119
+ self.mask_token_id = None