keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (297) hide show
  1. keras_hub/__init__.py +52 -0
  2. keras_hub/api/__init__.py +27 -0
  3. keras_hub/api/layers/__init__.py +47 -0
  4. keras_hub/api/metrics/__init__.py +24 -0
  5. keras_hub/api/models/__init__.py +249 -0
  6. keras_hub/api/samplers/__init__.py +29 -0
  7. keras_hub/api/tokenizers/__init__.py +35 -0
  8. keras_hub/src/__init__.py +13 -0
  9. keras_hub/src/api_export.py +53 -0
  10. keras_hub/src/layers/__init__.py +13 -0
  11. keras_hub/src/layers/modeling/__init__.py +13 -0
  12. keras_hub/src/layers/modeling/alibi_bias.py +143 -0
  13. keras_hub/src/layers/modeling/cached_multi_head_attention.py +137 -0
  14. keras_hub/src/layers/modeling/f_net_encoder.py +200 -0
  15. keras_hub/src/layers/modeling/masked_lm_head.py +239 -0
  16. keras_hub/src/layers/modeling/position_embedding.py +123 -0
  17. keras_hub/src/layers/modeling/reversible_embedding.py +311 -0
  18. keras_hub/src/layers/modeling/rotary_embedding.py +169 -0
  19. keras_hub/src/layers/modeling/sine_position_encoding.py +108 -0
  20. keras_hub/src/layers/modeling/token_and_position_embedding.py +150 -0
  21. keras_hub/src/layers/modeling/transformer_decoder.py +496 -0
  22. keras_hub/src/layers/modeling/transformer_encoder.py +262 -0
  23. keras_hub/src/layers/modeling/transformer_layer_utils.py +106 -0
  24. keras_hub/src/layers/preprocessing/__init__.py +13 -0
  25. keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +220 -0
  26. keras_hub/src/layers/preprocessing/multi_segment_packer.py +319 -0
  27. keras_hub/src/layers/preprocessing/preprocessing_layer.py +62 -0
  28. keras_hub/src/layers/preprocessing/random_deletion.py +271 -0
  29. keras_hub/src/layers/preprocessing/random_swap.py +267 -0
  30. keras_hub/src/layers/preprocessing/start_end_packer.py +219 -0
  31. keras_hub/src/metrics/__init__.py +13 -0
  32. keras_hub/src/metrics/bleu.py +394 -0
  33. keras_hub/src/metrics/edit_distance.py +197 -0
  34. keras_hub/src/metrics/perplexity.py +181 -0
  35. keras_hub/src/metrics/rouge_base.py +204 -0
  36. keras_hub/src/metrics/rouge_l.py +97 -0
  37. keras_hub/src/metrics/rouge_n.py +125 -0
  38. keras_hub/src/models/__init__.py +13 -0
  39. keras_hub/src/models/albert/__init__.py +20 -0
  40. keras_hub/src/models/albert/albert_backbone.py +267 -0
  41. keras_hub/src/models/albert/albert_classifier.py +202 -0
  42. keras_hub/src/models/albert/albert_masked_lm.py +129 -0
  43. keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +194 -0
  44. keras_hub/src/models/albert/albert_preprocessor.py +206 -0
  45. keras_hub/src/models/albert/albert_presets.py +70 -0
  46. keras_hub/src/models/albert/albert_tokenizer.py +119 -0
  47. keras_hub/src/models/backbone.py +311 -0
  48. keras_hub/src/models/bart/__init__.py +20 -0
  49. keras_hub/src/models/bart/bart_backbone.py +261 -0
  50. keras_hub/src/models/bart/bart_preprocessor.py +276 -0
  51. keras_hub/src/models/bart/bart_presets.py +74 -0
  52. keras_hub/src/models/bart/bart_seq_2_seq_lm.py +490 -0
  53. keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +262 -0
  54. keras_hub/src/models/bart/bart_tokenizer.py +124 -0
  55. keras_hub/src/models/bert/__init__.py +23 -0
  56. keras_hub/src/models/bert/bert_backbone.py +227 -0
  57. keras_hub/src/models/bert/bert_classifier.py +183 -0
  58. keras_hub/src/models/bert/bert_masked_lm.py +131 -0
  59. keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +198 -0
  60. keras_hub/src/models/bert/bert_preprocessor.py +184 -0
  61. keras_hub/src/models/bert/bert_presets.py +147 -0
  62. keras_hub/src/models/bert/bert_tokenizer.py +112 -0
  63. keras_hub/src/models/bloom/__init__.py +20 -0
  64. keras_hub/src/models/bloom/bloom_attention.py +186 -0
  65. keras_hub/src/models/bloom/bloom_backbone.py +173 -0
  66. keras_hub/src/models/bloom/bloom_causal_lm.py +298 -0
  67. keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +176 -0
  68. keras_hub/src/models/bloom/bloom_decoder.py +206 -0
  69. keras_hub/src/models/bloom/bloom_preprocessor.py +185 -0
  70. keras_hub/src/models/bloom/bloom_presets.py +121 -0
  71. keras_hub/src/models/bloom/bloom_tokenizer.py +116 -0
  72. keras_hub/src/models/causal_lm.py +383 -0
  73. keras_hub/src/models/classifier.py +109 -0
  74. keras_hub/src/models/csp_darknet/__init__.py +13 -0
  75. keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +410 -0
  76. keras_hub/src/models/csp_darknet/csp_darknet_image_classifier.py +133 -0
  77. keras_hub/src/models/deberta_v3/__init__.py +24 -0
  78. keras_hub/src/models/deberta_v3/deberta_v3_backbone.py +210 -0
  79. keras_hub/src/models/deberta_v3/deberta_v3_classifier.py +228 -0
  80. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm.py +135 -0
  81. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +191 -0
  82. keras_hub/src/models/deberta_v3/deberta_v3_preprocessor.py +206 -0
  83. keras_hub/src/models/deberta_v3/deberta_v3_presets.py +82 -0
  84. keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +155 -0
  85. keras_hub/src/models/deberta_v3/disentangled_attention_encoder.py +227 -0
  86. keras_hub/src/models/deberta_v3/disentangled_self_attention.py +412 -0
  87. keras_hub/src/models/deberta_v3/relative_embedding.py +94 -0
  88. keras_hub/src/models/densenet/__init__.py +13 -0
  89. keras_hub/src/models/densenet/densenet_backbone.py +210 -0
  90. keras_hub/src/models/densenet/densenet_image_classifier.py +131 -0
  91. keras_hub/src/models/distil_bert/__init__.py +26 -0
  92. keras_hub/src/models/distil_bert/distil_bert_backbone.py +187 -0
  93. keras_hub/src/models/distil_bert/distil_bert_classifier.py +208 -0
  94. keras_hub/src/models/distil_bert/distil_bert_masked_lm.py +137 -0
  95. keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +194 -0
  96. keras_hub/src/models/distil_bert/distil_bert_preprocessor.py +175 -0
  97. keras_hub/src/models/distil_bert/distil_bert_presets.py +57 -0
  98. keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +114 -0
  99. keras_hub/src/models/electra/__init__.py +20 -0
  100. keras_hub/src/models/electra/electra_backbone.py +247 -0
  101. keras_hub/src/models/electra/electra_preprocessor.py +154 -0
  102. keras_hub/src/models/electra/electra_presets.py +95 -0
  103. keras_hub/src/models/electra/electra_tokenizer.py +104 -0
  104. keras_hub/src/models/f_net/__init__.py +20 -0
  105. keras_hub/src/models/f_net/f_net_backbone.py +236 -0
  106. keras_hub/src/models/f_net/f_net_classifier.py +154 -0
  107. keras_hub/src/models/f_net/f_net_masked_lm.py +132 -0
  108. keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +196 -0
  109. keras_hub/src/models/f_net/f_net_preprocessor.py +177 -0
  110. keras_hub/src/models/f_net/f_net_presets.py +43 -0
  111. keras_hub/src/models/f_net/f_net_tokenizer.py +95 -0
  112. keras_hub/src/models/falcon/__init__.py +20 -0
  113. keras_hub/src/models/falcon/falcon_attention.py +156 -0
  114. keras_hub/src/models/falcon/falcon_backbone.py +164 -0
  115. keras_hub/src/models/falcon/falcon_causal_lm.py +291 -0
  116. keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +173 -0
  117. keras_hub/src/models/falcon/falcon_preprocessor.py +187 -0
  118. keras_hub/src/models/falcon/falcon_presets.py +30 -0
  119. keras_hub/src/models/falcon/falcon_tokenizer.py +110 -0
  120. keras_hub/src/models/falcon/falcon_transformer_decoder.py +255 -0
  121. keras_hub/src/models/feature_pyramid_backbone.py +73 -0
  122. keras_hub/src/models/gemma/__init__.py +20 -0
  123. keras_hub/src/models/gemma/gemma_attention.py +250 -0
  124. keras_hub/src/models/gemma/gemma_backbone.py +316 -0
  125. keras_hub/src/models/gemma/gemma_causal_lm.py +448 -0
  126. keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +167 -0
  127. keras_hub/src/models/gemma/gemma_decoder_block.py +241 -0
  128. keras_hub/src/models/gemma/gemma_preprocessor.py +191 -0
  129. keras_hub/src/models/gemma/gemma_presets.py +248 -0
  130. keras_hub/src/models/gemma/gemma_tokenizer.py +103 -0
  131. keras_hub/src/models/gemma/rms_normalization.py +40 -0
  132. keras_hub/src/models/gpt2/__init__.py +20 -0
  133. keras_hub/src/models/gpt2/gpt2_backbone.py +199 -0
  134. keras_hub/src/models/gpt2/gpt2_causal_lm.py +437 -0
  135. keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +173 -0
  136. keras_hub/src/models/gpt2/gpt2_preprocessor.py +187 -0
  137. keras_hub/src/models/gpt2/gpt2_presets.py +82 -0
  138. keras_hub/src/models/gpt2/gpt2_tokenizer.py +110 -0
  139. keras_hub/src/models/gpt_neo_x/__init__.py +13 -0
  140. keras_hub/src/models/gpt_neo_x/gpt_neo_x_attention.py +251 -0
  141. keras_hub/src/models/gpt_neo_x/gpt_neo_x_backbone.py +175 -0
  142. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm.py +201 -0
  143. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +141 -0
  144. keras_hub/src/models/gpt_neo_x/gpt_neo_x_decoder.py +258 -0
  145. keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +145 -0
  146. keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +88 -0
  147. keras_hub/src/models/image_classifier.py +90 -0
  148. keras_hub/src/models/llama/__init__.py +20 -0
  149. keras_hub/src/models/llama/llama_attention.py +225 -0
  150. keras_hub/src/models/llama/llama_backbone.py +188 -0
  151. keras_hub/src/models/llama/llama_causal_lm.py +327 -0
  152. keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +170 -0
  153. keras_hub/src/models/llama/llama_decoder.py +246 -0
  154. keras_hub/src/models/llama/llama_layernorm.py +48 -0
  155. keras_hub/src/models/llama/llama_preprocessor.py +189 -0
  156. keras_hub/src/models/llama/llama_presets.py +80 -0
  157. keras_hub/src/models/llama/llama_tokenizer.py +84 -0
  158. keras_hub/src/models/llama3/__init__.py +20 -0
  159. keras_hub/src/models/llama3/llama3_backbone.py +84 -0
  160. keras_hub/src/models/llama3/llama3_causal_lm.py +46 -0
  161. keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +173 -0
  162. keras_hub/src/models/llama3/llama3_preprocessor.py +21 -0
  163. keras_hub/src/models/llama3/llama3_presets.py +69 -0
  164. keras_hub/src/models/llama3/llama3_tokenizer.py +63 -0
  165. keras_hub/src/models/masked_lm.py +101 -0
  166. keras_hub/src/models/mistral/__init__.py +20 -0
  167. keras_hub/src/models/mistral/mistral_attention.py +238 -0
  168. keras_hub/src/models/mistral/mistral_backbone.py +203 -0
  169. keras_hub/src/models/mistral/mistral_causal_lm.py +328 -0
  170. keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +175 -0
  171. keras_hub/src/models/mistral/mistral_layer_norm.py +48 -0
  172. keras_hub/src/models/mistral/mistral_preprocessor.py +190 -0
  173. keras_hub/src/models/mistral/mistral_presets.py +48 -0
  174. keras_hub/src/models/mistral/mistral_tokenizer.py +82 -0
  175. keras_hub/src/models/mistral/mistral_transformer_decoder.py +265 -0
  176. keras_hub/src/models/mix_transformer/__init__.py +13 -0
  177. keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +181 -0
  178. keras_hub/src/models/mix_transformer/mix_transformer_classifier.py +133 -0
  179. keras_hub/src/models/mix_transformer/mix_transformer_layers.py +300 -0
  180. keras_hub/src/models/opt/__init__.py +20 -0
  181. keras_hub/src/models/opt/opt_backbone.py +173 -0
  182. keras_hub/src/models/opt/opt_causal_lm.py +301 -0
  183. keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +177 -0
  184. keras_hub/src/models/opt/opt_preprocessor.py +188 -0
  185. keras_hub/src/models/opt/opt_presets.py +72 -0
  186. keras_hub/src/models/opt/opt_tokenizer.py +116 -0
  187. keras_hub/src/models/pali_gemma/__init__.py +23 -0
  188. keras_hub/src/models/pali_gemma/pali_gemma_backbone.py +277 -0
  189. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm.py +313 -0
  190. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +147 -0
  191. keras_hub/src/models/pali_gemma/pali_gemma_decoder_block.py +160 -0
  192. keras_hub/src/models/pali_gemma/pali_gemma_presets.py +78 -0
  193. keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +79 -0
  194. keras_hub/src/models/pali_gemma/pali_gemma_vit.py +566 -0
  195. keras_hub/src/models/phi3/__init__.py +20 -0
  196. keras_hub/src/models/phi3/phi3_attention.py +260 -0
  197. keras_hub/src/models/phi3/phi3_backbone.py +224 -0
  198. keras_hub/src/models/phi3/phi3_causal_lm.py +218 -0
  199. keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +173 -0
  200. keras_hub/src/models/phi3/phi3_decoder.py +260 -0
  201. keras_hub/src/models/phi3/phi3_layernorm.py +48 -0
  202. keras_hub/src/models/phi3/phi3_preprocessor.py +190 -0
  203. keras_hub/src/models/phi3/phi3_presets.py +50 -0
  204. keras_hub/src/models/phi3/phi3_rotary_embedding.py +137 -0
  205. keras_hub/src/models/phi3/phi3_tokenizer.py +94 -0
  206. keras_hub/src/models/preprocessor.py +207 -0
  207. keras_hub/src/models/resnet/__init__.py +13 -0
  208. keras_hub/src/models/resnet/resnet_backbone.py +612 -0
  209. keras_hub/src/models/resnet/resnet_image_classifier.py +136 -0
  210. keras_hub/src/models/roberta/__init__.py +20 -0
  211. keras_hub/src/models/roberta/roberta_backbone.py +184 -0
  212. keras_hub/src/models/roberta/roberta_classifier.py +209 -0
  213. keras_hub/src/models/roberta/roberta_masked_lm.py +136 -0
  214. keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +198 -0
  215. keras_hub/src/models/roberta/roberta_preprocessor.py +192 -0
  216. keras_hub/src/models/roberta/roberta_presets.py +43 -0
  217. keras_hub/src/models/roberta/roberta_tokenizer.py +132 -0
  218. keras_hub/src/models/seq_2_seq_lm.py +54 -0
  219. keras_hub/src/models/t5/__init__.py +20 -0
  220. keras_hub/src/models/t5/t5_backbone.py +261 -0
  221. keras_hub/src/models/t5/t5_layer_norm.py +35 -0
  222. keras_hub/src/models/t5/t5_multi_head_attention.py +324 -0
  223. keras_hub/src/models/t5/t5_presets.py +95 -0
  224. keras_hub/src/models/t5/t5_tokenizer.py +100 -0
  225. keras_hub/src/models/t5/t5_transformer_layer.py +178 -0
  226. keras_hub/src/models/task.py +419 -0
  227. keras_hub/src/models/vgg/__init__.py +13 -0
  228. keras_hub/src/models/vgg/vgg_backbone.py +158 -0
  229. keras_hub/src/models/vgg/vgg_image_classifier.py +124 -0
  230. keras_hub/src/models/vit_det/__init__.py +13 -0
  231. keras_hub/src/models/vit_det/vit_det_backbone.py +204 -0
  232. keras_hub/src/models/vit_det/vit_layers.py +565 -0
  233. keras_hub/src/models/whisper/__init__.py +20 -0
  234. keras_hub/src/models/whisper/whisper_audio_feature_extractor.py +260 -0
  235. keras_hub/src/models/whisper/whisper_backbone.py +305 -0
  236. keras_hub/src/models/whisper/whisper_cached_multi_head_attention.py +153 -0
  237. keras_hub/src/models/whisper/whisper_decoder.py +141 -0
  238. keras_hub/src/models/whisper/whisper_encoder.py +106 -0
  239. keras_hub/src/models/whisper/whisper_preprocessor.py +326 -0
  240. keras_hub/src/models/whisper/whisper_presets.py +148 -0
  241. keras_hub/src/models/whisper/whisper_tokenizer.py +163 -0
  242. keras_hub/src/models/xlm_roberta/__init__.py +26 -0
  243. keras_hub/src/models/xlm_roberta/xlm_roberta_backbone.py +81 -0
  244. keras_hub/src/models/xlm_roberta/xlm_roberta_classifier.py +225 -0
  245. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm.py +141 -0
  246. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +195 -0
  247. keras_hub/src/models/xlm_roberta/xlm_roberta_preprocessor.py +205 -0
  248. keras_hub/src/models/xlm_roberta/xlm_roberta_presets.py +43 -0
  249. keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +191 -0
  250. keras_hub/src/models/xlnet/__init__.py +13 -0
  251. keras_hub/src/models/xlnet/relative_attention.py +459 -0
  252. keras_hub/src/models/xlnet/xlnet_backbone.py +222 -0
  253. keras_hub/src/models/xlnet/xlnet_content_and_query_embedding.py +133 -0
  254. keras_hub/src/models/xlnet/xlnet_encoder.py +378 -0
  255. keras_hub/src/samplers/__init__.py +13 -0
  256. keras_hub/src/samplers/beam_sampler.py +207 -0
  257. keras_hub/src/samplers/contrastive_sampler.py +231 -0
  258. keras_hub/src/samplers/greedy_sampler.py +50 -0
  259. keras_hub/src/samplers/random_sampler.py +77 -0
  260. keras_hub/src/samplers/sampler.py +237 -0
  261. keras_hub/src/samplers/serialization.py +97 -0
  262. keras_hub/src/samplers/top_k_sampler.py +92 -0
  263. keras_hub/src/samplers/top_p_sampler.py +113 -0
  264. keras_hub/src/tests/__init__.py +13 -0
  265. keras_hub/src/tests/test_case.py +608 -0
  266. keras_hub/src/tokenizers/__init__.py +13 -0
  267. keras_hub/src/tokenizers/byte_pair_tokenizer.py +638 -0
  268. keras_hub/src/tokenizers/byte_tokenizer.py +299 -0
  269. keras_hub/src/tokenizers/sentence_piece_tokenizer.py +267 -0
  270. keras_hub/src/tokenizers/sentence_piece_tokenizer_trainer.py +150 -0
  271. keras_hub/src/tokenizers/tokenizer.py +235 -0
  272. keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +355 -0
  273. keras_hub/src/tokenizers/word_piece_tokenizer.py +544 -0
  274. keras_hub/src/tokenizers/word_piece_tokenizer_trainer.py +176 -0
  275. keras_hub/src/utils/__init__.py +13 -0
  276. keras_hub/src/utils/keras_utils.py +130 -0
  277. keras_hub/src/utils/pipeline_model.py +293 -0
  278. keras_hub/src/utils/preset_utils.py +621 -0
  279. keras_hub/src/utils/python_utils.py +21 -0
  280. keras_hub/src/utils/tensor_utils.py +206 -0
  281. keras_hub/src/utils/timm/__init__.py +13 -0
  282. keras_hub/src/utils/timm/convert.py +37 -0
  283. keras_hub/src/utils/timm/convert_resnet.py +171 -0
  284. keras_hub/src/utils/transformers/__init__.py +13 -0
  285. keras_hub/src/utils/transformers/convert.py +101 -0
  286. keras_hub/src/utils/transformers/convert_bert.py +173 -0
  287. keras_hub/src/utils/transformers/convert_distilbert.py +184 -0
  288. keras_hub/src/utils/transformers/convert_gemma.py +187 -0
  289. keras_hub/src/utils/transformers/convert_gpt2.py +186 -0
  290. keras_hub/src/utils/transformers/convert_llama3.py +136 -0
  291. keras_hub/src/utils/transformers/convert_pali_gemma.py +303 -0
  292. keras_hub/src/utils/transformers/safetensor_utils.py +97 -0
  293. keras_hub/src/version_utils.py +23 -0
  294. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +34 -0
  295. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +297 -0
  296. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/WHEEL +5 -0
  297. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/top_level.txt +1 -0
keras_hub/__init__.py ADDED
@@ -0,0 +1,52 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """DO NOT EDIT.
15
+
16
+ This file was autogenerated. Do not edit it by hand,
17
+ since your modifications would be overwritten.
18
+ """
19
+
20
+ import os
21
+
22
+ # sentencepiece segfaults on some version of tensorflow if tf is imported first.
23
+ try:
24
+ import sentencepiece
25
+ except ImportError:
26
+ pass
27
+
28
+ # Import everything from /api/ into keras.
29
+ from keras_hub.api import * # noqa: F403
30
+ from keras_hub.api import __version__ # Import * ignores names start with "_".
31
+
32
+ # Add everything in /api/ to the module search path.
33
+ __path__.append(os.path.join(os.path.dirname(__file__), "api")) # noqa: F405
34
+
35
+ # Don't pollute namespace.
36
+ del os
37
+
38
+
39
+ # Never autocomplete `.src` or `.api` on an imported keras object.
40
+ def __dir__():
41
+ keys = dict.fromkeys((globals().keys()))
42
+ keys.pop("src")
43
+ keys.pop("api")
44
+ return list(keys)
45
+
46
+
47
+ # Don't import `.src` or `.api` during `from keras import *`.
48
+ __all__ = [
49
+ name
50
+ for name in globals().keys()
51
+ if not (name.startswith("_") or name in ("src", "api"))
52
+ ]
@@ -0,0 +1,27 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """DO NOT EDIT.
15
+
16
+ This file was autogenerated. Do not edit it by hand,
17
+ since your modifications would be overwritten.
18
+ """
19
+
20
+ from keras_hub.api import layers
21
+ from keras_hub.api import metrics
22
+ from keras_hub.api import models
23
+ from keras_hub.api import samplers
24
+ from keras_hub.api import tokenizers
25
+ from keras_hub.src.utils.preset_utils import upload_preset
26
+ from keras_hub.src.version_utils import __version__
27
+ from keras_hub.src.version_utils import version
@@ -0,0 +1,47 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """DO NOT EDIT.
15
+
16
+ This file was autogenerated. Do not edit it by hand,
17
+ since your modifications would be overwritten.
18
+ """
19
+
20
+ from keras_hub.src.layers.modeling.alibi_bias import AlibiBias
21
+ from keras_hub.src.layers.modeling.cached_multi_head_attention import (
22
+ CachedMultiHeadAttention,
23
+ )
24
+ from keras_hub.src.layers.modeling.f_net_encoder import FNetEncoder
25
+ from keras_hub.src.layers.modeling.masked_lm_head import MaskedLMHead
26
+ from keras_hub.src.layers.modeling.position_embedding import PositionEmbedding
27
+ from keras_hub.src.layers.modeling.reversible_embedding import (
28
+ ReversibleEmbedding,
29
+ )
30
+ from keras_hub.src.layers.modeling.rotary_embedding import RotaryEmbedding
31
+ from keras_hub.src.layers.modeling.sine_position_encoding import (
32
+ SinePositionEncoding,
33
+ )
34
+ from keras_hub.src.layers.modeling.token_and_position_embedding import (
35
+ TokenAndPositionEmbedding,
36
+ )
37
+ from keras_hub.src.layers.modeling.transformer_decoder import TransformerDecoder
38
+ from keras_hub.src.layers.modeling.transformer_encoder import TransformerEncoder
39
+ from keras_hub.src.layers.preprocessing.masked_lm_mask_generator import (
40
+ MaskedLMMaskGenerator,
41
+ )
42
+ from keras_hub.src.layers.preprocessing.multi_segment_packer import (
43
+ MultiSegmentPacker,
44
+ )
45
+ from keras_hub.src.layers.preprocessing.random_deletion import RandomDeletion
46
+ from keras_hub.src.layers.preprocessing.random_swap import RandomSwap
47
+ from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
@@ -0,0 +1,24 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """DO NOT EDIT.
15
+
16
+ This file was autogenerated. Do not edit it by hand,
17
+ since your modifications would be overwritten.
18
+ """
19
+
20
+ from keras_hub.src.metrics.bleu import Bleu
21
+ from keras_hub.src.metrics.edit_distance import EditDistance
22
+ from keras_hub.src.metrics.perplexity import Perplexity
23
+ from keras_hub.src.metrics.rouge_l import RougeL
24
+ from keras_hub.src.metrics.rouge_n import RougeN
@@ -0,0 +1,249 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """DO NOT EDIT.
15
+
16
+ This file was autogenerated. Do not edit it by hand,
17
+ since your modifications would be overwritten.
18
+ """
19
+
20
+ from keras_hub.src.models.albert.albert_backbone import AlbertBackbone
21
+ from keras_hub.src.models.albert.albert_classifier import AlbertClassifier
22
+ from keras_hub.src.models.albert.albert_masked_lm import AlbertMaskedLM
23
+ from keras_hub.src.models.albert.albert_masked_lm_preprocessor import (
24
+ AlbertMaskedLMPreprocessor,
25
+ )
26
+ from keras_hub.src.models.albert.albert_preprocessor import AlbertPreprocessor
27
+ from keras_hub.src.models.albert.albert_tokenizer import AlbertTokenizer
28
+ from keras_hub.src.models.backbone import Backbone
29
+ from keras_hub.src.models.bart.bart_backbone import BartBackbone
30
+ from keras_hub.src.models.bart.bart_preprocessor import BartPreprocessor
31
+ from keras_hub.src.models.bart.bart_seq_2_seq_lm import BartSeq2SeqLM
32
+ from keras_hub.src.models.bart.bart_seq_2_seq_lm_preprocessor import (
33
+ BartSeq2SeqLMPreprocessor,
34
+ )
35
+ from keras_hub.src.models.bart.bart_tokenizer import BartTokenizer
36
+ from keras_hub.src.models.bert.bert_backbone import BertBackbone
37
+ from keras_hub.src.models.bert.bert_classifier import BertClassifier
38
+ from keras_hub.src.models.bert.bert_masked_lm import BertMaskedLM
39
+ from keras_hub.src.models.bert.bert_masked_lm_preprocessor import (
40
+ BertMaskedLMPreprocessor,
41
+ )
42
+ from keras_hub.src.models.bert.bert_preprocessor import BertPreprocessor
43
+ from keras_hub.src.models.bert.bert_tokenizer import BertTokenizer
44
+ from keras_hub.src.models.bloom.bloom_backbone import BloomBackbone
45
+ from keras_hub.src.models.bloom.bloom_causal_lm import BloomCausalLM
46
+ from keras_hub.src.models.bloom.bloom_causal_lm_preprocessor import (
47
+ BloomCausalLMPreprocessor,
48
+ )
49
+ from keras_hub.src.models.bloom.bloom_preprocessor import BloomPreprocessor
50
+ from keras_hub.src.models.bloom.bloom_tokenizer import BloomTokenizer
51
+ from keras_hub.src.models.causal_lm import CausalLM
52
+ from keras_hub.src.models.classifier import Classifier
53
+ from keras_hub.src.models.csp_darknet.csp_darknet_backbone import (
54
+ CSPDarkNetBackbone,
55
+ )
56
+ from keras_hub.src.models.csp_darknet.csp_darknet_image_classifier import (
57
+ CSPDarkNetImageClassifier,
58
+ )
59
+ from keras_hub.src.models.deberta_v3.deberta_v3_backbone import (
60
+ DebertaV3Backbone,
61
+ )
62
+ from keras_hub.src.models.deberta_v3.deberta_v3_classifier import (
63
+ DebertaV3Classifier,
64
+ )
65
+ from keras_hub.src.models.deberta_v3.deberta_v3_masked_lm import (
66
+ DebertaV3MaskedLM,
67
+ )
68
+ from keras_hub.src.models.deberta_v3.deberta_v3_masked_lm_preprocessor import (
69
+ DebertaV3MaskedLMPreprocessor,
70
+ )
71
+ from keras_hub.src.models.deberta_v3.deberta_v3_preprocessor import (
72
+ DebertaV3Preprocessor,
73
+ )
74
+ from keras_hub.src.models.deberta_v3.deberta_v3_tokenizer import (
75
+ DebertaV3Tokenizer,
76
+ )
77
+ from keras_hub.src.models.densenet.densenet_backbone import DenseNetBackbone
78
+ from keras_hub.src.models.densenet.densenet_image_classifier import (
79
+ DenseNetImageClassifier,
80
+ )
81
+ from keras_hub.src.models.distil_bert.distil_bert_backbone import (
82
+ DistilBertBackbone,
83
+ )
84
+ from keras_hub.src.models.distil_bert.distil_bert_classifier import (
85
+ DistilBertClassifier,
86
+ )
87
+ from keras_hub.src.models.distil_bert.distil_bert_masked_lm import (
88
+ DistilBertMaskedLM,
89
+ )
90
+ from keras_hub.src.models.distil_bert.distil_bert_masked_lm_preprocessor import (
91
+ DistilBertMaskedLMPreprocessor,
92
+ )
93
+ from keras_hub.src.models.distil_bert.distil_bert_preprocessor import (
94
+ DistilBertPreprocessor,
95
+ )
96
+ from keras_hub.src.models.distil_bert.distil_bert_tokenizer import (
97
+ DistilBertTokenizer,
98
+ )
99
+ from keras_hub.src.models.electra.electra_backbone import ElectraBackbone
100
+ from keras_hub.src.models.electra.electra_preprocessor import (
101
+ ElectraPreprocessor,
102
+ )
103
+ from keras_hub.src.models.electra.electra_tokenizer import ElectraTokenizer
104
+ from keras_hub.src.models.f_net.f_net_backbone import FNetBackbone
105
+ from keras_hub.src.models.f_net.f_net_classifier import FNetClassifier
106
+ from keras_hub.src.models.f_net.f_net_masked_lm import FNetMaskedLM
107
+ from keras_hub.src.models.f_net.f_net_masked_lm_preprocessor import (
108
+ FNetMaskedLMPreprocessor,
109
+ )
110
+ from keras_hub.src.models.f_net.f_net_preprocessor import FNetPreprocessor
111
+ from keras_hub.src.models.f_net.f_net_tokenizer import FNetTokenizer
112
+ from keras_hub.src.models.falcon.falcon_backbone import FalconBackbone
113
+ from keras_hub.src.models.falcon.falcon_causal_lm import FalconCausalLM
114
+ from keras_hub.src.models.falcon.falcon_causal_lm_preprocessor import (
115
+ FalconCausalLMPreprocessor,
116
+ )
117
+ from keras_hub.src.models.falcon.falcon_preprocessor import FalconPreprocessor
118
+ from keras_hub.src.models.falcon.falcon_tokenizer import FalconTokenizer
119
+ from keras_hub.src.models.feature_pyramid_backbone import FeaturePyramidBackbone
120
+ from keras_hub.src.models.gemma.gemma_backbone import GemmaBackbone
121
+ from keras_hub.src.models.gemma.gemma_causal_lm import GemmaCausalLM
122
+ from keras_hub.src.models.gemma.gemma_causal_lm_preprocessor import (
123
+ GemmaCausalLMPreprocessor,
124
+ )
125
+ from keras_hub.src.models.gemma.gemma_preprocessor import GemmaPreprocessor
126
+ from keras_hub.src.models.gemma.gemma_tokenizer import GemmaTokenizer
127
+ from keras_hub.src.models.gpt2.gpt2_backbone import GPT2Backbone
128
+ from keras_hub.src.models.gpt2.gpt2_causal_lm import GPT2CausalLM
129
+ from keras_hub.src.models.gpt2.gpt2_causal_lm_preprocessor import (
130
+ GPT2CausalLMPreprocessor,
131
+ )
132
+ from keras_hub.src.models.gpt2.gpt2_preprocessor import GPT2Preprocessor
133
+ from keras_hub.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
134
+ from keras_hub.src.models.gpt_neo_x.gpt_neo_x_backbone import GPTNeoXBackbone
135
+ from keras_hub.src.models.gpt_neo_x.gpt_neo_x_causal_lm import GPTNeoXCausalLM
136
+ from keras_hub.src.models.gpt_neo_x.gpt_neo_x_causal_lm_preprocessor import (
137
+ GPTNeoXCausalLMPreprocessor,
138
+ )
139
+ from keras_hub.src.models.gpt_neo_x.gpt_neo_x_preprocessor import (
140
+ GPTNeoXPreprocessor,
141
+ )
142
+ from keras_hub.src.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer
143
+ from keras_hub.src.models.image_classifier import ImageClassifier
144
+ from keras_hub.src.models.llama3.llama3_backbone import Llama3Backbone
145
+ from keras_hub.src.models.llama3.llama3_causal_lm import Llama3CausalLM
146
+ from keras_hub.src.models.llama3.llama3_causal_lm_preprocessor import (
147
+ Llama3CausalLMPreprocessor,
148
+ )
149
+ from keras_hub.src.models.llama3.llama3_preprocessor import Llama3Preprocessor
150
+ from keras_hub.src.models.llama3.llama3_tokenizer import Llama3Tokenizer
151
+ from keras_hub.src.models.llama.llama_backbone import LlamaBackbone
152
+ from keras_hub.src.models.llama.llama_causal_lm import LlamaCausalLM
153
+ from keras_hub.src.models.llama.llama_causal_lm_preprocessor import (
154
+ LlamaCausalLMPreprocessor,
155
+ )
156
+ from keras_hub.src.models.llama.llama_preprocessor import LlamaPreprocessor
157
+ from keras_hub.src.models.llama.llama_tokenizer import LlamaTokenizer
158
+ from keras_hub.src.models.masked_lm import MaskedLM
159
+ from keras_hub.src.models.mistral.mistral_backbone import MistralBackbone
160
+ from keras_hub.src.models.mistral.mistral_causal_lm import MistralCausalLM
161
+ from keras_hub.src.models.mistral.mistral_causal_lm_preprocessor import (
162
+ MistralCausalLMPreprocessor,
163
+ )
164
+ from keras_hub.src.models.mistral.mistral_preprocessor import (
165
+ MistralPreprocessor,
166
+ )
167
+ from keras_hub.src.models.mistral.mistral_tokenizer import MistralTokenizer
168
+ from keras_hub.src.models.mix_transformer.mix_transformer_backbone import (
169
+ MiTBackbone,
170
+ )
171
+ from keras_hub.src.models.mix_transformer.mix_transformer_classifier import (
172
+ MiTImageClassifier,
173
+ )
174
+ from keras_hub.src.models.opt.opt_backbone import OPTBackbone
175
+ from keras_hub.src.models.opt.opt_causal_lm import OPTCausalLM
176
+ from keras_hub.src.models.opt.opt_causal_lm_preprocessor import (
177
+ OPTCausalLMPreprocessor,
178
+ )
179
+ from keras_hub.src.models.opt.opt_preprocessor import OPTPreprocessor
180
+ from keras_hub.src.models.opt.opt_tokenizer import OPTTokenizer
181
+ from keras_hub.src.models.pali_gemma.pali_gemma_backbone import (
182
+ PaliGemmaBackbone,
183
+ )
184
+ from keras_hub.src.models.pali_gemma.pali_gemma_causal_lm import (
185
+ PaliGemmaCausalLM,
186
+ )
187
+ from keras_hub.src.models.pali_gemma.pali_gemma_causal_lm_preprocessor import (
188
+ PaliGemmaCausalLMPreprocessor,
189
+ )
190
+ from keras_hub.src.models.pali_gemma.pali_gemma_tokenizer import (
191
+ PaliGemmaTokenizer,
192
+ )
193
+ from keras_hub.src.models.phi3.phi3_backbone import Phi3Backbone
194
+ from keras_hub.src.models.phi3.phi3_causal_lm import Phi3CausalLM
195
+ from keras_hub.src.models.phi3.phi3_causal_lm_preprocessor import (
196
+ Phi3CausalLMPreprocessor,
197
+ )
198
+ from keras_hub.src.models.phi3.phi3_preprocessor import Phi3Preprocessor
199
+ from keras_hub.src.models.phi3.phi3_tokenizer import Phi3Tokenizer
200
+ from keras_hub.src.models.preprocessor import Preprocessor
201
+ from keras_hub.src.models.resnet.resnet_backbone import ResNetBackbone
202
+ from keras_hub.src.models.resnet.resnet_image_classifier import (
203
+ ResNetImageClassifier,
204
+ )
205
+ from keras_hub.src.models.roberta.roberta_backbone import RobertaBackbone
206
+ from keras_hub.src.models.roberta.roberta_classifier import RobertaClassifier
207
+ from keras_hub.src.models.roberta.roberta_masked_lm import RobertaMaskedLM
208
+ from keras_hub.src.models.roberta.roberta_masked_lm_preprocessor import (
209
+ RobertaMaskedLMPreprocessor,
210
+ )
211
+ from keras_hub.src.models.roberta.roberta_preprocessor import (
212
+ RobertaPreprocessor,
213
+ )
214
+ from keras_hub.src.models.roberta.roberta_tokenizer import RobertaTokenizer
215
+ from keras_hub.src.models.seq_2_seq_lm import Seq2SeqLM
216
+ from keras_hub.src.models.t5.t5_backbone import T5Backbone
217
+ from keras_hub.src.models.t5.t5_tokenizer import T5Tokenizer
218
+ from keras_hub.src.models.task import Task
219
+ from keras_hub.src.models.vgg.vgg_backbone import VGGBackbone
220
+ from keras_hub.src.models.vgg.vgg_image_classifier import VGGImageClassifier
221
+ from keras_hub.src.models.vit_det.vit_det_backbone import ViTDetBackbone
222
+ from keras_hub.src.models.whisper.whisper_audio_feature_extractor import (
223
+ WhisperAudioFeatureExtractor,
224
+ )
225
+ from keras_hub.src.models.whisper.whisper_backbone import WhisperBackbone
226
+ from keras_hub.src.models.whisper.whisper_preprocessor import (
227
+ WhisperPreprocessor,
228
+ )
229
+ from keras_hub.src.models.whisper.whisper_tokenizer import WhisperTokenizer
230
+ from keras_hub.src.models.xlm_roberta.xlm_roberta_backbone import (
231
+ XLMRobertaBackbone,
232
+ )
233
+ from keras_hub.src.models.xlm_roberta.xlm_roberta_classifier import (
234
+ XLMRobertaClassifier,
235
+ )
236
+ from keras_hub.src.models.xlm_roberta.xlm_roberta_masked_lm import (
237
+ XLMRobertaMaskedLM,
238
+ )
239
+ from keras_hub.src.models.xlm_roberta.xlm_roberta_masked_lm_preprocessor import (
240
+ XLMRobertaMaskedLMPreprocessor,
241
+ )
242
+ from keras_hub.src.models.xlm_roberta.xlm_roberta_preprocessor import (
243
+ XLMRobertaPreprocessor,
244
+ )
245
+ from keras_hub.src.models.xlm_roberta.xlm_roberta_tokenizer import (
246
+ XLMRobertaTokenizer,
247
+ )
248
+ from keras_hub.src.models.xlnet.xlnet_backbone import XLNetBackbone
249
+ from keras_hub.src.tokenizers.tokenizer import Tokenizer
@@ -0,0 +1,29 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """DO NOT EDIT.
15
+
16
+ This file was autogenerated. Do not edit it by hand,
17
+ since your modifications would be overwritten.
18
+ """
19
+
20
+ from keras_hub.src.samplers.beam_sampler import BeamSampler
21
+ from keras_hub.src.samplers.contrastive_sampler import ContrastiveSampler
22
+ from keras_hub.src.samplers.greedy_sampler import GreedySampler
23
+ from keras_hub.src.samplers.random_sampler import RandomSampler
24
+ from keras_hub.src.samplers.sampler import Sampler
25
+ from keras_hub.src.samplers.serialization import deserialize
26
+ from keras_hub.src.samplers.serialization import get
27
+ from keras_hub.src.samplers.serialization import serialize
28
+ from keras_hub.src.samplers.top_k_sampler import TopKSampler
29
+ from keras_hub.src.samplers.top_p_sampler import TopPSampler
@@ -0,0 +1,35 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ """DO NOT EDIT.
15
+
16
+ This file was autogenerated. Do not edit it by hand,
17
+ since your modifications would be overwritten.
18
+ """
19
+
20
+ from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
21
+ from keras_hub.src.tokenizers.byte_tokenizer import ByteTokenizer
22
+ from keras_hub.src.tokenizers.sentence_piece_tokenizer import (
23
+ SentencePieceTokenizer,
24
+ )
25
+ from keras_hub.src.tokenizers.sentence_piece_tokenizer_trainer import (
26
+ compute_sentence_piece_proto,
27
+ )
28
+ from keras_hub.src.tokenizers.tokenizer import Tokenizer
29
+ from keras_hub.src.tokenizers.unicode_codepoint_tokenizer import (
30
+ UnicodeCodepointTokenizer,
31
+ )
32
+ from keras_hub.src.tokenizers.word_piece_tokenizer import WordPieceTokenizer
33
+ from keras_hub.src.tokenizers.word_piece_tokenizer_trainer import (
34
+ compute_word_piece_vocabulary,
35
+ )
@@ -0,0 +1,13 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
@@ -0,0 +1,53 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import types
16
+
17
+ import keras
18
+
19
+ try:
20
+ import namex
21
+ except ImportError:
22
+ namex = None
23
+
24
+
25
+ def maybe_register_serializable(symbol):
26
+ if isinstance(symbol, types.FunctionType) or hasattr(symbol, "get_config"):
27
+ # We register twice, first with the old name, second with the new name,
28
+ # so loading still works under the old name.
29
+ # TODO replace keras_nlp with keras-hub after rename.
30
+ compat_name = "keras_nlp"
31
+ keras.saving.register_keras_serializable(package=compat_name)(symbol)
32
+ keras.saving.register_keras_serializable(package="keras_hub")(symbol)
33
+
34
+
35
+ if namex:
36
+
37
+ class keras_hub_export(namex.export):
38
+ def __init__(self, path):
39
+ super().__init__(package="keras_hub", path=path)
40
+
41
+ def __call__(self, symbol):
42
+ maybe_register_serializable(symbol)
43
+ return super().__call__(symbol)
44
+
45
+ else:
46
+
47
+ class keras_hub_export:
48
+ def __init__(self, path):
49
+ pass
50
+
51
+ def __call__(self, symbol):
52
+ maybe_register_serializable(symbol)
53
+ return symbol
@@ -0,0 +1,13 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
@@ -0,0 +1,13 @@
1
+ # Copyright 2024 The KerasHub Authors
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.