keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl → 0.16.0.dev2024092017__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. keras_hub/__init__.py +0 -6
  2. keras_hub/api/__init__.py +2 -0
  3. keras_hub/api/bounding_box/__init__.py +36 -0
  4. keras_hub/api/layers/__init__.py +14 -0
  5. keras_hub/api/models/__init__.py +97 -48
  6. keras_hub/api/tokenizers/__init__.py +30 -0
  7. keras_hub/api/utils/__init__.py +22 -0
  8. keras_hub/src/api_export.py +15 -9
  9. keras_hub/src/bounding_box/__init__.py +13 -0
  10. keras_hub/src/bounding_box/converters.py +529 -0
  11. keras_hub/src/bounding_box/formats.py +162 -0
  12. keras_hub/src/bounding_box/iou.py +263 -0
  13. keras_hub/src/bounding_box/to_dense.py +95 -0
  14. keras_hub/src/bounding_box/to_ragged.py +99 -0
  15. keras_hub/src/bounding_box/utils.py +194 -0
  16. keras_hub/src/bounding_box/validate_format.py +99 -0
  17. keras_hub/src/layers/preprocessing/audio_converter.py +121 -0
  18. keras_hub/src/layers/preprocessing/image_converter.py +130 -0
  19. keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +2 -0
  20. keras_hub/src/layers/preprocessing/multi_segment_packer.py +9 -8
  21. keras_hub/src/layers/preprocessing/preprocessing_layer.py +2 -29
  22. keras_hub/src/layers/preprocessing/random_deletion.py +33 -31
  23. keras_hub/src/layers/preprocessing/random_swap.py +33 -31
  24. keras_hub/src/layers/preprocessing/resizing_image_converter.py +101 -0
  25. keras_hub/src/layers/preprocessing/start_end_packer.py +3 -2
  26. keras_hub/src/models/albert/__init__.py +1 -2
  27. keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +6 -86
  28. keras_hub/src/models/albert/{albert_classifier.py → albert_text_classifier.py} +34 -10
  29. keras_hub/src/models/albert/{albert_preprocessor.py → albert_text_classifier_preprocessor.py} +14 -70
  30. keras_hub/src/models/albert/albert_tokenizer.py +17 -36
  31. keras_hub/src/models/backbone.py +12 -34
  32. keras_hub/src/models/bart/__init__.py +1 -2
  33. keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +21 -148
  34. keras_hub/src/models/bart/bart_tokenizer.py +12 -39
  35. keras_hub/src/models/bert/__init__.py +1 -5
  36. keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +6 -87
  37. keras_hub/src/models/bert/bert_presets.py +1 -4
  38. keras_hub/src/models/bert/{bert_classifier.py → bert_text_classifier.py} +19 -12
  39. keras_hub/src/models/bert/{bert_preprocessor.py → bert_text_classifier_preprocessor.py} +14 -70
  40. keras_hub/src/models/bert/bert_tokenizer.py +17 -35
  41. keras_hub/src/models/bloom/__init__.py +1 -2
  42. keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +6 -91
  43. keras_hub/src/models/bloom/bloom_tokenizer.py +12 -41
  44. keras_hub/src/models/causal_lm.py +10 -29
  45. keras_hub/src/models/causal_lm_preprocessor.py +195 -0
  46. keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +54 -15
  47. keras_hub/src/models/deberta_v3/__init__.py +1 -4
  48. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +14 -77
  49. keras_hub/src/models/deberta_v3/{deberta_v3_classifier.py → deberta_v3_text_classifier.py} +16 -11
  50. keras_hub/src/models/deberta_v3/{deberta_v3_preprocessor.py → deberta_v3_text_classifier_preprocessor.py} +23 -64
  51. keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +30 -25
  52. keras_hub/src/models/densenet/densenet_backbone.py +46 -22
  53. keras_hub/src/models/distil_bert/__init__.py +1 -4
  54. keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +14 -76
  55. keras_hub/src/models/distil_bert/{distil_bert_classifier.py → distil_bert_text_classifier.py} +17 -12
  56. keras_hub/src/models/distil_bert/{distil_bert_preprocessor.py → distil_bert_text_classifier_preprocessor.py} +23 -63
  57. keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +19 -35
  58. keras_hub/src/models/efficientnet/__init__.py +13 -0
  59. keras_hub/src/models/efficientnet/efficientnet_backbone.py +569 -0
  60. keras_hub/src/models/efficientnet/fusedmbconv.py +229 -0
  61. keras_hub/src/models/efficientnet/mbconv.py +238 -0
  62. keras_hub/src/models/electra/__init__.py +1 -2
  63. keras_hub/src/models/electra/electra_tokenizer.py +17 -32
  64. keras_hub/src/models/f_net/__init__.py +1 -2
  65. keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +12 -78
  66. keras_hub/src/models/f_net/{f_net_classifier.py → f_net_text_classifier.py} +17 -10
  67. keras_hub/src/models/f_net/{f_net_preprocessor.py → f_net_text_classifier_preprocessor.py} +19 -63
  68. keras_hub/src/models/f_net/f_net_tokenizer.py +17 -35
  69. keras_hub/src/models/falcon/__init__.py +1 -2
  70. keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +6 -89
  71. keras_hub/src/models/falcon/falcon_tokenizer.py +12 -35
  72. keras_hub/src/models/gemma/__init__.py +1 -2
  73. keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +6 -90
  74. keras_hub/src/models/gemma/gemma_decoder_block.py +1 -1
  75. keras_hub/src/models/gemma/gemma_tokenizer.py +12 -23
  76. keras_hub/src/models/gpt2/__init__.py +1 -2
  77. keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +6 -89
  78. keras_hub/src/models/gpt2/gpt2_preprocessor.py +12 -90
  79. keras_hub/src/models/gpt2/gpt2_tokenizer.py +12 -34
  80. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +6 -91
  81. keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +12 -34
  82. keras_hub/src/models/image_classifier.py +0 -5
  83. keras_hub/src/models/image_classifier_preprocessor.py +83 -0
  84. keras_hub/src/models/llama/__init__.py +1 -2
  85. keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +6 -85
  86. keras_hub/src/models/llama/llama_tokenizer.py +12 -25
  87. keras_hub/src/models/llama3/__init__.py +1 -2
  88. keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +6 -89
  89. keras_hub/src/models/llama3/llama3_tokenizer.py +12 -33
  90. keras_hub/src/models/masked_lm.py +0 -2
  91. keras_hub/src/models/masked_lm_preprocessor.py +156 -0
  92. keras_hub/src/models/mistral/__init__.py +1 -2
  93. keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +6 -91
  94. keras_hub/src/models/mistral/mistral_tokenizer.py +12 -23
  95. keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +2 -2
  96. keras_hub/src/models/mobilenet/__init__.py +13 -0
  97. keras_hub/src/models/mobilenet/mobilenet_backbone.py +530 -0
  98. keras_hub/src/models/mobilenet/mobilenet_image_classifier.py +114 -0
  99. keras_hub/src/models/opt/__init__.py +1 -2
  100. keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +6 -93
  101. keras_hub/src/models/opt/opt_tokenizer.py +12 -41
  102. keras_hub/src/models/pali_gemma/__init__.py +1 -4
  103. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +28 -28
  104. keras_hub/src/models/pali_gemma/pali_gemma_image_converter.py +25 -0
  105. keras_hub/src/models/pali_gemma/pali_gemma_presets.py +5 -5
  106. keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +11 -3
  107. keras_hub/src/models/phi3/__init__.py +1 -2
  108. keras_hub/src/models/phi3/phi3_causal_lm.py +3 -9
  109. keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +6 -89
  110. keras_hub/src/models/phi3/phi3_tokenizer.py +12 -36
  111. keras_hub/src/models/preprocessor.py +72 -83
  112. keras_hub/src/models/resnet/__init__.py +6 -0
  113. keras_hub/src/models/resnet/resnet_backbone.py +390 -42
  114. keras_hub/src/models/resnet/resnet_image_classifier.py +33 -6
  115. keras_hub/src/models/resnet/resnet_image_classifier_preprocessor.py +28 -0
  116. keras_hub/src/models/{llama3/llama3_preprocessor.py → resnet/resnet_image_converter.py} +7 -5
  117. keras_hub/src/models/resnet/resnet_presets.py +95 -0
  118. keras_hub/src/models/retinanet/__init__.py +13 -0
  119. keras_hub/src/models/retinanet/anchor_generator.py +175 -0
  120. keras_hub/src/models/retinanet/box_matcher.py +259 -0
  121. keras_hub/src/models/retinanet/non_max_supression.py +578 -0
  122. keras_hub/src/models/roberta/__init__.py +1 -2
  123. keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +22 -74
  124. keras_hub/src/models/roberta/{roberta_classifier.py → roberta_text_classifier.py} +16 -11
  125. keras_hub/src/models/roberta/{roberta_preprocessor.py → roberta_text_classifier_preprocessor.py} +21 -53
  126. keras_hub/src/models/roberta/roberta_tokenizer.py +13 -52
  127. keras_hub/src/models/seq_2_seq_lm_preprocessor.py +269 -0
  128. keras_hub/src/models/stable_diffusion_v3/__init__.py +13 -0
  129. keras_hub/src/models/stable_diffusion_v3/clip_encoder_block.py +103 -0
  130. keras_hub/src/models/stable_diffusion_v3/clip_preprocessor.py +93 -0
  131. keras_hub/src/models/stable_diffusion_v3/clip_text_encoder.py +149 -0
  132. keras_hub/src/models/stable_diffusion_v3/clip_tokenizer.py +167 -0
  133. keras_hub/src/models/stable_diffusion_v3/mmdit.py +427 -0
  134. keras_hub/src/models/stable_diffusion_v3/mmdit_block.py +317 -0
  135. keras_hub/src/models/stable_diffusion_v3/t5_xxl_preprocessor.py +74 -0
  136. keras_hub/src/models/stable_diffusion_v3/t5_xxl_text_encoder.py +155 -0
  137. keras_hub/src/models/stable_diffusion_v3/vae_attention.py +126 -0
  138. keras_hub/src/models/stable_diffusion_v3/vae_image_decoder.py +186 -0
  139. keras_hub/src/models/t5/__init__.py +1 -2
  140. keras_hub/src/models/t5/t5_tokenizer.py +13 -23
  141. keras_hub/src/models/task.py +71 -116
  142. keras_hub/src/models/{classifier.py → text_classifier.py} +19 -13
  143. keras_hub/src/models/text_classifier_preprocessor.py +138 -0
  144. keras_hub/src/models/whisper/__init__.py +1 -2
  145. keras_hub/src/models/whisper/{whisper_audio_feature_extractor.py → whisper_audio_converter.py} +20 -18
  146. keras_hub/src/models/whisper/whisper_backbone.py +0 -3
  147. keras_hub/src/models/whisper/whisper_presets.py +10 -10
  148. keras_hub/src/models/whisper/whisper_tokenizer.py +20 -16
  149. keras_hub/src/models/xlm_roberta/__init__.py +1 -4
  150. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +26 -72
  151. keras_hub/src/models/xlm_roberta/{xlm_roberta_classifier.py → xlm_roberta_text_classifier.py} +16 -11
  152. keras_hub/src/models/xlm_roberta/{xlm_roberta_preprocessor.py → xlm_roberta_text_classifier_preprocessor.py} +26 -53
  153. keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +25 -10
  154. keras_hub/src/tests/test_case.py +46 -0
  155. keras_hub/src/tokenizers/byte_pair_tokenizer.py +30 -17
  156. keras_hub/src/tokenizers/byte_tokenizer.py +14 -15
  157. keras_hub/src/tokenizers/sentence_piece_tokenizer.py +20 -7
  158. keras_hub/src/tokenizers/tokenizer.py +67 -32
  159. keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +14 -15
  160. keras_hub/src/tokenizers/word_piece_tokenizer.py +34 -47
  161. keras_hub/src/utils/imagenet/__init__.py +13 -0
  162. keras_hub/src/utils/imagenet/imagenet_utils.py +1067 -0
  163. keras_hub/src/utils/keras_utils.py +0 -50
  164. keras_hub/src/utils/preset_utils.py +230 -68
  165. keras_hub/src/utils/tensor_utils.py +187 -69
  166. keras_hub/src/utils/timm/convert_resnet.py +19 -16
  167. keras_hub/src/utils/timm/preset_loader.py +66 -0
  168. keras_hub/src/utils/transformers/convert_albert.py +193 -0
  169. keras_hub/src/utils/transformers/convert_bart.py +373 -0
  170. keras_hub/src/utils/transformers/convert_bert.py +7 -17
  171. keras_hub/src/utils/transformers/convert_distilbert.py +10 -20
  172. keras_hub/src/utils/transformers/convert_gemma.py +5 -19
  173. keras_hub/src/utils/transformers/convert_gpt2.py +5 -18
  174. keras_hub/src/utils/transformers/convert_llama3.py +7 -18
  175. keras_hub/src/utils/transformers/convert_mistral.py +129 -0
  176. keras_hub/src/utils/transformers/convert_pali_gemma.py +7 -29
  177. keras_hub/src/utils/transformers/preset_loader.py +77 -0
  178. keras_hub/src/utils/transformers/safetensor_utils.py +2 -2
  179. keras_hub/src/version_utils.py +1 -1
  180. keras_hub_nightly-0.16.0.dev2024092017.dist-info/METADATA +202 -0
  181. keras_hub_nightly-0.16.0.dev2024092017.dist-info/RECORD +334 -0
  182. {keras_hub_nightly-0.15.0.dev20240823171555.dist-info → keras_hub_nightly-0.16.0.dev2024092017.dist-info}/WHEEL +1 -1
  183. keras_hub/src/models/bart/bart_preprocessor.py +0 -276
  184. keras_hub/src/models/bloom/bloom_preprocessor.py +0 -185
  185. keras_hub/src/models/electra/electra_preprocessor.py +0 -154
  186. keras_hub/src/models/falcon/falcon_preprocessor.py +0 -187
  187. keras_hub/src/models/gemma/gemma_preprocessor.py +0 -191
  188. keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +0 -145
  189. keras_hub/src/models/llama/llama_preprocessor.py +0 -189
  190. keras_hub/src/models/mistral/mistral_preprocessor.py +0 -190
  191. keras_hub/src/models/opt/opt_preprocessor.py +0 -188
  192. keras_hub/src/models/phi3/phi3_preprocessor.py +0 -190
  193. keras_hub/src/models/whisper/whisper_preprocessor.py +0 -326
  194. keras_hub/src/utils/timm/convert.py +0 -37
  195. keras_hub/src/utils/transformers/convert.py +0 -101
  196. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +0 -34
  197. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +0 -297
  198. {keras_hub_nightly-0.15.0.dev20240823171555.dist-info → keras_hub_nightly-0.16.0.dev2024092017.dist-info}/top_level.txt +0 -0
@@ -16,14 +16,21 @@
16
16
  import keras
17
17
 
18
18
  from keras_hub.src.api_export import keras_hub_export
19
- from keras_hub.src.models.classifier import Classifier
20
19
  from keras_hub.src.models.f_net.f_net_backbone import FNetBackbone
21
20
  from keras_hub.src.models.f_net.f_net_backbone import f_net_kernel_initializer
22
- from keras_hub.src.models.f_net.f_net_preprocessor import FNetPreprocessor
23
-
24
-
25
- @keras_hub_export("keras_hub.models.FNetClassifier")
26
- class FNetClassifier(Classifier):
21
+ from keras_hub.src.models.f_net.f_net_text_classifier_preprocessor import (
22
+ FNetTextClassifierPreprocessor,
23
+ )
24
+ from keras_hub.src.models.text_classifier import TextClassifier
25
+
26
+
27
+ @keras_hub_export(
28
+ [
29
+ "keras_hub.models.FNetTextClassifier",
30
+ "keras_hub.models.FNetClassifier",
31
+ ]
32
+ )
33
+ class FNetTextClassifier(TextClassifier):
27
34
  """An end-to-end f_net model for classification tasks.
28
35
 
29
36
  This model attaches a classification head to a
@@ -42,7 +49,7 @@ class FNetClassifier(Classifier):
42
49
  Args:
43
50
  backbone: A `keras_hub.models.FNetBackbone` instance.
44
51
  num_classes: int. Number of classes to predict.
45
- preprocessor: A `keras_hub.models.FNetPreprocessor` or `None`. If
52
+ preprocessor: A `keras_hub.models.FNetTextClassifierPreprocessor` or `None`. If
46
53
  `None`, this model will not apply preprocessing, and inputs should
47
54
  be preprocessed before calling the model.
48
55
  activation: Optional `str` or callable. The
@@ -61,7 +68,7 @@ class FNetClassifier(Classifier):
61
68
  labels = [0, 3]
62
69
 
63
70
  # Pretrained classifier.
64
- classifier = keras_hub.models.FNetClassifier.from_preset(
71
+ classifier = keras_hub.models.FNetTextClassifier.from_preset(
65
72
  "f_net_base_en",
66
73
  num_classes=4,
67
74
  )
@@ -89,7 +96,7 @@ class FNetClassifier(Classifier):
89
96
  labels = [0, 3]
90
97
 
91
98
  # Pretrained classifier without preprocessing.
92
- classifier = keras_hub.models.FNetClassifier.from_preset(
99
+ classifier = keras_hub.models.FNetTextClassifier.from_preset(
93
100
  "f_net_base_en",
94
101
  num_classes=4,
95
102
  preprocessor=None,
@@ -99,7 +106,7 @@ class FNetClassifier(Classifier):
99
106
  """
100
107
 
101
108
  backbone_cls = FNetBackbone
102
- preprocessor_cls = FNetPreprocessor
109
+ preprocessor_cls = FNetTextClassifierPreprocessor
103
110
 
104
111
  def __init__(
105
112
  self,
@@ -16,18 +16,21 @@
16
16
  import keras
17
17
 
18
18
  from keras_hub.src.api_export import keras_hub_export
19
- from keras_hub.src.layers.preprocessing.multi_segment_packer import (
20
- MultiSegmentPacker,
21
- )
19
+ from keras_hub.src.models.f_net.f_net_backbone import FNetBackbone
22
20
  from keras_hub.src.models.f_net.f_net_tokenizer import FNetTokenizer
23
- from keras_hub.src.models.preprocessor import Preprocessor
24
- from keras_hub.src.utils.keras_utils import (
25
- convert_inputs_to_list_of_tensor_segments,
21
+ from keras_hub.src.models.text_classifier_preprocessor import (
22
+ TextClassifierPreprocessor,
26
23
  )
24
+ from keras_hub.src.utils.tensor_utils import preprocessing_function
27
25
 
28
26
 
29
- @keras_hub_export("keras_hub.models.FNetPreprocessor")
30
- class FNetPreprocessor(Preprocessor):
27
+ @keras_hub_export(
28
+ [
29
+ "keras_hub.models.FNetTextClassifierPreprocessor",
30
+ "keras_hub.models.FNetPreprocessor",
31
+ ]
32
+ )
33
+ class FNetTextClassifierPreprocessor(TextClassifierPreprocessor):
31
34
  """An FNet preprocessing layer which tokenizes and packs inputs.
32
35
 
33
36
  This preprocessing layer will do three things:
@@ -68,7 +71,7 @@ class FNetPreprocessor(Preprocessor):
68
71
 
69
72
  Directly calling the from_preset().
70
73
  ```python
71
- preprocessor = keras_hub.models.FNetPreprocessor.from_preset(
74
+ preprocessor = keras_hub.models.TextClassifierPreprocessor.from_preset(
72
75
  "f_net_base_en"
73
76
  )
74
77
 
@@ -87,7 +90,7 @@ class FNetPreprocessor(Preprocessor):
87
90
 
88
91
  Mapping with `tf.data.Dataset`.
89
92
  ```python
90
- preprocessor = keras_hub.models.FNetPreprocessor.from_preset(
93
+ preprocessor = keras_hub.models.TextClassifierPreprocessor.from_preset(
91
94
  "f_net_base_en"
92
95
  )
93
96
  first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
@@ -118,60 +121,13 @@ class FNetPreprocessor(Preprocessor):
118
121
  ```
119
122
  """
120
123
 
124
+ backbone_cls = FNetBackbone
121
125
  tokenizer_cls = FNetTokenizer
122
126
 
123
- def __init__(
124
- self,
125
- tokenizer,
126
- sequence_length=512,
127
- truncate="round_robin",
128
- **kwargs,
129
- ):
130
- super().__init__(**kwargs)
131
- self.tokenizer = tokenizer
132
- self.packer = None
133
- self.truncate = truncate
134
- self.sequence_length = sequence_length
135
-
136
- def build(self, input_shape):
137
- # Defer packer creation to `build()` so that we can be sure tokenizer
138
- # assets have loaded when restoring a saved model.
139
- self.packer = MultiSegmentPacker(
140
- start_value=self.tokenizer.cls_token_id,
141
- end_value=self.tokenizer.sep_token_id,
142
- pad_value=self.tokenizer.pad_token_id,
143
- truncate=self.truncate,
144
- sequence_length=self.sequence_length,
145
- )
146
- self.built = True
147
-
148
- def get_config(self):
149
- config = super().get_config()
150
- config.update(
151
- {
152
- "sequence_length": self.sequence_length,
153
- "truncate": self.truncate,
154
- }
155
- )
156
- return config
157
-
127
+ @preprocessing_function
158
128
  def call(self, x, y=None, sample_weight=None):
159
- x = convert_inputs_to_list_of_tensor_segments(x)
160
- x = [self.tokenizer(segment) for segment in x]
161
- token_ids, segment_ids = self.packer(x)
162
- x = {
163
- "token_ids": token_ids,
164
- "segment_ids": segment_ids,
165
- }
129
+ # FNet has not padding mask.
130
+ output = super().call(x, y=y, sample_weight=sample_weight)
131
+ x, y, sample_weight = keras.utils.unpack_x_y_sample_weight(output)
132
+ del x["padding_mask"]
166
133
  return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
167
-
168
- @property
169
- def sequence_length(self):
170
- """The padded length of model input sequences."""
171
- return self._sequence_length
172
-
173
- @sequence_length.setter
174
- def sequence_length(self, value):
175
- self._sequence_length = value
176
- if self.packer is not None:
177
- self.packer.sequence_length = value
@@ -14,12 +14,18 @@
14
14
 
15
15
 
16
16
  from keras_hub.src.api_export import keras_hub_export
17
+ from keras_hub.src.models.f_net.f_net_backbone import FNetBackbone
17
18
  from keras_hub.src.tokenizers.sentence_piece_tokenizer import (
18
19
  SentencePieceTokenizer,
19
20
  )
20
21
 
21
22
 
22
- @keras_hub_export("keras_hub.models.FNetTokenizer")
23
+ @keras_hub_export(
24
+ [
25
+ "keras_hub.tokenizers.FNetTokenizer",
26
+ "keras_hub.models.FNetTokenizer",
27
+ ]
28
+ )
23
29
  class FNetTokenizer(SentencePieceTokenizer):
24
30
  """FNet tokenizer layer based on SentencePiece.
25
31
 
@@ -29,10 +35,6 @@ class FNetTokenizer(SentencePieceTokenizer):
29
35
  FNet models and provides a `from_preset()` method to automatically
30
36
  download a matching vocabulary for a FNet preset.
31
37
 
32
- This tokenizer does not provide truncation or padding of inputs. It can be
33
- combined with a `keras_hub.models.FNetPreprocessor` layer for input
34
- packing.
35
-
36
38
  If input is a batch of strings (rank > 0), the layer will output a
37
39
  `tf.RaggedTensor` where the last dimension of the output is ragged.
38
40
 
@@ -61,35 +63,15 @@ class FNetTokenizer(SentencePieceTokenizer):
61
63
  ```
62
64
  """
63
65
 
66
+ backbone_cls = FNetBackbone
67
+
64
68
  def __init__(self, proto, **kwargs):
65
- self.cls_token = "[CLS]"
66
- self.sep_token = "[SEP]"
67
- self.pad_token = "<pad>"
68
- self.mask_token = "[MASK]"
69
+ self._add_special_token("[CLS]", "cls_token")
70
+ self._add_special_token("[SEP]", "sep_token")
71
+ self._add_special_token("<pad>", "pad_token")
72
+ self._add_special_token("[MASK]", "mask_token")
73
+ # Also add `tokenizer.start_token` and `tokenizer.end_token` for
74
+ # compatibility with other tokenizers.
75
+ self._add_special_token("[CLS]", "start_token")
76
+ self._add_special_token("[SEP]", "end_token")
69
77
  super().__init__(proto=proto, **kwargs)
70
-
71
- def set_proto(self, proto):
72
- super().set_proto(proto)
73
- if proto is not None:
74
- for token in [
75
- self.cls_token,
76
- self.sep_token,
77
- self.pad_token,
78
- self.mask_token,
79
- ]:
80
- if token not in self.get_vocabulary():
81
- raise ValueError(
82
- f"Cannot find token `'{token}'` in the provided "
83
- f"`vocabulary`. Please provide `'{token}'` in your "
84
- "`vocabulary` or use a pretrained `vocabulary` name."
85
- )
86
-
87
- self.cls_token_id = self.token_to_id(self.cls_token)
88
- self.sep_token_id = self.token_to_id(self.sep_token)
89
- self.pad_token_id = self.token_to_id(self.pad_token)
90
- self.mask_token_id = self.token_to_id(self.mask_token)
91
- else:
92
- self.cls_token_id = None
93
- self.sep_token_id = None
94
- self.pad_token_id = None
95
- self.mask_token_id = None
@@ -14,7 +14,6 @@
14
14
 
15
15
  from keras_hub.src.models.falcon.falcon_backbone import FalconBackbone
16
16
  from keras_hub.src.models.falcon.falcon_presets import backbone_presets
17
- from keras_hub.src.models.falcon.falcon_tokenizer import FalconTokenizer
18
17
  from keras_hub.src.utils.preset_utils import register_presets
19
18
 
20
- register_presets(backbone_presets, (FalconBackbone, FalconTokenizer))
19
+ register_presets(backbone_presets, FalconBackbone)
@@ -12,19 +12,14 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- import keras
16
- from absl import logging
17
-
18
15
  from keras_hub.src.api_export import keras_hub_export
19
- from keras_hub.src.models.falcon.falcon_preprocessor import FalconPreprocessor
20
- from keras_hub.src.utils.keras_utils import (
21
- convert_inputs_to_list_of_tensor_segments,
22
- )
23
- from keras_hub.src.utils.tensor_utils import strip_to_ragged
16
+ from keras_hub.src.models.causal_lm_preprocessor import CausalLMPreprocessor
17
+ from keras_hub.src.models.falcon.falcon_backbone import FalconBackbone
18
+ from keras_hub.src.models.falcon.falcon_tokenizer import FalconTokenizer
24
19
 
25
20
 
26
21
  @keras_hub_export("keras_hub.models.FalconCausalLMPreprocessor")
27
- class FalconCausalLMPreprocessor(FalconPreprocessor):
22
+ class FalconCausalLMPreprocessor(CausalLMPreprocessor):
28
23
  """Falcon Causal LM preprocessor.
29
24
 
30
25
  This preprocessing layer is meant for use with
@@ -91,83 +86,5 @@ class FalconCausalLMPreprocessor(FalconPreprocessor):
91
86
  ```
92
87
  """
93
88
 
94
- def call(
95
- self,
96
- x,
97
- y=None,
98
- sample_weight=None,
99
- sequence_length=None,
100
- ):
101
- if y is not None or sample_weight is not None:
102
- logging.warning(
103
- "`FalconCausalLMPreprocessor` generates `y` and `sample_weight` "
104
- "based on your input data, but your data already contains `y` "
105
- "or `sample_weight`. Your `y` and `sample_weight` will be "
106
- "ignored."
107
- )
108
- sequence_length = sequence_length or self.sequence_length
109
-
110
- x = convert_inputs_to_list_of_tensor_segments(x)[0]
111
- x = self.tokenizer(x)
112
- # Pad with one extra token to account for the truncation below.
113
- token_ids, padding_mask = self.packer(
114
- x,
115
- sequence_length=sequence_length + 1,
116
- add_start_value=self.add_start_token,
117
- add_end_value=self.add_end_token,
118
- )
119
- # The last token does not have a next token, so we truncate it out.
120
- x = {
121
- "token_ids": token_ids[..., :-1],
122
- "padding_mask": padding_mask[..., :-1],
123
- }
124
- # Target `y` will be the next token.
125
- y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:]
126
- return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
127
-
128
- def generate_preprocess(
129
- self,
130
- x,
131
- sequence_length=None,
132
- ):
133
- """Convert strings to integer token input for generation.
134
-
135
- Similar to calling the layer for training, this method takes in strings
136
- or tensor strings, tokenizes and packs the input, and computes a padding
137
- mask masking all inputs not filled in with a padded value.
138
-
139
- Unlike calling the layer for training, this method does not compute
140
- labels and will never append a `tokenizer.end_token_id` to the end of
141
- the sequence (as generation is expected to continue at the end of the
142
- inputted prompt).
143
- """
144
- if not self.built:
145
- self.build(None)
146
-
147
- x = convert_inputs_to_list_of_tensor_segments(x)[0]
148
- x = self.tokenizer(x)
149
- token_ids, padding_mask = self.packer(
150
- x, sequence_length=sequence_length, add_end_value=False
151
- )
152
- return {
153
- "token_ids": token_ids,
154
- "padding_mask": padding_mask,
155
- }
156
-
157
- def generate_postprocess(
158
- self,
159
- x,
160
- ):
161
- """Convert integer token output to strings for generation.
162
-
163
- This method reverses `generate_preprocess()`, by first removing all
164
- padding and start/end tokens, and then converting the integer sequence
165
- back to a string.
166
- """
167
- if not self.built:
168
- self.build(None)
169
-
170
- token_ids, padding_mask = x["token_ids"], x["padding_mask"]
171
- ids_to_strip = (self.tokenizer.end_token_id,)
172
- token_ids = strip_to_ragged(token_ids, padding_mask, ids_to_strip)
173
- return self.tokenizer.detokenize(token_ids)
89
+ backbone_cls = FalconBackbone
90
+ tokenizer_cls = FalconTokenizer
@@ -14,10 +14,16 @@
14
14
 
15
15
 
16
16
  from keras_hub.src.api_export import keras_hub_export
17
+ from keras_hub.src.models.falcon.falcon_backbone import FalconBackbone
17
18
  from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
18
19
 
19
20
 
20
- @keras_hub_export("keras_hub.models.FalconTokenizer")
21
+ @keras_hub_export(
22
+ [
23
+ "keras_hub.tokenizers.FalconTokenizer",
24
+ "keras_hub.models.FalconTokenizer",
25
+ ]
26
+ )
21
27
  class FalconTokenizer(BytePairTokenizer):
22
28
  """Falcon tokenizer based on BytePairTokenizer.
23
29
 
@@ -27,8 +33,6 @@ class FalconTokenizer(BytePairTokenizer):
27
33
  models and provides a `from_preset()` method to automatically download
28
34
  a matching vocabulary for a Falcon preset.
29
35
 
30
- This tokenizer does not provide truncation or padding of inputs.
31
-
32
36
  If input is a batch of strings (rank > 0), the layer will output a
33
37
  `tf.RaggedTensor` where the last dimension of the output is ragged.
34
38
 
@@ -65,46 +69,19 @@ class FalconTokenizer(BytePairTokenizer):
65
69
  ```
66
70
  """
67
71
 
72
+ backbone_cls = FalconBackbone
73
+
68
74
  def __init__(
69
75
  self,
70
76
  vocabulary=None,
71
77
  merges=None,
72
78
  **kwargs,
73
79
  ):
74
- # Falcon uses the same start as end token, i.e., "<|endoftext|>".
75
- self.end_token = self.start_token = "<|endoftext|>"
76
-
80
+ self._add_special_token("<|endoftext|>", "end_token")
81
+ self._add_special_token("<|endoftext|>", "start_token")
82
+ self.pad_token_id = 0
77
83
  super().__init__(
78
84
  vocabulary=vocabulary,
79
85
  merges=merges,
80
- unsplittable_tokens=[self.end_token],
81
86
  **kwargs,
82
87
  )
83
-
84
- def set_vocabulary_and_merges(self, vocabulary, merges):
85
- super().set_vocabulary_and_merges(vocabulary, merges)
86
-
87
- if vocabulary is not None:
88
- # Check for necessary special tokens.
89
- if self.end_token not in self.get_vocabulary():
90
- raise ValueError(
91
- f"Cannot find token `'{self.end_token}'` in the provided "
92
- f"`vocabulary`. Please provide `'{self.end_token}'` in "
93
- "your `vocabulary` or use a pretrained `vocabulary` name."
94
- )
95
-
96
- self.end_token_id = self.token_to_id(self.end_token)
97
- self.start_token_id = self.end_token_id
98
- self.pad_token_id = 0
99
- else:
100
- self.end_token_id = None
101
- self.start_token_id = None
102
- self.pad_token_id = None
103
-
104
- def get_config(self):
105
- config = super().get_config()
106
- # In the constructor, we pass the list of special tokens to the
107
- # `unsplittable_tokens` arg of the superclass' constructor. Hence, we
108
- # delete it from the config here.
109
- del config["unsplittable_tokens"]
110
- return config
@@ -14,7 +14,6 @@
14
14
 
15
15
  from keras_hub.src.models.gemma.gemma_backbone import GemmaBackbone
16
16
  from keras_hub.src.models.gemma.gemma_presets import backbone_presets
17
- from keras_hub.src.models.gemma.gemma_tokenizer import GemmaTokenizer
18
17
  from keras_hub.src.utils.preset_utils import register_presets
19
18
 
20
- register_presets(backbone_presets, (GemmaBackbone, GemmaTokenizer))
19
+ register_presets(backbone_presets, GemmaBackbone)
@@ -12,19 +12,14 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- import keras
16
- from absl import logging
17
-
18
15
  from keras_hub.src.api_export import keras_hub_export
19
- from keras_hub.src.models.gemma.gemma_preprocessor import GemmaPreprocessor
20
- from keras_hub.src.utils.keras_utils import (
21
- convert_inputs_to_list_of_tensor_segments,
22
- )
23
- from keras_hub.src.utils.tensor_utils import strip_to_ragged
16
+ from keras_hub.src.models.causal_lm_preprocessor import CausalLMPreprocessor
17
+ from keras_hub.src.models.gemma.gemma_backbone import GemmaBackbone
18
+ from keras_hub.src.models.gemma.gemma_tokenizer import GemmaTokenizer
24
19
 
25
20
 
26
21
  @keras_hub_export("keras_hub.models.GemmaCausalLMPreprocessor")
27
- class GemmaCausalLMPreprocessor(GemmaPreprocessor):
22
+ class GemmaCausalLMPreprocessor(CausalLMPreprocessor):
28
23
  """Gemma Causal LM preprocessor.
29
24
 
30
25
  This preprocessing layer is meant for use with
@@ -84,84 +79,5 @@ class GemmaCausalLMPreprocessor(GemmaPreprocessor):
84
79
  ```
85
80
  """
86
81
 
87
- def call(
88
- self,
89
- x,
90
- y=None,
91
- sample_weight=None,
92
- sequence_length=None,
93
- ):
94
- if y is not None or sample_weight is not None:
95
- logging.warning(
96
- "`GemmaCausalLMPreprocessor` generates `y` and `sample_weight` "
97
- "based on your input data, but your data already contains `y` "
98
- "or `sample_weight`. Your `y` and `sample_weight` will be "
99
- "ignored."
100
- )
101
- sequence_length = sequence_length or self.sequence_length
102
-
103
- x = convert_inputs_to_list_of_tensor_segments(x)[0]
104
- x = self.tokenizer(x)
105
- # Pad with one extra token to account for the truncation below.
106
- token_ids, padding_mask = self.packer(
107
- x,
108
- sequence_length=sequence_length + 1,
109
- add_start_value=self.add_start_token,
110
- add_end_value=self.add_end_token,
111
- )
112
- # The last token does not have a next token, so we truncate it out.
113
- x = {
114
- "token_ids": token_ids[..., :-1],
115
- "padding_mask": padding_mask[..., :-1],
116
- }
117
- # Target `y` will be the next token.
118
- y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:]
119
- return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
120
-
121
- def generate_preprocess(
122
- self,
123
- x,
124
- sequence_length=None,
125
- ):
126
- """Convert strings to integer token input for generation.
127
-
128
- Similar to calling the layer for training, this method takes in strings
129
- or tensor strings, tokenizes and packs the input, and computes a padding
130
- mask masking all inputs not filled in with a padded value.
131
-
132
- Unlike calling the layer for training, this method does not compute
133
- labels and will never append a `tokenizer.end_token_id` to the end of
134
- the sequence (as generation is expected to continue at the end of the
135
- inputted prompt).
136
- """
137
- if not self.built:
138
- self.build(None)
139
-
140
- x = convert_inputs_to_list_of_tensor_segments(x)[0]
141
- x = self.tokenizer(x)
142
- token_ids, padding_mask = self.packer(
143
- x, sequence_length=sequence_length, add_end_value=False
144
- )
145
- return {
146
- "token_ids": token_ids,
147
- "padding_mask": padding_mask,
148
- }
149
-
150
- def generate_postprocess(self, x):
151
- """Convert integer token output to strings for generation.
152
-
153
- This method reverses `generate_preprocess()`, by first removing all
154
- padding and start/end tokens, and then converting the integer sequence
155
- back to a string.
156
- """
157
- if not self.built:
158
- self.build(None)
159
-
160
- token_ids, padding_mask = x["token_ids"], x["padding_mask"]
161
- ids_to_strip = (
162
- self.tokenizer.start_token_id,
163
- self.tokenizer.end_token_id,
164
- self.tokenizer.pad_token_id,
165
- )
166
- token_ids = strip_to_ragged(token_ids, padding_mask, ids_to_strip)
167
- return self.tokenizer.detokenize(token_ids)
82
+ backbone_cls = GemmaBackbone
83
+ tokenizer_cls = GemmaTokenizer
@@ -68,7 +68,7 @@ class GemmaDecoderBlock(keras.layers.Layer):
68
68
  self.post_attention_norm = RMSNormalization(
69
69
  epsilon=self.layer_norm_epsilon,
70
70
  dtype=self.dtype_policy,
71
- name="pre_attention_norm",
71
+ name="post_attention_norm",
72
72
  )
73
73
 
74
74
  self.attention = CachedGemmaAttention(
@@ -13,12 +13,18 @@
13
13
  # limitations under the License.
14
14
 
15
15
  from keras_hub.src.api_export import keras_hub_export
16
+ from keras_hub.src.models.gemma.gemma_backbone import GemmaBackbone
16
17
  from keras_hub.src.tokenizers.sentence_piece_tokenizer import (
17
18
  SentencePieceTokenizer,
18
19
  )
19
20
 
20
21
 
21
- @keras_hub_export("keras_hub.models.GemmaTokenizer")
22
+ @keras_hub_export(
23
+ [
24
+ "keras_hub.tokenizers.GemmaTokenizer",
25
+ "keras_hub.models.GemmaTokenizer",
26
+ ]
27
+ )
22
28
  class GemmaTokenizer(SentencePieceTokenizer):
23
29
  """Gemma tokenizer layer based on SentencePiece.
24
30
 
@@ -77,27 +83,10 @@ class GemmaTokenizer(SentencePieceTokenizer):
77
83
  ```
78
84
  """
79
85
 
80
- def __init__(self, proto, **kwargs):
81
- self.start_token = "<bos>"
82
- self.end_token = "<eos>"
83
- self.pad_token = "<pad>"
86
+ backbone_cls = GemmaBackbone
84
87
 
88
+ def __init__(self, proto, **kwargs):
89
+ self._add_special_token("<bos>", "start_token")
90
+ self._add_special_token("<eos>", "end_token")
91
+ self._add_special_token("<pad>", "pad_token")
85
92
  super().__init__(proto=proto, **kwargs)
86
-
87
- def set_proto(self, proto):
88
- super().set_proto(proto)
89
- if proto is not None:
90
- for token in [self.end_token, self.pad_token]:
91
- if token not in self.get_vocabulary():
92
- raise ValueError(
93
- f"Cannot find token `'{token}'` in the provided "
94
- f"`vocabulary`. Please provide `'{token}'` in your "
95
- "`vocabulary` or use a pretrained `vocabulary` name."
96
- )
97
- self.start_token_id = self.token_to_id(self.start_token)
98
- self.end_token_id = self.token_to_id(self.end_token)
99
- self.pad_token_id = self.token_to_id(self.pad_token)
100
- else:
101
- self.start_token_id = None
102
- self.end_token_id = None
103
- self.pad_token_id = None
@@ -14,7 +14,6 @@
14
14
 
15
15
  from keras_hub.src.models.gpt2.gpt2_backbone import GPT2Backbone
16
16
  from keras_hub.src.models.gpt2.gpt2_presets import backbone_presets
17
- from keras_hub.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
18
17
  from keras_hub.src.utils.preset_utils import register_presets
19
18
 
20
- register_presets(backbone_presets, (GPT2Backbone, GPT2Tokenizer))
19
+ register_presets(backbone_presets, GPT2Backbone)