keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl → 0.16.0.dev2024092017__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. keras_hub/__init__.py +0 -6
  2. keras_hub/api/__init__.py +2 -0
  3. keras_hub/api/bounding_box/__init__.py +36 -0
  4. keras_hub/api/layers/__init__.py +14 -0
  5. keras_hub/api/models/__init__.py +97 -48
  6. keras_hub/api/tokenizers/__init__.py +30 -0
  7. keras_hub/api/utils/__init__.py +22 -0
  8. keras_hub/src/api_export.py +15 -9
  9. keras_hub/src/bounding_box/__init__.py +13 -0
  10. keras_hub/src/bounding_box/converters.py +529 -0
  11. keras_hub/src/bounding_box/formats.py +162 -0
  12. keras_hub/src/bounding_box/iou.py +263 -0
  13. keras_hub/src/bounding_box/to_dense.py +95 -0
  14. keras_hub/src/bounding_box/to_ragged.py +99 -0
  15. keras_hub/src/bounding_box/utils.py +194 -0
  16. keras_hub/src/bounding_box/validate_format.py +99 -0
  17. keras_hub/src/layers/preprocessing/audio_converter.py +121 -0
  18. keras_hub/src/layers/preprocessing/image_converter.py +130 -0
  19. keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +2 -0
  20. keras_hub/src/layers/preprocessing/multi_segment_packer.py +9 -8
  21. keras_hub/src/layers/preprocessing/preprocessing_layer.py +2 -29
  22. keras_hub/src/layers/preprocessing/random_deletion.py +33 -31
  23. keras_hub/src/layers/preprocessing/random_swap.py +33 -31
  24. keras_hub/src/layers/preprocessing/resizing_image_converter.py +101 -0
  25. keras_hub/src/layers/preprocessing/start_end_packer.py +3 -2
  26. keras_hub/src/models/albert/__init__.py +1 -2
  27. keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +6 -86
  28. keras_hub/src/models/albert/{albert_classifier.py → albert_text_classifier.py} +34 -10
  29. keras_hub/src/models/albert/{albert_preprocessor.py → albert_text_classifier_preprocessor.py} +14 -70
  30. keras_hub/src/models/albert/albert_tokenizer.py +17 -36
  31. keras_hub/src/models/backbone.py +12 -34
  32. keras_hub/src/models/bart/__init__.py +1 -2
  33. keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +21 -148
  34. keras_hub/src/models/bart/bart_tokenizer.py +12 -39
  35. keras_hub/src/models/bert/__init__.py +1 -5
  36. keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +6 -87
  37. keras_hub/src/models/bert/bert_presets.py +1 -4
  38. keras_hub/src/models/bert/{bert_classifier.py → bert_text_classifier.py} +19 -12
  39. keras_hub/src/models/bert/{bert_preprocessor.py → bert_text_classifier_preprocessor.py} +14 -70
  40. keras_hub/src/models/bert/bert_tokenizer.py +17 -35
  41. keras_hub/src/models/bloom/__init__.py +1 -2
  42. keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +6 -91
  43. keras_hub/src/models/bloom/bloom_tokenizer.py +12 -41
  44. keras_hub/src/models/causal_lm.py +10 -29
  45. keras_hub/src/models/causal_lm_preprocessor.py +195 -0
  46. keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +54 -15
  47. keras_hub/src/models/deberta_v3/__init__.py +1 -4
  48. keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +14 -77
  49. keras_hub/src/models/deberta_v3/{deberta_v3_classifier.py → deberta_v3_text_classifier.py} +16 -11
  50. keras_hub/src/models/deberta_v3/{deberta_v3_preprocessor.py → deberta_v3_text_classifier_preprocessor.py} +23 -64
  51. keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +30 -25
  52. keras_hub/src/models/densenet/densenet_backbone.py +46 -22
  53. keras_hub/src/models/distil_bert/__init__.py +1 -4
  54. keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +14 -76
  55. keras_hub/src/models/distil_bert/{distil_bert_classifier.py → distil_bert_text_classifier.py} +17 -12
  56. keras_hub/src/models/distil_bert/{distil_bert_preprocessor.py → distil_bert_text_classifier_preprocessor.py} +23 -63
  57. keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +19 -35
  58. keras_hub/src/models/efficientnet/__init__.py +13 -0
  59. keras_hub/src/models/efficientnet/efficientnet_backbone.py +569 -0
  60. keras_hub/src/models/efficientnet/fusedmbconv.py +229 -0
  61. keras_hub/src/models/efficientnet/mbconv.py +238 -0
  62. keras_hub/src/models/electra/__init__.py +1 -2
  63. keras_hub/src/models/electra/electra_tokenizer.py +17 -32
  64. keras_hub/src/models/f_net/__init__.py +1 -2
  65. keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +12 -78
  66. keras_hub/src/models/f_net/{f_net_classifier.py → f_net_text_classifier.py} +17 -10
  67. keras_hub/src/models/f_net/{f_net_preprocessor.py → f_net_text_classifier_preprocessor.py} +19 -63
  68. keras_hub/src/models/f_net/f_net_tokenizer.py +17 -35
  69. keras_hub/src/models/falcon/__init__.py +1 -2
  70. keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +6 -89
  71. keras_hub/src/models/falcon/falcon_tokenizer.py +12 -35
  72. keras_hub/src/models/gemma/__init__.py +1 -2
  73. keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +6 -90
  74. keras_hub/src/models/gemma/gemma_decoder_block.py +1 -1
  75. keras_hub/src/models/gemma/gemma_tokenizer.py +12 -23
  76. keras_hub/src/models/gpt2/__init__.py +1 -2
  77. keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +6 -89
  78. keras_hub/src/models/gpt2/gpt2_preprocessor.py +12 -90
  79. keras_hub/src/models/gpt2/gpt2_tokenizer.py +12 -34
  80. keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +6 -91
  81. keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +12 -34
  82. keras_hub/src/models/image_classifier.py +0 -5
  83. keras_hub/src/models/image_classifier_preprocessor.py +83 -0
  84. keras_hub/src/models/llama/__init__.py +1 -2
  85. keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +6 -85
  86. keras_hub/src/models/llama/llama_tokenizer.py +12 -25
  87. keras_hub/src/models/llama3/__init__.py +1 -2
  88. keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +6 -89
  89. keras_hub/src/models/llama3/llama3_tokenizer.py +12 -33
  90. keras_hub/src/models/masked_lm.py +0 -2
  91. keras_hub/src/models/masked_lm_preprocessor.py +156 -0
  92. keras_hub/src/models/mistral/__init__.py +1 -2
  93. keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +6 -91
  94. keras_hub/src/models/mistral/mistral_tokenizer.py +12 -23
  95. keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +2 -2
  96. keras_hub/src/models/mobilenet/__init__.py +13 -0
  97. keras_hub/src/models/mobilenet/mobilenet_backbone.py +530 -0
  98. keras_hub/src/models/mobilenet/mobilenet_image_classifier.py +114 -0
  99. keras_hub/src/models/opt/__init__.py +1 -2
  100. keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +6 -93
  101. keras_hub/src/models/opt/opt_tokenizer.py +12 -41
  102. keras_hub/src/models/pali_gemma/__init__.py +1 -4
  103. keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +28 -28
  104. keras_hub/src/models/pali_gemma/pali_gemma_image_converter.py +25 -0
  105. keras_hub/src/models/pali_gemma/pali_gemma_presets.py +5 -5
  106. keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +11 -3
  107. keras_hub/src/models/phi3/__init__.py +1 -2
  108. keras_hub/src/models/phi3/phi3_causal_lm.py +3 -9
  109. keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +6 -89
  110. keras_hub/src/models/phi3/phi3_tokenizer.py +12 -36
  111. keras_hub/src/models/preprocessor.py +72 -83
  112. keras_hub/src/models/resnet/__init__.py +6 -0
  113. keras_hub/src/models/resnet/resnet_backbone.py +390 -42
  114. keras_hub/src/models/resnet/resnet_image_classifier.py +33 -6
  115. keras_hub/src/models/resnet/resnet_image_classifier_preprocessor.py +28 -0
  116. keras_hub/src/models/{llama3/llama3_preprocessor.py → resnet/resnet_image_converter.py} +7 -5
  117. keras_hub/src/models/resnet/resnet_presets.py +95 -0
  118. keras_hub/src/models/retinanet/__init__.py +13 -0
  119. keras_hub/src/models/retinanet/anchor_generator.py +175 -0
  120. keras_hub/src/models/retinanet/box_matcher.py +259 -0
  121. keras_hub/src/models/retinanet/non_max_supression.py +578 -0
  122. keras_hub/src/models/roberta/__init__.py +1 -2
  123. keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +22 -74
  124. keras_hub/src/models/roberta/{roberta_classifier.py → roberta_text_classifier.py} +16 -11
  125. keras_hub/src/models/roberta/{roberta_preprocessor.py → roberta_text_classifier_preprocessor.py} +21 -53
  126. keras_hub/src/models/roberta/roberta_tokenizer.py +13 -52
  127. keras_hub/src/models/seq_2_seq_lm_preprocessor.py +269 -0
  128. keras_hub/src/models/stable_diffusion_v3/__init__.py +13 -0
  129. keras_hub/src/models/stable_diffusion_v3/clip_encoder_block.py +103 -0
  130. keras_hub/src/models/stable_diffusion_v3/clip_preprocessor.py +93 -0
  131. keras_hub/src/models/stable_diffusion_v3/clip_text_encoder.py +149 -0
  132. keras_hub/src/models/stable_diffusion_v3/clip_tokenizer.py +167 -0
  133. keras_hub/src/models/stable_diffusion_v3/mmdit.py +427 -0
  134. keras_hub/src/models/stable_diffusion_v3/mmdit_block.py +317 -0
  135. keras_hub/src/models/stable_diffusion_v3/t5_xxl_preprocessor.py +74 -0
  136. keras_hub/src/models/stable_diffusion_v3/t5_xxl_text_encoder.py +155 -0
  137. keras_hub/src/models/stable_diffusion_v3/vae_attention.py +126 -0
  138. keras_hub/src/models/stable_diffusion_v3/vae_image_decoder.py +186 -0
  139. keras_hub/src/models/t5/__init__.py +1 -2
  140. keras_hub/src/models/t5/t5_tokenizer.py +13 -23
  141. keras_hub/src/models/task.py +71 -116
  142. keras_hub/src/models/{classifier.py → text_classifier.py} +19 -13
  143. keras_hub/src/models/text_classifier_preprocessor.py +138 -0
  144. keras_hub/src/models/whisper/__init__.py +1 -2
  145. keras_hub/src/models/whisper/{whisper_audio_feature_extractor.py → whisper_audio_converter.py} +20 -18
  146. keras_hub/src/models/whisper/whisper_backbone.py +0 -3
  147. keras_hub/src/models/whisper/whisper_presets.py +10 -10
  148. keras_hub/src/models/whisper/whisper_tokenizer.py +20 -16
  149. keras_hub/src/models/xlm_roberta/__init__.py +1 -4
  150. keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +26 -72
  151. keras_hub/src/models/xlm_roberta/{xlm_roberta_classifier.py → xlm_roberta_text_classifier.py} +16 -11
  152. keras_hub/src/models/xlm_roberta/{xlm_roberta_preprocessor.py → xlm_roberta_text_classifier_preprocessor.py} +26 -53
  153. keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +25 -10
  154. keras_hub/src/tests/test_case.py +46 -0
  155. keras_hub/src/tokenizers/byte_pair_tokenizer.py +30 -17
  156. keras_hub/src/tokenizers/byte_tokenizer.py +14 -15
  157. keras_hub/src/tokenizers/sentence_piece_tokenizer.py +20 -7
  158. keras_hub/src/tokenizers/tokenizer.py +67 -32
  159. keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +14 -15
  160. keras_hub/src/tokenizers/word_piece_tokenizer.py +34 -47
  161. keras_hub/src/utils/imagenet/__init__.py +13 -0
  162. keras_hub/src/utils/imagenet/imagenet_utils.py +1067 -0
  163. keras_hub/src/utils/keras_utils.py +0 -50
  164. keras_hub/src/utils/preset_utils.py +230 -68
  165. keras_hub/src/utils/tensor_utils.py +187 -69
  166. keras_hub/src/utils/timm/convert_resnet.py +19 -16
  167. keras_hub/src/utils/timm/preset_loader.py +66 -0
  168. keras_hub/src/utils/transformers/convert_albert.py +193 -0
  169. keras_hub/src/utils/transformers/convert_bart.py +373 -0
  170. keras_hub/src/utils/transformers/convert_bert.py +7 -17
  171. keras_hub/src/utils/transformers/convert_distilbert.py +10 -20
  172. keras_hub/src/utils/transformers/convert_gemma.py +5 -19
  173. keras_hub/src/utils/transformers/convert_gpt2.py +5 -18
  174. keras_hub/src/utils/transformers/convert_llama3.py +7 -18
  175. keras_hub/src/utils/transformers/convert_mistral.py +129 -0
  176. keras_hub/src/utils/transformers/convert_pali_gemma.py +7 -29
  177. keras_hub/src/utils/transformers/preset_loader.py +77 -0
  178. keras_hub/src/utils/transformers/safetensor_utils.py +2 -2
  179. keras_hub/src/version_utils.py +1 -1
  180. keras_hub_nightly-0.16.0.dev2024092017.dist-info/METADATA +202 -0
  181. keras_hub_nightly-0.16.0.dev2024092017.dist-info/RECORD +334 -0
  182. {keras_hub_nightly-0.15.0.dev20240823171555.dist-info → keras_hub_nightly-0.16.0.dev2024092017.dist-info}/WHEEL +1 -1
  183. keras_hub/src/models/bart/bart_preprocessor.py +0 -276
  184. keras_hub/src/models/bloom/bloom_preprocessor.py +0 -185
  185. keras_hub/src/models/electra/electra_preprocessor.py +0 -154
  186. keras_hub/src/models/falcon/falcon_preprocessor.py +0 -187
  187. keras_hub/src/models/gemma/gemma_preprocessor.py +0 -191
  188. keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +0 -145
  189. keras_hub/src/models/llama/llama_preprocessor.py +0 -189
  190. keras_hub/src/models/mistral/mistral_preprocessor.py +0 -190
  191. keras_hub/src/models/opt/opt_preprocessor.py +0 -188
  192. keras_hub/src/models/phi3/phi3_preprocessor.py +0 -190
  193. keras_hub/src/models/whisper/whisper_preprocessor.py +0 -326
  194. keras_hub/src/utils/timm/convert.py +0 -37
  195. keras_hub/src/utils/transformers/convert.py +0 -101
  196. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +0 -34
  197. keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +0 -297
  198. {keras_hub_nightly-0.15.0.dev20240823171555.dist-info → keras_hub_nightly-0.16.0.dev2024092017.dist-info}/top_level.txt +0 -0
@@ -1,185 +0,0 @@
1
- # Copyright 2024 The KerasHub Authors
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # https://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
-
15
-
16
- import keras
17
-
18
- from keras_hub.src.api_export import keras_hub_export
19
- from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
20
- from keras_hub.src.models.bloom.bloom_tokenizer import BloomTokenizer
21
- from keras_hub.src.models.preprocessor import Preprocessor
22
- from keras_hub.src.utils.keras_utils import (
23
- convert_inputs_to_list_of_tensor_segments,
24
- )
25
-
26
-
27
- @keras_hub_export("keras_hub.models.BloomPreprocessor")
28
- class BloomPreprocessor(Preprocessor):
29
- """BLOOM preprocessing layer which tokenizes and packs inputs.
30
-
31
- This preprocessing layer will do 2 things:
32
-
33
- - Tokenize the inputs using the `tokenizer`.
34
- - Construct a dictionary with keys `"token_ids"`, `"padding_mask"`, that can
35
- be passed directly to a `keras_hub.models.BloomBackbone`.
36
-
37
- This layer can be used directly with `tf.data.Dataset.map` to preprocess
38
- string data in the `(x, y, sample_weight)` format used by
39
- `keras.Model.fit`.
40
-
41
- The call method of this layer accepts three arguments, `x`, `y`, and
42
- `sample_weight`. `x` can be a python string or tensor representing a single
43
- segment, a list of python strings representing a batch of single segments,
44
- or a list of tensors representing multiple segments to be packed together.
45
- `y` and `sample_weight` are both optional, can have any format, and will be
46
- passed through unaltered.
47
-
48
- Args:
49
- tokenizer: A `keras_hub.models.BloomTokenizer` instance.
50
- sequence_length: The length of the packed inputs.
51
- add_start_token: If `True`, the preprocessor will prepend the tokenizer
52
- start token to each input sequence.
53
- add_end_token: If `True`, the preprocessor will append the tokenizer
54
- end token to each input sequence.
55
-
56
- Call arguments:
57
- x: A string, `tf.Tensor` or list of python strings.
58
- y: Any label data. Will be passed through unaltered.
59
- sample_weight: Any label weight data. Will be passed through unaltered.
60
- sequence_length: Pass to override the configured `sequence_length` of
61
- the layer.
62
-
63
- Examples:
64
-
65
- Directly calling the layer on data.
66
- ```python
67
- preprocessor = keras_hub.models.BloomPreprocessor.from_preset(
68
- "bloom_560m_multi"
69
- )
70
- # Tokenize and pack a single sentence.
71
- preprocessor("The quick brown fox jumped.")
72
-
73
- # Tokenize a batch of single sentences.
74
- preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
75
-
76
- # Custom vocabulary.
77
- features = ["a quick fox.", "a fox quick."]
78
- vocab = {"<pad>": 0, "<s>":1, "</s>":2, "a": 3, "Ġquick": 4, "Ġfox": 5}
79
- merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"]
80
- merges += ["Ġ f", "o x", "Ġf ox"]
81
- tokenizer = keras_hub.models.BloomTokenizer(
82
- vocabulary=vocab,
83
- merges=merges,
84
- )
85
- preprocessor = keras_hub.models.BloomPreprocessor(tokenizer=tokenizer)
86
- preprocessor("The quick brown fox jumped.")
87
- ```
88
-
89
- Mapping with `tf.data.Dataset`.
90
- ```python
91
- preprocessor = keras_hub.models.BloomPreprocessor.from_preset(
92
- "bloom_560m_multi"
93
- )
94
-
95
- text = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
96
- label = tf.constant([1, 1])
97
-
98
- # Map labeled single sentences.
99
- ds = tf.data.Dataset.from_tensor_slices((text, label))
100
- ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
101
-
102
- # Map unlabeled single sentences.
103
- ds = tf.data.Dataset.from_tensor_slices(text)
104
- ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
105
- ```
106
- """
107
-
108
- tokenizer_cls = BloomTokenizer
109
-
110
- def __init__(
111
- self,
112
- tokenizer,
113
- sequence_length=1024,
114
- add_start_token=True,
115
- add_end_token=True,
116
- **kwargs,
117
- ):
118
- super().__init__(**kwargs)
119
- self.tokenizer = tokenizer
120
- self.packer = None
121
- self.sequence_length = sequence_length
122
- self.add_start_token = add_start_token
123
- self.add_end_token = add_end_token
124
-
125
- def build(self, input_shape):
126
- # Defer packer creation to `build()` so that we can be sure tokenizer
127
- # assets have loaded when restoring a saved model.
128
- self.packer = StartEndPacker(
129
- start_value=self.tokenizer.start_token_id,
130
- end_value=self.tokenizer.end_token_id,
131
- pad_value=self.tokenizer.pad_token_id,
132
- sequence_length=self.sequence_length,
133
- return_padding_mask=True,
134
- )
135
- self.built = True
136
-
137
- def call(
138
- self,
139
- x,
140
- y=None,
141
- sample_weight=None,
142
- sequence_length=None,
143
- ):
144
- x = convert_inputs_to_list_of_tensor_segments(x)
145
- if len(x) != 1:
146
- raise ValueError(
147
- "BLOOM requires each input feature to contain only "
148
- f"one segment, but received {len(x)}. If you are using BLOOM "
149
- "for a multi-segment classification task, please refer to "
150
- "classification models like BERT or RoBERTa."
151
- )
152
- sequence_length = sequence_length or self.sequence_length
153
- token_ids, padding_mask = self.packer(
154
- self.tokenizer(x[0]),
155
- sequence_length=sequence_length,
156
- add_start_value=self.add_start_token,
157
- add_end_value=self.add_end_token,
158
- )
159
- x = {
160
- "token_ids": token_ids,
161
- "padding_mask": padding_mask,
162
- }
163
- return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
164
-
165
- def get_config(self):
166
- config = super().get_config()
167
- config.update(
168
- {
169
- "sequence_length": self.sequence_length,
170
- "add_start_token": self.add_start_token,
171
- "add_end_token": self.add_end_token,
172
- }
173
- )
174
- return config
175
-
176
- @property
177
- def sequence_length(self):
178
- """The padded length of model input sequences."""
179
- return self._sequence_length
180
-
181
- @sequence_length.setter
182
- def sequence_length(self, value):
183
- self._sequence_length = value
184
- if self.packer is not None:
185
- self.packer.sequence_length = value
@@ -1,154 +0,0 @@
1
- # Copyright 2024 The KerasHub Authors
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # https://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
-
15
- import keras
16
-
17
- from keras_hub.src.api_export import keras_hub_export
18
- from keras_hub.src.layers.preprocessing.multi_segment_packer import (
19
- MultiSegmentPacker,
20
- )
21
- from keras_hub.src.models.electra.electra_tokenizer import ElectraTokenizer
22
- from keras_hub.src.models.preprocessor import Preprocessor
23
- from keras_hub.src.utils.keras_utils import (
24
- convert_inputs_to_list_of_tensor_segments,
25
- )
26
-
27
-
28
- @keras_hub_export("keras_hub.models.ElectraPreprocessor")
29
- class ElectraPreprocessor(Preprocessor):
30
- """A ELECTRA preprocessing layer which tokenizes and packs inputs.
31
-
32
- This preprocessing layer will do three things:
33
-
34
- 1. Tokenize any number of input segments using the `tokenizer`.
35
- 2. Pack the inputs together using a `keras_hub.layers.MultiSegmentPacker`.
36
- with the appropriate `"[CLS]"`, `"[SEP]"` and `"[PAD]"` tokens.
37
- 3. Construct a dictionary of with keys `"token_ids"` and `"padding_mask"`,
38
- that can be passed directly to a ELECTRA model.
39
-
40
- This layer can be used directly with `tf.data.Dataset.map` to preprocess
41
- string data in the `(x, y, sample_weight)` format used by
42
- `keras.Model.fit`.
43
-
44
- Args:
45
- tokenizer: A `keras_hub.models.ElectraTokenizer` instance.
46
- sequence_length: The length of the packed inputs.
47
- truncate: string. The algorithm to truncate a list of batched segments
48
- to fit within `sequence_length`. The value can be either
49
- `round_robin` or `waterfall`:
50
- - `"round_robin"`: Available space is assigned one token at a
51
- time in a round-robin fashion to the inputs that still need
52
- some, until the limit is reached.
53
- - `"waterfall"`: The allocation of the budget is done using a
54
- "waterfall" algorithm that allocates quota in a
55
- left-to-right manner and fills up the buckets until we run
56
- out of budget. It supports an arbitrary number of segments.
57
-
58
- Call arguments:
59
- x: A tensor of single string sequences, or a tuple of multiple
60
- tensor sequences to be packed together. Inputs may be batched or
61
- unbatched. For single sequences, raw python inputs will be converted
62
- to tensors. For multiple sequences, pass tensors directly.
63
- y: Any label data. Will be passed through unaltered.
64
- sample_weight: Any label weight data. Will be passed through unaltered.
65
-
66
- Examples:
67
-
68
- Directly calling the layer on data.
69
- ```python
70
- preprocessor = keras_hub.models.ElectraPreprocessor.from_preset(
71
- "electra_base_discriminator_en"
72
- )
73
- preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
74
-
75
- # Custom vocabulary.
76
- vocab = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
77
- vocab += ["The", "quick", "brown", "fox", "jumped", "."]
78
- tokenizer = keras_hub.models.ElectraTokenizer(vocabulary=vocab)
79
- preprocessor = keras_hub.models.ElectraPreprocessor(tokenizer)
80
- preprocessor("The quick brown fox jumped.")
81
- ```
82
-
83
- Mapping with `tf.data.Dataset`.
84
- ```python
85
- preprocessor = keras_hub.models.ElectraPreprocessor.from_preset(
86
- "electra_base_discriminator_en"
87
- )
88
-
89
- first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
90
- second = tf.constant(["The fox tripped.", "Oh look, a whale."])
91
- label = tf.constant([1, 1])
92
- # Map labeled single sentences.
93
- ds = tf.data.Dataset.from_tensor_slices((first, label))
94
- ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
95
-
96
-
97
- # Map unlabeled single sentences.
98
- ds = tf.data.Dataset.from_tensor_slices(first)
99
- ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
100
-
101
- # Map labeled sentence pairs.
102
- ds = tf.data.Dataset.from_tensor_slices(((first, second), label))
103
- ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
104
- # Map unlabeled sentence pairs.
105
- ds = tf.data.Dataset.from_tensor_slices((first, second))
106
-
107
- # Watch out for tf.data's default unpacking of tuples here!
108
- # Best to invoke the `preprocessor` directly in this case.
109
- ds = ds.map(
110
- lambda first, second: preprocessor(x=(first, second)),
111
- num_parallel_calls=tf.data.AUTOTUNE,
112
- )
113
- ```
114
- """
115
-
116
- tokenizer_cls = ElectraTokenizer
117
-
118
- def __init__(
119
- self,
120
- tokenizer,
121
- sequence_length=512,
122
- truncate="round_robin",
123
- **kwargs,
124
- ):
125
- super().__init__(**kwargs)
126
- self.tokenizer = tokenizer
127
- self.packer = MultiSegmentPacker(
128
- start_value=self.tokenizer.cls_token_id,
129
- end_value=self.tokenizer.sep_token_id,
130
- pad_value=self.tokenizer.pad_token_id,
131
- truncate=truncate,
132
- sequence_length=sequence_length,
133
- )
134
-
135
- def get_config(self):
136
- config = super().get_config()
137
- config.update(
138
- {
139
- "sequence_length": self.packer.sequence_length,
140
- "truncate": self.packer.truncate,
141
- }
142
- )
143
- return config
144
-
145
- def call(self, x, y=None, sample_weight=None):
146
- x = convert_inputs_to_list_of_tensor_segments(x)
147
- x = [self.tokenizer(segment) for segment in x]
148
- token_ids, segment_ids = self.packer(x)
149
- x = {
150
- "token_ids": token_ids,
151
- "segment_ids": segment_ids,
152
- "padding_mask": token_ids != self.tokenizer.pad_token_id,
153
- }
154
- return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
@@ -1,187 +0,0 @@
1
- # Copyright 2024 The KerasHub Authors
2
- #
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
- #
7
- # https://www.apache.org/licenses/LICENSE-2.0
8
- #
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
-
15
-
16
- import keras
17
-
18
- from keras_hub.src.api_export import keras_hub_export
19
- from keras_hub.src.layers.preprocessing.start_end_packer import StartEndPacker
20
- from keras_hub.src.models.falcon.falcon_tokenizer import FalconTokenizer
21
- from keras_hub.src.models.preprocessor import Preprocessor
22
- from keras_hub.src.utils.keras_utils import (
23
- convert_inputs_to_list_of_tensor_segments,
24
- )
25
-
26
-
27
- @keras_hub_export("keras_hub.models.FalconPreprocessor")
28
- class FalconPreprocessor(Preprocessor):
29
- """Falcon preprocessing layer which tokenizes and packs inputs.
30
-
31
- This preprocessing layer will do 2 things:
32
-
33
- - Tokenize the inputs using the `tokenizer`.
34
- - Construct a dictionary with keys `"token_ids"`, `"padding_mask"`, that can
35
- be passed directly to a `keras_hub.models.FalconBackbone`.
36
-
37
- This layer can be used directly with `tf.data.Dataset.map` to preprocess
38
- string data in the `(x, y, sample_weight)` format used by
39
- `keras.Model.fit`.
40
-
41
- The call method of this layer accepts three arguments, `x`, `y`, and
42
- `sample_weight`. `x` can be a python string or tensor representing a single
43
- segment, a list of python strings representing a batch of single segments,
44
- or a list of tensors representing multiple segments to be packed together.
45
- `y` and `sample_weight` are both optional, can have any format, and will be
46
- passed through unaltered.
47
-
48
- `FalconPreprocessor` forces the input to have only one segment, as Falcon is
49
- mainly used for generation tasks. For tasks having multi-segment inputs
50
- like "glue/mnli", please use a model designed for classification purposes
51
- such as BERT or RoBERTa.
52
-
53
- Args:
54
- tokenizer: A `keras_hub.models.FalconTokenizer` instance.
55
- sequence_length: The length of the packed inputs.
56
- add_start_token: If `True`, the preprocessor will prepend the tokenizer
57
- start token to each input sequence.
58
- add_end_token: If `True`, the preprocessor will append the tokenizer
59
- end token to each input sequence.
60
-
61
- Call arguments:
62
- x: A string, `tf.Tensor` or list of python strings.
63
- y: Any label data. Will be passed through unaltered.
64
- sample_weight: Any label weight data. Will be passed through unaltered.
65
- sequence_length: Pass to override the configured `sequence_length` of
66
- the layer.
67
-
68
- Examples:
69
-
70
- Directly calling the layer on data.
71
- ```python
72
- preprocessor = keras_hub.models.FalconPreprocessor.from_preset("falcon_rw_1b")
73
-
74
- # Tokenize and pack a single sentence.
75
- preprocessor("The quick brown fox jumped.")
76
-
77
- # Tokenize a batch of single sentences.
78
- preprocessor(["The quick brown fox jumped.", "Call me Ishmael."])
79
-
80
- # Custom vocabulary.
81
- features = ["a quick fox.", "a fox quick."]
82
- vocab = {"<|endoftext|>": 0, "a": 4, "Ġquick": 5, "Ġfox": 6}
83
- merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"]
84
- merges += ["Ġ f", "o x", "Ġf ox"]
85
- tokenizer = keras_hub.models.FalconTokenizer(
86
- vocabulary=vocab,
87
- merges=merges,
88
- )
89
- preprocessor = keras_hub.models.FalconPreprocessor(tokenizer=tokenizer)
90
- preprocessor("The quick brown fox jumped.")
91
- ```
92
-
93
- Mapping with `tf.data.Dataset`.
94
- ```python
95
- preprocessor = keras_hub.models.FalconPreprocessor.from_preset("falcon_rw_1b")
96
-
97
- text = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
98
- label = tf.constant([1, 1])
99
-
100
- # Map labeled single sentences.
101
- ds = tf.data.Dataset.from_tensor_slices((text, label))
102
- ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
103
-
104
- # Map unlabeled single sentences.
105
- ds = tf.data.Dataset.from_tensor_slices(text)
106
- ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE)
107
- ```
108
- """
109
-
110
- tokenizer_cls = FalconTokenizer
111
-
112
- def __init__(
113
- self,
114
- tokenizer,
115
- sequence_length=1024,
116
- add_start_token=True,
117
- add_end_token=True,
118
- **kwargs,
119
- ):
120
- super().__init__(**kwargs)
121
- self.tokenizer = tokenizer
122
- self.packer = None
123
- self.sequence_length = sequence_length
124
- self.add_start_token = add_start_token
125
- self.add_end_token = add_end_token
126
-
127
- def build(self, input_shape):
128
- # Defer packer creation to `build()` so that we can be sure tokenizer
129
- # assets have loaded when restoring a saved model.
130
- self.packer = StartEndPacker(
131
- start_value=self.tokenizer.start_token_id,
132
- end_value=self.tokenizer.end_token_id,
133
- pad_value=self.tokenizer.pad_token_id,
134
- sequence_length=self.sequence_length,
135
- return_padding_mask=True,
136
- )
137
- self.built = True
138
-
139
- def call(
140
- self,
141
- x,
142
- y=None,
143
- sample_weight=None,
144
- sequence_length=None,
145
- ):
146
- x = convert_inputs_to_list_of_tensor_segments(x)
147
- if len(x) != 1:
148
- raise ValueError(
149
- "Falcon requires each input feature to contain only "
150
- f"one segment, but received {len(x)}. If you are using Falcon "
151
- "for a multi-segment classification task, please refer to "
152
- "classification models like BERT or RoBERTa."
153
- )
154
- sequence_length = sequence_length or self.sequence_length
155
- token_ids, padding_mask = self.packer(
156
- self.tokenizer(x[0]),
157
- sequence_length=sequence_length,
158
- add_start_value=self.add_start_token,
159
- add_end_value=self.add_end_token,
160
- )
161
- x = {
162
- "token_ids": token_ids,
163
- "padding_mask": padding_mask,
164
- }
165
- return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
166
-
167
- def get_config(self):
168
- config = super().get_config()
169
- config.update(
170
- {
171
- "sequence_length": self.sequence_length,
172
- "add_start_token": self.add_start_token,
173
- "add_end_token": self.add_end_token,
174
- }
175
- )
176
- return config
177
-
178
- @property
179
- def sequence_length(self):
180
- """The padded length of model input sequences."""
181
- return self._sequence_length
182
-
183
- @sequence_length.setter
184
- def sequence_length(self, value):
185
- self._sequence_length = value
186
- if self.packer is not None:
187
- self.packer.sequence_length = value