keras-hub-nightly 0.15.0.dev20240823171555__py3-none-any.whl → 0.16.0.dev2024092017__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- keras_hub/__init__.py +0 -6
- keras_hub/api/__init__.py +2 -0
- keras_hub/api/bounding_box/__init__.py +36 -0
- keras_hub/api/layers/__init__.py +14 -0
- keras_hub/api/models/__init__.py +97 -48
- keras_hub/api/tokenizers/__init__.py +30 -0
- keras_hub/api/utils/__init__.py +22 -0
- keras_hub/src/api_export.py +15 -9
- keras_hub/src/bounding_box/__init__.py +13 -0
- keras_hub/src/bounding_box/converters.py +529 -0
- keras_hub/src/bounding_box/formats.py +162 -0
- keras_hub/src/bounding_box/iou.py +263 -0
- keras_hub/src/bounding_box/to_dense.py +95 -0
- keras_hub/src/bounding_box/to_ragged.py +99 -0
- keras_hub/src/bounding_box/utils.py +194 -0
- keras_hub/src/bounding_box/validate_format.py +99 -0
- keras_hub/src/layers/preprocessing/audio_converter.py +121 -0
- keras_hub/src/layers/preprocessing/image_converter.py +130 -0
- keras_hub/src/layers/preprocessing/masked_lm_mask_generator.py +2 -0
- keras_hub/src/layers/preprocessing/multi_segment_packer.py +9 -8
- keras_hub/src/layers/preprocessing/preprocessing_layer.py +2 -29
- keras_hub/src/layers/preprocessing/random_deletion.py +33 -31
- keras_hub/src/layers/preprocessing/random_swap.py +33 -31
- keras_hub/src/layers/preprocessing/resizing_image_converter.py +101 -0
- keras_hub/src/layers/preprocessing/start_end_packer.py +3 -2
- keras_hub/src/models/albert/__init__.py +1 -2
- keras_hub/src/models/albert/albert_masked_lm_preprocessor.py +6 -86
- keras_hub/src/models/albert/{albert_classifier.py → albert_text_classifier.py} +34 -10
- keras_hub/src/models/albert/{albert_preprocessor.py → albert_text_classifier_preprocessor.py} +14 -70
- keras_hub/src/models/albert/albert_tokenizer.py +17 -36
- keras_hub/src/models/backbone.py +12 -34
- keras_hub/src/models/bart/__init__.py +1 -2
- keras_hub/src/models/bart/bart_seq_2_seq_lm_preprocessor.py +21 -148
- keras_hub/src/models/bart/bart_tokenizer.py +12 -39
- keras_hub/src/models/bert/__init__.py +1 -5
- keras_hub/src/models/bert/bert_masked_lm_preprocessor.py +6 -87
- keras_hub/src/models/bert/bert_presets.py +1 -4
- keras_hub/src/models/bert/{bert_classifier.py → bert_text_classifier.py} +19 -12
- keras_hub/src/models/bert/{bert_preprocessor.py → bert_text_classifier_preprocessor.py} +14 -70
- keras_hub/src/models/bert/bert_tokenizer.py +17 -35
- keras_hub/src/models/bloom/__init__.py +1 -2
- keras_hub/src/models/bloom/bloom_causal_lm_preprocessor.py +6 -91
- keras_hub/src/models/bloom/bloom_tokenizer.py +12 -41
- keras_hub/src/models/causal_lm.py +10 -29
- keras_hub/src/models/causal_lm_preprocessor.py +195 -0
- keras_hub/src/models/csp_darknet/csp_darknet_backbone.py +54 -15
- keras_hub/src/models/deberta_v3/__init__.py +1 -4
- keras_hub/src/models/deberta_v3/deberta_v3_masked_lm_preprocessor.py +14 -77
- keras_hub/src/models/deberta_v3/{deberta_v3_classifier.py → deberta_v3_text_classifier.py} +16 -11
- keras_hub/src/models/deberta_v3/{deberta_v3_preprocessor.py → deberta_v3_text_classifier_preprocessor.py} +23 -64
- keras_hub/src/models/deberta_v3/deberta_v3_tokenizer.py +30 -25
- keras_hub/src/models/densenet/densenet_backbone.py +46 -22
- keras_hub/src/models/distil_bert/__init__.py +1 -4
- keras_hub/src/models/distil_bert/distil_bert_masked_lm_preprocessor.py +14 -76
- keras_hub/src/models/distil_bert/{distil_bert_classifier.py → distil_bert_text_classifier.py} +17 -12
- keras_hub/src/models/distil_bert/{distil_bert_preprocessor.py → distil_bert_text_classifier_preprocessor.py} +23 -63
- keras_hub/src/models/distil_bert/distil_bert_tokenizer.py +19 -35
- keras_hub/src/models/efficientnet/__init__.py +13 -0
- keras_hub/src/models/efficientnet/efficientnet_backbone.py +569 -0
- keras_hub/src/models/efficientnet/fusedmbconv.py +229 -0
- keras_hub/src/models/efficientnet/mbconv.py +238 -0
- keras_hub/src/models/electra/__init__.py +1 -2
- keras_hub/src/models/electra/electra_tokenizer.py +17 -32
- keras_hub/src/models/f_net/__init__.py +1 -2
- keras_hub/src/models/f_net/f_net_masked_lm_preprocessor.py +12 -78
- keras_hub/src/models/f_net/{f_net_classifier.py → f_net_text_classifier.py} +17 -10
- keras_hub/src/models/f_net/{f_net_preprocessor.py → f_net_text_classifier_preprocessor.py} +19 -63
- keras_hub/src/models/f_net/f_net_tokenizer.py +17 -35
- keras_hub/src/models/falcon/__init__.py +1 -2
- keras_hub/src/models/falcon/falcon_causal_lm_preprocessor.py +6 -89
- keras_hub/src/models/falcon/falcon_tokenizer.py +12 -35
- keras_hub/src/models/gemma/__init__.py +1 -2
- keras_hub/src/models/gemma/gemma_causal_lm_preprocessor.py +6 -90
- keras_hub/src/models/gemma/gemma_decoder_block.py +1 -1
- keras_hub/src/models/gemma/gemma_tokenizer.py +12 -23
- keras_hub/src/models/gpt2/__init__.py +1 -2
- keras_hub/src/models/gpt2/gpt2_causal_lm_preprocessor.py +6 -89
- keras_hub/src/models/gpt2/gpt2_preprocessor.py +12 -90
- keras_hub/src/models/gpt2/gpt2_tokenizer.py +12 -34
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +6 -91
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_tokenizer.py +12 -34
- keras_hub/src/models/image_classifier.py +0 -5
- keras_hub/src/models/image_classifier_preprocessor.py +83 -0
- keras_hub/src/models/llama/__init__.py +1 -2
- keras_hub/src/models/llama/llama_causal_lm_preprocessor.py +6 -85
- keras_hub/src/models/llama/llama_tokenizer.py +12 -25
- keras_hub/src/models/llama3/__init__.py +1 -2
- keras_hub/src/models/llama3/llama3_causal_lm_preprocessor.py +6 -89
- keras_hub/src/models/llama3/llama3_tokenizer.py +12 -33
- keras_hub/src/models/masked_lm.py +0 -2
- keras_hub/src/models/masked_lm_preprocessor.py +156 -0
- keras_hub/src/models/mistral/__init__.py +1 -2
- keras_hub/src/models/mistral/mistral_causal_lm_preprocessor.py +6 -91
- keras_hub/src/models/mistral/mistral_tokenizer.py +12 -23
- keras_hub/src/models/mix_transformer/mix_transformer_backbone.py +2 -2
- keras_hub/src/models/mobilenet/__init__.py +13 -0
- keras_hub/src/models/mobilenet/mobilenet_backbone.py +530 -0
- keras_hub/src/models/mobilenet/mobilenet_image_classifier.py +114 -0
- keras_hub/src/models/opt/__init__.py +1 -2
- keras_hub/src/models/opt/opt_causal_lm_preprocessor.py +6 -93
- keras_hub/src/models/opt/opt_tokenizer.py +12 -41
- keras_hub/src/models/pali_gemma/__init__.py +1 -4
- keras_hub/src/models/pali_gemma/pali_gemma_causal_lm_preprocessor.py +28 -28
- keras_hub/src/models/pali_gemma/pali_gemma_image_converter.py +25 -0
- keras_hub/src/models/pali_gemma/pali_gemma_presets.py +5 -5
- keras_hub/src/models/pali_gemma/pali_gemma_tokenizer.py +11 -3
- keras_hub/src/models/phi3/__init__.py +1 -2
- keras_hub/src/models/phi3/phi3_causal_lm.py +3 -9
- keras_hub/src/models/phi3/phi3_causal_lm_preprocessor.py +6 -89
- keras_hub/src/models/phi3/phi3_tokenizer.py +12 -36
- keras_hub/src/models/preprocessor.py +72 -83
- keras_hub/src/models/resnet/__init__.py +6 -0
- keras_hub/src/models/resnet/resnet_backbone.py +390 -42
- keras_hub/src/models/resnet/resnet_image_classifier.py +33 -6
- keras_hub/src/models/resnet/resnet_image_classifier_preprocessor.py +28 -0
- keras_hub/src/models/{llama3/llama3_preprocessor.py → resnet/resnet_image_converter.py} +7 -5
- keras_hub/src/models/resnet/resnet_presets.py +95 -0
- keras_hub/src/models/retinanet/__init__.py +13 -0
- keras_hub/src/models/retinanet/anchor_generator.py +175 -0
- keras_hub/src/models/retinanet/box_matcher.py +259 -0
- keras_hub/src/models/retinanet/non_max_supression.py +578 -0
- keras_hub/src/models/roberta/__init__.py +1 -2
- keras_hub/src/models/roberta/roberta_masked_lm_preprocessor.py +22 -74
- keras_hub/src/models/roberta/{roberta_classifier.py → roberta_text_classifier.py} +16 -11
- keras_hub/src/models/roberta/{roberta_preprocessor.py → roberta_text_classifier_preprocessor.py} +21 -53
- keras_hub/src/models/roberta/roberta_tokenizer.py +13 -52
- keras_hub/src/models/seq_2_seq_lm_preprocessor.py +269 -0
- keras_hub/src/models/stable_diffusion_v3/__init__.py +13 -0
- keras_hub/src/models/stable_diffusion_v3/clip_encoder_block.py +103 -0
- keras_hub/src/models/stable_diffusion_v3/clip_preprocessor.py +93 -0
- keras_hub/src/models/stable_diffusion_v3/clip_text_encoder.py +149 -0
- keras_hub/src/models/stable_diffusion_v3/clip_tokenizer.py +167 -0
- keras_hub/src/models/stable_diffusion_v3/mmdit.py +427 -0
- keras_hub/src/models/stable_diffusion_v3/mmdit_block.py +317 -0
- keras_hub/src/models/stable_diffusion_v3/t5_xxl_preprocessor.py +74 -0
- keras_hub/src/models/stable_diffusion_v3/t5_xxl_text_encoder.py +155 -0
- keras_hub/src/models/stable_diffusion_v3/vae_attention.py +126 -0
- keras_hub/src/models/stable_diffusion_v3/vae_image_decoder.py +186 -0
- keras_hub/src/models/t5/__init__.py +1 -2
- keras_hub/src/models/t5/t5_tokenizer.py +13 -23
- keras_hub/src/models/task.py +71 -116
- keras_hub/src/models/{classifier.py → text_classifier.py} +19 -13
- keras_hub/src/models/text_classifier_preprocessor.py +138 -0
- keras_hub/src/models/whisper/__init__.py +1 -2
- keras_hub/src/models/whisper/{whisper_audio_feature_extractor.py → whisper_audio_converter.py} +20 -18
- keras_hub/src/models/whisper/whisper_backbone.py +0 -3
- keras_hub/src/models/whisper/whisper_presets.py +10 -10
- keras_hub/src/models/whisper/whisper_tokenizer.py +20 -16
- keras_hub/src/models/xlm_roberta/__init__.py +1 -4
- keras_hub/src/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor.py +26 -72
- keras_hub/src/models/xlm_roberta/{xlm_roberta_classifier.py → xlm_roberta_text_classifier.py} +16 -11
- keras_hub/src/models/xlm_roberta/{xlm_roberta_preprocessor.py → xlm_roberta_text_classifier_preprocessor.py} +26 -53
- keras_hub/src/models/xlm_roberta/xlm_roberta_tokenizer.py +25 -10
- keras_hub/src/tests/test_case.py +46 -0
- keras_hub/src/tokenizers/byte_pair_tokenizer.py +30 -17
- keras_hub/src/tokenizers/byte_tokenizer.py +14 -15
- keras_hub/src/tokenizers/sentence_piece_tokenizer.py +20 -7
- keras_hub/src/tokenizers/tokenizer.py +67 -32
- keras_hub/src/tokenizers/unicode_codepoint_tokenizer.py +14 -15
- keras_hub/src/tokenizers/word_piece_tokenizer.py +34 -47
- keras_hub/src/utils/imagenet/__init__.py +13 -0
- keras_hub/src/utils/imagenet/imagenet_utils.py +1067 -0
- keras_hub/src/utils/keras_utils.py +0 -50
- keras_hub/src/utils/preset_utils.py +230 -68
- keras_hub/src/utils/tensor_utils.py +187 -69
- keras_hub/src/utils/timm/convert_resnet.py +19 -16
- keras_hub/src/utils/timm/preset_loader.py +66 -0
- keras_hub/src/utils/transformers/convert_albert.py +193 -0
- keras_hub/src/utils/transformers/convert_bart.py +373 -0
- keras_hub/src/utils/transformers/convert_bert.py +7 -17
- keras_hub/src/utils/transformers/convert_distilbert.py +10 -20
- keras_hub/src/utils/transformers/convert_gemma.py +5 -19
- keras_hub/src/utils/transformers/convert_gpt2.py +5 -18
- keras_hub/src/utils/transformers/convert_llama3.py +7 -18
- keras_hub/src/utils/transformers/convert_mistral.py +129 -0
- keras_hub/src/utils/transformers/convert_pali_gemma.py +7 -29
- keras_hub/src/utils/transformers/preset_loader.py +77 -0
- keras_hub/src/utils/transformers/safetensor_utils.py +2 -2
- keras_hub/src/version_utils.py +1 -1
- keras_hub_nightly-0.16.0.dev2024092017.dist-info/METADATA +202 -0
- keras_hub_nightly-0.16.0.dev2024092017.dist-info/RECORD +334 -0
- {keras_hub_nightly-0.15.0.dev20240823171555.dist-info → keras_hub_nightly-0.16.0.dev2024092017.dist-info}/WHEEL +1 -1
- keras_hub/src/models/bart/bart_preprocessor.py +0 -276
- keras_hub/src/models/bloom/bloom_preprocessor.py +0 -185
- keras_hub/src/models/electra/electra_preprocessor.py +0 -154
- keras_hub/src/models/falcon/falcon_preprocessor.py +0 -187
- keras_hub/src/models/gemma/gemma_preprocessor.py +0 -191
- keras_hub/src/models/gpt_neo_x/gpt_neo_x_preprocessor.py +0 -145
- keras_hub/src/models/llama/llama_preprocessor.py +0 -189
- keras_hub/src/models/mistral/mistral_preprocessor.py +0 -190
- keras_hub/src/models/opt/opt_preprocessor.py +0 -188
- keras_hub/src/models/phi3/phi3_preprocessor.py +0 -190
- keras_hub/src/models/whisper/whisper_preprocessor.py +0 -326
- keras_hub/src/utils/timm/convert.py +0 -37
- keras_hub/src/utils/transformers/convert.py +0 -101
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/METADATA +0 -34
- keras_hub_nightly-0.15.0.dev20240823171555.dist-info/RECORD +0 -297
- {keras_hub_nightly-0.15.0.dev20240823171555.dist-info → keras_hub_nightly-0.16.0.dev2024092017.dist-info}/top_level.txt +0 -0
@@ -16,14 +16,21 @@
|
|
16
16
|
import keras
|
17
17
|
|
18
18
|
from keras_hub.src.api_export import keras_hub_export
|
19
|
-
from keras_hub.src.models.classifier import Classifier
|
20
19
|
from keras_hub.src.models.f_net.f_net_backbone import FNetBackbone
|
21
20
|
from keras_hub.src.models.f_net.f_net_backbone import f_net_kernel_initializer
|
22
|
-
from keras_hub.src.models.f_net.
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
21
|
+
from keras_hub.src.models.f_net.f_net_text_classifier_preprocessor import (
|
22
|
+
FNetTextClassifierPreprocessor,
|
23
|
+
)
|
24
|
+
from keras_hub.src.models.text_classifier import TextClassifier
|
25
|
+
|
26
|
+
|
27
|
+
@keras_hub_export(
|
28
|
+
[
|
29
|
+
"keras_hub.models.FNetTextClassifier",
|
30
|
+
"keras_hub.models.FNetClassifier",
|
31
|
+
]
|
32
|
+
)
|
33
|
+
class FNetTextClassifier(TextClassifier):
|
27
34
|
"""An end-to-end f_net model for classification tasks.
|
28
35
|
|
29
36
|
This model attaches a classification head to a
|
@@ -42,7 +49,7 @@ class FNetClassifier(Classifier):
|
|
42
49
|
Args:
|
43
50
|
backbone: A `keras_hub.models.FNetBackbone` instance.
|
44
51
|
num_classes: int. Number of classes to predict.
|
45
|
-
preprocessor: A `keras_hub.models.
|
52
|
+
preprocessor: A `keras_hub.models.FNetTextClassifierPreprocessor` or `None`. If
|
46
53
|
`None`, this model will not apply preprocessing, and inputs should
|
47
54
|
be preprocessed before calling the model.
|
48
55
|
activation: Optional `str` or callable. The
|
@@ -61,7 +68,7 @@ class FNetClassifier(Classifier):
|
|
61
68
|
labels = [0, 3]
|
62
69
|
|
63
70
|
# Pretrained classifier.
|
64
|
-
classifier = keras_hub.models.
|
71
|
+
classifier = keras_hub.models.FNetTextClassifier.from_preset(
|
65
72
|
"f_net_base_en",
|
66
73
|
num_classes=4,
|
67
74
|
)
|
@@ -89,7 +96,7 @@ class FNetClassifier(Classifier):
|
|
89
96
|
labels = [0, 3]
|
90
97
|
|
91
98
|
# Pretrained classifier without preprocessing.
|
92
|
-
classifier = keras_hub.models.
|
99
|
+
classifier = keras_hub.models.FNetTextClassifier.from_preset(
|
93
100
|
"f_net_base_en",
|
94
101
|
num_classes=4,
|
95
102
|
preprocessor=None,
|
@@ -99,7 +106,7 @@ class FNetClassifier(Classifier):
|
|
99
106
|
"""
|
100
107
|
|
101
108
|
backbone_cls = FNetBackbone
|
102
|
-
preprocessor_cls =
|
109
|
+
preprocessor_cls = FNetTextClassifierPreprocessor
|
103
110
|
|
104
111
|
def __init__(
|
105
112
|
self,
|
@@ -16,18 +16,21 @@
|
|
16
16
|
import keras
|
17
17
|
|
18
18
|
from keras_hub.src.api_export import keras_hub_export
|
19
|
-
from keras_hub.src.
|
20
|
-
MultiSegmentPacker,
|
21
|
-
)
|
19
|
+
from keras_hub.src.models.f_net.f_net_backbone import FNetBackbone
|
22
20
|
from keras_hub.src.models.f_net.f_net_tokenizer import FNetTokenizer
|
23
|
-
from keras_hub.src.models.
|
24
|
-
|
25
|
-
convert_inputs_to_list_of_tensor_segments,
|
21
|
+
from keras_hub.src.models.text_classifier_preprocessor import (
|
22
|
+
TextClassifierPreprocessor,
|
26
23
|
)
|
24
|
+
from keras_hub.src.utils.tensor_utils import preprocessing_function
|
27
25
|
|
28
26
|
|
29
|
-
@keras_hub_export(
|
30
|
-
|
27
|
+
@keras_hub_export(
|
28
|
+
[
|
29
|
+
"keras_hub.models.FNetTextClassifierPreprocessor",
|
30
|
+
"keras_hub.models.FNetPreprocessor",
|
31
|
+
]
|
32
|
+
)
|
33
|
+
class FNetTextClassifierPreprocessor(TextClassifierPreprocessor):
|
31
34
|
"""An FNet preprocessing layer which tokenizes and packs inputs.
|
32
35
|
|
33
36
|
This preprocessing layer will do three things:
|
@@ -68,7 +71,7 @@ class FNetPreprocessor(Preprocessor):
|
|
68
71
|
|
69
72
|
Directly calling the from_preset().
|
70
73
|
```python
|
71
|
-
preprocessor = keras_hub.models.
|
74
|
+
preprocessor = keras_hub.models.TextClassifierPreprocessor.from_preset(
|
72
75
|
"f_net_base_en"
|
73
76
|
)
|
74
77
|
|
@@ -87,7 +90,7 @@ class FNetPreprocessor(Preprocessor):
|
|
87
90
|
|
88
91
|
Mapping with `tf.data.Dataset`.
|
89
92
|
```python
|
90
|
-
preprocessor = keras_hub.models.
|
93
|
+
preprocessor = keras_hub.models.TextClassifierPreprocessor.from_preset(
|
91
94
|
"f_net_base_en"
|
92
95
|
)
|
93
96
|
first = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."])
|
@@ -118,60 +121,13 @@ class FNetPreprocessor(Preprocessor):
|
|
118
121
|
```
|
119
122
|
"""
|
120
123
|
|
124
|
+
backbone_cls = FNetBackbone
|
121
125
|
tokenizer_cls = FNetTokenizer
|
122
126
|
|
123
|
-
|
124
|
-
self,
|
125
|
-
tokenizer,
|
126
|
-
sequence_length=512,
|
127
|
-
truncate="round_robin",
|
128
|
-
**kwargs,
|
129
|
-
):
|
130
|
-
super().__init__(**kwargs)
|
131
|
-
self.tokenizer = tokenizer
|
132
|
-
self.packer = None
|
133
|
-
self.truncate = truncate
|
134
|
-
self.sequence_length = sequence_length
|
135
|
-
|
136
|
-
def build(self, input_shape):
|
137
|
-
# Defer packer creation to `build()` so that we can be sure tokenizer
|
138
|
-
# assets have loaded when restoring a saved model.
|
139
|
-
self.packer = MultiSegmentPacker(
|
140
|
-
start_value=self.tokenizer.cls_token_id,
|
141
|
-
end_value=self.tokenizer.sep_token_id,
|
142
|
-
pad_value=self.tokenizer.pad_token_id,
|
143
|
-
truncate=self.truncate,
|
144
|
-
sequence_length=self.sequence_length,
|
145
|
-
)
|
146
|
-
self.built = True
|
147
|
-
|
148
|
-
def get_config(self):
|
149
|
-
config = super().get_config()
|
150
|
-
config.update(
|
151
|
-
{
|
152
|
-
"sequence_length": self.sequence_length,
|
153
|
-
"truncate": self.truncate,
|
154
|
-
}
|
155
|
-
)
|
156
|
-
return config
|
157
|
-
|
127
|
+
@preprocessing_function
|
158
128
|
def call(self, x, y=None, sample_weight=None):
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
x
|
163
|
-
"token_ids": token_ids,
|
164
|
-
"segment_ids": segment_ids,
|
165
|
-
}
|
129
|
+
# FNet has not padding mask.
|
130
|
+
output = super().call(x, y=y, sample_weight=sample_weight)
|
131
|
+
x, y, sample_weight = keras.utils.unpack_x_y_sample_weight(output)
|
132
|
+
del x["padding_mask"]
|
166
133
|
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
|
167
|
-
|
168
|
-
@property
|
169
|
-
def sequence_length(self):
|
170
|
-
"""The padded length of model input sequences."""
|
171
|
-
return self._sequence_length
|
172
|
-
|
173
|
-
@sequence_length.setter
|
174
|
-
def sequence_length(self, value):
|
175
|
-
self._sequence_length = value
|
176
|
-
if self.packer is not None:
|
177
|
-
self.packer.sequence_length = value
|
@@ -14,12 +14,18 @@
|
|
14
14
|
|
15
15
|
|
16
16
|
from keras_hub.src.api_export import keras_hub_export
|
17
|
+
from keras_hub.src.models.f_net.f_net_backbone import FNetBackbone
|
17
18
|
from keras_hub.src.tokenizers.sentence_piece_tokenizer import (
|
18
19
|
SentencePieceTokenizer,
|
19
20
|
)
|
20
21
|
|
21
22
|
|
22
|
-
@keras_hub_export(
|
23
|
+
@keras_hub_export(
|
24
|
+
[
|
25
|
+
"keras_hub.tokenizers.FNetTokenizer",
|
26
|
+
"keras_hub.models.FNetTokenizer",
|
27
|
+
]
|
28
|
+
)
|
23
29
|
class FNetTokenizer(SentencePieceTokenizer):
|
24
30
|
"""FNet tokenizer layer based on SentencePiece.
|
25
31
|
|
@@ -29,10 +35,6 @@ class FNetTokenizer(SentencePieceTokenizer):
|
|
29
35
|
FNet models and provides a `from_preset()` method to automatically
|
30
36
|
download a matching vocabulary for a FNet preset.
|
31
37
|
|
32
|
-
This tokenizer does not provide truncation or padding of inputs. It can be
|
33
|
-
combined with a `keras_hub.models.FNetPreprocessor` layer for input
|
34
|
-
packing.
|
35
|
-
|
36
38
|
If input is a batch of strings (rank > 0), the layer will output a
|
37
39
|
`tf.RaggedTensor` where the last dimension of the output is ragged.
|
38
40
|
|
@@ -61,35 +63,15 @@ class FNetTokenizer(SentencePieceTokenizer):
|
|
61
63
|
```
|
62
64
|
"""
|
63
65
|
|
66
|
+
backbone_cls = FNetBackbone
|
67
|
+
|
64
68
|
def __init__(self, proto, **kwargs):
|
65
|
-
self.
|
66
|
-
self.
|
67
|
-
self.
|
68
|
-
self.
|
69
|
+
self._add_special_token("[CLS]", "cls_token")
|
70
|
+
self._add_special_token("[SEP]", "sep_token")
|
71
|
+
self._add_special_token("<pad>", "pad_token")
|
72
|
+
self._add_special_token("[MASK]", "mask_token")
|
73
|
+
# Also add `tokenizer.start_token` and `tokenizer.end_token` for
|
74
|
+
# compatibility with other tokenizers.
|
75
|
+
self._add_special_token("[CLS]", "start_token")
|
76
|
+
self._add_special_token("[SEP]", "end_token")
|
69
77
|
super().__init__(proto=proto, **kwargs)
|
70
|
-
|
71
|
-
def set_proto(self, proto):
|
72
|
-
super().set_proto(proto)
|
73
|
-
if proto is not None:
|
74
|
-
for token in [
|
75
|
-
self.cls_token,
|
76
|
-
self.sep_token,
|
77
|
-
self.pad_token,
|
78
|
-
self.mask_token,
|
79
|
-
]:
|
80
|
-
if token not in self.get_vocabulary():
|
81
|
-
raise ValueError(
|
82
|
-
f"Cannot find token `'{token}'` in the provided "
|
83
|
-
f"`vocabulary`. Please provide `'{token}'` in your "
|
84
|
-
"`vocabulary` or use a pretrained `vocabulary` name."
|
85
|
-
)
|
86
|
-
|
87
|
-
self.cls_token_id = self.token_to_id(self.cls_token)
|
88
|
-
self.sep_token_id = self.token_to_id(self.sep_token)
|
89
|
-
self.pad_token_id = self.token_to_id(self.pad_token)
|
90
|
-
self.mask_token_id = self.token_to_id(self.mask_token)
|
91
|
-
else:
|
92
|
-
self.cls_token_id = None
|
93
|
-
self.sep_token_id = None
|
94
|
-
self.pad_token_id = None
|
95
|
-
self.mask_token_id = None
|
@@ -14,7 +14,6 @@
|
|
14
14
|
|
15
15
|
from keras_hub.src.models.falcon.falcon_backbone import FalconBackbone
|
16
16
|
from keras_hub.src.models.falcon.falcon_presets import backbone_presets
|
17
|
-
from keras_hub.src.models.falcon.falcon_tokenizer import FalconTokenizer
|
18
17
|
from keras_hub.src.utils.preset_utils import register_presets
|
19
18
|
|
20
|
-
register_presets(backbone_presets,
|
19
|
+
register_presets(backbone_presets, FalconBackbone)
|
@@ -12,19 +12,14 @@
|
|
12
12
|
# See the License for the specific language governing permissions and
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
|
-
import keras
|
16
|
-
from absl import logging
|
17
|
-
|
18
15
|
from keras_hub.src.api_export import keras_hub_export
|
19
|
-
from keras_hub.src.models.
|
20
|
-
from keras_hub.src.
|
21
|
-
|
22
|
-
)
|
23
|
-
from keras_hub.src.utils.tensor_utils import strip_to_ragged
|
16
|
+
from keras_hub.src.models.causal_lm_preprocessor import CausalLMPreprocessor
|
17
|
+
from keras_hub.src.models.falcon.falcon_backbone import FalconBackbone
|
18
|
+
from keras_hub.src.models.falcon.falcon_tokenizer import FalconTokenizer
|
24
19
|
|
25
20
|
|
26
21
|
@keras_hub_export("keras_hub.models.FalconCausalLMPreprocessor")
|
27
|
-
class FalconCausalLMPreprocessor(
|
22
|
+
class FalconCausalLMPreprocessor(CausalLMPreprocessor):
|
28
23
|
"""Falcon Causal LM preprocessor.
|
29
24
|
|
30
25
|
This preprocessing layer is meant for use with
|
@@ -91,83 +86,5 @@ class FalconCausalLMPreprocessor(FalconPreprocessor):
|
|
91
86
|
```
|
92
87
|
"""
|
93
88
|
|
94
|
-
|
95
|
-
|
96
|
-
x,
|
97
|
-
y=None,
|
98
|
-
sample_weight=None,
|
99
|
-
sequence_length=None,
|
100
|
-
):
|
101
|
-
if y is not None or sample_weight is not None:
|
102
|
-
logging.warning(
|
103
|
-
"`FalconCausalLMPreprocessor` generates `y` and `sample_weight` "
|
104
|
-
"based on your input data, but your data already contains `y` "
|
105
|
-
"or `sample_weight`. Your `y` and `sample_weight` will be "
|
106
|
-
"ignored."
|
107
|
-
)
|
108
|
-
sequence_length = sequence_length or self.sequence_length
|
109
|
-
|
110
|
-
x = convert_inputs_to_list_of_tensor_segments(x)[0]
|
111
|
-
x = self.tokenizer(x)
|
112
|
-
# Pad with one extra token to account for the truncation below.
|
113
|
-
token_ids, padding_mask = self.packer(
|
114
|
-
x,
|
115
|
-
sequence_length=sequence_length + 1,
|
116
|
-
add_start_value=self.add_start_token,
|
117
|
-
add_end_value=self.add_end_token,
|
118
|
-
)
|
119
|
-
# The last token does not have a next token, so we truncate it out.
|
120
|
-
x = {
|
121
|
-
"token_ids": token_ids[..., :-1],
|
122
|
-
"padding_mask": padding_mask[..., :-1],
|
123
|
-
}
|
124
|
-
# Target `y` will be the next token.
|
125
|
-
y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:]
|
126
|
-
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
|
127
|
-
|
128
|
-
def generate_preprocess(
|
129
|
-
self,
|
130
|
-
x,
|
131
|
-
sequence_length=None,
|
132
|
-
):
|
133
|
-
"""Convert strings to integer token input for generation.
|
134
|
-
|
135
|
-
Similar to calling the layer for training, this method takes in strings
|
136
|
-
or tensor strings, tokenizes and packs the input, and computes a padding
|
137
|
-
mask masking all inputs not filled in with a padded value.
|
138
|
-
|
139
|
-
Unlike calling the layer for training, this method does not compute
|
140
|
-
labels and will never append a `tokenizer.end_token_id` to the end of
|
141
|
-
the sequence (as generation is expected to continue at the end of the
|
142
|
-
inputted prompt).
|
143
|
-
"""
|
144
|
-
if not self.built:
|
145
|
-
self.build(None)
|
146
|
-
|
147
|
-
x = convert_inputs_to_list_of_tensor_segments(x)[0]
|
148
|
-
x = self.tokenizer(x)
|
149
|
-
token_ids, padding_mask = self.packer(
|
150
|
-
x, sequence_length=sequence_length, add_end_value=False
|
151
|
-
)
|
152
|
-
return {
|
153
|
-
"token_ids": token_ids,
|
154
|
-
"padding_mask": padding_mask,
|
155
|
-
}
|
156
|
-
|
157
|
-
def generate_postprocess(
|
158
|
-
self,
|
159
|
-
x,
|
160
|
-
):
|
161
|
-
"""Convert integer token output to strings for generation.
|
162
|
-
|
163
|
-
This method reverses `generate_preprocess()`, by first removing all
|
164
|
-
padding and start/end tokens, and then converting the integer sequence
|
165
|
-
back to a string.
|
166
|
-
"""
|
167
|
-
if not self.built:
|
168
|
-
self.build(None)
|
169
|
-
|
170
|
-
token_ids, padding_mask = x["token_ids"], x["padding_mask"]
|
171
|
-
ids_to_strip = (self.tokenizer.end_token_id,)
|
172
|
-
token_ids = strip_to_ragged(token_ids, padding_mask, ids_to_strip)
|
173
|
-
return self.tokenizer.detokenize(token_ids)
|
89
|
+
backbone_cls = FalconBackbone
|
90
|
+
tokenizer_cls = FalconTokenizer
|
@@ -14,10 +14,16 @@
|
|
14
14
|
|
15
15
|
|
16
16
|
from keras_hub.src.api_export import keras_hub_export
|
17
|
+
from keras_hub.src.models.falcon.falcon_backbone import FalconBackbone
|
17
18
|
from keras_hub.src.tokenizers.byte_pair_tokenizer import BytePairTokenizer
|
18
19
|
|
19
20
|
|
20
|
-
@keras_hub_export(
|
21
|
+
@keras_hub_export(
|
22
|
+
[
|
23
|
+
"keras_hub.tokenizers.FalconTokenizer",
|
24
|
+
"keras_hub.models.FalconTokenizer",
|
25
|
+
]
|
26
|
+
)
|
21
27
|
class FalconTokenizer(BytePairTokenizer):
|
22
28
|
"""Falcon tokenizer based on BytePairTokenizer.
|
23
29
|
|
@@ -27,8 +33,6 @@ class FalconTokenizer(BytePairTokenizer):
|
|
27
33
|
models and provides a `from_preset()` method to automatically download
|
28
34
|
a matching vocabulary for a Falcon preset.
|
29
35
|
|
30
|
-
This tokenizer does not provide truncation or padding of inputs.
|
31
|
-
|
32
36
|
If input is a batch of strings (rank > 0), the layer will output a
|
33
37
|
`tf.RaggedTensor` where the last dimension of the output is ragged.
|
34
38
|
|
@@ -65,46 +69,19 @@ class FalconTokenizer(BytePairTokenizer):
|
|
65
69
|
```
|
66
70
|
"""
|
67
71
|
|
72
|
+
backbone_cls = FalconBackbone
|
73
|
+
|
68
74
|
def __init__(
|
69
75
|
self,
|
70
76
|
vocabulary=None,
|
71
77
|
merges=None,
|
72
78
|
**kwargs,
|
73
79
|
):
|
74
|
-
|
75
|
-
self.
|
76
|
-
|
80
|
+
self._add_special_token("<|endoftext|>", "end_token")
|
81
|
+
self._add_special_token("<|endoftext|>", "start_token")
|
82
|
+
self.pad_token_id = 0
|
77
83
|
super().__init__(
|
78
84
|
vocabulary=vocabulary,
|
79
85
|
merges=merges,
|
80
|
-
unsplittable_tokens=[self.end_token],
|
81
86
|
**kwargs,
|
82
87
|
)
|
83
|
-
|
84
|
-
def set_vocabulary_and_merges(self, vocabulary, merges):
|
85
|
-
super().set_vocabulary_and_merges(vocabulary, merges)
|
86
|
-
|
87
|
-
if vocabulary is not None:
|
88
|
-
# Check for necessary special tokens.
|
89
|
-
if self.end_token not in self.get_vocabulary():
|
90
|
-
raise ValueError(
|
91
|
-
f"Cannot find token `'{self.end_token}'` in the provided "
|
92
|
-
f"`vocabulary`. Please provide `'{self.end_token}'` in "
|
93
|
-
"your `vocabulary` or use a pretrained `vocabulary` name."
|
94
|
-
)
|
95
|
-
|
96
|
-
self.end_token_id = self.token_to_id(self.end_token)
|
97
|
-
self.start_token_id = self.end_token_id
|
98
|
-
self.pad_token_id = 0
|
99
|
-
else:
|
100
|
-
self.end_token_id = None
|
101
|
-
self.start_token_id = None
|
102
|
-
self.pad_token_id = None
|
103
|
-
|
104
|
-
def get_config(self):
|
105
|
-
config = super().get_config()
|
106
|
-
# In the constructor, we pass the list of special tokens to the
|
107
|
-
# `unsplittable_tokens` arg of the superclass' constructor. Hence, we
|
108
|
-
# delete it from the config here.
|
109
|
-
del config["unsplittable_tokens"]
|
110
|
-
return config
|
@@ -14,7 +14,6 @@
|
|
14
14
|
|
15
15
|
from keras_hub.src.models.gemma.gemma_backbone import GemmaBackbone
|
16
16
|
from keras_hub.src.models.gemma.gemma_presets import backbone_presets
|
17
|
-
from keras_hub.src.models.gemma.gemma_tokenizer import GemmaTokenizer
|
18
17
|
from keras_hub.src.utils.preset_utils import register_presets
|
19
18
|
|
20
|
-
register_presets(backbone_presets,
|
19
|
+
register_presets(backbone_presets, GemmaBackbone)
|
@@ -12,19 +12,14 @@
|
|
12
12
|
# See the License for the specific language governing permissions and
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
|
-
import keras
|
16
|
-
from absl import logging
|
17
|
-
|
18
15
|
from keras_hub.src.api_export import keras_hub_export
|
19
|
-
from keras_hub.src.models.
|
20
|
-
from keras_hub.src.
|
21
|
-
|
22
|
-
)
|
23
|
-
from keras_hub.src.utils.tensor_utils import strip_to_ragged
|
16
|
+
from keras_hub.src.models.causal_lm_preprocessor import CausalLMPreprocessor
|
17
|
+
from keras_hub.src.models.gemma.gemma_backbone import GemmaBackbone
|
18
|
+
from keras_hub.src.models.gemma.gemma_tokenizer import GemmaTokenizer
|
24
19
|
|
25
20
|
|
26
21
|
@keras_hub_export("keras_hub.models.GemmaCausalLMPreprocessor")
|
27
|
-
class GemmaCausalLMPreprocessor(
|
22
|
+
class GemmaCausalLMPreprocessor(CausalLMPreprocessor):
|
28
23
|
"""Gemma Causal LM preprocessor.
|
29
24
|
|
30
25
|
This preprocessing layer is meant for use with
|
@@ -84,84 +79,5 @@ class GemmaCausalLMPreprocessor(GemmaPreprocessor):
|
|
84
79
|
```
|
85
80
|
"""
|
86
81
|
|
87
|
-
|
88
|
-
|
89
|
-
x,
|
90
|
-
y=None,
|
91
|
-
sample_weight=None,
|
92
|
-
sequence_length=None,
|
93
|
-
):
|
94
|
-
if y is not None or sample_weight is not None:
|
95
|
-
logging.warning(
|
96
|
-
"`GemmaCausalLMPreprocessor` generates `y` and `sample_weight` "
|
97
|
-
"based on your input data, but your data already contains `y` "
|
98
|
-
"or `sample_weight`. Your `y` and `sample_weight` will be "
|
99
|
-
"ignored."
|
100
|
-
)
|
101
|
-
sequence_length = sequence_length or self.sequence_length
|
102
|
-
|
103
|
-
x = convert_inputs_to_list_of_tensor_segments(x)[0]
|
104
|
-
x = self.tokenizer(x)
|
105
|
-
# Pad with one extra token to account for the truncation below.
|
106
|
-
token_ids, padding_mask = self.packer(
|
107
|
-
x,
|
108
|
-
sequence_length=sequence_length + 1,
|
109
|
-
add_start_value=self.add_start_token,
|
110
|
-
add_end_value=self.add_end_token,
|
111
|
-
)
|
112
|
-
# The last token does not have a next token, so we truncate it out.
|
113
|
-
x = {
|
114
|
-
"token_ids": token_ids[..., :-1],
|
115
|
-
"padding_mask": padding_mask[..., :-1],
|
116
|
-
}
|
117
|
-
# Target `y` will be the next token.
|
118
|
-
y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:]
|
119
|
-
return keras.utils.pack_x_y_sample_weight(x, y, sample_weight)
|
120
|
-
|
121
|
-
def generate_preprocess(
|
122
|
-
self,
|
123
|
-
x,
|
124
|
-
sequence_length=None,
|
125
|
-
):
|
126
|
-
"""Convert strings to integer token input for generation.
|
127
|
-
|
128
|
-
Similar to calling the layer for training, this method takes in strings
|
129
|
-
or tensor strings, tokenizes and packs the input, and computes a padding
|
130
|
-
mask masking all inputs not filled in with a padded value.
|
131
|
-
|
132
|
-
Unlike calling the layer for training, this method does not compute
|
133
|
-
labels and will never append a `tokenizer.end_token_id` to the end of
|
134
|
-
the sequence (as generation is expected to continue at the end of the
|
135
|
-
inputted prompt).
|
136
|
-
"""
|
137
|
-
if not self.built:
|
138
|
-
self.build(None)
|
139
|
-
|
140
|
-
x = convert_inputs_to_list_of_tensor_segments(x)[0]
|
141
|
-
x = self.tokenizer(x)
|
142
|
-
token_ids, padding_mask = self.packer(
|
143
|
-
x, sequence_length=sequence_length, add_end_value=False
|
144
|
-
)
|
145
|
-
return {
|
146
|
-
"token_ids": token_ids,
|
147
|
-
"padding_mask": padding_mask,
|
148
|
-
}
|
149
|
-
|
150
|
-
def generate_postprocess(self, x):
|
151
|
-
"""Convert integer token output to strings for generation.
|
152
|
-
|
153
|
-
This method reverses `generate_preprocess()`, by first removing all
|
154
|
-
padding and start/end tokens, and then converting the integer sequence
|
155
|
-
back to a string.
|
156
|
-
"""
|
157
|
-
if not self.built:
|
158
|
-
self.build(None)
|
159
|
-
|
160
|
-
token_ids, padding_mask = x["token_ids"], x["padding_mask"]
|
161
|
-
ids_to_strip = (
|
162
|
-
self.tokenizer.start_token_id,
|
163
|
-
self.tokenizer.end_token_id,
|
164
|
-
self.tokenizer.pad_token_id,
|
165
|
-
)
|
166
|
-
token_ids = strip_to_ragged(token_ids, padding_mask, ids_to_strip)
|
167
|
-
return self.tokenizer.detokenize(token_ids)
|
82
|
+
backbone_cls = GemmaBackbone
|
83
|
+
tokenizer_cls = GemmaTokenizer
|
@@ -68,7 +68,7 @@ class GemmaDecoderBlock(keras.layers.Layer):
|
|
68
68
|
self.post_attention_norm = RMSNormalization(
|
69
69
|
epsilon=self.layer_norm_epsilon,
|
70
70
|
dtype=self.dtype_policy,
|
71
|
-
name="
|
71
|
+
name="post_attention_norm",
|
72
72
|
)
|
73
73
|
|
74
74
|
self.attention = CachedGemmaAttention(
|
@@ -13,12 +13,18 @@
|
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
15
|
from keras_hub.src.api_export import keras_hub_export
|
16
|
+
from keras_hub.src.models.gemma.gemma_backbone import GemmaBackbone
|
16
17
|
from keras_hub.src.tokenizers.sentence_piece_tokenizer import (
|
17
18
|
SentencePieceTokenizer,
|
18
19
|
)
|
19
20
|
|
20
21
|
|
21
|
-
@keras_hub_export(
|
22
|
+
@keras_hub_export(
|
23
|
+
[
|
24
|
+
"keras_hub.tokenizers.GemmaTokenizer",
|
25
|
+
"keras_hub.models.GemmaTokenizer",
|
26
|
+
]
|
27
|
+
)
|
22
28
|
class GemmaTokenizer(SentencePieceTokenizer):
|
23
29
|
"""Gemma tokenizer layer based on SentencePiece.
|
24
30
|
|
@@ -77,27 +83,10 @@ class GemmaTokenizer(SentencePieceTokenizer):
|
|
77
83
|
```
|
78
84
|
"""
|
79
85
|
|
80
|
-
|
81
|
-
self.start_token = "<bos>"
|
82
|
-
self.end_token = "<eos>"
|
83
|
-
self.pad_token = "<pad>"
|
86
|
+
backbone_cls = GemmaBackbone
|
84
87
|
|
88
|
+
def __init__(self, proto, **kwargs):
|
89
|
+
self._add_special_token("<bos>", "start_token")
|
90
|
+
self._add_special_token("<eos>", "end_token")
|
91
|
+
self._add_special_token("<pad>", "pad_token")
|
85
92
|
super().__init__(proto=proto, **kwargs)
|
86
|
-
|
87
|
-
def set_proto(self, proto):
|
88
|
-
super().set_proto(proto)
|
89
|
-
if proto is not None:
|
90
|
-
for token in [self.end_token, self.pad_token]:
|
91
|
-
if token not in self.get_vocabulary():
|
92
|
-
raise ValueError(
|
93
|
-
f"Cannot find token `'{token}'` in the provided "
|
94
|
-
f"`vocabulary`. Please provide `'{token}'` in your "
|
95
|
-
"`vocabulary` or use a pretrained `vocabulary` name."
|
96
|
-
)
|
97
|
-
self.start_token_id = self.token_to_id(self.start_token)
|
98
|
-
self.end_token_id = self.token_to_id(self.end_token)
|
99
|
-
self.pad_token_id = self.token_to_id(self.pad_token)
|
100
|
-
else:
|
101
|
-
self.start_token_id = None
|
102
|
-
self.end_token_id = None
|
103
|
-
self.pad_token_id = None
|
@@ -14,7 +14,6 @@
|
|
14
14
|
|
15
15
|
from keras_hub.src.models.gpt2.gpt2_backbone import GPT2Backbone
|
16
16
|
from keras_hub.src.models.gpt2.gpt2_presets import backbone_presets
|
17
|
-
from keras_hub.src.models.gpt2.gpt2_tokenizer import GPT2Tokenizer
|
18
17
|
from keras_hub.src.utils.preset_utils import register_presets
|
19
18
|
|
20
|
-
register_presets(backbone_presets,
|
19
|
+
register_presets(backbone_presets, GPT2Backbone)
|