keras-hub-nightly 0.16.1.dev202410020340__py3-none-any.whl → 0.19.0.dev202501260345__py3-none-any.whl

keras_hub/src/utils/timm/convert_efficientnet.py

@@ -0,0 +1,447 @@
+import math
+
+import numpy as np
+
+from keras_hub.src.models.efficientnet.efficientnet_backbone import (
+    EfficientNetBackbone,
+)
+
+backbone_cls = EfficientNetBackbone
+
+
+VARIANT_MAP = {
+    "b0": {
+        "stackwise_width_coefficients": [1.0] * 7,
+        "stackwise_depth_coefficients": [1.0] * 7,
+        "stackwise_squeeze_and_excite_ratios": [0.25] * 7,
+    },
+    "b1": {
+        "stackwise_width_coefficients": [1.0] * 7,
+        "stackwise_depth_coefficients": [1.1] * 7,
+        "stackwise_squeeze_and_excite_ratios": [0.25] * 7,
+    },
+    "b2": {
+        "stackwise_width_coefficients": [1.1] * 7,
+        "stackwise_depth_coefficients": [1.2] * 7,
+        "stackwise_squeeze_and_excite_ratios": [0.25] * 7,
+    },
+    "b3": {
+        "stackwise_width_coefficients": [1.2] * 7,
+        "stackwise_depth_coefficients": [1.4] * 7,
+        "stackwise_squeeze_and_excite_ratios": [0.25] * 7,
+    },
+    "b4": {
+        "stackwise_width_coefficients": [1.4] * 7,
+        "stackwise_depth_coefficients": [1.8] * 7,
+        "stackwise_squeeze_and_excite_ratios": [0.25] * 7,
+    },
+    "b5": {
+        "stackwise_width_coefficients": [1.6] * 7,
+        "stackwise_depth_coefficients": [2.2] * 7,
+        "stackwise_squeeze_and_excite_ratios": [0.25] * 7,
+    },
+    "lite0": {
+        "stackwise_width_coefficients": [1.0] * 7,
+        "stackwise_depth_coefficients": [1.0] * 7,
+        "stackwise_squeeze_and_excite_ratios": [0] * 7,
+        "activation": "relu6",
+    },
+    "el": {
+        "stackwise_width_coefficients": [1.2] * 6,
+        "stackwise_depth_coefficients": [1.4] * 6,
+        "stackwise_kernel_sizes": [3, 3, 3, 5, 5, 5],
+        "stackwise_num_repeats": [1, 2, 4, 5, 4, 2],
+        "stackwise_input_filters": [32, 24, 32, 48, 96, 144],
+        "stackwise_output_filters": [24, 32, 48, 96, 144, 192],
+        "stackwise_expansion_ratios": [4, 8, 8, 8, 8, 8],
+        "stackwise_strides": [1, 2, 2, 2, 1, 2],
+        "stackwise_squeeze_and_excite_ratios": [0] * 6,
+        "stackwise_block_types": ["fused"] * 3 + ["unfused"] * 3,
+        "stackwise_force_input_filters": [24, 0, 0, 0, 0, 0],
+        "stackwise_nores_option": [True] + [False] * 5,
+        "activation": "relu",
+    },
+    "em": {
+        "stackwise_width_coefficients": [1.0] * 6,
+        "stackwise_depth_coefficients": [1.1] * 6,
+        "stackwise_kernel_sizes": [3, 3, 3, 5, 5, 5],
+        "stackwise_num_repeats": [1, 2, 4, 5, 4, 2],
+        "stackwise_input_filters": [32, 24, 32, 48, 96, 144],
+        "stackwise_output_filters": [24, 32, 48, 96, 144, 192],
+        "stackwise_expansion_ratios": [4, 8, 8, 8, 8, 8],
+        "stackwise_strides": [1, 2, 2, 2, 1, 2],
+        "stackwise_squeeze_and_excite_ratios": [0] * 6,
+        "stackwise_block_types": ["fused"] * 3 + ["unfused"] * 3,
+        "stackwise_force_input_filters": [24, 0, 0, 0, 0, 0],
+        "stackwise_nores_option": [True] + [False] * 5,
+        "activation": "relu",
+    },
+    "es": {
+        "stackwise_width_coefficients": [1.0] * 6,
+        "stackwise_depth_coefficients": [1.0] * 6,
+        "stackwise_kernel_sizes": [3, 3, 3, 5, 5, 5],
+        "stackwise_num_repeats": [1, 2, 4, 5, 4, 2],
+        "stackwise_input_filters": [32, 24, 32, 48, 96, 144],
+        "stackwise_output_filters": [24, 32, 48, 96, 144, 192],
+        "stackwise_expansion_ratios": [4, 8, 8, 8, 8, 8],
+        "stackwise_strides": [1, 2, 2, 2, 1, 2],
+        "stackwise_squeeze_and_excite_ratios": [0] * 6,
+        "stackwise_block_types": ["fused"] * 3 + ["unfused"] * 3,
+        "stackwise_force_input_filters": [24, 0, 0, 0, 0, 0],
+        "stackwise_nores_option": [True] + [False] * 5,
+        "activation": "relu",
+    },
+    "rw_m": {
+        "stackwise_width_coefficients": [1.2] * 6,
+        "stackwise_depth_coefficients": [1.2] * 4 + [1.6] * 2,
+        "stackwise_kernel_sizes": [3, 3, 3, 3, 3, 3],
+        "stackwise_num_repeats": [2, 4, 4, 6, 9, 15],
+        "stackwise_input_filters": [24, 24, 48, 64, 128, 160],
+        "stackwise_output_filters": [24, 48, 64, 128, 160, 272],
+        "stackwise_expansion_ratios": [1, 4, 4, 4, 6, 6],
+        "stackwise_strides": [1, 2, 2, 2, 1, 2],
+        "stackwise_squeeze_and_excite_ratios": [0, 0, 0, 0.25, 0.25, 0.25],
+        "stackwise_block_types": ["fused"] * 3 + ["unfused"] * 3,
+        "stackwise_force_input_filters": [0, 0, 0, 0, 0, 0],
+        "stackwise_nores_option": [False] * 6,
+        "activation": "silu",
+        "num_features": 1792,
+    },
+    "rw_s": {
+        "stackwise_width_coefficients": [1.0] * 6,
+        "stackwise_depth_coefficients": [1.0] * 6,
+        "stackwise_kernel_sizes": [3, 3, 3, 3, 3, 3],
+        "stackwise_num_repeats": [2, 4, 4, 6, 9, 15],
+        "stackwise_input_filters": [24, 24, 48, 64, 128, 160],
+        "stackwise_output_filters": [24, 48, 64, 128, 160, 272],
+        "stackwise_expansion_ratios": [1, 4, 4, 4, 6, 6],
+        "stackwise_strides": [1, 2, 2, 2, 1, 2],
+        "stackwise_squeeze_and_excite_ratios": [0, 0, 0, 0.25, 0.25, 0.25],
+        "stackwise_block_types": ["fused"] * 3 + ["unfused"] * 3,
+        "stackwise_force_input_filters": [0, 0, 0, 0, 0, 0],
+        "stackwise_nores_option": [False] * 6,
+        "activation": "silu",
+        "num_features": 1792,
+    },
+    "rw_t": {
+        "stackwise_width_coefficients": [0.8] * 6,
+        "stackwise_depth_coefficients": [0.9] * 6,
+        "stackwise_kernel_sizes": [3, 3, 3, 3, 3, 3],
+        "stackwise_num_repeats": [2, 4, 4, 6, 9, 15],
+        "stackwise_input_filters": [24, 24, 48, 64, 128, 160],
+        "stackwise_output_filters": [24, 48, 64, 128, 160, 256],
+        "stackwise_expansion_ratios": [1, 4, 4, 4, 6, 6],
+        "stackwise_strides": [1, 2, 2, 2, 1, 2],
+        "stackwise_squeeze_and_excite_ratios": [0, 0, 0, 0.25, 0.25, 0.25],
+        "stackwise_block_types": ["cba"] + ["fused"] * 2 + ["unfused"] * 3,
+        "stackwise_force_input_filters": [0, 0, 0, 0, 0, 0],
+        "stackwise_nores_option": [False] * 6,
+        "activation": "silu",
+    },
+}
+
+
+def convert_backbone_config(timm_config):
+    timm_architecture = timm_config["architecture"]
+
+    base_kwargs = {
+        "stackwise_kernel_sizes": [3, 3, 5, 3, 5, 5, 3],
+        "stackwise_num_repeats": [1, 2, 2, 3, 3, 4, 1],
+        "stackwise_input_filters": [32, 16, 24, 40, 80, 112, 192],
+        "stackwise_output_filters": [16, 24, 40, 80, 112, 192, 320],
+        "stackwise_expansion_ratios": [1, 6, 6, 6, 6, 6, 6],
+        "stackwise_strides": [1, 2, 2, 2, 1, 2, 1],
+        "stackwise_block_types": ["v1"] * 7,
+        "min_depth": None,
+        "include_stem_padding": True,
+        "use_depth_divisor_as_min_depth": True,
+        "cap_round_filter_decrease": True,
+        "stem_conv_padding": "valid",
+        "batch_norm_momentum": 0.9,
+        "batch_norm_epsilon": 1e-5,
+        "dropout": 0,
+        "projection_activation": None,
+    }
+
+    variant = "_".join(timm_architecture.split("_")[1:])
+
+    if variant not in VARIANT_MAP:
+        raise ValueError(
+            f"Currently, the architecture {timm_architecture} is not supported."
+        )
+
+    base_kwargs.update(VARIANT_MAP[variant])
+
+    return base_kwargs
+
+
+def convert_weights(backbone, loader, timm_config):
+    timm_architecture = timm_config["architecture"]
+    variant = "_".join(timm_architecture.split("_")[1:])
+
+    def port_conv2d(keras_layer, hf_weight_prefix, port_bias=True):
+        loader.port_weight(
+            keras_layer.kernel,
+            hf_weight_key=f"{hf_weight_prefix}.weight",
+            hook_fn=lambda x, _: np.transpose(x, (2, 3, 1, 0)),
+        )
+
+        if port_bias:
+            loader.port_weight(
+                keras_layer.bias,
+                hf_weight_key=f"{hf_weight_prefix}.bias",
+            )
+
+    def port_depthwise_conv2d(
+        keras_layer,
+        hf_weight_prefix,
+        port_bias=True,
+        depth_multiplier=1,
+    ):
+        def convert_pt_conv2d_kernel(pt_kernel):
+            out_channels, in_channels_per_group, height, width = pt_kernel.shape
+            # PT Convs are depthwise convs if and only if
+            # `in_channels_per_group == 1`
+            assert in_channels_per_group == 1
+            pt_kernel = np.transpose(pt_kernel, (2, 3, 0, 1))
+            in_channels = out_channels // depth_multiplier
+            return np.reshape(
+                pt_kernel, (height, width, in_channels, depth_multiplier)
+            )
+
+        loader.port_weight(
+            keras_layer.kernel,
+            hf_weight_key=f"{hf_weight_prefix}.weight",
+            hook_fn=lambda x, _: convert_pt_conv2d_kernel(x),
+        )
+
+        if port_bias:
+            loader.port_weight(
+                keras_layer.bias,
+                hf_weight_key=f"{hf_weight_prefix}.bias",
+            )
+
+    def port_batch_normalization(keras_layer, hf_weight_prefix):
+        loader.port_weight(
+            keras_layer.gamma,
+            hf_weight_key=f"{hf_weight_prefix}.weight",
+        )
+        loader.port_weight(
+            keras_layer.beta,
+            hf_weight_key=f"{hf_weight_prefix}.bias",
+        )
+        loader.port_weight(
+            keras_layer.moving_mean,
+            hf_weight_key=f"{hf_weight_prefix}.running_mean",
+        )
+        loader.port_weight(
+            keras_layer.moving_variance,
+            hf_weight_key=f"{hf_weight_prefix}.running_var",
+        )
+        # do we need num batches tracked?
+
+    # Stem
+    port_conv2d(backbone.get_layer("stem_conv"), "conv_stem", port_bias=False)
+    port_batch_normalization(backbone.get_layer("stem_bn"), "bn1")
+
+    # Stages
+    num_stacks = len(backbone.stackwise_kernel_sizes)
+
+    for stack_index in range(num_stacks):
+        block_type = backbone.stackwise_block_types[stack_index]
+        expansion_ratio = backbone.stackwise_expansion_ratios[stack_index]
+        repeats = backbone.stackwise_num_repeats[stack_index]
+        stack_depth_coefficient = backbone.stackwise_depth_coefficients[
+            stack_index
+        ]
+
+        repeats = int(math.ceil(stack_depth_coefficient * repeats))
+
+        se_ratio = VARIANT_MAP[variant]["stackwise_squeeze_and_excite_ratios"][
+            stack_index
+        ]
+
+        for block_idx in range(repeats):
+            conv_pw_count = 0
+            bn_count = 1
+
+            # 97 is the start of the lowercase alphabet.
+            letter_identifier = chr(block_idx + 97)
+
+            keras_block_prefix = f"block{stack_index + 1}{letter_identifier}_"
+            hf_block_prefix = f"blocks.{stack_index}.{block_idx}."
+
+            if block_type == "v1":
+                conv_pw_name_map = ["conv_pw", "conv_pwl"]
+                # Initial Expansion Conv
+                if expansion_ratio != 1:
+                    port_conv2d(
+                        backbone.get_layer(keras_block_prefix + "expand_conv"),
+                        hf_block_prefix + conv_pw_name_map[conv_pw_count],
+                        port_bias=False,
+                    )
+                    conv_pw_count += 1
+                    port_batch_normalization(
+                        backbone.get_layer(keras_block_prefix + "expand_bn"),
+                        hf_block_prefix + f"bn{bn_count}",
+                    )
+                    bn_count += 1
+
+                # Depthwise Conv
+                port_depthwise_conv2d(
+                    backbone.get_layer(keras_block_prefix + "dwconv"),
+                    hf_block_prefix + "conv_dw",
+                    port_bias=False,
+                )
+                port_batch_normalization(
+                    backbone.get_layer(keras_block_prefix + "dwconv_bn"),
+                    hf_block_prefix + f"bn{bn_count}",
+                )
+                bn_count += 1
+
+                if 0 < se_ratio <= 1:
+                    # Squeeze and Excite
+                    port_conv2d(
+                        backbone.get_layer(keras_block_prefix + "se_reduce"),
+                        hf_block_prefix + "se.conv_reduce",
+                    )
+                    port_conv2d(
+                        backbone.get_layer(keras_block_prefix + "se_expand"),
+                        hf_block_prefix + "se.conv_expand",
+                    )
+
+                # Output/Projection
+                port_conv2d(
+                    backbone.get_layer(keras_block_prefix + "project"),
+                    hf_block_prefix + conv_pw_name_map[conv_pw_count],
+                    port_bias=False,
+                )
+                conv_pw_count += 1
+                port_batch_normalization(
+                    backbone.get_layer(keras_block_prefix + "project_bn"),
+                    hf_block_prefix + f"bn{bn_count}",
+                )
+                bn_count += 1
+            elif block_type == "fused":
+                fused_block_layer = backbone.get_layer(keras_block_prefix)
+
+                # Initial Expansion Conv
+                port_conv2d(
+                    fused_block_layer.conv1,
+                    hf_block_prefix + "conv_exp",
+                    port_bias=False,
+                )
+                conv_pw_count += 1
+                port_batch_normalization(
+                    fused_block_layer.bn1,
+                    hf_block_prefix + f"bn{bn_count}",
+                )
+                bn_count += 1
+
+                if 0 < se_ratio <= 1:
+                    # Squeeze and Excite
+                    port_conv2d(
+                        fused_block_layer.se_conv1,
+                        hf_block_prefix + "se.conv_reduce",
+                    )
+                    port_conv2d(
+                        fused_block_layer.se_conv2,
+                        hf_block_prefix + "se.conv_expand",
+                    )
+
+                # Output/Projection
+                port_conv2d(
+                    fused_block_layer.output_conv,
+                    hf_block_prefix + "conv_pwl",
+                    port_bias=False,
+                )
+                conv_pw_count += 1
+                port_batch_normalization(
+                    fused_block_layer.bn2,
+                    hf_block_prefix + f"bn{bn_count}",
+                )
+                bn_count += 1
+
+            elif block_type == "unfused":
+                unfused_block_layer = backbone.get_layer(keras_block_prefix)
+                # Initial Expansion Conv
+                if expansion_ratio != 1:
+                    port_conv2d(
+                        unfused_block_layer.conv1,
+                        hf_block_prefix + "conv_pw",
+                        port_bias=False,
+                    )
+                    conv_pw_count += 1
+                    port_batch_normalization(
+                        unfused_block_layer.bn1,
+                        hf_block_prefix + f"bn{bn_count}",
+                    )
+                    bn_count += 1
+
+                # Depthwise Conv
+                port_depthwise_conv2d(
+                    unfused_block_layer.depthwise,
+                    hf_block_prefix + "conv_dw",
+                    port_bias=False,
+                )
+                port_batch_normalization(
+                    unfused_block_layer.bn2,
+                    hf_block_prefix + f"bn{bn_count}",
+                )
+                bn_count += 1
+
+                if 0 < se_ratio <= 1:
+                    # Squeeze and Excite
+                    port_conv2d(
+                        unfused_block_layer.se_conv1,
+                        hf_block_prefix + "se.conv_reduce",
+                    )
+                    port_conv2d(
+                        unfused_block_layer.se_conv2,
+                        hf_block_prefix + "se.conv_expand",
+                    )
+
+                # Output/Projection
+                port_conv2d(
+                    unfused_block_layer.output_conv,
+                    hf_block_prefix + "conv_pwl",
+                    port_bias=False,
+                )
+                conv_pw_count += 1
+                port_batch_normalization(
+                    unfused_block_layer.bn3,
+                    hf_block_prefix + f"bn{bn_count}",
+                )
+                bn_count += 1
+            elif block_type == "cba":
+                cba_block_layer = backbone.get_layer(keras_block_prefix)
+                # Initial Expansion Conv
+                port_conv2d(
+                    cba_block_layer.conv1,
+                    hf_block_prefix + "conv",
+                    port_bias=False,
+                )
+                conv_pw_count += 1
+                port_batch_normalization(
+                    cba_block_layer.bn1,
+                    hf_block_prefix + f"bn{bn_count}",
+                )
+                bn_count += 1
+
+    # Head/Top
+    port_conv2d(backbone.get_layer("top_conv"), "conv_head", port_bias=False)
+    port_batch_normalization(backbone.get_layer("top_bn"), "bn2")
+
+
+def convert_head(task, loader, timm_config):
+    classifier_prefix = timm_config["pretrained_cfg"]["classifier"]
+    prefix = f"{classifier_prefix}."
+    loader.port_weight(
+        task.output_dense.kernel,
+        hf_weight_key=prefix + "weight",
+        hook_fn=lambda x, _: np.transpose(np.squeeze(x)),
+    )
+    loader.port_weight(
+        task.output_dense.bias,
+        hf_weight_key=prefix + "bias",
+    )

keras_hub/src/utils/timm/convert_resnet.py

@@ -89,7 +89,7 @@ def convert_weights(backbone, loader, timm_config):
         for block_idx in range(backbone.stackwise_num_blocks[stack_index]):
             if version == "v1":
                 keras_name = f"stack{stack_index}_block{block_idx}"
-                hf_name = f"layer{stack_index+1}.{block_idx}"
+                hf_name = f"layer{stack_index + 1}.{block_idx}"
             else:
                 keras_name = f"stack{stack_index}_block{block_idx}"
                 hf_name = f"stages.{stack_index}.blocks.{block_idx}"

keras_hub/src/utils/timm/convert_vgg.py

@@ -0,0 +1,85 @@
+from typing import Any
+
+import numpy as np
+
+from keras_hub.src.models.vgg.vgg_backbone import VGGBackbone
+from keras_hub.src.models.vgg.vgg_image_classifier import VGGImageClassifier
+
+backbone_cls = VGGBackbone
+
+
+REPEATS_BY_SIZE = {
+    "vgg11": [1, 1, 2, 2, 2],
+    "vgg13": [2, 2, 2, 2, 2],
+    "vgg16": [2, 2, 3, 3, 3],
+    "vgg19": [2, 2, 4, 4, 4],
+}
+
+
+def convert_backbone_config(timm_config):
+    architecture = timm_config["architecture"]
+    stackwise_num_repeats = REPEATS_BY_SIZE[architecture]
+    return dict(
+        stackwise_num_repeats=stackwise_num_repeats,
+        stackwise_num_filters=[64, 128, 256, 512, 512],
+    )
+
+
+def convert_conv2d(
+    model,
+    loader,
+    keras_layer_name: str,
+    hf_layer_name: str,
+):
+    loader.port_weight(
+        model.get_layer(keras_layer_name).kernel,
+        hf_weight_key=f"{hf_layer_name}.weight",
+        hook_fn=lambda x, _: np.transpose(x, (2, 3, 1, 0)),
+    )
+    loader.port_weight(
+        model.get_layer(keras_layer_name).bias,
+        hf_weight_key=f"{hf_layer_name}.bias",
+    )
+
+
+def convert_weights(
+    backbone: VGGBackbone,
+    loader,
+    timm_config: dict[Any],
+):
+    architecture = timm_config["architecture"]
+    stackwise_num_repeats = REPEATS_BY_SIZE[architecture]
+
+    hf_index_to_keras_layer_name = {}
+    layer_index = 0
+    for block_index, repeats_in_block in enumerate(stackwise_num_repeats):
+        for repeat_index in range(repeats_in_block):
+            hf_index = layer_index
+            layer_index += 2  # Conv + activation layers.
+            layer_name = f"block{block_index + 1}_conv{repeat_index + 1}"
+            hf_index_to_keras_layer_name[hf_index] = layer_name
+        layer_index += 1  # Pooling layer after blocks.
+
+    for hf_index, keras_layer_name in hf_index_to_keras_layer_name.items():
+        convert_conv2d(
+            backbone, loader, keras_layer_name, f"features.{hf_index}"
+        )
+
+
+def convert_head(
+    task: VGGImageClassifier,
+    loader,
+    timm_config: dict[Any],
+):
+    convert_conv2d(task.head, loader, "fc1", "pre_logits.fc1")
+    convert_conv2d(task.head, loader, "fc2", "pre_logits.fc2")
+
+    loader.port_weight(
+        task.head.get_layer("predictions").kernel,
+        hf_weight_key="head.fc.weight",
+        hook_fn=lambda x, _: np.transpose(np.squeeze(x)),
+    )
+    loader.port_weight(
+        task.head.get_layer("predictions").bias,
+        hf_weight_key="head.fc.bias",
+    )

keras_hub/src/utils/timm/preset_loader.py

@@ -4,7 +4,9 @@ from keras_hub.src.models.image_classifier import ImageClassifier
 from keras_hub.src.utils.preset_utils import PresetLoader
 from keras_hub.src.utils.preset_utils import jax_memory_cleanup
 from keras_hub.src.utils.timm import convert_densenet
+from keras_hub.src.utils.timm import convert_efficientnet
 from keras_hub.src.utils.timm import convert_resnet
+from keras_hub.src.utils.timm import convert_vgg
 from keras_hub.src.utils.transformers.safetensor_utils import SafetensorLoader
 
 
@@ -14,8 +16,12 @@ class TimmPresetLoader(PresetLoader):
         architecture = self.config["architecture"]
         if "resnet" in architecture:
             self.converter = convert_resnet
-        if "densenet" in architecture:
+        elif "densenet" in architecture:
             self.converter = convert_densenet
+        elif "vgg" in architecture:
+            self.converter = convert_vgg
+        elif "efficientnet" in architecture:
+            self.converter = convert_efficientnet
         else:
             raise ValueError(
                 "KerasHub has no converter for timm models "
@@ -52,20 +58,19 @@ class TimmPresetLoader(PresetLoader):
         pretrained_cfg = self.config.get("pretrained_cfg", None)
         if not pretrained_cfg or "input_size" not in pretrained_cfg:
             return None
-        # This assumes the same basic setup for all timm preprocessing, and that
-        # all our image conversion will be via a `ResizingImageConverter. We may
+        # This assumes the same basic setup for all timm preprocessing, We may
         # need to extend this as we cover more model types.
         input_size = pretrained_cfg["input_size"]
         mean = pretrained_cfg["mean"]
-        variance = [s**2 for s in pretrained_cfg["std"]]
+        std = pretrained_cfg["std"]
+        scale = [1.0 / 255.0 / s for s in std]
+        offset = [-m / s for m, s in zip(mean, std)]
         interpolation = pretrained_cfg["interpolation"]
         if interpolation not in ("bilinear", "nearest", "bicubic"):
             interpolation = "bilinear"  # Unsupported interpolation type.
         return cls(
-            width=input_size[1],
-            height=input_size[2],
-            scale=1 / 255.0,
-            mean=mean,
-            variance=variance,
+            image_size=input_size[1:],
+            scale=scale,
+            offset=offset,
             interpolation=interpolation,
         )

keras_hub/src/utils/transformers/convert_llama3.py

@@ -107,10 +107,26 @@ def convert_tokenizer(cls, preset, **kwargs):
     vocab = tokenizer_config["model"]["vocab"]
     merges = tokenizer_config["model"]["merges"]
 
-    bot = tokenizer_config["added_tokens"][0]  # begin of text
-    eot = tokenizer_config["added_tokens"][1]  # end of text
-
-    vocab[bot["content"]] = bot["id"]
-    vocab[eot["content"]] = eot["id"]
+    # Load all special tokens with the exception of "reserved" ones.
+    special_tokens = set()
+    for token in tokenizer_config["added_tokens"]:
+        if not token["content"].startswith("<|reserved_special_token_"):
+            vocab[token["content"]] = token["id"]
+            special_tokens.add(token["content"])
+
+    # Load text start and stop tokens from the config.
+    # Llama3 uses the <|end_of_text|> end token for regular models
+    # but uses <|eot_id|> for instruction-tuned  variants.
+    tokenizer_config2 = load_json(preset, "tokenizer_config.json")
+    bos_token = tokenizer_config2["bos_token"]
+    eos_token = tokenizer_config2["eos_token"]
+
+    kwargs.update(
+        {
+            "bos_token": bos_token,
+            "eos_token": eos_token,
+            "misc_special_tokens": special_tokens,
+        }
+    )
 
     return cls(vocabulary=vocab, merges=merges, **kwargs)