tpu-inference 0.12.0.dev20251222__py3-none-any.whl

tpu_inference/models/vllm/vllm_model_wrapper.py

@@ -0,0 +1,307 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import functools
+from collections.abc import Sequence
+from contextlib import nullcontext
+from typing import Any, List, Optional, Tuple
+from unittest.mock import patch
+
+import jax
+import torch
+import torch.nn
+import torchax
+import vllm.envs as vllm_envs
+from flax.typing import PRNGKey
+from jax.sharding import Mesh, NamedSharding, PartitionSpec
+from torchax.interop import jax_view, torch_view
+from torchax.ops.mappings import TORCH_DTYPE_TO_JAX
+from vllm.config import VllmConfig
+from vllm.forward_context import set_forward_context
+from vllm.lora.layers import BaseLayerWithLoRA
+from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
+from vllm.model_executor.model_loader import get_model as vllm_get_model
+from vllm.model_executor.models import supports_lora, supports_multimodal
+from vllm.sequence import IntermediateTensors
+
+from tpu_inference.layers.common.attention_metadata import AttentionMetadata
+from tpu_inference.layers.vllm.quantization import get_tpu_quantization_config
+from tpu_inference.layers.vllm.sharding import shard_model_to_tpu
+from tpu_inference.logger import init_logger
+from tpu_inference.models.jax.jax_intermediate_tensor import \
+    JaxIntermediateTensors
+from tpu_inference.models.vllm.vllm_model_wrapper_context import (
+    get_vllm_model_wrapper_context, set_vllm_model_wrapper_context)
+from tpu_inference.runner.lora_utils import replace_lora_metadata
+
+logger = init_logger(__name__)
+
+
+class _VllmRunner(torch.nn.Module):
+
+    def __init__(self, vllm_model: torch.nn.Module):
+        super().__init__()
+        self.vllm_model = vllm_model
+
+    def forward(self, **kwargs) -> torch.Tensor:
+        if "hidden_state" in kwargs:
+            return self.compute_logits(kwargs["hidden_state"])
+        else:
+            return self.compute_hidden_state(
+                kwargs["input_ids"],
+                kwargs["positions"],
+                kwargs["intermediate_tensors"],
+                kwargs["inputs_embeds"],
+            )
+
+    def compute_hidden_state(
+        self,
+        input_ids: torch.Tensor,
+        positions: torch.Tensor,
+        intermediate_tensors: Optional[IntermediateTensors],
+        inputs_embeds: Optional[torch.Tensor],
+    ) -> torch.Tensor:
+        hidden_state = self.vllm_model(input_ids, positions,
+                                       intermediate_tensors, inputs_embeds)
+        return hidden_state
+
+    def compute_logits(self, hidden_state: torch.Tensor) -> torch.Tensor:
+        return self.vllm_model.compute_logits(hidden_state)
+
+
+class VllmModelWrapper:
+    """ Wraps a vLLM Pytorch model and let it run on the JAX engine. """
+
+    rng: PRNGKey
+    mesh: Mesh
+    model: _VllmRunner
+
+    def __init__(self, vllm_config: VllmConfig, rng: PRNGKey, mesh: Mesh):
+        self.vllm_config = vllm_config
+        self.rng = rng
+        self.mesh = mesh
+
+        self.vllm_config.quant_config = get_tpu_quantization_config(
+            self.vllm_config, self.mesh)
+
+    def load_weights(self):
+        # Set up to load the model into CPU first.
+        # Cache device slice config since device config cannot be deepcopied
+        modified_slice_config = False
+        if hasattr(
+                self.vllm_config.device_config,
+                'slice') and self.vllm_config.device_config.slice is not None:
+            slice_config = self.vllm_config.device_config.slice
+            modified_slice_config = True
+            self.vllm_config.device_config.slice = None
+        self.vllm_config.compilation_config.static_forward_context.clear()
+
+        vllm_config_for_load = copy.deepcopy(self.vllm_config)
+        if modified_slice_config:
+            self.vllm_config.device_config.slice = slice_config
+        assert self.vllm_config.model_config.dtype in TORCH_DTYPE_TO_JAX, "The model_config.dtype must be a PyTorch dtype."
+        vllm_config_for_load.device_config.device = "cpu"
+        # Clearing the cached compilation config, otherwise vllm model init will fail
+
+        # When expert parallelism is enabled, vLLM loads weight in sharding
+        # aware manner. Since tpu-inference has its own sharding logic, this
+        # may casue errors. Therefore, we disable it during weight loading.
+        vllm_config_for_load.parallel_config.enable_expert_parallel = False
+
+        use_random_weights = (
+            vllm_config_for_load.load_config.load_format == "dummy")
+        if use_random_weights:
+            logger.info(
+                "Initializing vLLM model with random weights, weight loading skipped."
+            )
+        # The DummyModelLoader in vLLM calls torch._sync for torch_xla path when
+        # it detects the tpu platform, but we don't need it and it causes crash
+        # without proper setup.
+        load_context = patch(
+            "torch._sync",
+            return_value=None) if use_random_weights else nullcontext()
+
+        # By default load weights to the CPU device first. If we are running
+        # under Pathways, this would cause weights to be loaded on a CPU-only
+        # node, so we'll need to remove this context.
+        jax_context = jax.default_device(
+            jax.devices("cpu")
+            [0]) if not vllm_envs.VLLM_TPU_USING_PATHWAYS else nullcontext()
+
+        # Load the vLLM model and wrap it into a new model whose forward
+        # function can calculate the hidden_state and logits.
+        with load_context, jax_context:
+            vllm_model = vllm_get_model(vllm_config=vllm_config_for_load)
+        lora_manager = None
+        if vllm_config_for_load.lora_config is not None:
+            # Replace layers in the model with LoRA layers.
+            with torchax.default_env():
+                # Argument "device" in load_lora_model is used to set the device
+                # used in punica wrapper.
+                lora_manager, vllm_model = load_lora_model(
+                    vllm_model, vllm_config_for_load, device="jax")
+            replace_set_lora(vllm_model)
+
+        static_forward_context = vllm_config_for_load.compilation_config.static_forward_context
+        self.vllm_config.compilation_config.static_forward_context = static_forward_context
+
+        self.model = _VllmRunner(vllm_model)
+        params_and_buffers = shard_model_to_tpu(self.model, self.mesh)
+
+        # Returning to the jax land, so we need to wrap it into a JaxValue.
+        return jax_view(params_and_buffers), lora_manager
+
+    def jit_step_func(self):
+
+        @functools.partial(
+            jax.jit,
+            donate_argnames=("kv_caches", ),
+            compiler_options={
+                "xla_tpu_all_gather_collective_matmul_mode":
+                "post_spmd_conservative",
+                "xla_tpu_reduce_scatter_collective_matmul_mode":
+                "post_spmd_conservative"
+            },
+            static_argnames=("layer_name_to_kvcache_index", "is_first_rank",
+                             "is_last_rank"),
+        )
+        def step_fun(
+            params_and_buffers,  # This has been wrapped into torchax TorchValue
+            kv_caches: List[jax.Array],
+            input_ids: jax.Array,
+            attn_metadata: AttentionMetadata,
+            input_embeds: jax.Array,
+            input_positions: jax.Array,
+            layer_name_to_kvcache_index: Sequence[Tuple[str, int]],
+            lora_metadata,
+            intermediate_tensors: JaxIntermediateTensors = None,
+            is_first_rank: bool = True,
+            is_last_rank: bool = True,
+            *args,
+        ) -> Tuple[List[jax.Array], jax.Array]:
+            layer_name_to_kvcache_index = dict(layer_name_to_kvcache_index)
+            lora_metadata = torch_view(lora_metadata)
+            with torchax.default_env(), set_vllm_model_wrapper_context(
+                    kv_caches=kv_caches,
+                    mesh=self.mesh,
+                    layer_name_to_kvcache_index=layer_name_to_kvcache_index
+            ), set_forward_context(attn_metadata=attn_metadata,
+                                   vllm_config=self.vllm_config):
+                # We need to wrap args from jax land into TorchValue with
+                # torch_view in order to call the Torch function.
+                original_lora_metadata = replace_lora_metadata(
+                    self.model, lora_metadata, self.vllm_config.lora_config)
+                if not is_first_rank:
+                    intermediate_tensors = intermediate_tensors.to_torch()
+                output_from_torch = torch.func.functional_call(
+                    self.model,
+                    torch_view(params_and_buffers),
+                    kwargs={
+                        "input_ids": torch_view(input_ids),
+                        "positions": torch_view(input_positions),
+                        "intermediate_tensors": intermediate_tensors,
+                        "inputs_embeds": None,
+                    },
+                    tie_weights=False,
+                )
+                replace_lora_metadata(self.model, original_lora_metadata,
+                                      self.vllm_config.lora_config)
+                vllm_model_wrapper_context = get_vllm_model_wrapper_context()
+                new_kv_caches = vllm_model_wrapper_context.kv_caches
+            # Wrap the output(hidden states or intermediate tensor)
+            # from torch land into a JaxValue for the jax code to consume.
+            if not is_last_rank:
+                output = JaxIntermediateTensors.from_torch(output_from_torch)
+            else:
+                output = jax_view(output_from_torch)
+            return new_kv_caches, output, []
+
+        return step_fun
+
+    def jit_compute_logits_func(self):
+
+        @functools.partial(
+            jax.jit,
+            out_shardings=(NamedSharding(self.mesh,
+                                         PartitionSpec("data", "model"))),
+        )
+        def compute_logits_func(
+            params_and_buffers: Any,
+            hidden_states: jax.Array,
+            lora_metadata,
+        ) -> jax.Array:
+            lora_metadata = torch_view(lora_metadata)
+            with torchax.default_env(), set_vllm_model_wrapper_context(
+                    kv_caches=None, mesh=self.mesh):
+                original_lora_metadata = replace_lora_metadata(
+                    self.model, lora_metadata, self.vllm_config.lora_config)
+                logits = torch.func.functional_call(
+                    self.model,
+                    torch_view(params_and_buffers),
+                    kwargs={
+                        "hidden_state": torch_view(hidden_states),
+                    },
+                    tie_weights=False,
+                )
+                replace_lora_metadata(self.model, original_lora_metadata,
+                                      self.vllm_config.lora_config)
+            return jax_view(logits)
+
+        return compute_logits_func
+
+
+def load_lora_model(model: torch.nn.Module, vllm_config: VllmConfig,
+                    device: str) -> torch.nn.Module:
+    if not supports_lora(model):
+        raise ValueError(
+            f"{model.__class__.__name__} does not support LoRA yet.")
+
+    if supports_multimodal(model):
+        logger.warning("Regarding multimodal models, vLLM currently "
+                       "only supports adding LoRA to language model.")
+
+    # Add LoRA Manager to the Model Runner
+    lora_manager = LRUCacheWorkerLoRAManager(
+        vllm_config,
+        device,
+        model.embedding_modules,
+    )
+    return lora_manager, lora_manager.create_lora_manager(model)
+
+
+# The reason why replace the method is that the set_lora and reset_lora need to
+# run under torchax env.
+def replace_set_lora(model):
+
+    def _tpu_set_lora(
+        self,
+        index: int,
+        lora_a: torch.Tensor,
+        lora_b: torch.Tensor,
+    ):
+        with torchax.default_env():
+            self._original_set_lora(index, lora_a, lora_b)
+
+    def _tpu_reset_lora(self, index: int):
+        with torchax.default_env():
+            self._original_reset_lora(index)
+
+    for _, module in model.named_modules():
+        if isinstance(module, BaseLayerWithLoRA):
+            module._original_set_lora = module.set_lora
+            module._original_reset_lora = module.reset_lora
+            module.set_lora = _tpu_set_lora.__get__(module, module.__class__)
+            module.reset_lora = _tpu_reset_lora.__get__(
+                module, module.__class__)

tpu_inference/models/vllm/vllm_model_wrapper_context.py

@@ -0,0 +1,59 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Dict, List, Optional
+
+import jax
+from jax.sharding import Mesh
+
+
+@dataclass
+class VllmModelWrapperContext:
+    kv_caches: List[jax.Array]
+    mesh: Mesh
+    layer_name_to_kvcache_index: Dict[str, int]
+
+
+_vllm_model_wrapper_context: Optional[VllmModelWrapperContext] = None
+
+
+def get_vllm_model_wrapper_context() -> VllmModelWrapperContext:
+    assert _vllm_model_wrapper_context is not None, (
+        "VllmModelWrapperContext is not set. "
+        "Please use `set_vllm_model_wrapper_context` to set the VllmModelWrapperContext."
+    )
+    return _vllm_model_wrapper_context
+
+
+@contextmanager
+def set_vllm_model_wrapper_context(
+    *,
+    kv_caches: List[jax.Array],
+    mesh: Mesh,
+    layer_name_to_kvcache_index: Dict[str, int] = None,
+):
+    global _vllm_model_wrapper_context
+    prev_context = _vllm_model_wrapper_context
+    _vllm_model_wrapper_context = VllmModelWrapperContext(
+        kv_caches=kv_caches,
+        mesh=mesh,
+        layer_name_to_kvcache_index=layer_name_to_kvcache_index,
+    )
+
+    try:
+        yield
+    finally:
+        _vllm_model_wrapper_context = prev_context

tpu_inference/platforms/__init__.py

@@ -0,0 +1,16 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ruff: noqa
+from tpu_inference.platforms.tpu_platform import TpuPlatform

tpu_inference/platforms/tpu_platform.py

@@ -0,0 +1,258 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import TYPE_CHECKING, Optional, Tuple, Union, cast
+
+import jax.numpy as jnp
+import torch
+import vllm.envs as vllm_envs
+from tpu_info import device
+from vllm.inputs import ProcessorInputs, PromptType
+from vllm.platforms.interface import Platform, PlatformEnum
+
+from tpu_inference import envs
+from tpu_inference.layers.common.sharding import ShardingConfigManager
+from tpu_inference.logger import init_logger
+
+if TYPE_CHECKING:
+    from vllm.attention.backends.registry import AttentionBackendEnum
+    from vllm.attention.selector import AttentionSelectorConfig
+    from vllm.config import BlockSize, ModelConfig, VllmConfig
+    from vllm.pooling_params import PoolingParams
+    from vllm.sampling_params import SamplingParams, SamplingType
+else:
+    BlockSize = None
+    ModelConfig = None
+    VllmConfig = None
+    PoolingParams = None
+    AttentionBackendEnum = None
+    SamplingParams = None
+    SamplingType = None
+
+logger = init_logger(__name__)
+
+
+class TpuPlatform(Platform):
+    _enum = PlatformEnum.TPU
+    device_name: str = "tpu"
+    device_type: str = "tpu"
+    dispatch_key: str = "XLA"
+    ray_device_key: str = "TPU"
+    device_control_env_var: str = "TPU_VISIBLE_CHIPS"
+    simple_compile_backend: str = "openxla"
+
+    supported_quantization: list[str] = [
+        "tpu_int8", "compressed-tensors", "awq", "fp8", "mxfp4"
+    ]
+
+    additional_env_vars: list[str] = [
+        "PHASED_PROFILING_DIR", "TPU_CHIPS_PER_HOST_BOUNDS", "TPU_HOST_BOUNDS",
+        "TPU_MULTIHOST_BACKEND", "VLLM_MLA_DISABLE", "TPU_BACKEND_TYPE",
+        "NEW_MODEL_DESIGN"
+    ]
+
+    @classmethod
+    def get_attn_backend_cls(cls, selected_backend: "AttentionBackendEnum",
+                             attn_selector_config: "AttentionSelectorConfig",
+                             **kwargs) -> str:
+        from vllm.attention.backends.registry import AttentionBackendEnum
+
+        if selected_backend != AttentionBackendEnum.PALLAS:
+            logger.info("Cannot use %s backend on TPU.", selected_backend)
+
+        logger.info("Using Pallas V1 backend.")
+        return "tpu_inference.layers.vllm.attention.PallasAttentionBackend"
+
+    @classmethod
+    def get_device_name(cls, device_id: int = 0) -> str:
+        try:
+            if vllm_envs.VLLM_TPU_USING_PATHWAYS:
+                # Causes mutliprocess accessing IFRT when calling jax.devices()
+                return "TPU v6 lite"
+            else:
+                chip_type, _ = device.get_local_chips()
+                return f"TPU {chip_type.name}"
+        except Exception as e:
+            logger.warning(f"Error getting device name: {e}")
+            return 'TPU'
+
+    @classmethod
+    def fp8_dtype(cls) -> torch.dtype:
+        if cls.get_device_name().lower() == "tpu v6e":
+            logger.info(
+                "Automatically using fp8_e5m2 for FP8 KV cache on TPU v6e.")
+            return torch.float8_e5m2
+        return torch.float8_e4m3fn
+
+    @classmethod
+    def get_device_total_memory(cls, device_id: int = 0) -> int:
+        raise NotImplementedError
+
+    @classmethod
+    def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
+        return False
+
+    @classmethod
+    def get_punica_wrapper(cls) -> str:
+        return "tpu_inference.lora.torch_punica_tpu.PunicaWrapperTPU"
+
+    @classmethod
+    def get_infinity_values(cls, dtype: jnp.dtype) -> Tuple[float, float]:
+        return jnp.finfo(dtype).min, jnp.finfo(dtype).max
+
+    @classmethod
+    def can_update_inplace(cls):
+        return False
+
+    @classmethod
+    def get_lora_vocab_padding_size(cls) -> int:
+        return 1
+
+    @classmethod
+    def inference_mode(cls):
+        return True
+
+    @classmethod
+    def _initialize_sharding_config(cls, vllm_config: VllmConfig) -> None:
+
+        sharding_config = ShardingConfigManager.from_vllm_config(vllm_config)
+        vllm_config.sharding_config = sharding_config
+        logger.info(f"Initialized sharding configuration: {sharding_config}")
+
+    @classmethod
+    def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
+
+        if vllm_envs.VLLM_TPU_USING_PATHWAYS:
+            assert not vllm_envs.VLLM_ENABLE_V1_MULTIPROCESSING, (
+                "VLLM_ENABLE_V1_MULTIPROCESSING must be 0 when using Pathways(JAX_PLATFORMS=proxy)"
+            )
+        cls._initialize_sharding_config(vllm_config)
+
+        from vllm.config import CompilationMode
+
+        cache_config = vllm_config.cache_config
+        # For v0, the default block size is 16.
+        if cache_config and cache_config.block_size is None:
+            cache_config.block_size = cast(BlockSize, 16)
+
+        compilation_config = vllm_config.compilation_config
+
+        # TPU only supports DYNAMO_TRACE_ONCE compilation level
+        # NOTE(xiang): the compilation_config is not used by jax.
+        if compilation_config.mode != CompilationMode.DYNAMO_TRACE_ONCE:
+            compilation_config.mode = CompilationMode.DYNAMO_TRACE_ONCE
+
+        if compilation_config.backend == "":
+            compilation_config.backend = "openxla"
+
+        # TODO(cuiq): remove this dependency.
+        if vllm_config.model_config:
+            from vllm.v1.attention.backends.pallas import \
+                PallasAttentionBackend
+            cache_config.block_size = PallasAttentionBackend.get_page_size(
+                vllm_config)  # type: ignore[assignment]
+            min_page_size = PallasAttentionBackend.get_min_page_size(
+                vllm_config)
+            if min_page_size > cache_config.block_size:
+                logger.warning(
+                    "Increase the page size from %s to %s to avoid SMEM OOM",
+                    cache_config.block_size,
+                    min_page_size,
+                )
+                cache_config.block_size = min_page_size  # type: ignore[assignment]
+
+        parallel_config = vllm_config.parallel_config
+        scheduler_config = vllm_config.scheduler_config
+        parallel_config.worker_cls = \
+                        "tpu_inference.worker.tpu_worker.TPUWorker"
+
+        multihost_backend = envs.TPU_MULTIHOST_BACKEND
+        if not multihost_backend:  # Single host
+            if parallel_config.pipeline_parallel_size == 1:
+                logger.info("Force using UniProcExecutor for JAX on \
+                        single host without pipeline parallelism.")
+                parallel_config.distributed_executor_backend = "uni"
+            else:
+                logger.info("Force using MultiprocExecutor for JAX on \
+                        single host with pipeline parallelism.")
+                parallel_config.distributed_executor_backend = "mp"
+        elif multihost_backend == "ray":
+            from tpu_inference.executors.ray_distributed_executor import \
+                RayDistributedExecutor
+            parallel_config.distributed_executor_backend = RayDistributedExecutor
+            logger.info(
+                "Force using RayDistributedExecutor for JAX on multihost.")
+        else:
+            logger.warning(
+                f"Unknown TPU multihost backend: {multihost_backend}. "
+                "Using uniproc_executor.")
+            parallel_config.distributed_executor_backend = "uni"
+
+        if scheduler_config.is_multimodal_model and not \
+            scheduler_config.disable_chunked_mm_input:
+            logger.warning("TPU does not support running Multimodal models"\
+            " without setting `--disable_chunked_mm_input`. " \
+            "Forcing --disable_chunked_mm_input.")
+            scheduler_config.disable_chunked_mm_input = True
+
+        kv_transfer_config = vllm_config.kv_transfer_config
+        if kv_transfer_config is not None:
+            assert kv_transfer_config.kv_connector == "TPUConnector"
+        # Late initialization to avoid circular import.
+        # Only perform qwix quantization if it is jax model.
+        if vllm_config.model_config is not None:
+            from tpu_inference.models.jax.utils.qwix.qwix_utils import \
+                update_vllm_config_for_qwix_quantization
+            if vllm_config.model_config:
+                update_vllm_config_for_qwix_quantization(vllm_config)
+
+        from tpu_inference.core.sched.dp_scheduler import \
+            update_vllm_config_for_dp_scheduler
+        update_vllm_config_for_dp_scheduler(vllm_config)
+
+    @classmethod
+    def is_pin_memory_available(cls):
+        logger.warning("Pin memory is not supported on TPU.")
+        return False
+
+    @classmethod
+    def get_device_communicator_cls(cls) -> str:
+        return "vllm.distributed.device_communicators.tpu_communicator.TpuCommunicator"  # noqa
+
+    @classmethod
+    def use_all_gather(cls) -> bool:
+        return True
+
+    @classmethod
+    def supports_v1(cls, model_config: ModelConfig) -> bool:
+        # V1 support on TPU is experimental
+        return True
+
+    @classmethod
+    def validate_request(
+        cls,
+        prompt: PromptType,
+        params: Union["SamplingParams", PoolingParams],
+        processed_inputs: ProcessorInputs,
+    ) -> None:
+        """Raises if this request is unsupported on this platform"""
+        from vllm.sampling_params import SamplingParams, SamplingType
+
+        if isinstance(params, SamplingParams):
+            if params.sampling_type == SamplingType.RANDOM_SEED:
+                raise ValueError("JAX does not support per-request seed.")
+
+    @classmethod
+    def is_kv_cache_dtype_supported(cls, kv_cache_dtype: str,
+                                    model_config: ModelConfig) -> bool:
+        return True
+
+    @classmethod
+    def use_sync_weight_loader(cls) -> bool:
+        """
+        Returns if the current platform needs to sync weight loader.
+        """
+        return True
+
+    @classmethod
+    def support_hybrid_kv_cache(cls) -> bool:
+        return True

tpu_inference/runner/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.