tpu-inference 0.12.0.dev20251213__py3-none-any.whl → 0.13.2.dev20251230__py3-none-any.whl

tpu_inference/layers/vllm/quantization/unquantized.py

@@ -1,19 +1,29 @@
-from typing import Any, Optional, Union
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Optional
 
 import jax
 import jax.numpy as jnp
 import torch
-from jax.experimental.layout import Format, Layout
 from jax.sharding import Mesh, NamedSharding, PartitionSpec
 from torch.nn.parameter import Parameter
 from torchax.interop import jax_view, torch_view
 from torchax.ops.mappings import t2j
 from vllm.attention.layer import Attention
-from vllm.logger import init_logger
 from vllm.model_executor.layers.fused_moe.layer import (
     FusedMoE, FusedMoEConfig, UnquantizedFusedMoEMethod)
-from vllm.model_executor.layers.fused_moe.modular_kernel import (
-    FusedMoEPermuteExpertsUnpermute, FusedMoEPrepareAndFinalize)
 from vllm.model_executor.layers.linear import (LinearBase,
                                                UnquantizedLinearMethod)
 from vllm.model_executor.layers.quantization import \
@@ -21,27 +31,31 @@ from vllm.model_executor.layers.quantization import \
 from vllm.model_executor.layers.quantization.base_config import (
     QuantizationConfig, QuantizeMethodBase)
 
-from tpu_inference import envs
-from tpu_inference.kernels.fused_moe.v1.kernel import fused_ep_moe
 from tpu_inference.layers.common.quant_methods import (UNQUANTIZED,
                                                        get_tpu_quant_method)
-from tpu_inference.layers.vllm.fused_moe import fused_moe_func
-from tpu_inference.layers.vllm.linear_common import (
-    reorder_concatenated_tensor_for_sharding,
-    slice_sharded_tensor_for_concatenation, torch_to_jax_param)
-from tpu_inference.layers.vllm.quantization.common import (
-    JaxCommonConfig, JaxCommonLinearConfig)
+from tpu_inference.layers.common.sharding import ShardingAxisName
+from tpu_inference.layers.common.utils import \
+    slice_sharded_tensor_for_concatenation
+from tpu_inference.layers.vllm.fused_moe import (FusedMoEBackend,
+                                                 fused_moe_apply,
+                                                 select_moe_backend)
+from tpu_inference.layers.vllm.process_weights.fused_moe_weights import (
+    FusedMoEWeights, process_moe_weights, shard_moe_weights)
+from tpu_inference.layers.vllm.process_weights.linear_weights import (
+    LinearWeights, process_lienar_weights, shard_linear_weights,
+    to_parameter_list)
+from tpu_inference.layers.vllm.quantization.configs import (
+    VllmQuantConfig, VllmQuantLinearConfig)
+from tpu_inference.logger import init_logger
+from tpu_inference.utils import get_mesh_shape_product
 
 P = PartitionSpec
-logger = init_logger(__name__)
-
 
-def align_to(a, b):
-    return (a + b - 1) // b * b
+logger = init_logger(__name__)
 
 
 @register_quantization_config(get_tpu_quant_method(UNQUANTIZED))
-class VllmUnquantizedConfig(QuantizationConfig, JaxCommonConfig):
+class VllmUnquantizedConfig(QuantizationConfig, VllmQuantConfig):
 
     @classmethod
     def get_name(cls) -> str:
@@ -78,35 +92,54 @@ class VllmUnquantizedConfig(QuantizationConfig, JaxCommonConfig):
 
 class VllmUnquantizedLinearMethod(UnquantizedLinearMethod):
 
-    def __init__(self, jax_config: JaxCommonLinearConfig):
-        self.jax_config = jax_config
+    def __init__(self, linear_config: VllmQuantLinearConfig):
+        self.linear_config = linear_config
 
     def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
-        weight = torch_to_jax_param(
-            layer.weight,
-            NamedSharding(self.jax_config.mesh,
-                          self.jax_config.weight_sharding),
-            self.jax_config.output_sizes,
-            self.jax_config.n_shards,
-            self.jax_config.fuse_matmuls,
-        )
+        weight = t2j(layer.weight, use_dlpack=False)
         delattr(layer, "weight")
-        layer.weight = weight
-
         if layer.bias is not None and not layer.skip_bias_add:
             if layer.return_bias:
                 logger.warning_once("Bias might return incorrect value.")
-
-            bias = torch_to_jax_param(
-                layer.bias,
-                NamedSharding(self.jax_config.mesh,
-                              self.jax_config.bias_sharding),
-                self.jax_config.output_sizes,
-                self.jax_config.n_shards,
-                self.jax_config.fuse_matmuls,
-            )
+            bias = t2j(layer.bias, use_dlpack=False)
             delattr(layer, "bias")
-            layer.bias = bias
+        else:
+            bias = None
+
+        @jax.jit
+        def process_unquantized_linear_weights(
+            weight: jax.Array,
+            bias: jax.Array | None,
+        ) -> LinearWeights:
+            return process_lienar_weights(
+                LinearWeights(
+                    weight=weight,
+                    weight_scale=None,
+                    zero_point=None,
+                    bias=bias,
+                ),
+                fused=self.linear_config.fuse_matmuls,
+                output_sizes=self.linear_config.output_sizes,
+                reorder_size=self.linear_config.n_shards,
+            )
+
+        weights = process_unquantized_linear_weights(weight, bias)
+        weights = torch_view(
+            shard_linear_weights(
+                weights,
+                mesh=self.linear_config.mesh,
+                weight_p_spec=self.linear_config.weight_sharding,
+                bias_p_spec=self.linear_config.bias_sharding,
+            ))
+
+        if self.linear_config.fuse_matmuls:
+            layer.weight = Parameter(weights.weight, requires_grad=False)
+            if bias is not None:
+                layer.bias = Parameter(weights.bias, requires_grad=False)
+        else:
+            layer.weight = to_parameter_list(weights.weight)
+            if bias is not None:
+                layer.bias = to_parameter_list(weights.bias)
 
     def apply(self,
               layer: torch.nn.Module,
@@ -115,16 +148,17 @@ class VllmUnquantizedLinearMethod(UnquantizedLinearMethod):
         assert isinstance(layer, LinearBase)
 
         with jax.named_scope(layer._get_name()):
-            if in_sharding := self.jax_config.get_input_sharding(x):
-                x.shard_(NamedSharding(self.jax_config.mesh, in_sharding))
+            if in_sharding := self.linear_config.get_input_sharding(x):
+                x.shard_(NamedSharding(self.linear_config.mesh, in_sharding))
 
-            if self.jax_config.fuse_matmuls:
+            if self.linear_config.fuse_matmuls:
                 out = self._apply_fused(layer, x, bias)
             else:
                 out = self._apply_split(layer, x, bias)
 
-            if out_sharding := self.jax_config.get_output_sharding(out):
-                out.shard_(NamedSharding(self.jax_config.mesh, out_sharding))
+            if out_sharding := self.linear_config.get_output_sharding(out):
+                out.shard_(NamedSharding(self.linear_config.mesh,
+                                         out_sharding))
 
         return out
 
@@ -140,7 +174,7 @@ class VllmUnquantizedLinearMethod(UnquantizedLinearMethod):
             outs += bias.jax()
 
         outs = slice_sharded_tensor_for_concatenation(
-            outs, self.jax_config.output_sizes, self.jax_config.n_shards)
+            outs, self.linear_config.output_sizes, self.linear_config.n_shards)
         out = jnp.concatenate(outs, axis=-1)
         return torch_view(out)
 
@@ -166,232 +200,99 @@ class VllmUnquantizedLinearMethod(UnquantizedLinearMethod):
 
 class VllmUnquantizedFusedMoEMethod(UnquantizedFusedMoEMethod):
 
-    def __init__(self,
-                 moe: FusedMoEConfig,
-                 mesh: Mesh,
-                 ep_axis_name: str = 'model'):
-        super().__init__(moe)
-        self.mesh = mesh
-        self.use_kernel = envs.USE_MOE_EP_KERNEL and moe.use_ep
-        self.ep_axis_name = ep_axis_name
-        # TODO: Use autotune table once we have it.
-        self.block_size = {
-            "bt": 64,
-            "bf": 1024,
-            "bd1": 1536,
-            "bd2": 1536,
-            "btc": 64,
-            "bfc": 1024,
-            "bd1c": 1536,
-            "bd2c": 1536,
-        }
-
-    def select_gemm_impl(
+    def __init__(
         self,
-        prepare_finalize: FusedMoEPrepareAndFinalize,
         moe: FusedMoEConfig,
-        layer: torch.nn.Module,
-    ) -> FusedMoEPermuteExpertsUnpermute:
-        raise NotImplementedError(
-            "Selecting gemm implementation is currently not supported.")
+        mesh: Mesh,
+        ep_axis_name: str = "model",
+    ):
+        super().__init__(moe)
+        self.mesh = mesh
+        self.moe_backend = select_moe_backend(self.moe)
+
+        self.extra_backend_kwargs = {}
+        if self.moe_backend == FusedMoEBackend.FUSED_MOE:
+            # When fused moe kernle is used, we pass extra arguments like
+            # tuned block sizes to the kernel.
+            self.extra_backend_kwargs = dict(
+                ep_axis_name=ep_axis_name,
+                # TODO: Use autotune table once we have it.
+                bt=64,
+                bf=1024,
+                bd1=1536,
+                bd2=1536,
+                btc=64,
+                bfc=1024,
+                bd1c=1536,
+                bd2c=1536,
+            )
 
     def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
         assert isinstance(layer, FusedMoE)
+
         w13_weight = t2j(layer.w13_weight, use_dlpack=False)
         w2_weight = t2j(layer.w2_weight, use_dlpack=False)
 
-        num_experts, hidden_size, intermediate_size = w2_weight.shape
-
         if self.moe.has_bias:
             w13_bias = t2j(layer.w13_bias, use_dlpack=False)
             w2_bias = t2j(layer.w2_bias, use_dlpack=False)
-
-        if layer.activation == "swigluoai":
-            # When using swigluoai, vLLM splits gmm output in a interleaved way.
-            # However, interleaved split is not performant on TPU. Therefore,
-            # we preprocess the weight so that splitting gmm output by middle
-            # can still get the same result.
-            w1_weight = w13_weight[:, ::2, :]
-            w3_weight = w13_weight[:, 1::2, :]
-            w13_weight = jnp.concat([w1_weight, w3_weight], axis=1)
-
-            if self.moe.has_bias:
-                w1_bias = w13_bias[:, ::2]
-                w3_bias = w13_bias[:, 1::2]
-                w13_bias = jnp.concat([w1_bias, w3_bias], axis=1)
-
-        if self.use_kernel:
-            # Kernel expects:
-            # w13: (num_experts, 2, hidden_size, intermediate_size)
-            # w2: (num_experts, intermediate_size, hidden_size)
-            # Current format:
-            # w13_weight: (num_experts, 2*intermediate_size, hidden_size)
-            # w2_weight: (num_experts, hidden_size, intermediate_size)
-            num_experts = w13_weight.shape[0]
-            intermediate_size = w13_weight.shape[1] // 2
-            hidden_size = w13_weight.shape[2]
-
-            padded_intermediate_size = align_to(intermediate_size, 256)
-            padded_hidden_size = align_to(hidden_size, 256)
-
-            w13_weight = w13_weight.reshape(num_experts, 2, intermediate_size,
-                                            hidden_size)
-            w13_weight = jnp.transpose(w13_weight, (0, 1, 3, 2))
-
-            # Transpose w2_weight to (num_experts, intermediate_size, hidden_size)
-            w2_weight = jnp.transpose(w2_weight, (0, 2, 1))
-
-            w13_weight = jnp.pad(
-                w13_weight,
-                ((0, 0), (0, 0), (0, padded_hidden_size - hidden_size),
-                 (0, padded_intermediate_size - intermediate_size)),
-                constant_values=0)
-
-            w2_weight = jnp.pad(
-                w2_weight,
-                ((0, 0), (0, padded_intermediate_size - intermediate_size),
-                 (0, padded_hidden_size - hidden_size)),
-                constant_values=0)
-
-            # Apply EP sharding
-            ep_sharding = NamedSharding(self.mesh, P("model"))
-
-            w13_weight = jax.device_put(
-                w13_weight,
-                Format(Layout((0, 1, 2, 3)),
-                       NamedSharding(self.mesh, P("model", None, None, None))))
-            w2_weight = jax.device_put(
-                w2_weight,
-                Format(Layout((0, 1, 2)),
-                       NamedSharding(self.mesh, P("model", None, None))))
-
-            if self.moe.has_bias:
-                w13_bias = w13_bias.astype(jnp.float32).reshape(
-                    num_experts, 2, 1, intermediate_size)
-                w2_bias = w2_bias.astype(jnp.float32).reshape(
-                    num_experts, 1, hidden_size)
-
-                w13_bias = jnp.pad(
-                    w13_bias,
-                    ((0, 0), (0, 0), (0, 0),
-                     (0, padded_intermediate_size - intermediate_size)),
-                    constant_values=0)
-
-                w2_bias = jnp.pad(w2_bias,
-                                  ((0, 0), (0, 0),
-                                   (0, padded_hidden_size - hidden_size)),
-                                  constant_values=0)
-
-                # Apply EP sharding
-                w13_bias = jax.device_put(
-                    w13_bias, Format(Layout((0, 1, 2, 3)), ep_sharding))
-                w2_bias = jax.device_put(
-                    w2_bias, Format(Layout((0, 1, 2)), ep_sharding))
         else:
+            w13_bias = w2_bias = None
+
+        @jax.jit
+        def process_unquantized_moe_weights(
+            w13_weight: jax.Array,
+            w13_bias: jax.Array | None,
+            w2_weight: jax.Array,
+            w2_bias: jax.Array | None,
+        ) -> FusedMoEWeights:
+
+            w13_interleave = layer.activation == "swigluoai"
+            w13_reorder_size = get_mesh_shape_product(
+                self.mesh, ShardingAxisName.MLP_TENSOR)
+
+            return process_moe_weights(
+                FusedMoEWeights(
+                    w13_weight=w13_weight,
+                    w13_weight_scale=None,
+                    w13_bias=w13_bias,
+                    w2_weight=w2_weight,
+                    w2_weight_scale=None,
+                    w2_bias=w2_bias,
+                ),
+                moe_backend=self.moe_backend,
+                w13_reorder_size=w13_reorder_size,
+                w13_interleave=w13_interleave,
+            )
 
-            if layer.use_ep:
-                ep_sharding = NamedSharding(self.mesh, P("model"))
-                w13_weight = jax.device_put(
-                    w13_weight, Format(Layout((0, 1, 2)), ep_sharding))
-                w2_weight = jax.device_put(
-                    w2_weight, Format(Layout((0, 1, 2)), ep_sharding))
+        weights = process_unquantized_moe_weights(
+            w13_weight,
+            w13_bias,
+            w2_weight,
+            w2_bias,
+        )
+        weights = torch_view(
+            shard_moe_weights(weights, self.moe_backend, self.mesh))
 
-                if self.moe.has_bias:
-                    w13_bias = jax.device_put(
-                        w13_bias, Format(Layout((0, 1)), ep_sharding))
-                    w2_bias = jax.device_put(
-                        w2_bias, Format(Layout((0, 1)), ep_sharding))
-
-            else:
-                output_sizes = [intermediate_size, intermediate_size]
-                n_shards = self.mesh.shape["model"]
-                assert intermediate_size % n_shards == 0
-
-                w13_weight = reorder_concatenated_tensor_for_sharding(
-                    w13_weight, output_sizes, n_shards, dim=1)
-                w13_weight = jax.device_put(
-                    w13_weight,
-                    Format(Layout((0, 1, 2)),
-                           NamedSharding(self.mesh, P(None, "model", None))))
-                w2_weight = jax.device_put(
-                    w2_weight,
-                    Format(Layout((0, 1, 2)),
-                           NamedSharding(self.mesh, P(None, None, "model"))))
-
-                if self.moe.has_bias:
-                    w13_bias = reorder_concatenated_tensor_for_sharding(
-                        w13_bias, output_sizes, n_shards, dim=1)
-                    w13_bias = jax.device_put(
-                        w13_bias,
-                        Format(Layout((0, 1)),
-                               NamedSharding(self.mesh, P(None, "model"))))
-                    w2_bias = jax.device_put(
-                        w2_bias,
-                        Format(Layout((0, 1)),
-                               NamedSharding(self.mesh, P(None, None))))
-
-        layer.w13_weight = Parameter(torch_view(w13_weight),
-                                     requires_grad=False)
-        layer.w2_weight = Parameter(torch_view(w2_weight), requires_grad=False)
+        layer.w13_weight = Parameter(weights.w13_weight, requires_grad=False)
+        layer.w2_weight = Parameter(weights.w2_weight, requires_grad=False)
 
         if self.moe.has_bias:
-            layer.w13_bias = Parameter(torch_view(w13_bias),
-                                       requires_grad=False)
-            layer.w2_bias = Parameter(torch_view(w2_bias), requires_grad=False)
+            layer.w13_bias = Parameter(weights.w13_bias, requires_grad=False)
+            layer.w2_bias = Parameter(weights.w2_bias, requires_grad=False)
 
     def apply(
         self,
         layer: torch.nn.Module,
         x: torch.Tensor,
         router_logits: torch.Tensor,
-    ) -> Union[torch.Tensor, tuple[torch.Tensor, torch.Tensor]]:
-        assert isinstance(layer, FusedMoE)
-        if layer.scoring_func != "softmax":
-            raise NotImplementedError(
-                "Only softmax is supported for scoring_func")
-
-        x = jax_view(x)
-        w13_weight = jax_view(layer.w13_weight)
-        w2_weight = jax_view(layer.w2_weight)
-        w13_bias = w2_bias = None
-        if self.moe.has_bias:
-            w13_bias = jax_view(layer.w13_bias)
-            w2_bias = jax_view(layer.w2_bias)
-        gating_output = jax_view(router_logits)
-
-        if self.use_kernel:
-            actual_hidden_size = x.shape[-1]
-            padded_hidden_size = align_to(actual_hidden_size, 256)
-            x = jnp.pad(x,
-                        ((0, 0), (0, padded_hidden_size - actual_hidden_size)),
-                        constant_values=0)
-            output = fused_ep_moe(
-                mesh=self.mesh,
-                tokens=x,
-                w1=w13_weight,
-                w2=w2_weight,
-                b1=w13_bias,
-                b2=w2_bias,
-                gating_output=gating_output,
-                top_k=layer.top_k,
-                ep_axis_name=self.ep_axis_name,
-                renormalize_topk_logits=layer.renormalize,
-                act_fn=layer.activation,
-                **self.block_size,
-            )[:, :actual_hidden_size]
-        else:
-            output = fused_moe_func(
-                hidden_states=x,
-                w1=w13_weight,
-                w2=w2_weight,
-                w1_bias=w13_bias,
-                w2_bias=w2_bias,
-                gating_output=gating_output,
-                topk=layer.top_k,
-                renormalize=layer.renormalize,
-                mesh=self.mesh,
-                use_ep=layer.use_ep,
-                activation=layer.activation,
-            )
-
-        return torch_view(output)
+    ) -> torch.Tensor:
+
+        return fused_moe_apply(
+            layer,
+            x,
+            router_logits,
+            self.moe_backend,
+            self.mesh,
+            self.extra_backend_kwargs,
+        )

tpu_inference/lora/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tpu_inference/lora/torch_lora_ops.py

@@ -4,7 +4,6 @@
 import jax
 import jax.numpy as jnp
 import torch
-import torch.nn.functional as F
 from torchax.interop import call_jax
 
 
@@ -85,19 +84,15 @@ def bgmv_expand_slice(
         add_inputs (bool): Whether or not to add the input tensor to the output
             tensor.
     """
-    outputs = bgmv_torch(inputs, lora_b_weights, lora_indices_tensor)
+    outputs = bgmv_torch(inputs, lora_b_weights,
+                         lora_indices_tensor)  # [num_tokens, out_features]
 
-    outputs = F.pad(
-        outputs,
-        (
-            slice_offset,
-            output_tensor.shape[1] - (slice_offset + slice_size),
-            0,
-            0,
-        ),
-    )
+    # Create a padded tensor manually to avoid issues with F.pad on sharded tensors.
+    # This is a more robust way to handle padding in a distributed environment.
+    outputs_padded = torch.zeros_like(output_tensor)
+    outputs_padded[:, slice_offset:slice_offset + slice_size] = outputs
 
     if add_inputs:
-        return output_tensor + outputs
+        return output_tensor + outputs_padded
     else:
-        return outputs
+        return outputs_padded

tpu_inference/models/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tpu_inference/models/common/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.