ipex-llm 2.2.0b20250102__py3-none-win_amd64.whl → 2.2.0b20250104__py3-none-win_amd64.whl

ipex_llm/transformers/convert.py

@@ -1984,16 +1984,9 @@ def _optimize_post(model):
         modeling_module_name = model.__class__.__module__
         module = importlib.import_module(modeling_module_name)
         from ipex_llm.transformers.models.yuan import yuan_attention_forward
-        # from ipex_llm.transformers.models.yuan import yuan_mlp_forward
-        convert_forward(model,
-                        module.YuanAttention,
-                        yuan_attention_forward
-                        )
-        # disable able mlp_forward for quantize_kv on mtl.
-        # convert_forward(model,
-        #                 module.YuanMLP,
-        #                 yuan_mlp_forward
-        #                 )
+        convert_forward(model, module.YuanAttention, yuan_attention_forward)
+        # from ipex_llm.transformers.models.common import mlp_silu_forward
+        # convert_forward(model, module.YuanMLP, mlp_silu_forward)
     elif model.config.model_type == 'bert' and (
         not model.config.is_decoder and
         model.config.position_embedding_type == "absolute"

ipex_llm/transformers/models/yuan.py

@@ -20,17 +20,15 @@
 # https://huggingface.co/IEITYuan/Yuan2-2B-hf/blob/7ab7b3c18eb8e5232ce2a3f720d4e6f4b53a2806/README.md#%E5%A3%B0%E6%98%8E%E4%B8%8E%E5%8D%8F%E8%AE%AEterms-and-conditions
 #
 
-import math
 from typing import Optional, Tuple
 
 import torch
 
 from ipex_llm.utils.common import invalidInputError
 from ipex_llm.transformers.models.common import scaled_dot_product_attention
-from ipex_llm.transformers.models.utils import apply_rotary_pos_emb, \
-    mlp_fusion_check, fp16_fusion_check
+from ipex_llm.transformers.models.utils import apply_rotary_pos_emb
 from ipex_llm.transformers.models.utils import use_quantize_kv_cache
-from ipex_llm.transformers.models.utils import SILU, update_past_key_value
+from ipex_llm.transformers.models.utils import update_past_key_value
 from ipex_llm.transformers.models.utils import should_use_fuse_rope
 
 
@@ -98,52 +96,6 @@ def yuan_localized_filtering_forward(
     return lf_output
 
 
-def yuan_mlp_forward(
-    self,
-    x: torch.Tensor,
-    residual=None
-) -> torch.Tensor:
-    x_2d = x.view(-1, x.shape[-1])
-    bsz, hidden_size = x_2d.shape
-    qtype = getattr(self.up_proj, "qtype", None)
-    if mlp_fusion_check(x_2d, qtype, self.training):
-        import xe_linear
-        if not x_2d.is_contiguous():
-            x_2d = x_2d.contiguous()
-        out = self.down_proj(xe_linear.mlp_forward_xpu(
-            x_2d, self.up_proj.weight.data, self.gate_proj.weight.data,
-            x_2d.shape[0], x_2d.shape[1], self.up_proj.out_len,
-            SILU, qtype
-        ))
-        if residual is not None:
-            return out + residual
-        else:
-            return out
-    elif fp16_fusion_check(self.up_proj, x, self.training) and \
-            hidden_size == 4096 and bsz == 1:
-        hidden_states1 = torch.ops.torch_ipex.mm_silu(x, self.up_proj.weight)
-        hidden_states = torch.ops.torch_ipex.mm_resmul(
-            x, self.gate_proj.weight, hidden_states1
-        )
-        if residual is None:
-            hidden_states = torch.matmul(hidden_states, self.down_proj.weight)
-        else:
-            attn_output = torch.addmm(
-                residual.flatten(0, -2),
-                hidden_states.flatten(0, -2),
-                self.down_proj.weight,
-                beta=1,
-            )
-            hidden_states = attn_output.view(x.shape)
-        return hidden_states
-    else:
-        out = self.down_proj(self.act_fn(self.up_proj(x)) * self.gate_proj(x))
-        if residual is not None:
-            return out + residual
-        else:
-            return out
-
-
 def yuan_attention_forward(
     self,
     hidden_states: torch.Tensor,

ipex_llm/transformers/xpu_ops.py

@@ -0,0 +1,155 @@
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import torch
+import xe_linear
+import xe_batch
+import xe_addons
+
+
+@torch.library.register_fake("ipex_llm::forward_new")
+def _(x, weight, qtype, input_size):
+    return torch.empty_like(x)
+
+
+# @torch.library.register_fake("ipex_llm::dequant")
+# def _(x, weight, qtype):
+#     return ???
+
+
+@torch.library.register_fake("ipex_llm::mlp_forward_xpu")
+def _(x, weight1, weight2, batch_size, state_size, output_size, act_type, qtype):
+    return torch.empty_like(x)
+
+
+# @torch.library.register_fake("ipex_llm::rwkv_linear_attention_v4")
+# def _(time_decay, time_first, key, value, num_state, den_state, max_state)
+    # return ???
+
+
+# @torch.library.register_fake("ipex_llm::rwkv_linear_attention_v5")
+# def _(time_decay, time_first, receptance, key, value, state)
+    # return ???
+
+
+# @torch.library.register_fake("ipex_llm::rwkv_time_shift")
+# def _(hidden, shifted, mix):
+    # return ???
+
+
+# @torch.library.register_fake("ipex_llm::dequantize_rows")
+# def _(x, weight, qtype, state_size, output_size):
+    # return ???
+
+
+@torch.library.register_fake("ipex_llm::batch_forward")
+def _(x, weight, qtype):
+    return torch.empty_like(x)
+
+
+@torch.library.register_fake("ipex_llm::sdp")
+def _(query, key, value, mask):
+    return torch.empty(query.shape, dtype=query.dtype, device=query.device)
+
+
+@torch.library.register_fake("ipex_llm::sdp_fp8")
+def _(query, key, value, mask):
+    return torch.empty(query.shape, dtype=query.dtype, device=query.device)
+
+
+@torch.library.register_fake("ipex_llm::sdp_causal")
+def _(query, key, value, mask, scale):
+    return torch.empty(query.shape, dtype=query.dtype, device=query.device)
+
+
+@torch.library.register_fake("ipex_llm::sdp_fp8_causal")
+def _(query, key, value, mask, scale):
+    return torch.empty(query.shape, dtype=query.dtype, device=query.device)
+
+
+@torch.library.register_fake("ipex_llm::sdp_non_causal")
+def _(query, key, value, mask, scale):
+    return torch.empty(query.shape, dtype=query.dtype, device=query.device)
+
+
+@torch.library.register_fake("ipex_llm::sdp_fp8_non_causal")
+def _(query, key, value, mask, scale):
+    return torch.empty(query.shape, dtype=query.dtype, device=query.device)
+
+
+@torch.library.register_fake("ipex_llm::siglip_sdp_non_causal")
+def _(query, key, value, mask):
+    return torch.empty(query.shape, dtype=query.dtype, device=query.device)
+
+
+@torch.library.register_fake("ipex_llm::gemma2_sdp")
+def _(query, key, value, mask, f1, f2):
+    return torch.empty(query.shape, dtype=query.dtype, device=query.device)
+
+
+@torch.library.register_fake("ipex_llm::gemma2_sdp_causal")
+def _(query, key, value, mask, f1, f2):
+    return torch.empty(query.shape, dtype=query.dtype, device=query.device)
+
+
+@torch.library.register_fake("ipex_llm::rms_norm")
+def _(weight, x, eps):
+    return torch.empty_like(x)
+
+
+@torch.library.register_fake("ipex_llm::layer_norm")
+def _(x, weight, bias, eps):
+    return torch.empty_like(x)
+
+
+@torch.library.register_fake("ipex_llm::rotary_half_inplaced")
+def _(inv_freq, position_ids, query, key):
+    pass
+
+
+@torch.library.register_fake("ipex_llm::rotary_two_inplaced")
+def _(inv_freq, position_ids, query, key):
+    pass
+
+
+@torch.library.register_fake("ipex_llm::rotary_half_with_cache_inplaced")
+def _(query, key, cos, sin):
+    pass
+
+
+@torch.library.register_fake("ipex_llm::rotary_two_with_cache_inplaced")
+def _(query, key, cos, sin, half_layout):
+    pass
+
+
+@torch.library.register_fake("ipex_llm::mlp_silu_mul_inplaced")
+def _(gate, up):
+    pass
+
+
+@torch.library.register_fake("ipex_llm::quantize_key_value")
+def _(key, value, key_output, value_output):
+    pass
+
+
+@torch.library.register_fake("ipex_llm::dequantize_key_value")
+def _(key, value, key_output, value_output):
+    pass
+
+
+@torch.library.register_fake("ipex_llm::attn_softmax_inplaced")
+def _(attn):
+    pass

ipex_llm/utils/__init__.py

@@ -23,8 +23,7 @@ import transformers
 trans_version = transformers.__version__
 
 if trans_version >= "4.47.0":
-    # TODO
-    pass
+    from .benchmark_util_4_47 import BenchmarkWrapper
 elif trans_version >= "4.45.0":
     from .benchmark_util_4_45 import BenchmarkWrapper
 elif trans_version >= "4.44.0":