ipex-llm 2.3.0b20250410__py3-none-win_amd64.whl → 2.3.0b20250414__py3-none-win_amd64.whl

ipex_llm/transformers/convert.py

@@ -1089,6 +1089,15 @@ def ggml_convert_low_bit(model, qtype, optimize_model=True,
                          embedding_qtype=None,
                          mixed_precision=False,
                          disable_optimize_pre=False):
+    if qtype == ggml_tensor_qtype["sym_int4"] and torch.__version__ >= "2.6":
+        logger.warning("sym_int4 is deprecated, use woq_int4 instead, "
+                       "if you are loading saved sym_int4 low bit model, "
+                       "please resaved it with woq_int4")
+        qtype = ggml_tensor_qtype["woq_int4"]
+    elif qtype == ggml_tensor_qtype["woq_int4"] and torch.__version__ < "2.6":
+        logger.warning("woq_int4 is not supported with pytorch<2.6, "
+                       "use sym_int4 instead or use ipex-llm with pytorch>=2.6")
+        qtype = ggml_tensor_qtype["sym_int4"]
     if qtype in ggml_tensor_qtype.values():
         index = list(ggml_tensor_qtype.values()).index(qtype)
         logger.info(f"Converting the current model to "

ipex_llm/transformers/kv.py

@@ -22,7 +22,8 @@ import math
 
 from .models.utils import (
     init_fp8_kv_cache, append_fp8_kv_cache,
-    init_kv_cache, append_kv_cache, extend_kv_cache
+    init_kv_cache, append_kv_cache, extend_kv_cache,
+    init_unbalanced_fp8_kv_cache, append_unbalanced_fp8_kv_cache,
 )
 from typing import Optional, Dict, Tuple, Any, List
 from transformers.cache_utils import DynamicCache
@@ -151,6 +152,55 @@ class DynamicNormalCache(DynamicCache):
         return past_key_values
 
 
+class DynamicUnbalancedFp8Cache(DynamicCache):
+    def __init__(self, num_hidden_layers: Optional[int] = None) -> None:
+        # ignore num_hidden_layers to fix transformers >= 4.45
+        super().__init__()
+
+    def update(
+        self,
+        key_states: torch.Tensor,
+        value_states: torch.Tensor,
+        layer_idx: int,
+        cache_kwargs: Optional[Dict[str, Any]]=None,
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        # fix converting empty DynamicCache in transformers >= 4.45
+        if key_states == []:
+            return key_states, value_states
+
+        batch_size, num_heads, seq_len, k_head_dim = key_states.shape
+        _, _, _, v_head_dim = value_states.shape
+
+        if layer_idx == 0:
+            if hasattr(self, "_seen_tokens"):
+                # 4.39 uses `_seen_tokens`
+                self._seen_tokens += seq_len
+            else:
+                # 4.37 uses `seen_tokens`
+                self.seen_tokens += seq_len
+
+        # Update the cache
+        if len(self.key_cache) <= layer_idx:
+            k_cache, v_cache = init_unbalanced_fp8_kv_cache(
+                batch_size, num_heads, seq_len, k_head_dim, v_head_dim,
+                device=key_states.device,
+            )
+            k_cache, v_cache = append_unbalanced_fp8_kv_cache(k_cache, v_cache,
+                                                              key_states, value_states)
+
+            self.key_cache.append(k_cache)
+            self.value_cache.append(v_cache)
+        else:
+            k_cache = self.key_cache[layer_idx]
+            v_cache = self.value_cache[layer_idx]
+            k_cache, v_cache = append_unbalanced_fp8_kv_cache(k_cache, v_cache,
+                                                              key_states, value_states)
+            self.key_cache[layer_idx] = k_cache
+            self.value_cache[layer_idx] = v_cache
+
+        return self.key_cache[layer_idx], self.value_cache[layer_idx]
+
+
 # Copied from transformers.models.llama.modeling_llama.repeat_kv
 def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
     """

ipex_llm/transformers/models/common.py

@@ -273,11 +273,11 @@ def scaled_dot_product_attention(query: torch.Tensor, key: torch.Tensor,
             else:
                 attn_output = xe_addons.sdp_causal(query, key, value, mask, scale)
         elif seq_length != kv_length and seq_length <= 32:
-            # todo: add scale support
+            # todo: add further scale support
             if key.dtype == torch.uint8:
-                attn_output = xe_addons.sdp_fp8(query, key, value, mask)
+                attn_output = xe_addons.sdp_fp8(query, key, value, mask, scale)
             else:
-                attn_output = xe_addons.sdp(query, key, value, mask)
+                attn_output = xe_addons.sdp(query, key, value, mask, scale)
         else:
             if key.dtype == torch.uint8:
                 attn_output = xe_addons.sdp_fp8_non_causal(query, key, value, mask, scale)

ipex_llm/transformers/models/utils.py

@@ -138,6 +138,49 @@ def append_fp8_kv_cache(k_cache, v_cache, key, value):
     return new_k_cache, new_v_cache
 
 
+def init_unbalanced_fp8_kv_cache(batch_size, num_heads, current_length,
+                                 k_head_dim, v_head_dim, device):
+    # for case which k head dim is different from v head dim
+    max_length = current_length + FP8_KV_ALLOC_LENGTH
+
+    k_cache_storage = torch.empty(batch_size, num_heads, max_length, k_head_dim,
+                                  dtype=torch.uint8, device=device)
+    k_cache = k_cache_storage.as_strided((batch_size, num_heads, 0, k_head_dim),
+                                         k_cache_storage.stride(), storage_offset=0)
+
+    v_cache_storage = torch.empty(batch_size, num_heads, max_length, v_head_dim,
+                                  dtype=torch.uint8, device=device)
+    v_cache = v_cache_storage.as_strided((batch_size, num_heads, 0, v_head_dim),
+                                         v_cache_storage.stride(), storage_offset=0)
+    return k_cache, v_cache
+
+
+def append_unbalanced_fp8_kv_cache(k_cache, v_cache, key, value):
+    batch_size, num_heads, cur_length, k_head_dim = k_cache.shape
+    _, _, _, v_head_dim = v_cache.shape
+    new_length = cur_length + key.size(2)
+    new_k_size = (batch_size, num_heads, new_length, k_head_dim)
+    new_v_size = (batch_size, num_heads, new_length, v_head_dim)
+
+    if k_cache.stride(1) < new_length * k_cache.size(3):
+        new_k_cache, new_v_cache = init_unbalanced_fp8_kv_cache(batch_size, num_heads, new_length,
+                                                                k_head_dim, v_head_dim, key.device)
+        new_k_cache = new_k_cache.as_strided(new_k_size, new_k_cache.stride(), storage_offset=0)
+        new_v_cache = new_v_cache.as_strided(new_v_size, new_v_cache.stride(), storage_offset=0)
+        new_k_cache[:, :, :cur_length, :] = k_cache
+        new_v_cache[:, :, :cur_length, :] = v_cache
+    else:
+        new_k_cache = k_cache.as_strided(new_k_size, k_cache.stride(), storage_offset=0)
+        new_v_cache = v_cache.as_strided(new_v_size, v_cache.stride(), storage_offset=0)
+
+    import xe_addons
+    xe_addons.quantize_key_value(key, value,
+                                 new_k_cache[:, :, cur_length:new_length, :],
+                                 new_v_cache[:, :, cur_length:new_length, :])
+
+    return new_k_cache, new_v_cache
+
+
 def restore_fp8_kv_cache(k_cache, v_cache, dtype):
     key_states = torch.empty(k_cache.shape, device=k_cache.device, dtype=dtype)
     value_states = torch.empty(v_cache.shape, device=v_cache.device, dtype=dtype)

{ipex_llm-2.3.0b20250410.dist-info → ipex_llm-2.3.0b20250414.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipex-llm
-Version: 2.3.0b20250410
+Version: 2.3.0b20250414
 Summary: Large Language Model Develop Toolkit
 Home-page: https://github.com/intel-analytics/ipex-llm
 Author: BigDL Authors
@@ -27,7 +27,7 @@ Requires-Dist: intel-openmp ; (platform_machine == "x86_64" or platform_machine
 Requires-Dist: torch ==2.1.2+cpu ; (platform_system == "Linux") and extra == 'all'
 Requires-Dist: torch ==2.1.2 ; (platform_system == "Windows") and extra == 'all'
 Provides-Extra: cpp
-Requires-Dist: bigdl-core-cpp ==2.7.0b20250410 ; extra == 'cpp'
+Requires-Dist: bigdl-core-cpp ==2.7.0b20250414 ; extra == 'cpp'
 Requires-Dist: setuptools ; extra == 'cpp'
 Requires-Dist: onednn-devel ==2025.0.1 ; (platform_system == "Windows") and extra == 'cpp'
 Requires-Dist: onednn ==2025.0.1 ; (platform_system == "Windows") and extra == 'cpp'
@@ -60,7 +60,7 @@ Requires-Dist: transformers ==4.40.0 ; extra == 'npu'
 Requires-Dist: intel-openmp ; (platform_machine == "x86_64" or platform_machine == "AMD64") and extra == 'npu'
 Requires-Dist: torch ==2.1.2+cpu ; (platform_system == "Linux") and extra == 'npu'
 Requires-Dist: torch ==2.1.2 ; (platform_system == "Windows") and extra == 'npu'
-Requires-Dist: bigdl-core-npu ==2.7.0b20250410 ; (platform_system == "Windows") and extra == 'npu'
+Requires-Dist: bigdl-core-npu ==2.7.0b20250414 ; (platform_system == "Windows") and extra == 'npu'
 Provides-Extra: serving
 Requires-Dist: py-cpuinfo ; extra == 'serving'
 Requires-Dist: fschat[model_worker,webui] ==0.2.36 ; extra == 'serving'
@@ -80,9 +80,9 @@ Requires-Dist: setuptools <70.0.0 ; extra == 'xpu'
 Requires-Dist: torch ==2.1.0a0 ; extra == 'xpu'
 Requires-Dist: torchvision ==0.16.0a0 ; extra == 'xpu'
 Requires-Dist: intel-extension-for-pytorch ==2.1.10+xpu ; extra == 'xpu'
-Requires-Dist: bigdl-core-xe-21 ==2.7.0b20250410 ; extra == 'xpu'
-Requires-Dist: bigdl-core-xe-batch-21 ==2.7.0b20250410 ; extra == 'xpu'
-Requires-Dist: bigdl-core-xe-addons-21 ==2.7.0b20250410 ; extra == 'xpu'
+Requires-Dist: bigdl-core-xe-21 ==2.7.0b20250414 ; extra == 'xpu'
+Requires-Dist: bigdl-core-xe-batch-21 ==2.7.0b20250414 ; extra == 'xpu'
+Requires-Dist: bigdl-core-xe-addons-21 ==2.7.0b20250414 ; extra == 'xpu'
 Provides-Extra: xpu-2-1
 Requires-Dist: py-cpuinfo ; extra == 'xpu-2-1'
 Requires-Dist: protobuf ; extra == 'xpu-2-1'
@@ -97,9 +97,9 @@ Requires-Dist: setuptools <70.0.0 ; extra == 'xpu-2-1'
 Requires-Dist: torch ==2.1.0a0 ; extra == 'xpu-2-1'
 Requires-Dist: torchvision ==0.16.0a0 ; extra == 'xpu-2-1'
 Requires-Dist: intel-extension-for-pytorch ==2.1.10+xpu ; extra == 'xpu-2-1'
-Requires-Dist: bigdl-core-xe-21 ==2.7.0b20250410 ; extra == 'xpu-2-1'
-Requires-Dist: bigdl-core-xe-batch-21 ==2.7.0b20250410 ; extra == 'xpu-2-1'
-Requires-Dist: bigdl-core-xe-addons-21 ==2.7.0b20250410 ; extra == 'xpu-2-1'
+Requires-Dist: bigdl-core-xe-21 ==2.7.0b20250414 ; extra == 'xpu-2-1'
+Requires-Dist: bigdl-core-xe-batch-21 ==2.7.0b20250414 ; extra == 'xpu-2-1'
+Requires-Dist: bigdl-core-xe-addons-21 ==2.7.0b20250414 ; extra == 'xpu-2-1'
 Requires-Dist: intel-openmp ; (platform_machine == "x86_64" or platform_machine == "AMD64") and extra == 'xpu-2-1'
 Requires-Dist: dpcpp-cpp-rt ==2024.0.2 ; (platform_system == "Windows") and extra == 'xpu-2-1'
 Requires-Dist: mkl-dpcpp ==2024.0.0 ; (platform_system == "Windows") and extra == 'xpu-2-1'
@@ -117,7 +117,7 @@ Requires-Dist: setuptools ; extra == 'xpu-2-6'
 Requires-Dist: torch ==2.6.0+xpu ; extra == 'xpu-2-6'
 Requires-Dist: torchvision ==0.21.0+xpu ; extra == 'xpu-2-6'
 Requires-Dist: torchaudio ==2.6.0+xpu ; extra == 'xpu-2-6'
-Requires-Dist: bigdl-core-xe-all ==2.7.0b20250410 ; extra == 'xpu-2-6'
+Requires-Dist: bigdl-core-xe-all ==2.7.0b20250414 ; extra == 'xpu-2-6'
 Requires-Dist: onednn-devel ==2025.0.1 ; extra == 'xpu-2-6'
 Requires-Dist: onednn ==2025.0.1 ; extra == 'xpu-2-6'
 Requires-Dist: dpcpp-cpp-rt ==2025.0.2 ; extra == 'xpu-2-6'
@@ -132,7 +132,7 @@ Requires-Dist: tokenizers ==0.15.2 ; extra == 'xpu-2-6-arl'
 Requires-Dist: accelerate ==0.23.0 ; extra == 'xpu-2-6-arl'
 Requires-Dist: tabulate ; extra == 'xpu-2-6-arl'
 Requires-Dist: setuptools ; extra == 'xpu-2-6-arl'
-Requires-Dist: bigdl-core-xe-all ==2.7.0b20250410 ; extra == 'xpu-2-6-arl'
+Requires-Dist: bigdl-core-xe-all ==2.7.0b20250414 ; extra == 'xpu-2-6-arl'
 Requires-Dist: onednn-devel ==2025.0.1 ; extra == 'xpu-2-6-arl'
 Requires-Dist: onednn ==2025.0.1 ; extra == 'xpu-2-6-arl'
 Requires-Dist: dpcpp-cpp-rt ==2025.0.2 ; extra == 'xpu-2-6-arl'
@@ -155,9 +155,9 @@ Requires-Dist: tokenizers ==0.15.2 ; extra == 'xpu-arc'
 Requires-Dist: accelerate ==0.23.0 ; extra == 'xpu-arc'
 Requires-Dist: tabulate ; extra == 'xpu-arc'
 Requires-Dist: setuptools ; extra == 'xpu-arc'
-Requires-Dist: bigdl-core-xe-23 ==2.7.0b20250410 ; extra == 'xpu-arc'
-Requires-Dist: bigdl-core-xe-batch-23 ==2.7.0b20250410 ; extra == 'xpu-arc'
-Requires-Dist: bigdl-core-xe-addons-23 ==2.7.0b20250410 ; extra == 'xpu-arc'
+Requires-Dist: bigdl-core-xe-23 ==2.7.0b20250414 ; extra == 'xpu-arc'
+Requires-Dist: bigdl-core-xe-batch-23 ==2.7.0b20250414 ; extra == 'xpu-arc'
+Requires-Dist: bigdl-core-xe-addons-23 ==2.7.0b20250414 ; extra == 'xpu-arc'
 Requires-Dist: intel-openmp ; (platform_machine == "x86_64" or platform_machine == "AMD64") and extra == 'xpu-arc'
 Requires-Dist: torch ==2.3.1+cxx11.abi ; (platform_system == "Linux") and extra == 'xpu-arc'
 Requires-Dist: torchvision ==0.18.1+cxx11.abi ; (platform_system == "Linux") and extra == 'xpu-arc'
@@ -178,9 +178,9 @@ Requires-Dist: tokenizers ==0.15.2 ; extra == 'xpu-arl'
 Requires-Dist: accelerate ==0.23.0 ; extra == 'xpu-arl'
 Requires-Dist: tabulate ; extra == 'xpu-arl'
 Requires-Dist: setuptools ; extra == 'xpu-arl'
-Requires-Dist: bigdl-core-xe-23 ==2.7.0b20250410 ; extra == 'xpu-arl'
-Requires-Dist: bigdl-core-xe-batch-23 ==2.7.0b20250410 ; extra == 'xpu-arl'
-Requires-Dist: bigdl-core-xe-addons-23 ==2.7.0b20250410 ; extra == 'xpu-arl'
+Requires-Dist: bigdl-core-xe-23 ==2.7.0b20250414 ; extra == 'xpu-arl'
+Requires-Dist: bigdl-core-xe-batch-23 ==2.7.0b20250414 ; extra == 'xpu-arl'
+Requires-Dist: bigdl-core-xe-addons-23 ==2.7.0b20250414 ; extra == 'xpu-arl'
 Requires-Dist: intel-openmp ; (platform_machine == "x86_64" or platform_machine == "AMD64") and extra == 'xpu-arl'
 Requires-Dist: torch ==2.3.1+cxx11.abi ; (platform_system == "Linux") and extra == 'xpu-arl'
 Requires-Dist: torchvision ==0.18.1+cxx11.abi ; (platform_system == "Linux") and extra == 'xpu-arl'
@@ -201,9 +201,9 @@ Requires-Dist: tokenizers ==0.15.2 ; extra == 'xpu-lnl'
 Requires-Dist: accelerate ==0.23.0 ; extra == 'xpu-lnl'
 Requires-Dist: tabulate ; extra == 'xpu-lnl'
 Requires-Dist: setuptools ; extra == 'xpu-lnl'
-Requires-Dist: bigdl-core-xe-23 ==2.7.0b20250410 ; extra == 'xpu-lnl'
-Requires-Dist: bigdl-core-xe-batch-23 ==2.7.0b20250410 ; extra == 'xpu-lnl'
-Requires-Dist: bigdl-core-xe-addons-23 ==2.7.0b20250410 ; extra == 'xpu-lnl'
+Requires-Dist: bigdl-core-xe-23 ==2.7.0b20250414 ; extra == 'xpu-lnl'
+Requires-Dist: bigdl-core-xe-batch-23 ==2.7.0b20250414 ; extra == 'xpu-lnl'
+Requires-Dist: bigdl-core-xe-addons-23 ==2.7.0b20250414 ; extra == 'xpu-lnl'
 Requires-Dist: intel-openmp ; (platform_machine == "x86_64" or platform_machine == "AMD64") and extra == 'xpu-lnl'
 Requires-Dist: torch ==2.3.1+cxx11.abi ; (platform_system == "Linux") and extra == 'xpu-lnl'
 Requires-Dist: torchvision ==0.18.1+cxx11.abi ; (platform_system == "Linux") and extra == 'xpu-lnl'

{ipex_llm-2.3.0b20250410.dist-info → ipex_llm-2.3.0b20250414.dist-info}/RECORD

@@ -41,35 +41,35 @@ ipex_llm/langchain/llms/transformerspipelinellm.py,sha256=vm522YPPwWxxAPVvQBtxRf
 ipex_llm/langchain/vllm/__init__.py,sha256=T-EbRT6GJ_8RCu-iLmSzcftOimXSPQf2d5X72AUAy2Y,874
 ipex_llm/langchain/vllm/vllm.py,sha256=6dxc-ZISZQrJilEa_HA827l75Dv9rcHpY_G6FdJ8BVs,7793
 ipex_llm/libs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-ipex_llm/libs/bloom-api.dll,sha256=WfAd-vNHNBYmooU8EnEoQA2JkWpyBuIejPaTfyY_SZk,36352
-ipex_llm/libs/bloom.dll,sha256=9fM_JsT4LUA9XM26LvJ9ZiZdZw0GY_UftjdVsPbC-7M,507904
-ipex_llm/libs/gptneox-api.dll,sha256=ngWpc5XL-yQYfBsxN0QtqgGiWjoZUG-yUW7S_BCNpb4,24576
-ipex_llm/libs/gptneox.dll,sha256=yy1fkaCLDdIEIkAA0pfq8kbq81oMEk3Mm-Rkl0VX2yg,568320
-ipex_llm/libs/libbloom_avx.dll,sha256=Q7kv0z-0GyCEp5nj5Ern_NxFEWl7eDV6wmRgauaC5GA,536576
-ipex_llm/libs/libbloom_vnni.dll,sha256=cgx0LDge7gATzfF7NkIL4yLXFRtFm_DT3r72hge3V40,508416
-ipex_llm/libs/libgptneox_avx.dll,sha256=IBl-RDmNw0Qs2RHJzjX7jofILYQKL9IAWV-suC8m62A,596992
-ipex_llm/libs/libgptneox_vnni.dll,sha256=CvcoD4UfSIhOTO1WG4EFN6Vkv2XdZq5J5f_pYh4Nt7o,568832
-ipex_llm/libs/libllama_avx.dll,sha256=xjko1uyWQFGOJ7h_VEMjTrcsCNeZUUrunoFX5YnTfzM,591360
-ipex_llm/libs/libllama_vnni.dll,sha256=RcvAYB4CPffb-YlS8VMfliFGhL4CrIVvf-4bZb-r_2M,563200
-ipex_llm/libs/libstarcoder_avx.dll,sha256=OKtbH_1hJZ4jlkl3gZLapf7clikeC3rVV8RuZ_hWLhQ,627712
-ipex_llm/libs/libstarcoder_vnni.dll,sha256=kSAIo8O3YtBBgUwfXCeadECJK9nAxs1YvRa8SfIxE-Q,599552
-ipex_llm/libs/llama-api.dll,sha256=vxKuiZix6fx5ydDeWOdT1sPb1wV0ouIXuD7elzk8kpc,25600
-ipex_llm/libs/llama.dll,sha256=wtbG9_50kOTFAxef8Tb8P8FIzZ5JYqkmig9A9JxVp0A,562688
-ipex_llm/libs/main-bloom.exe,sha256=hvo6htyCEyZ8xZ2LOt90xbx43v5cDQtt7YrfskzncSU,103424
-ipex_llm/libs/main-gptneox.exe,sha256=qvp8UlpkBU09W29bUJkckfpN5hFjmN4HGL2i7G7KhpE,98816
-ipex_llm/libs/main-llama.exe,sha256=yB2z4ddSKWZnECsn5jm70gMyOq4t-QK1QRT7SlM8d_E,99840
-ipex_llm/libs/main-starcoder.exe,sha256=wGcRZBMqWTK64kmR36f94Jq1R1ElF8JH-xYl4UkxUxg,157696
-ipex_llm/libs/pipeline.dll,sha256=3x6sBRn72IP6CyhFdsq9dyFDJra8WRGZs0dstl5SYHo,73216
-ipex_llm/libs/quantize-bloom.exe,sha256=2au0XxXiYkWmg-z3RtqH958_2hAsMDsj_HP69DuykGI,126464
-ipex_llm/libs/quantize-bloom_vnni.exe,sha256=Pygds4P-hMVyc9vqCNHIKrRcETNwDlIU-sHGiKwDUBU,128000
-ipex_llm/libs/quantize-gptneox.exe,sha256=2F8d_iMGeWxtmoN5WpMnf2qhosPqm1aZd4FFlunqb7k,104448
-ipex_llm/libs/quantize-gptneox_vnni.exe,sha256=u-_BVBPRj5edcTzjQj_jd_dJ3ANDER73I8pnA_fI0v8,104960
-ipex_llm/libs/quantize-llama.exe,sha256=MdSzxiyjFEgNooqgUM8DMVNniiDRwnzuQ63IEu0b4As,110080
-ipex_llm/libs/quantize-llama_vnni.exe,sha256=fMg9YKVbJoB2GgdZLjpuPkOvecqF2joR2JovboQLzhg,110592
-ipex_llm/libs/quantize-starcoder.exe,sha256=cgHF7r8oMpxIDCoKYQfcVRQ5RLDA2n9ttqklvsfBTj0,127488
-ipex_llm/libs/quantize-starcoder_vnni.exe,sha256=BywuKBiNyJovLQ8uQxvmRgK5Mvr-VZMXh6ZkwIGXjDQ,128512
-ipex_llm/libs/starcoder-api.dll,sha256=tYqccLQiJkRgMST2UlkhYLGJQOYPott3mnwGZZxkuGU,21504
-ipex_llm/libs/starcoder.dll,sha256=Xt6gimIDqhNGkNnKkeOGiEgXU9kAJKrnMlJlbppzHrY,599040
+ipex_llm/libs/bloom-api.dll,sha256=RQH35fsCjbpJAqZ6vqpzIvurExHg77Dm_j9SF1eL8DQ,36352
+ipex_llm/libs/bloom.dll,sha256=riUIPG6kaX1CeCirSMMuPjnadxQrm89t3s8MCjW9VDg,507904
+ipex_llm/libs/gptneox-api.dll,sha256=3Z21zemTxGTPtZyLrJQlwaphhjnEUH_nD2_azE4GHjU,24576
+ipex_llm/libs/gptneox.dll,sha256=Y91HtjmiOHXOgttQod7KG9f9KkCdU9Jbu_V_2Ncz1Ic,568320
+ipex_llm/libs/libbloom_avx.dll,sha256=siYYEY3keJNXxD1q1qe2EcX0Cz6ezyr4E1jD4kQ8DMI,536576
+ipex_llm/libs/libbloom_vnni.dll,sha256=Q0Wtp6u-xcTW_TTsUHqn0X7353W_ere_k0GwsI0QVhE,508416
+ipex_llm/libs/libgptneox_avx.dll,sha256=AtpJopCdHu0RAVPyunHsQ9qQWLiJMIE7ba4gme7nC_k,596992
+ipex_llm/libs/libgptneox_vnni.dll,sha256=Gt4tFPro-zNdU4S22k96Tds-tVCwRgccuu3RRwWCsJ4,568832
+ipex_llm/libs/libllama_avx.dll,sha256=BrqmZm6ASzZ5hPiu7cjJKLxs2pnTUDS8vlaVwXtglKc,591360
+ipex_llm/libs/libllama_vnni.dll,sha256=44HGXEJgJYJ6kS_dngqYR9VBFXRiYEooxo4K_xLLvyA,563200
+ipex_llm/libs/libstarcoder_avx.dll,sha256=XkCQzKH7Yg3n1hLbYhNaCgRGa_vEjT4G048i7FqiPFI,627712
+ipex_llm/libs/libstarcoder_vnni.dll,sha256=hhggl9l81TYcEGpZymdRkAtGeyrgSUqdHR3Xn-yfARE,599552
+ipex_llm/libs/llama-api.dll,sha256=k50ow33IgEjCfmyhaeLEK77JRVhu8UacJGiVCdj56lI,25600
+ipex_llm/libs/llama.dll,sha256=c81XUKbRl5RRpvad8qji61I-TW2N2WIA7SPWTmJ6c7I,562688
+ipex_llm/libs/main-bloom.exe,sha256=-Pg9G61YLFlBuMloBWPZ0BwuyizTdP9Gmp9H8lBmlIE,103424
+ipex_llm/libs/main-gptneox.exe,sha256=FqJFdSuRLzHzISAS3Fq-hpNaD5AzByRhidVJGvx8XsM,98816
+ipex_llm/libs/main-llama.exe,sha256=gCMvDvxMGMcTITQVB7JwzDQl8hWiS7cMedpd5qH1Yso,99840
+ipex_llm/libs/main-starcoder.exe,sha256=Hp8bVAB3gkleaTILUFP373bj_-FfZBhgY98ifF4Seg8,157696
+ipex_llm/libs/pipeline.dll,sha256=klcRphRZj-Ver9bb-3YarFz1Aofx2eH_9ZFVj6005IY,73216
+ipex_llm/libs/quantize-bloom.exe,sha256=1RCyEGDHqMgEV3-Z5whNWeoPH2pbmIuoNdFxwwWVzBM,126464
+ipex_llm/libs/quantize-bloom_vnni.exe,sha256=9B3eXy6FuOxdw4R7226GWwZdYRkA_0wrrdoEB4iRvL4,128000
+ipex_llm/libs/quantize-gptneox.exe,sha256=ho-9GbDDXLe6OCZUQXLLKQBaxndRs0uoIp7blm7mkX4,104448
+ipex_llm/libs/quantize-gptneox_vnni.exe,sha256=YxBxpYZGZASjRE0AvfhWPFN5y07J7b_p7uQ6KBfcgoA,104960
+ipex_llm/libs/quantize-llama.exe,sha256=N4t0zFvqDydwEF-ziXUfiWtoBUU_0InDyBgufaUGXzU,110080
+ipex_llm/libs/quantize-llama_vnni.exe,sha256=23i9bbF_ezU4WLc2vSjdzPP2aTMdPyxiQMFnSAaLk1A,110592
+ipex_llm/libs/quantize-starcoder.exe,sha256=DYm5NLxvHgpeldjdK8e0LcvgnLauLo5gz78CGVEKQTI,127488
+ipex_llm/libs/quantize-starcoder_vnni.exe,sha256=uoMxJlhpV-BzhANKO6IJls6iIP7KG03HIq8KxZXp-A8,128512
+ipex_llm/libs/starcoder-api.dll,sha256=wKmuMWuK2rlCEPrSLIaaK_ylaECQEiU730vSyurxBG8,21504
+ipex_llm/libs/starcoder.dll,sha256=rWkn029EPDSsEjlRu18jLq_0fC4b6axEeHjgoezGSEM,599040
 ipex_llm/llamaindex/__init__.py,sha256=T-EbRT6GJ_8RCu-iLmSzcftOimXSPQf2d5X72AUAy2Y,874
 ipex_llm/llamaindex/llms/__init__.py,sha256=KP1lEdGqDuxPoxL1ZSH25Pm2kKMPJBWUTLR0ckSLMIU,1139
 ipex_llm/llamaindex/llms/bigdlllm.py,sha256=FQBzq1KOjfc6uofTXAha3O7TqpJkNfOFepXQmOVlbnI,26314
@@ -87,10 +87,10 @@ ipex_llm/serving/fastchat/tgi_api_protocol.py,sha256=brT3k3-V0NJrU4fRqUwWjC0O3iO
 ipex_llm/serving/fastchat/tgi_api_server.py,sha256=agNTAEiZPSuj3dEdIdYKwkoY0cXOUDX06DiM9VP2knQ,24418
 ipex_llm/serving/fastchat/vllm_worker.py,sha256=ZLz2Q9GxJO6r_LOiP6epgCRjBGk-K4EB1SNEWSJp5DA,11091
 ipex_llm/transformers/__init__.py,sha256=BreA3EY6hkNq0rVixb_sUuTLzMrcWXTt3yRsshCPHHQ,1214
-ipex_llm/transformers/convert.py,sha256=xqqZFGcdDRko2IYgfSgDRs8ef4THUR25IAhSyDV0VUs,106933
+ipex_llm/transformers/convert.py,sha256=niCLZiEQbiMz8UkfYDNwYl2hE1caj93FUR1qjgVh9ZM,107538
 ipex_llm/transformers/convert_ipex.py,sha256=_nSnUTQy-yfkKaqGdqnBdWztZf3NGmnbZ0TKaDrF4X4,14617
 ipex_llm/transformers/embedding.py,sha256=bdgk59DvD4ZZyxRzewXOR7g56nThgO6uhIwk8QL7f-s,9299
-ipex_llm/transformers/kv.py,sha256=k4TU18LlA-Sbq9WNNQnfuzu3RSFBwFhmaV3BcGN5bAo,19191
+ipex_llm/transformers/kv.py,sha256=src_HcVDKFwQ1V8hdTrFQw5RIwUewM9VOR47GVTPJG4,21187
 ipex_llm/transformers/lisa.py,sha256=F5WxbtXQ7RdKulj83h_2DnEIgKiKGZf7zvOmg6QBl2s,3289
 ipex_llm/transformers/loader.py,sha256=c9qfJSC6-in-mkd-iKb1igk3nHWUYS3QtyH2cOazmKc,6825
 ipex_llm/transformers/lookup.py,sha256=b6OlZ9OV10R9qeWw8mVryVpDxszkjwLkldvi7GPMJY8,19614
@@ -146,7 +146,7 @@ ipex_llm/transformers/models/chatglm.py,sha256=UHai1t2AUtGmF765_eHF8LUMVQzp_oCBx
 ipex_llm/transformers/models/chatglm2.py,sha256=KyAIX7zGVQDQuwwM3QMBNWZbTeMHEzKUIgAryT0voHc,14933
 ipex_llm/transformers/models/chatglm4.py,sha256=QvUehdaCePB3MNHyWg3dneDxmjtBdxYeKUyQUVcsgfM,16886
 ipex_llm/transformers/models/chatglm4v.py,sha256=Ba9Xtzwtzk_rzg5khGqDrlHfJsDwc5YcM5_yPoord7o,13324
-ipex_llm/transformers/models/common.py,sha256=0OTRaXekOPApRdQ8UKl5Du8DOtKJ6awnQIStvYvFQOI,13018
+ipex_llm/transformers/models/common.py,sha256=LVA9nL_qJ61NEkEn9T985PjrrWPGpDTCALknH4Qv5aw,13040
 ipex_llm/transformers/models/decilm.py,sha256=P-PBuDPf07GvKggLwJx_wPwIn6esN3rX8ai2JxRuZmE,5246
 ipex_llm/transformers/models/deepseek.py,sha256=w6tGeyJ9joD7lQBiZ6A01Z00g8hAXC1N2yGtJh8kyuk,13096
 ipex_llm/transformers/models/deepseek_v3.py,sha256=CTgwIKQlUPlUCbOxc9Id5GapWkXOP6pMtkguYrWpCio,10003
@@ -180,7 +180,7 @@ ipex_llm/transformers/models/rwkv5.py,sha256=OkRNj1pCAZg1z2Fw-I0DEnxLEdZyPeRSQ6m
 ipex_llm/transformers/models/sd.py,sha256=VvHV5u-0k2MgHu3NL9113hPj7DgfxqctuKzEEeNfRDU,5981
 ipex_llm/transformers/models/stablelm.py,sha256=fj-XtOnR6kggnFUQTMPCOOzolkPztN06WAv8QW-XRnI,7054
 ipex_llm/transformers/models/starcoder2.py,sha256=ONKvD7JCkRM0DI-R56x28QFBJ7CjD5hOZBQ_3WfOcNk,6626
-ipex_llm/transformers/models/utils.py,sha256=c3hh0YDHE-Qg7SQBXhnNXf85Nx7jopZFfa1KS-Pe6kQ,14734
+ipex_llm/transformers/models/utils.py,sha256=Rj7QK1s3QxsTT4HzGNR00Q7izG5xYiZEX29YoQIM1tA,16998
 ipex_llm/transformers/models/whisper.py,sha256=ju3WP8Eq-KvD7kb3Qy51r4FOfSX3NBxfp5RBcq__gzc,4241
 ipex_llm/transformers/models/yuan.py,sha256=JYAn_ZaSGK0NBJLEIxCACfAq084a66GFJkdd5NbpmMA,7732
 ipex_llm/transformers/npu_models/__init__.py,sha256=ulEUGLjaP48LCrVeury3UxLjXxKzRi0UpSG4bYu-7f8,585
@@ -256,11 +256,11 @@ ipex_llm/vllm/xpu/engine/__init__.py,sha256=pY_CpyuZd72fr6s32ejeKHKFW0K4vUU2rzZj
 ipex_llm/vllm/xpu/engine/engine.py,sha256=NvCMbp0X8NVrOqbwm4FTvXOptTRLzu9jQsy37ZHnTk8,9493
 ipex_llm/vllm/xpu/entrypoints/openai/api_server.py,sha256=IjiSze9vzBCAkLu_VwIcJwuO1jyFna7DLrj6aSL7RaQ,35220
 ipex_llm/vllm/xpu/entrypoints/openai/cli_args.py,sha256=hB398yYtKauASRzevctScdbFIjiiSGMAe1bwEuIHrhY,10893
-ipex_llm-2.3.0b20250410.data/scripts/ipex-llm-init.bat,sha256=HPtCYuDYwEatq7dAwOvdfVcHYCpAVdbj75K1qh0vQek,2578
-ipex_llm-2.3.0b20250410.data/scripts/llm-chat.ps1,sha256=6qrs-hGVAV8IKh7Jx8nq_XrnZcjd7qGU5wndArM7Yag,2769
-ipex_llm-2.3.0b20250410.data/scripts/llm-cli.ps1,sha256=3qBtTLs_EjYDnM8YyCpJhzLnGCKTEGssu9UNqfkjVXs,3009
-ipex_llm-2.3.0b20250410.dist-info/METADATA,sha256=Tr1wuth1NPT_yf_bvKcaIS2ZlYLd0mR_A2uZ7m3tJb0,13917
-ipex_llm-2.3.0b20250410.dist-info/WHEEL,sha256=6iYPr8vTHsyDK75jr9X0V3I9wPSVmtwr_8fdATBciGk,98
-ipex_llm-2.3.0b20250410.dist-info/entry_points.txt,sha256=TiUyBB2MRmfF3ko-pyAEzqeBCRnyhu27bNOAsWPp3e8,61
-ipex_llm-2.3.0b20250410.dist-info/top_level.txt,sha256=CGCMHM-SyqUabU4h8RqJ2KTYckQUO3LvIWwmUQ6Qbzw,9
-ipex_llm-2.3.0b20250410.dist-info/RECORD,,
+ipex_llm-2.3.0b20250414.data/scripts/ipex-llm-init.bat,sha256=HPtCYuDYwEatq7dAwOvdfVcHYCpAVdbj75K1qh0vQek,2578
+ipex_llm-2.3.0b20250414.data/scripts/llm-chat.ps1,sha256=6qrs-hGVAV8IKh7Jx8nq_XrnZcjd7qGU5wndArM7Yag,2769
+ipex_llm-2.3.0b20250414.data/scripts/llm-cli.ps1,sha256=3qBtTLs_EjYDnM8YyCpJhzLnGCKTEGssu9UNqfkjVXs,3009
+ipex_llm-2.3.0b20250414.dist-info/METADATA,sha256=4kiLch6GAWOIkpZGeb-YiWgSUq8Linmj_qrwVY-hZew,13917
+ipex_llm-2.3.0b20250414.dist-info/WHEEL,sha256=6iYPr8vTHsyDK75jr9X0V3I9wPSVmtwr_8fdATBciGk,98
+ipex_llm-2.3.0b20250414.dist-info/entry_points.txt,sha256=TiUyBB2MRmfF3ko-pyAEzqeBCRnyhu27bNOAsWPp3e8,61
+ipex_llm-2.3.0b20250414.dist-info/top_level.txt,sha256=CGCMHM-SyqUabU4h8RqJ2KTYckQUO3LvIWwmUQ6Qbzw,9
+ipex_llm-2.3.0b20250414.dist-info/RECORD,,