ipex-llm 2.3.0b20250515__py3-none-manylinux2010_x86_64.whl → 2.3.0b20250518__py3-none-manylinux2010_x86_64.whl

ipex_llm/transformers/models/common.py

@@ -373,3 +373,41 @@ def moe_softmax_topk(router_logits: torch.Tensor, top_k: int, norm_topk_prob: bo
         router_logits, top_k, norm_topk_prob
     )
     return selected_experts, routing_weights
+
+
+# q,k,v_proj should be ipex-llm quantized linears
+def merge_quantized_qkv(q_proj, k_proj, v_proj, module):
+    from ipex_llm.transformers.low_bit_linear import FP4Params
+    from ipex_llm.ggml.quantize import ggml_tensor_qtype
+    has_qtype = (hasattr(q_proj.weight, 'qtype')
+                 and hasattr(k_proj.weight, 'qtype')
+                 and hasattr(v_proj.weight, 'qtype'))
+    invalidInputError((has_qtype
+                       and q_proj.weight.qtype == k_proj.weight.qtype
+                       and q_proj.weight.qtype == v_proj.weight.qtype
+                       and q_proj.weight.qtype in ggml_tensor_qtype.values()),
+                      f"{q_proj.weight.qtype} is not supported, "
+                      f"only {ggml_tensor_qtype.values()} are supported now.")
+    origin_device = q_proj.weight.device
+    q_proj.weight = q_proj.weight.to('cpu')
+    k_proj.weight = k_proj.weight.to('cpu')
+    v_proj.weight = v_proj.weight.to('cpu')
+    linears = [q_proj, k_proj, v_proj]
+    new_weight = torch.cat(list(linear.weight.data for linear in linears), dim=0)
+    if q_proj.has_bias:
+        new_bias = torch.cat(list(linear.bias.data for linear in linears), dim=0)
+        q_proj.bias = torch.nn.Parameter(new_bias, requires_grad=False)
+    new_out_features = sum(layer.out_features for layer in linears)
+    new_params_low_bit = FP4Params(data=new_weight.data,
+                                   requires_grad=False,
+                                   quantized=True,
+                                   _shape=[new_out_features, q_proj.in_features],
+                                   convert_shape_only=False,
+                                   qtype=q_proj.weight.qtype,
+                                   in_features=q_proj.in_features,
+                                   enable_scale_search=False)
+    q_proj.out_features = new_out_features
+    q_proj.weight = new_params_low_bit.to(origin_device)
+    del module.q_proj.weight
+    module.qkv_proj = module.q_proj
+    del module.k_proj, module.v_proj, module.q_proj

{ipex_llm-2.3.0b20250515.dist-info → ipex_llm-2.3.0b20250518.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: ipex-llm
-Version: 2.3.0b20250515
+Version: 2.3.0b20250518
 Summary: Large Language Model Develop Toolkit
 Home-page: https://github.com/intel-analytics/ipex-llm
 Author: BigDL Authors
@@ -27,7 +27,7 @@ Requires-Dist: intel-openmp ; (platform_machine == "x86_64" or platform_machine
 Requires-Dist: torch ==2.1.2+cpu ; (platform_system == "Linux") and extra == 'all'
 Requires-Dist: torch ==2.1.2 ; (platform_system == "Windows") and extra == 'all'
 Provides-Extra: cpp
-Requires-Dist: bigdl-core-cpp ==2.7.0b20250515 ; extra == 'cpp'
+Requires-Dist: bigdl-core-cpp ==2.7.0b20250518 ; extra == 'cpp'
 Requires-Dist: setuptools ; extra == 'cpp'
 Requires-Dist: onednn-devel ==2025.0.1 ; (platform_system == "Windows") and extra == 'cpp'
 Requires-Dist: onednn ==2025.0.1 ; (platform_system == "Windows") and extra == 'cpp'
@@ -60,7 +60,7 @@ Requires-Dist: transformers ==4.40.0 ; extra == 'npu'
 Requires-Dist: intel-openmp ; (platform_machine == "x86_64" or platform_machine == "AMD64") and extra == 'npu'
 Requires-Dist: torch ==2.1.2+cpu ; (platform_system == "Linux") and extra == 'npu'
 Requires-Dist: torch ==2.1.2 ; (platform_system == "Windows") and extra == 'npu'
-Requires-Dist: bigdl-core-npu ==2.7.0b20250515 ; (platform_system == "Windows") and extra == 'npu'
+Requires-Dist: bigdl-core-npu ==2.7.0b20250518 ; (platform_system == "Windows") and extra == 'npu'
 Provides-Extra: serving
 Requires-Dist: py-cpuinfo ; extra == 'serving'
 Requires-Dist: fschat[model_worker,webui] ==0.2.36 ; extra == 'serving'
@@ -80,9 +80,9 @@ Requires-Dist: setuptools <70.0.0 ; extra == 'xpu'
 Requires-Dist: torch ==2.1.0a0 ; extra == 'xpu'
 Requires-Dist: torchvision ==0.16.0a0 ; extra == 'xpu'
 Requires-Dist: intel-extension-for-pytorch ==2.1.10+xpu ; extra == 'xpu'
-Requires-Dist: bigdl-core-xe-21 ==2.7.0b20250515 ; extra == 'xpu'
-Requires-Dist: bigdl-core-xe-batch-21 ==2.7.0b20250515 ; extra == 'xpu'
-Requires-Dist: bigdl-core-xe-addons-21 ==2.7.0b20250515 ; extra == 'xpu'
+Requires-Dist: bigdl-core-xe-21 ==2.7.0b20250518 ; extra == 'xpu'
+Requires-Dist: bigdl-core-xe-batch-21 ==2.7.0b20250518 ; extra == 'xpu'
+Requires-Dist: bigdl-core-xe-addons-21 ==2.7.0b20250518 ; extra == 'xpu'
 Provides-Extra: xpu-2-1
 Requires-Dist: py-cpuinfo ; extra == 'xpu-2-1'
 Requires-Dist: protobuf ; extra == 'xpu-2-1'
@@ -97,9 +97,9 @@ Requires-Dist: setuptools <70.0.0 ; extra == 'xpu-2-1'
 Requires-Dist: torch ==2.1.0a0 ; extra == 'xpu-2-1'
 Requires-Dist: torchvision ==0.16.0a0 ; extra == 'xpu-2-1'
 Requires-Dist: intel-extension-for-pytorch ==2.1.10+xpu ; extra == 'xpu-2-1'
-Requires-Dist: bigdl-core-xe-21 ==2.7.0b20250515 ; extra == 'xpu-2-1'
-Requires-Dist: bigdl-core-xe-batch-21 ==2.7.0b20250515 ; extra == 'xpu-2-1'
-Requires-Dist: bigdl-core-xe-addons-21 ==2.7.0b20250515 ; extra == 'xpu-2-1'
+Requires-Dist: bigdl-core-xe-21 ==2.7.0b20250518 ; extra == 'xpu-2-1'
+Requires-Dist: bigdl-core-xe-batch-21 ==2.7.0b20250518 ; extra == 'xpu-2-1'
+Requires-Dist: bigdl-core-xe-addons-21 ==2.7.0b20250518 ; extra == 'xpu-2-1'
 Requires-Dist: intel-openmp ; (platform_machine == "x86_64" or platform_machine == "AMD64") and extra == 'xpu-2-1'
 Requires-Dist: dpcpp-cpp-rt ==2024.0.2 ; (platform_system == "Windows") and extra == 'xpu-2-1'
 Requires-Dist: mkl-dpcpp ==2024.0.0 ; (platform_system == "Windows") and extra == 'xpu-2-1'
@@ -117,7 +117,7 @@ Requires-Dist: setuptools ; extra == 'xpu-2-6'
 Requires-Dist: torch ==2.6.0+xpu ; extra == 'xpu-2-6'
 Requires-Dist: torchvision ==0.21.0+xpu ; extra == 'xpu-2-6'
 Requires-Dist: torchaudio ==2.6.0+xpu ; extra == 'xpu-2-6'
-Requires-Dist: bigdl-core-xe-all ==2.7.0b20250515 ; extra == 'xpu-2-6'
+Requires-Dist: bigdl-core-xe-all ==2.7.0b20250518 ; extra == 'xpu-2-6'
 Requires-Dist: onednn-devel ==2025.0.1 ; extra == 'xpu-2-6'
 Requires-Dist: onednn ==2025.0.1 ; extra == 'xpu-2-6'
 Requires-Dist: dpcpp-cpp-rt ==2025.0.2 ; extra == 'xpu-2-6'
@@ -132,7 +132,7 @@ Requires-Dist: tokenizers ==0.15.2 ; extra == 'xpu-2-6-arl'
 Requires-Dist: accelerate ==0.23.0 ; extra == 'xpu-2-6-arl'
 Requires-Dist: tabulate ; extra == 'xpu-2-6-arl'
 Requires-Dist: setuptools ; extra == 'xpu-2-6-arl'
-Requires-Dist: bigdl-core-xe-all ==2.7.0b20250515 ; extra == 'xpu-2-6-arl'
+Requires-Dist: bigdl-core-xe-all ==2.7.0b20250518 ; extra == 'xpu-2-6-arl'
 Requires-Dist: onednn-devel ==2025.0.1 ; extra == 'xpu-2-6-arl'
 Requires-Dist: onednn ==2025.0.1 ; extra == 'xpu-2-6-arl'
 Requires-Dist: dpcpp-cpp-rt ==2025.0.2 ; extra == 'xpu-2-6-arl'

{ipex_llm-2.3.0b20250515.dist-info → ipex_llm-2.3.0b20250518.dist-info}/RECORD

@@ -153,7 +153,7 @@ ipex_llm/transformers/models/chatglm.py,sha256=DQM63oPIVMMTBQN4O4hPF4WY1aSiTWq4B
 ipex_llm/transformers/models/chatglm2.py,sha256=KyAIX7zGVQDQuwwM3QMBNWZbTeMHEzKUIgAryT0voHc,14933
 ipex_llm/transformers/models/chatglm4.py,sha256=QvUehdaCePB3MNHyWg3dneDxmjtBdxYeKUyQUVcsgfM,16886
 ipex_llm/transformers/models/chatglm4v.py,sha256=Ba9Xtzwtzk_rzg5khGqDrlHfJsDwc5YcM5_yPoord7o,13324
-ipex_llm/transformers/models/common.py,sha256=TnSEPaXixHg2kjbvNnHtM4raN5ayMzwkI7nyW4J0Rwk,16043
+ipex_llm/transformers/models/common.py,sha256=NjwDmhrTcLQ4ceP5VKndl8YJ0uFdkftVCi3pn3oTOic,18080
 ipex_llm/transformers/models/decilm.py,sha256=P-PBuDPf07GvKggLwJx_wPwIn6esN3rX8ai2JxRuZmE,5246
 ipex_llm/transformers/models/deepseek.py,sha256=BJocxhznzvM99IZeAWuhyHEBm6Z2-AwhLFYMdjMJuc4,13083
 ipex_llm/transformers/models/deepseek_v3.py,sha256=CTgwIKQlUPlUCbOxc9Id5GapWkXOP6pMtkguYrWpCio,10003
@@ -265,11 +265,11 @@ ipex_llm/vllm/xpu/engine/__init__.py,sha256=sOvwLx_Zj0jiRCGj9W3DgGTfcSU3hABYhgIQ
 ipex_llm/vllm/xpu/engine/engine.py,sha256=XAprw7VifjfnR915TZOaKcxe3QCFsVBgxzS8qOdn1yg,14462
 ipex_llm/vllm/xpu/entrypoints/openai/api_server.py,sha256=uWHyyHLw-B8wXBnQw9_MCG81tKK9Jb0dyq1xfYHgoNw,45905
 ipex_llm/vllm/xpu/entrypoints/openai/cli_args.py,sha256=hB398yYtKauASRzevctScdbFIjiiSGMAe1bwEuIHrhY,10893
-ipex_llm-2.3.0b20250515.data/scripts/ipex-llm-init,sha256=fLQsT2dRL6H5bThb4GuIWotAuqoLsIxFwA-0c2qmaO8,6672
-ipex_llm-2.3.0b20250515.data/scripts/llm-chat,sha256=TdUnUmNapzuoe1c8IzrdVOQwWEg8IqsMSBRlOD3daZM,2249
-ipex_llm-2.3.0b20250515.data/scripts/llm-cli,sha256=RXGPlLElHxcKzoUxljEMBIAXbzCDysXL-Nxw-xF-7LU,2457
-ipex_llm-2.3.0b20250515.dist-info/METADATA,sha256=_lbNuaDcLJThqZdxaHamNmc-qDmC7S7F7E_MgDqH_lk,8865
-ipex_llm-2.3.0b20250515.dist-info/WHEEL,sha256=PPJcBMAZibF_2GFE9NmOJGqiaSMPiNFbJd6QaJjdA6Y,109
-ipex_llm-2.3.0b20250515.dist-info/entry_points.txt,sha256=TiUyBB2MRmfF3ko-pyAEzqeBCRnyhu27bNOAsWPp3e8,61
-ipex_llm-2.3.0b20250515.dist-info/top_level.txt,sha256=CGCMHM-SyqUabU4h8RqJ2KTYckQUO3LvIWwmUQ6Qbzw,9
-ipex_llm-2.3.0b20250515.dist-info/RECORD,,
+ipex_llm-2.3.0b20250518.data/scripts/ipex-llm-init,sha256=fLQsT2dRL6H5bThb4GuIWotAuqoLsIxFwA-0c2qmaO8,6672
+ipex_llm-2.3.0b20250518.data/scripts/llm-chat,sha256=TdUnUmNapzuoe1c8IzrdVOQwWEg8IqsMSBRlOD3daZM,2249
+ipex_llm-2.3.0b20250518.data/scripts/llm-cli,sha256=RXGPlLElHxcKzoUxljEMBIAXbzCDysXL-Nxw-xF-7LU,2457
+ipex_llm-2.3.0b20250518.dist-info/METADATA,sha256=EyH2NoP3wcaSi92HT3Ss1zNnvVhsRwkHtPeFhAqMvD8,8865
+ipex_llm-2.3.0b20250518.dist-info/WHEEL,sha256=PPJcBMAZibF_2GFE9NmOJGqiaSMPiNFbJd6QaJjdA6Y,109
+ipex_llm-2.3.0b20250518.dist-info/entry_points.txt,sha256=TiUyBB2MRmfF3ko-pyAEzqeBCRnyhu27bNOAsWPp3e8,61
+ipex_llm-2.3.0b20250518.dist-info/top_level.txt,sha256=CGCMHM-SyqUabU4h8RqJ2KTYckQUO3LvIWwmUQ6Qbzw,9
+ipex_llm-2.3.0b20250518.dist-info/RECORD,,