bigdl-core-npu 2.6.0b20241117__cp310-cp310-win_amd64.whl → 2.6.0b20241118__cp310-cp310-win_amd64.whl

{bigdl_core_npu-2.6.0b20241117.dist-info → bigdl_core_npu-2.6.0b20241118.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: bigdl-core-npu
-Version: 2.6.0b20241117
+Version: 2.6.0b20241118
 Summary: Intel® NPU Acceleration Library
 Home-page: https://github.com/intel/intel-npu-acceleration-library
 Author: Alessandro Palla

{bigdl_core_npu-2.6.0b20241117.dist-info → bigdl_core_npu-2.6.0b20241118.dist-info}/RECORD

@@ -1,5 +1,5 @@
 intel_npu_acceleration_library/__init__.py,sha256=ZKTIhGMDjF7P6pF-yX8KWcSXbeHWRk24AO_orsa18f8,536
-intel_npu_acceleration_library/_version.py,sha256=aQGSlQhbDKAmt0SXhnnSMKrIdQJdlD4QOf0ueOSAAh8,112
+intel_npu_acceleration_library/_version.py,sha256=irQJohc3KFxvKBKCL8Lkw8VSyeUK_28o0EIhT0xLrH0,112
 intel_npu_acceleration_library/compiler.py,sha256=3IdgqjamSC8MLexDBJypIeZRiWIcTFnvQSU1LPXUr7Y,6225
 intel_npu_acceleration_library/device.py,sha256=9bn8eVXJa5cXIqgfLsQAdkMVtVUQABb8z0-mQik5jRg,7424
 intel_npu_acceleration_library/dtypes.py,sha256=gdd06Wsc9zIZFHlauUEx4xcK9WGTn1Mu6GkuYDJeA-E,4683
@@ -8,19 +8,19 @@ intel_npu_acceleration_library/optimizations.py,sha256=9NY8QoDFbs2LY12jbx6As8g2v
 intel_npu_acceleration_library/quantization.py,sha256=6N_04h1KX6TNbw-ceANV0Pmk4_lQ2Y9C7Pwn5x-zQzo,5566
 intel_npu_acceleration_library/backend/__init__.py,sha256=2NP6Ypr1dGUNXmLGW5GD9xrh0U9KJgqxTd_c7su1RUY,857
 intel_npu_acceleration_library/backend/base.py,sha256=hbHqxSOfWH5BaA5PY6_zaf1Zdg5NrQK6WOfe-hr279k,8605
-intel_npu_acceleration_library/backend/bindings.py,sha256=wXn-5HCYGUFOpDJwuOWf7cHSrwsfe9hdwD0tF4wcXps,9378
+intel_npu_acceleration_library/backend/bindings.py,sha256=6APMmDZnYBwsjcZIO5bgFz8IwfhR4CH1evWWc2IylwM,10005
 intel_npu_acceleration_library/backend/compression.py,sha256=Avz_zm2s_ELy5peVQ8zFGn8njBfh9nEGR16mflotBic,630
 intel_npu_acceleration_library/backend/convolution.py,sha256=cN3k78X3Y4Cbf7er-MFq0sJ4OwIvquj8PajpdEDmCo4,2018
-intel_npu_acceleration_library/backend/factory.py,sha256=ZvvatSKSNRUzht1ufCjuW0VvvfZBu38AdI-nKHEfqVQ,39064
+intel_npu_acceleration_library/backend/factory.py,sha256=5NQyfBxjc0lkFrzmzwDSy6q7K-W78uTpGoWWfUDJGjg,40953
 intel_npu_acceleration_library/backend/linear.py,sha256=RiLUh5FOSxRWHB5kYx7mOPOOrS_vxIeBJ5t3yC6wOiQ,1908
 intel_npu_acceleration_library/backend/matmul.py,sha256=mfGi73-mIbUcXp4kyvCGW0Y9kb4Xp1ppbGNpdJFohuA,1819
 intel_npu_acceleration_library/backend/mlp.py,sha256=BuKVwSI726v3nHQQvtMBbXyWxRTq-WoLZtTxeSeWaaY,2330
-intel_npu_acceleration_library/backend/ops.py,sha256=HzfE1Le9s7DWycWx6beeETe0N8yGEIttZfeM6FcCzME,4976
+intel_npu_acceleration_library/backend/ops.py,sha256=3yS-f-VPErHFt_oWZrgplNmSWnRrjm_wdxXNsqEBN7M,5070
 intel_npu_acceleration_library/backend/qlinear.py,sha256=oeawOjRBA_kQRqfQ1Vn_e3aJQa2b4pQ8y0gMwvHBTzk,2362
 intel_npu_acceleration_library/backend/qmatmul.py,sha256=pJkFJaBxZk3Oh5w_f6ywRNeGOfloEOCj0mCGnvim9Ew,2250
 intel_npu_acceleration_library/backend/runtime.py,sha256=COsTslfHknjrix4kETlnKvEmLO2lyBtXpAzEQTZJSvk,7195
 intel_npu_acceleration_library/backend/sdpa.py,sha256=HNlL9jEA9OH3KnZqOkLcaKwt8tfCe5apUQxlWw0UhlA,3818
-intel_npu_acceleration_library/backend/tensor.py,sha256=ok9wZ1QkD2IAbTn4R16vM6yUjHNf9jr8N1HUG--iT3g,35396
+intel_npu_acceleration_library/backend/tensor.py,sha256=swTymMVcXLPFXOlo6b_H3VGO1xf76Fz30RKyJPrVhPY,35468
 intel_npu_acceleration_library/backend/utils.py,sha256=WJ2agtqYxBuDd21ngE55io9VX-MOGg-AjnE63UHpCiU,2174
 intel_npu_acceleration_library/external/openvino/__init__.py,sha256=is1yO5Cdjdykxfhk7CnsU7w-pcn2iaIcmTLktjQPAT0,2254
 intel_npu_acceleration_library/external/openvino/_pyopenvino.cp310-win_amd64.pyd,sha256=N_bRmYtuWUocGX7sl4A7nmjX19w1QyEWuHsKxbUKEGc,3376096
@@ -196,7 +196,7 @@ intel_npu_acceleration_library/external/openvino/torch/__init__.py,sha256=RXLzsf
 intel_npu_acceleration_library/functional/__init__.py,sha256=WWKwKOh6Sgovv7mKctA872TbLP98Pg5m5-MREvUmlAA,204
 intel_npu_acceleration_library/functional/scaled_dot_product_attention.py,sha256=yGUcg4tDQOLuUnP1g74cl-ec8TRr2SuAMcNLlN6qLvE,1620
 intel_npu_acceleration_library/lib/Release/cache.json,sha256=CyrSqZUWo0Ec4_7ydOiuKIC0Gm8AybrGdozUqUuHxBw,8840377
-intel_npu_acceleration_library/lib/Release/intel_npu_acceleration_library.dll,sha256=03zAYlmxO62cTf0e8JFw_2vyZnGHFKn4sDTxqiXOMj4,326144
+intel_npu_acceleration_library/lib/Release/intel_npu_acceleration_library.dll,sha256=UQD_CUlEx-LGsl1CV6dNZgSk0xFzswdVJB7oBhCBSYg,332800
 intel_npu_acceleration_library/lib/Release/openvino.dll,sha256=faNybSE2qCTi0EiAAnPlxHqr3WU0RI9lMDhuDwPNWlk,13090280
 intel_npu_acceleration_library/lib/Release/openvino_auto_batch_plugin.dll,sha256=Pqg45nRmDeOo8qkvZdwuHxCc109-AvzTwH6ow_LF6bI,198120
 intel_npu_acceleration_library/lib/Release/openvino_auto_plugin.dll,sha256=auHJGs1XwZgyo46Ze-Cg7CDstaQRK6e_uBNZAMqx3F0,473576
@@ -227,7 +227,7 @@ intel_npu_acceleration_library/nn/functional.py,sha256=UfAKBc0u6RtyaMo14ldH2GpEn
 intel_npu_acceleration_library/nn/linear.py,sha256=Q06SoGQeLaI86nA_ky2GnFC6H2Fw1zyMDILKnpYC2eo,5739
 intel_npu_acceleration_library/nn/llm.py,sha256=P6dz36Yf6BHtzWcftaghC6QaMI_WeRfQwrCbO7fD6hk,15002
 intel_npu_acceleration_library/nn/module.py,sha256=EYxoTq6I_YgBDgTF76GPDxHrT8SupOTDGMzQaomBeq8,12667
-bigdl_core_npu-2.6.0b20241117.dist-info/METADATA,sha256=XJZQZgB7WOLjKHfgiVcOrEL6MaHbu_zfDouXl8Fmr2w,1543
-bigdl_core_npu-2.6.0b20241117.dist-info/WHEEL,sha256=zH8lgyF2gFod6lpCM_fWcr62DERZcd3iX2BoPN2-YV0,101
-bigdl_core_npu-2.6.0b20241117.dist-info/top_level.txt,sha256=CH3qQoleRBC1eThu8mCEMxYNKdzJuXCtmeCXRKskt7A,31
-bigdl_core_npu-2.6.0b20241117.dist-info/RECORD,,
+bigdl_core_npu-2.6.0b20241118.dist-info/METADATA,sha256=CKr80VzsJrqtqLYza-vQPcFOM67iQa5LgL1jaesCAJA,1543
+bigdl_core_npu-2.6.0b20241118.dist-info/WHEEL,sha256=zH8lgyF2gFod6lpCM_fWcr62DERZcd3iX2BoPN2-YV0,101
+bigdl_core_npu-2.6.0b20241118.dist-info/top_level.txt,sha256=CH3qQoleRBC1eThu8mCEMxYNKdzJuXCtmeCXRKskt7A,31
+bigdl_core_npu-2.6.0b20241118.dist-info/RECORD,,

intel_npu_acceleration_library/_version.py

@@ -3,4 +3,4 @@
 # SPDX-License-Identifier: Apache 2.0
 #
 
-__version__ = "2.6.0b20241117"
+__version__ = "2.6.0b20241118"

intel_npu_acceleration_library/backend/bindings.py

@@ -116,15 +116,18 @@ def init_network_factory(lib: ctypes.CDLL):
 
     lib.setNNFactoryWeights.argtypes = [handler, ctypes.c_int, handler, ctypes.c_bool]
 
-    lib.op_shape_size.argtypes = [handler]
+    lib.op_shape_size.argtypes = [handler, ctypes.c_int]
     lib.op_shape_size.restype = ctypes.c_int
 
-    lib.op_shape.argtypes = [handler, ctypes.c_int]
+    lib.op_shape.argtypes = [handler, ctypes.c_int, ctypes.c_int]
     lib.op_shape.restype = ctypes.c_int
 
-    lib.op_dtype.argtypes = [handler]
+    lib.op_dtype.argtypes = [handler, ctypes.c_int]
     lib.op_dtype.restype = ctypes.c_int
 
+    lib.op_output_size.argtypes = [handler]
+    lib.op_output_size.restype = ctypes.c_int
+
     lib.parameter.argtypes = [handler, ctypes.c_int, c_u32_array, ctypes.c_char_p]
     lib.parameter.restype = handler
 
@@ -243,6 +246,15 @@ def init_network_factory(lib: ctypes.CDLL):
     ]
     lib.multi_concat.restype = handler
 
+    lib.variadic_split.argtypes = [
+        handler,
+        handler,
+        ctypes.c_int,
+        c_u32_array,
+        ctypes.c_int,
+    ]
+    lib.variadic_split.restype = handler
+
     lib.dq_split_linear.argtypes = [
         handler,
         handler,
@@ -269,6 +281,19 @@ def init_network_factory(lib: ctypes.CDLL):
     ]
     lib.dq_split_linear_prefill.restype = handler
 
+    lib.gw_linear_prefill.argtypes = [
+        handler,
+        handler,
+        ctypes.c_int,
+        ctypes.c_int,
+        ctypes.c_int,
+        ctypes.c_bool,
+        ctypes.c_char_p,
+        ctypes.c_char_p,
+        ctypes.c_bool,
+    ]
+    lib.gw_linear_prefill.restype = handler
+
     for op in get_supported_ops():
         fn = getattr(lib, op.name)
         fn.argtypes = [handler] * (op.inputs + 1) + list(op.parameters)

intel_npu_acceleration_library/backend/factory.py

@@ -71,17 +71,24 @@ class NNFactory(BaseNPUBackendWithPrefetch):
                 Tensor: Tensor object
             """
             # Convert Tensor objects to their underlying node
-            args = tuple(arg.node if isinstance(arg, Tensor) else arg for arg in args)
             kwargs = {
                 k: v.node if isinstance(v, Tensor) else v for k, v in kwargs.items()
             }
 
+            if fn.__qualname__ == 'NNFactory.reshape':
+                output_idx = args[0].output_idx
+                kwargs["output_idx"] = output_idx
+            args = tuple(arg.node if isinstance(arg, Tensor) else arg for arg in args)
+            
+
             input_nodes = [arg for arg in args if isinstance(arg, ctypes._Pointer)] + [
                 v for v in kwargs.values() if isinstance(v, ctypes._Pointer)
             ]
             # Call the function
             node = fn(self, *args, **kwargs)
 
+            output_len = backend_lib.op_output_size(node)
+
             # remove input nodes from output_nodes
             self.output_nodes = [
                 node for node in self.output_nodes if node not in input_nodes
@@ -91,7 +98,13 @@ class NNFactory(BaseNPUBackendWithPrefetch):
                 self.output_nodes.append(node)
 
             # Wrap the node in a Tensor object
-            return Tensor(factory=self, node=node)
+            if output_len == 1:
+                return Tensor(factory=self, node=node, output_idx=0)
+            else:
+                output_tensor_list = []
+                for i in range(output_len):
+                    output_tensor_list.append(Tensor(factory=self, node=node, output_idx=i))
+                return output_tensor_list
 
         return cast(F, wrapper)
 
@@ -398,6 +411,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
         wt_dtype: npt.DTypeLike = np.float16,
         scale_factor: bool = True,
         is_prefill: bool = False,
+        use_dq: bool = True,
     ) -> ctypes._Pointer:
         """Generate a linear layer for dynamic quantization linear layer.
 
@@ -415,7 +429,10 @@ class NNFactory(BaseNPUBackendWithPrefetch):
         Returns:
             ctypes._Pointer: output node
         """
-        func = backend_lib.dq_split_linear_prefill if is_prefill else backend_lib.dq_split_linear
+        if is_prefill:
+            func = backend_lib.dq_split_linear_prefill if use_dq else backend_lib.gw_linear_prefill
+        else:
+            func = backend_lib.dq_split_linear
         return func(self._mm, input_node, n_splits,
                     input_channels, outout_channels, bias,
                     self.get_backend_dtype(act_dtype),
@@ -424,7 +441,9 @@ class NNFactory(BaseNPUBackendWithPrefetch):
 
     @return_tensor
     def reshape(
-        self, input_node: ctypes._Pointer, shape: Sequence[int]
+        self, input_node: ctypes._Pointer, shape: Sequence[int],
+        special_zero: bool = True,
+        output_idx: int = 0
     ) -> ctypes._Pointer:
         """Generate a reshape layer.
 
@@ -436,7 +455,8 @@ class NNFactory(BaseNPUBackendWithPrefetch):
             ctypes._Pointer: output node
         """
         shape_node = self.constant(shape).node  # type: ignore
-        return backend_lib.reshape(self._mm, input_node, shape_node)
+        return backend_lib.reshape(self._mm, input_node, shape_node,
+                                   special_zero, output_idx)
 
     @return_tensor
     def broadcast(
@@ -588,7 +608,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
             ctypes._Pointer: output node
         """
         if axis < 0:
-            shape_size = backend_lib.op_shape_size(input_node_1)
+            shape_size = backend_lib.op_shape_size(input_node_1, 0)
             axis = (axis + shape_size) % shape_size
         axis = np.int64(axis)
         return backend_lib.concat(self._mm, input_node_1, input_node_2, axis)
@@ -607,7 +627,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
             ctypes._Pointer: output node
         """
         if axis < 0:
-            shape_size = backend_lib.op_shape_size(input_nodes[0])
+            shape_size = backend_lib.op_shape_size(input_nodes[0], 0)
             axis = (axis + shape_size) % shape_size
         axis = np.int64(axis)
 
@@ -632,7 +652,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
             ctypes._Pointer: output node
         """
         if reduction_axes is None:
-            shape_size = backend_lib.op_shape_size(input_node)
+            shape_size = backend_lib.op_shape_size(input_node, 0)
             reduction_axes = list(range(shape_size - 1, -1, -1))
         axis_node = self.constant(reduction_axes).node  # type: ignore
         return backend_lib.reduce_max(self._mm, input_node, axis_node, keep_dims)
@@ -655,7 +675,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
             ctypes._Pointer: output node
         """
         if reduction_axes is None:
-            shape_size = backend_lib.op_shape_size(input_node)
+            shape_size = backend_lib.op_shape_size(input_node, 0)
             reduction_axes = list(range(shape_size - 1, -1, -1))
         axis_node = self.constant(reduction_axes).node  # type: ignore
         return backend_lib.reduce_mean(self._mm, input_node, axis_node, keep_dims)
@@ -678,7 +698,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
             ctypes._Pointer: output node
         """
         if reduction_axes is None:
-            shape_size = backend_lib.op_shape_size(input_node)
+            shape_size = backend_lib.op_shape_size(input_node, 0)
             reduction_axes = list(range(shape_size - 1, -1, -1))
         axis_node = self.constant(reduction_axes).node  # type: ignore
         return backend_lib.reduce_min(self._mm, input_node, axis_node, keep_dims)
@@ -701,7 +721,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
             ctypes._Pointer: output node
         """
         if reduction_axes is None:
-            shape_size = backend_lib.op_shape_size(input_node)
+            shape_size = backend_lib.op_shape_size(input_node, 0)
             reduction_axes = list(range(shape_size - 1, -1, -1))
         axis_node = self.constant(reduction_axes).node  # type: ignore
         return backend_lib.reduce_prod(self._mm, input_node, axis_node, keep_dims)
@@ -724,7 +744,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
             ctypes._Pointer: output node
         """
         if reduction_axes is None:
-            shape_size = backend_lib.op_shape_size(input_node)
+            shape_size = backend_lib.op_shape_size(input_node, 0)
             reduction_axes = list(range(shape_size - 1, -1, -1))
         axis_node = self.constant(reduction_axes).node  # type: ignore
         return backend_lib.reduce_sum(self._mm, input_node, axis_node, keep_dims)
@@ -744,7 +764,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
             ctypes._Pointer: output node
         """
         if axis < 0:
-            shape_size = backend_lib.op_shape_size(input_node)
+            shape_size = backend_lib.op_shape_size(input_node, 0)
             axis = (axis + shape_size) % shape_size
         axis_node = self.constant(axis).node  # type: ignore
         return backend_lib.normL2(self._mm, input_node, axis_node, eps)
@@ -767,14 +787,14 @@ class NNFactory(BaseNPUBackendWithPrefetch):
         Returns:
             ctypes._Pointer: output node
         """
-        input_shape_size = backend_lib.op_shape_size(input_node)
+        input_shape_size = backend_lib.op_shape_size(input_node, 0)
         input_shape = [
-            backend_lib.op_shape(input_node, i) for i in range(input_shape_size)
+            backend_lib.op_shape(input_node, i, 0) for i in range(input_shape_size)
         ]
         if isinstance(exponent, ctypes._Pointer):
-            exponent_shape_size = backend_lib.op_shape_size(input_node)
+            exponent_shape_size = backend_lib.op_shape_size(input_node, 0)
             exponent_shape = [
-                backend_lib.op_shape(exponent, i) for i in range(exponent_shape_size)
+                backend_lib.op_shape(exponent, i, 0) for i in range(exponent_shape_size)
             ]
         else:
             exponent_shape = list(exponent.shape)
@@ -783,6 +803,39 @@ class NNFactory(BaseNPUBackendWithPrefetch):
         #     raise ValueError("Input tensor shapes are not equal")
 
         return backend_lib.power(self._mm, input_node, exponent)
+    
+    @return_tensor
+    def variadic_split(
+        self,
+        input: ctypes._Pointer,
+        axis: int,
+        split_lengths: Sequence[int],
+    ) -> ctypes._Pointer:
+        """Generate an average pooling layer.
+
+        Args:
+            input (ctypes._Pointer): layer input node
+            axis (int): split axis
+            split_lengths (Sequence[int]): A list containing the sizes of each output tensor
+            along the split "axis". Size of "split_lengths" should be equal to the number of
+            outputs. The sum of split_lengths must match data.shape[axis]
+
+        Raises:
+            NotImplementedError: divisor_override is not supported
+
+        Returns:
+            ctypes._Pointer: output node
+        """
+
+        split_lens_ptr = np.array(split_lengths, dtype=np.uint32)
+
+        return backend_lib.variadic_split(
+            self._mm,
+            input,
+            axis,
+            split_lens_ptr,
+            split_lens_ptr.size,
+        )
 
     @return_tensor
     def avg_pooling(
@@ -938,7 +991,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
                                                         value, attn_mask,
                                                         is_causal)
 
-    def get_tensor_shape(self, node):
+    def get_tensor_shape(self, node, output_idx=0):
         """Get tensor shape.
 
         Args:
@@ -947,10 +1000,10 @@ class NNFactory(BaseNPUBackendWithPrefetch):
         Returns:
             tuple[int]: tensor shape
         """
-        size = backend_lib.op_shape_size(node)
-        return tuple([backend_lib.op_shape(node, idx) for idx in range(size)])
+        size = backend_lib.op_shape_size(node, output_idx)
+        return tuple([backend_lib.op_shape(node, idx, output_idx) for idx in range(size)])
 
-    def get_tensor_dtype(self, node):
+    def get_tensor_dtype(self, node, output_idx=0):
         """Get tensor dtype.
 
         Args:
@@ -962,7 +1015,7 @@ class NNFactory(BaseNPUBackendWithPrefetch):
         Returns:
             str: tensor dtype
         """
-        dtype_int = backend_lib.op_dtype(node)
+        dtype_int = backend_lib.op_dtype(node, output_idx)
 
         if dtype_int == 2:
             return np.bool

intel_npu_acceleration_library/backend/ops.py

@@ -98,7 +98,7 @@ def get_supported_ops() -> List[SupportedOp]:
             inputs=3,
             parameters=[ctypes.c_int],
         ),
-        SupportedOp(name="reshape", inputs=2),
+        SupportedOp(name="reshape", inputs=2, parameters=[ctypes.c_bool, ctypes.c_int]),
         SupportedOp(name="transpose", inputs=2),
         SupportedOp(name="squeeze", inputs=1),
         SupportedOp(name="unsqueeze", inputs=2),
@@ -137,5 +137,6 @@ def get_supported_ops() -> List[SupportedOp]:
         SupportedOp(name="power", inputs=2),
         SupportedOp(name="broadcast", inputs=2),
         SupportedOp(name="log_softmax", inputs=1, parameters=[ctypes.c_int64]),
+        SupportedOp(name="rotate_half", inputs=1),
     ]
     return supported_ops

intel_npu_acceleration_library/backend/tensor.py

@@ -157,6 +157,7 @@ class Tensor:
 
     factory: "NNFactory"  # type: ignore # noqa: F821
     node: ctypes._Pointer
+    output_idx: int
 
     @property
     def shape(self) -> Sequence[int]:
@@ -166,8 +167,8 @@ class Tensor:
         Returns:
             Sequence[int]: The shape of the tensor.
         """
-        shape_size = backend_lib.op_shape_size(self.node)
-        return [backend_lib.op_shape(self.node, i) for i in range(shape_size)]
+        shape_size = backend_lib.op_shape_size(self.node, self.output_idx)
+        return [backend_lib.op_shape(self.node, i, self.output_idx) for i in range(shape_size)]
 
     @property
     def dtype(self) -> NPUDtype:
@@ -177,7 +178,7 @@ class Tensor:
         Returns:
             type: The data type of the tensor.
         """
-        dtype_int = backend_lib.op_dtype(self.node)
+        dtype_int = backend_lib.op_dtype(self.node, self.output_idx)
 
         if dtype_int == 2:
             return np.bool