blksprs 1.0__tar.gz → 1.2__tar.gz

{blksprs-1.0 → blksprs-1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blksprs
-Version: 1.0
+Version: 1.2
 Summary: A lightweight library for operations on blocksparse matrices in PyTorch.
 Author-email: Felix Schön <schoen@kr.tuwien.ac.at>
 Project-URL: Homepage, https://github.com/FelixSchoen/blksprs
@@ -23,16 +23,21 @@ Requires-Dist: pdoc3; extra == "deploy"
 
 ## Overview
 
-A lightweight library for operations on blocksparse matrices in PyTorch.
+A lightweight and efficient library for operations on block-sparse matrices in PyTorch using Triton.
 
 Currently supported operations (includes gradient calculation):
 
-- Sparse matrix multiplication (_supports any combination of sparse and dense matrices due to support for `sparse = sparse @ sparse` matmul_)
+- Sparse matrix multiplication (_supports any combination of sparse and dense matrices due to support
+  for `sparse = sparse @ sparse` matmul_)
 - Softmax
 - Transposition
-- Conversion from and to sparse form
+- Gather
+- Scatter (_supports either no reduction or summation, gradients are only available for summation_)
+- Conversion to and from sparse form
+- Conversion to different sparsity layouts and different sparsity block sizes
 
-As with this library sparse matrices are represented using a tuple of `(matrix, sparsity_layout, sparsity_block_size)`, any element-wise operations can be applied in regular torch-like fashion.
+As with this library sparse matrices are represented using a tuple of `(matrix, sparsity_layout, sparsity_block_size)`,
+any element-wise operations can be applied in regular torch-like fashion.
 These include, e.g.,
 
 - Element-wise addition and subtraction
@@ -40,24 +45,45 @@ These include, e.g.,
 - Element-wise exponentiation
 - ...
 
+Note that in order to correctly apply element-wise operations between two sparse tensors their sparsity layouts have to
+match.
+
+Furthermore, the library provides a set of utility functions for the creation of sparsity layouts based on existing
+dense tensors.
+
 ## Installation
 
+Note that due to the dependency on [Triton](https://github.com/triton-lang/triton) this library is only compatible with
+the Linux platform.
+
 We recommend installing blksprs from [PyPI](https://pypi.org/project/blksprs/) using pip:
 
 ```pip install blksprs```
 
+### Dependencies
+
+- [PyTorch](https://pytorch.org/) (built with v2.4.0)
+- _[Triton](https://github.com/triton-lang/triton) (included with PyTorch)_
+
 ## Changelog
 
 See [`CHANGELOG.md`](https://github.com/FelixSchoen/blksprs/blob/main/CHANGELOG.md) for a detailed changelog.
 
 ## Usage
 
+We provide an example below to demonstrate the usage of the library.
+For more detailed examples, please refer to
+the [test cases](https://github.com/FelixSchoen/blksprs/blob/main/test/cases/test_blocksparse.py) which cover all
+implemented operations and functions.
+The example below can also be found in
+the [test cases](https://github.com/FelixSchoen/blksprs/blob/main/test/cases/test_readme.py).
+
 ```python
 import torch
 
-from blksprs.layouting.sparsity_layout import create_sparsity_layout
+from blksprs.layouting.sparsity_layout import build_sparsity_layout
 from blksprs.ops.conversion import to_sparse, to_dense
-from blksprs.ops.matmul_sss import matmul_sss
+from blksprs.ops.matmul import matmul
 from blksprs.ops.row_wise_sum import row_wise_sum
 from blksprs.ops.softmax import softmax
 from blksprs.ops.transpose import transpose
@@ -65,7 +91,7 @@ from blksprs.utils.tools import do_shape_blocksparse, undo_shape_blocksparse
 
 
 def test_readme():
-    # Set up parameters
+    # Set up parameters (batch size, number of heads, dimensions for matrices (m, k) and (n, k))
     b, h, m, n, k = 2, 4, 64, 64, 16
 
     # Percentage of blocks that will be sparse in the output for demonstration purposes
@@ -78,7 +104,6 @@ def test_readme():
     # If it is set to ``none`` a value will be chosen automatically
     triton_block_size = None
 
-
     # Initialise random (dense) tensors
     x = torch.randn(size=(b, h, m, k), device="cuda")
     y = torch.randn(size=(b, h, n, k), device="cuda").transpose(-1, -2).contiguous()
@@ -88,8 +113,8 @@ def test_readme():
     y_dense, y_shape_original = do_shape_blocksparse(y)
 
     # Create sparsity layouts from existing tensors
-    sparsity_layout_x = create_sparsity_layout(x_dense, sparsity_block_size, triton_block_size=triton_block_size)
-    sparsity_layout_y = create_sparsity_layout(y_dense, sparsity_block_size, triton_block_size=triton_block_size)
+    sparsity_layout_x = build_sparsity_layout(x_dense, sparsity_block_size, triton_block_size=triton_block_size)
+    sparsity_layout_y = build_sparsity_layout(y_dense, sparsity_block_size, triton_block_size=triton_block_size)
 
     # Create random sparsity layout for output tensor
     sparsity_layout_o = _get_random_sparsity_layout(b * h, m, n, sparsity_block_size, sparsity_percentage)
@@ -99,8 +124,8 @@ def test_readme():
     y_sparse = to_sparse(y_dense, sparsity_layout_y, sparsity_block_size, triton_block_size=triton_block_size)
 
     # Perform matrix multiplication
-    o_sparse = matmul_sss(x_sparse, y_sparse, sparsity_layout_x, sparsity_layout_y, sparsity_layout_o,
-                          sparsity_block_size, triton_block_size=triton_block_size)
+    o_sparse = matmul(x_sparse, sparsity_layout_x, y_sparse, sparsity_layout_y, sparsity_layout_o, sparsity_block_size,
+                      triton_block_size=triton_block_size)
     o_dense = to_dense(o_sparse, sparsity_layout_o, sparsity_block_size, triton_block_size=triton_block_size)
 
     # Sanity check
@@ -115,7 +140,7 @@ def test_readme():
     assert torch.allclose(o_dense, o_torch_round_trip, atol=2e-2)  # Note that small numerical differences are expected
 
     # Assert that the output has the correct sparsity layout
-    actual_sparsity_layout_o = create_sparsity_layout(o_dense, sparsity_block_size, triton_block_size=triton_block_size)
+    actual_sparsity_layout_o = build_sparsity_layout(o_dense, sparsity_block_size, triton_block_size=triton_block_size)
     assert torch.allclose(actual_sparsity_layout_o, sparsity_layout_o)
 
     # Convert output tensor back to original shape

{blksprs-1.0 → blksprs-1.2}/README.md

@@ -2,16 +2,21 @@
 
 ## Overview
 
-A lightweight library for operations on blocksparse matrices in PyTorch.
+A lightweight and efficient library for operations on block-sparse matrices in PyTorch using Triton.
 
 Currently supported operations (includes gradient calculation):
 
-- Sparse matrix multiplication (_supports any combination of sparse and dense matrices due to support for `sparse = sparse @ sparse` matmul_)
+- Sparse matrix multiplication (_supports any combination of sparse and dense matrices due to support
+  for `sparse = sparse @ sparse` matmul_)
 - Softmax
 - Transposition
-- Conversion from and to sparse form
+- Gather
+- Scatter (_supports either no reduction or summation, gradients are only available for summation_)
+- Conversion to and from sparse form
+- Conversion to different sparsity layouts and different sparsity block sizes
 
-As with this library sparse matrices are represented using a tuple of `(matrix, sparsity_layout, sparsity_block_size)`, any element-wise operations can be applied in regular torch-like fashion.
+As with this library sparse matrices are represented using a tuple of `(matrix, sparsity_layout, sparsity_block_size)`,
+any element-wise operations can be applied in regular torch-like fashion.
 These include, e.g.,
 
 - Element-wise addition and subtraction
@@ -19,24 +24,45 @@ These include, e.g.,
 - Element-wise exponentiation
 - ...
 
+Note that in order to correctly apply element-wise operations between two sparse tensors their sparsity layouts have to
+match.
+
+Furthermore, the library provides a set of utility functions for the creation of sparsity layouts based on existing
+dense tensors.
+
 ## Installation
 
+Note that due to the dependency on [Triton](https://github.com/triton-lang/triton) this library is only compatible with
+the Linux platform.
+
 We recommend installing blksprs from [PyPI](https://pypi.org/project/blksprs/) using pip:
 
 ```pip install blksprs```
 
+### Dependencies
+
+- [PyTorch](https://pytorch.org/) (built with v2.4.0)
+- _[Triton](https://github.com/triton-lang/triton) (included with PyTorch)_
+
 ## Changelog
 
 See [`CHANGELOG.md`](https://github.com/FelixSchoen/blksprs/blob/main/CHANGELOG.md) for a detailed changelog.
 
 ## Usage
 
+We provide an example below to demonstrate the usage of the library.
+For more detailed examples, please refer to
+the [test cases](https://github.com/FelixSchoen/blksprs/blob/main/test/cases/test_blocksparse.py) which cover all
+implemented operations and functions.
+The example below can also be found in
+the [test cases](https://github.com/FelixSchoen/blksprs/blob/main/test/cases/test_readme.py).
+
 ```python
 import torch
 
-from blksprs.layouting.sparsity_layout import create_sparsity_layout
+from blksprs.layouting.sparsity_layout import build_sparsity_layout
 from blksprs.ops.conversion import to_sparse, to_dense
-from blksprs.ops.matmul_sss import matmul_sss
+from blksprs.ops.matmul import matmul
 from blksprs.ops.row_wise_sum import row_wise_sum
 from blksprs.ops.softmax import softmax
 from blksprs.ops.transpose import transpose
@@ -44,7 +70,7 @@ from blksprs.utils.tools import do_shape_blocksparse, undo_shape_blocksparse
 
 
 def test_readme():
-    # Set up parameters
+    # Set up parameters (batch size, number of heads, dimensions for matrices (m, k) and (n, k))
     b, h, m, n, k = 2, 4, 64, 64, 16
 
     # Percentage of blocks that will be sparse in the output for demonstration purposes
@@ -57,7 +83,6 @@ def test_readme():
     # If it is set to ``none`` a value will be chosen automatically
     triton_block_size = None
 
-
     # Initialise random (dense) tensors
     x = torch.randn(size=(b, h, m, k), device="cuda")
     y = torch.randn(size=(b, h, n, k), device="cuda").transpose(-1, -2).contiguous()
@@ -67,8 +92,8 @@ def test_readme():
     y_dense, y_shape_original = do_shape_blocksparse(y)
 
     # Create sparsity layouts from existing tensors
-    sparsity_layout_x = create_sparsity_layout(x_dense, sparsity_block_size, triton_block_size=triton_block_size)
-    sparsity_layout_y = create_sparsity_layout(y_dense, sparsity_block_size, triton_block_size=triton_block_size)
+    sparsity_layout_x = build_sparsity_layout(x_dense, sparsity_block_size, triton_block_size=triton_block_size)
+    sparsity_layout_y = build_sparsity_layout(y_dense, sparsity_block_size, triton_block_size=triton_block_size)
 
     # Create random sparsity layout for output tensor
     sparsity_layout_o = _get_random_sparsity_layout(b * h, m, n, sparsity_block_size, sparsity_percentage)
@@ -78,8 +103,8 @@ def test_readme():
     y_sparse = to_sparse(y_dense, sparsity_layout_y, sparsity_block_size, triton_block_size=triton_block_size)
 
     # Perform matrix multiplication
-    o_sparse = matmul_sss(x_sparse, y_sparse, sparsity_layout_x, sparsity_layout_y, sparsity_layout_o,
-                          sparsity_block_size, triton_block_size=triton_block_size)
+    o_sparse = matmul(x_sparse, sparsity_layout_x, y_sparse, sparsity_layout_y, sparsity_layout_o, sparsity_block_size,
+                      triton_block_size=triton_block_size)
     o_dense = to_dense(o_sparse, sparsity_layout_o, sparsity_block_size, triton_block_size=triton_block_size)
 
     # Sanity check
@@ -94,7 +119,7 @@ def test_readme():
     assert torch.allclose(o_dense, o_torch_round_trip, atol=2e-2)  # Note that small numerical differences are expected
 
     # Assert that the output has the correct sparsity layout
-    actual_sparsity_layout_o = create_sparsity_layout(o_dense, sparsity_block_size, triton_block_size=triton_block_size)
+    actual_sparsity_layout_o = build_sparsity_layout(o_dense, sparsity_block_size, triton_block_size=triton_block_size)
     assert torch.allclose(actual_sparsity_layout_o, sparsity_layout_o)
 
     # Convert output tensor back to original shape

blksprs-1.2/blksprs/layouting/distribution_layout.py

@@ -0,0 +1,114 @@
+import torch
+import triton
+from torch import Tensor
+from triton import language as tl
+
+from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.validation import validate_triton_block_size, validate_dimensions, validate_device, \
+    validate_contiguous
+
+
+def build_distribution_layout(indices: Tensor, sparsity_layout_indices: Tensor,
+                              size_target: torch.Size,
+                              sparsity_block_size: int, triton_block_size: int = None) -> Tensor:
+    """Builds the sparsity layout of either the source of a gather or the target of a scatter operation.
+
+    Args:
+        indices (Tensor): The block-sparse indices tensor in compressed form used for the gather or scatter operation.
+        sparsity_layout_indices (Tensor): The sparsity layout of the indices block-sparse tensor.
+        size_target (torch.Size): The size of the block-sparse target tensor in regular form.
+        sparsity_block_size (int): The size of the sparsity blocks.
+        triton_block_size (int, optional): The block size to use for the triton kernel (default ``None``).
+
+    Returns:
+        Tensor: The sparsity layout of the source or target tensor.
+
+    """
+    validate_dimensions(indices)
+    validate_contiguous(indices)
+    validate_device(indices)
+
+    sparsity_lut_i = torch.nonzero(sparsity_layout_indices).contiguous()
+
+    output = torch.zeros(size_target[0], size_target[1] // sparsity_block_size, size_target[2] // sparsity_block_size,
+                         device=indices.device, dtype=torch.int32)
+
+    i_b, i_r, i_c = indices.size()
+    i_b_s, i_r_s, i_c_s = indices.stride()
+    s_l_i_b, s_l_i_r, s_l_i_c = sparsity_layout_indices.size()
+    s_l_i_b_s, s_l_i_r_s, s_l_i_c_s = sparsity_layout_indices.stride()
+    s_lut_i_r, s_lut_i_c = sparsity_lut_i.size()
+    s_lut_i_r_s, s_lut_i_c_s = sparsity_lut_i.stride()
+    o_b, o_r, o_c = output.size()
+    o_b_s, o_r_s, o_c_s = output.stride()
+
+    if triton_block_size is None:
+        triton_block_size = get_triton_block_size(sparsity_block_size)
+
+    validate_triton_block_size(triton_block_size, sparsity_block_size)
+
+    triton_grid = lambda meta: [i_b,
+                                triton.cdiv(i_r, meta["TRITON_BLOCK_SIZE"]),
+                                triton.cdiv(i_c, meta["TRITON_BLOCK_SIZE"])]
+
+    (kernel_distribution_layout[triton_grid]
+     (indices,
+      i_b, i_b_s, i_r_s, i_c_s,
+      sparsity_layout_indices,
+      s_l_i_b, s_l_i_b_s, s_l_i_r, s_l_i_r_s, s_l_i_c, s_l_i_c_s,
+      sparsity_lut_i,
+      s_lut_i_r, s_lut_i_r_s, s_lut_i_c, s_lut_i_c_s,
+      output,
+      o_b, o_b_s, o_r, o_r_s, o_c, o_c_s,
+      sparsity_block_size,
+      triton_block_size))
+
+    return output
+
+
+@triton.jit
+def kernel_distribution_layout(i,
+                               i_b, i_b_s, i_r_s, i_c_s,
+                               s_l_i,
+                               s_l_i_b, s_l_i_b_s, s_l_i_r, s_l_i_r_s, s_l_i_c, s_l_i_c_s,
+                               s_lut_i,
+                               s_lut_i_r, s_lut_i_r_s, s_lut_i_c, s_lut_i_c_s,
+                               o,
+                               o_b, o_b_s, o_r, o_r_s, o_c, o_c_s,
+                               sparsity_block_size,
+                               TRITON_BLOCK_SIZE: tl.constexpr) -> None:
+    # Get triton block indices
+    pid_blk = tl.program_id(axis=0)
+    pid_row = tl.program_id(axis=1)
+    pid_col = tl.program_id(axis=2)
+
+    # Get position of current sparsity block consisting of its batch, row, and column index
+    spa_bat_i_idx = (pid_blk * s_lut_i_r_s + 0 * s_lut_i_c_s)
+    spa_bat_i_msk = (spa_bat_i_idx < s_lut_i_r * s_lut_i_r_s)
+    spa_bat_i = tl.load(s_lut_i + spa_bat_i_idx, mask=spa_bat_i_msk)
+
+    spa_row_i_idx = (pid_blk * s_lut_i_r_s + 1 * s_lut_i_c_s)
+    spa_row_i_msk = (spa_row_i_idx < s_lut_i_r * s_lut_i_r_s)
+    spa_row_i = tl.load(s_lut_i + spa_row_i_idx, mask=spa_row_i_msk)
+
+    blk_i_idx = (pid_blk * i_b_s +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_c_s)[None, :])
+    blk_i_msk = (blk_i_idx < i_b * i_b_s)
+    blk_i = tl.load(i + blk_i_idx, mask=blk_i_msk)
+
+    blk_i = blk_i // sparsity_block_size
+    blk_v = tl.full((TRITON_BLOCK_SIZE, TRITON_BLOCK_SIZE), 1, dtype=tl.int32)
+
+    blk_o_idx = ((spa_bat_i * o_b_s) +
+                 (spa_row_i * o_r_s) +
+                 (blk_i * o_c_s))
+    blk_o_msk = (blk_o_idx < o_b * o_b_s)
+    tl.store(o + blk_o_idx, blk_v, mask=blk_o_msk)
+
+    # if tl.min(blk_x) != 0 or tl.max(blk_x) != 0:
+    #     blk_o_idx = (pid_bat * o_b_s +
+    #                  (((pid_row * TRITON_BLOCK_SIZE) // sparsity_block_size) * o_r_s +
+    #                   ((pid_col * TRITON_BLOCK_SIZE) // sparsity_block_size) * o_c_s))
+    #     blk_o_msk = (blk_o_idx < o_b * o_b_s)
+    #     tl.store(o + blk_o_idx, 1, mask=blk_o_msk)

blksprs-1.2/blksprs/layouting/sparsity_layout.py

@@ -0,0 +1,190 @@
+import math
+
+import torch
+import triton
+from torch import Tensor
+from triton import language as tl
+
+from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.validation import validate_triton_block_size, validate_dimensions, validate_device, \
+    validate_contiguous, validate_sparsity, validate_sparsity_block_size
+
+
+def build_sparsity_layout(x: Tensor, sparsity_block_size: int, triton_block_size: int = None) -> Tensor:
+    """Builds the sparsity layout of a dense tensor in regular form covering its sparse blocks.
+
+    Args:
+        x (Tensor): A block-sparse (or dense) tensor in regular form.
+        sparsity_block_size (int): The size of the sparsity blocks.
+        triton_block_size (int, optional): The block size to use for the triton kernel (default ``None``).
+
+    Returns:
+        Tensor: The sparsity layout of the input block-sparse (or dense) tensor.
+
+    """
+    validate_dimensions(x)
+    validate_contiguous(x)
+    validate_device(x)
+
+    output = torch.zeros(x.size(0), x.size(1) // sparsity_block_size, x.size(2) // sparsity_block_size,
+                         device=x.device, dtype=torch.int32)
+
+    x_b, x_r, x_c = x.size()
+    x_b_s, x_r_s, x_c_s = x.stride()
+    o_b, o_r, o_c = output.size()
+    o_b_s, o_r_s, o_c_s = output.stride()
+
+    if triton_block_size is None:
+        triton_block_size = get_triton_block_size(sparsity_block_size)
+
+    validate_triton_block_size(triton_block_size, sparsity_block_size)
+
+    triton_grid = lambda meta: [x_b,
+                                triton.cdiv(x_r, meta["TRITON_BLOCK_SIZE"]),
+                                triton.cdiv(x_c, meta["TRITON_BLOCK_SIZE"])]
+
+    (kernel_sparsity_layout[triton_grid]
+     (x,
+      x_b, x_b_s, x_r_s, x_c_s,
+      output,
+      o_b, o_b_s, o_r_s, o_c_s,
+      sparsity_block_size,
+      triton_block_size))
+
+    return output
+
+
+@triton.jit
+def kernel_sparsity_layout(x,
+                           x_b, x_b_s, x_r_s, x_c_s,
+                           o,
+                           o_b, o_b_s, o_r_s, o_c_s,
+                           sparsity_block_size,
+                           TRITON_BLOCK_SIZE: tl.constexpr) -> None:
+    # Get triton block indices
+    pid_bat = tl.program_id(axis=0)
+    pid_row = tl.program_id(axis=1)
+    pid_col = tl.program_id(axis=2)
+
+    # Load x values
+    blk_x_idx = (pid_bat * x_b_s +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_c_s)[None, :])
+    blk_x_msk = (blk_x_idx < x_b * x_b_s)
+    blk_x = tl.load(x + blk_x_idx, mask=blk_x_msk)
+
+    # Store sparsity layout value
+    if tl.min(blk_x) != 0 or tl.max(blk_x) != 0:
+        blk_o_idx = (pid_bat * o_b_s +
+                     (((pid_row * TRITON_BLOCK_SIZE) // sparsity_block_size) * o_r_s +
+                      ((pid_col * TRITON_BLOCK_SIZE) // sparsity_block_size) * o_c_s))
+        blk_o_msk = (blk_o_idx < o_b * o_b_s)
+        tl.store(o + blk_o_idx, 1, mask=blk_o_msk)
+
+
+def build_sparsity_layout_adaption(x: Tensor, sparsity_layout_from: Tensor,
+                                   sparsity_block_size_from: int, sparsity_block_size_to: int,
+                                   triton_block_size: int = None) -> Tensor:
+    """Builds the sparsity layout of a block-sparse tensor in compressed form if a different sparsity block size were
+        used.
+        
+    Args:
+        x (Tensor): A block-sparse tensor in compressed form.
+        sparsity_layout_from (Tensor): The sparsity layout of the input block-sparse tensor.
+        sparsity_block_size_from (int): The size of the sparsity blocks of the input tensor.
+        sparsity_block_size_to (int): The desired size of the sparsity blocks for the resulting layout.
+        triton_block_size (int, optional): The block size to use for the triton kernel (default ``None``).
+
+    Returns:
+        Tensor: The sparsity layout in regular form using the new sparsity block size of the input block-sparse tensor
+            in compressed form.
+    
+    """
+    validate_dimensions(x)
+    validate_contiguous(x, sparsity_layout_from)
+    validate_device(x)
+    validate_sparsity(sparsity_block_size_from, (x, sparsity_layout_from))
+    validate_sparsity_block_size(sparsity_block_size_from, x)
+    validate_sparsity_block_size(sparsity_block_size_to)
+    min_sparsity_block_size = min(sparsity_block_size_from, sparsity_block_size_to)
+    validate_triton_block_size(triton_block_size, min_sparsity_block_size)
+
+    sparsity_lut = torch.nonzero(sparsity_layout_from).contiguous()
+
+    validate_contiguous(sparsity_layout_from, sparsity_lut)
+
+    o_b = sparsity_layout_from.size(0)
+    o_r = math.ceil(sparsity_layout_from.size(1) * sparsity_block_size_from // sparsity_block_size_to)
+    o_c = math.ceil(sparsity_layout_from.size(2) * sparsity_block_size_from // sparsity_block_size_to)
+
+    output = torch.zeros(o_b, o_r, o_c, device=x.device, dtype=torch.int32)
+
+    x_b, x_r, x_c = x.size()
+    x_b_s, x_r_s, x_c_s = x.stride()
+    s_lut_r, s_lut_c = sparsity_lut.size()
+    s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
+    o_b_s, o_r_s, o_c_s = output.stride()
+
+    if triton_block_size is None:
+        triton_block_size = get_triton_block_size(sparsity_block_size_from)
+
+    triton_grid = lambda meta: [x_b,
+                                triton.cdiv(x_r, meta["TRITON_BLOCK_SIZE"]),
+                                triton.cdiv(x_c, meta["TRITON_BLOCK_SIZE"])]
+
+    (kernel_sparsity_layout_adaption[triton_grid]
+     (x,
+      x_b, x_b_s, x_r_s, x_c_s,
+      sparsity_lut, s_lut_r, s_lut_r_s, s_lut_c_s,
+      output,
+      o_b, o_b_s, o_r_s, o_c_s,
+      sparsity_block_size_from,
+      sparsity_block_size_to,
+      triton_block_size))
+
+    return output
+
+
+@triton.jit
+def kernel_sparsity_layout_adaption(x,
+                                    x_b, x_b_s, x_r_s, x_c_s,
+                                    s_lut, s_lut_r, s_lut_r_s, s_lut_c_s,
+                                    o,
+                                    o_b, o_b_s, o_r_s, o_c_s,
+                                    sparsity_block_size_from,
+                                    sparsity_block_size_to,
+                                    TRITON_BLOCK_SIZE: tl.constexpr) -> None:
+    # Get triton block indices
+    pid_blk = tl.program_id(axis=0)
+    pid_row = tl.program_id(axis=1)
+    pid_col = tl.program_id(axis=2)
+
+    # Get sparsity index of current output block consisting of its batch, row, and column index
+    spa_bat_idx = (pid_blk * s_lut_r_s + 0 * s_lut_c_s)
+    spa_bat_msk = (spa_bat_idx < s_lut_r * s_lut_r_s)
+    spa_bat = tl.load(s_lut + spa_bat_idx, mask=spa_bat_msk)
+
+    spa_row_idx = (pid_blk * s_lut_r_s + 1 * s_lut_c_s)
+    spa_row_msk = (spa_row_idx < s_lut_r * s_lut_r_s)
+    spa_row = tl.load(s_lut + spa_row_idx, mask=spa_row_msk)
+
+    spa_col_idx = (pid_blk * s_lut_r_s + 2 * s_lut_c_s)
+    spa_col_msk = (spa_col_idx < s_lut_r * s_lut_r_s)
+    spa_col = tl.load(s_lut + spa_col_idx, mask=spa_col_msk)
+
+    # Load x values
+    blk_x_idx = ((pid_blk * x_b_s) +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_c_s)[None, :])
+    blk_x_msk = (blk_x_idx < x_b * x_b_s)
+    blk_x = tl.load(x + blk_x_idx, mask=blk_x_msk)
+
+    # Store sparsity layout value
+    if tl.min(blk_x) != 0 or tl.max(blk_x) != 0:
+        blk_o_idx = ((spa_bat * o_b_s) +
+                     (((spa_row * sparsity_block_size_from + pid_row * TRITON_BLOCK_SIZE)
+                       // sparsity_block_size_to) * o_r_s) +
+                     (((spa_col * sparsity_block_size_from + pid_col * TRITON_BLOCK_SIZE)
+                       // sparsity_block_size_to) * o_c_s))
+        blk_o_msk = (blk_o_idx < o_b * o_b_s)
+        tl.store(o + blk_o_idx, 1, mask=blk_o_msk)

blksprs-1.2/blksprs/misc/broadcast_addition.py

@@ -0,0 +1,132 @@
+import torch
+import triton
+from torch import Tensor
+from triton import language as tl
+
+from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.validation import validate_contiguous, validate_device, \
+    validate_sparsity_block_size, validate_triton_block_size
+
+
+def broadcast_addition(x: Tensor, y: Tensor, sparsity_layout_output: Tensor,
+                       sparsity_block_size: int, triton_block_size: int = None) -> Tensor:
+    """Performs a broadcast and subsequent addition of two dense tensors x and y. Returns a block-sparse tensor in
+        compressed form.
+
+    Args:
+        x (Tensor): A dense input tensor.
+        y (Tensor): A dense input tensor.
+        sparsity_layout_output (Tensor): The sparsity layout of the output tensor.
+        sparsity_block_size (int): The size of the sparsity blocks.
+        triton_block_size (int, optional): The block size to use for the triton kernel (default ``None``).
+
+    Returns:
+        Tensor: The result of the operation as a block-sparse tensor in compressed form. Each element o(i, j) of the
+            output tensor corresponds to x(i) + y(j).
+
+    """
+    validate_device(x, y)
+    validate_contiguous(x, y)
+    if x.size(-1) != y.size(-1):
+        raise ValueError("Dimensions of tensors must match")
+    validate_sparsity_block_size(sparsity_block_size)
+    validate_triton_block_size(triton_block_size, sparsity_block_size)
+
+    sparsity_lut_o = torch.nonzero(sparsity_layout_output).contiguous()
+
+    n_sparse_blocks = torch.sum(sparsity_layout_output.to(torch.int)).item()
+
+    validate_contiguous(sparsity_layout_output, sparsity_lut_o)
+
+    output = torch.zeros(n_sparse_blocks, sparsity_block_size, sparsity_block_size, device=x.device)
+
+    x_b, x_c = x.size()
+    x_b_s, x_c_s = x.stride()
+    y_b, y_c = y.size()
+    y_b_s, y_c_s = y.stride()
+    o_b, o_r, o_c = output.size()
+    o_b_s, o_r_s, o_c_s = output.stride()
+    s_lut_o_r, s_lut_o_c = sparsity_lut_o.size()
+    s_lut_o_r_s, s_lut_o_c_s = sparsity_lut_o.stride()
+
+    if triton_block_size is None:
+        triton_block_size = get_triton_block_size(sparsity_block_size)
+
+    triton_grid = lambda meta: [o_b,
+                                triton.cdiv(o_r, meta["TRITON_BLOCK_SIZE"]),
+                                triton.cdiv(o_c, meta["TRITON_BLOCK_SIZE"])]
+
+    (kernel_broadcast_addition[triton_grid]
+     (x,
+      x_b, x_b_s, x_c_s,
+      y,
+      y_b, y_b_s, y_c_s,
+      output,
+      o_b, o_b_s, o_r_s, o_c_s,
+      sparsity_lut_o, s_lut_o_r, s_lut_o_r_s, s_lut_o_c_s,
+      sparsity_block_size,
+      triton_block_size))
+
+    return output
+
+
+def broadcast_subtraction(x: Tensor, y: Tensor, sparsity_layout_output: Tensor,
+                          sparsity_block_size: int, triton_block_size: int = None) -> Tensor:
+    """Wrapper for ``broadcast_addition`` with negated y.
+
+    """
+    return broadcast_addition(x, torch.neg(y), sparsity_layout_output, sparsity_block_size, triton_block_size)
+
+
+@triton.jit
+def kernel_broadcast_addition(x,
+                              x_b, x_b_s, x_c_s,
+                              y,
+                              y_b, y_b_s, y_c_s,
+                              o,
+                              o_b, o_b_s, o_r_s, o_c_s,
+                              s_lut_o, s_lut_o_r, s_lut_o_r_s, s_lut_o_c_s,
+                              sparsity_block_size,
+                              TRITON_BLOCK_SIZE: tl.constexpr) -> None:
+    # Get triton block indices
+    pid_blk = tl.program_id(axis=0)
+    pid_row = tl.program_id(axis=1)
+    pid_col = tl.program_id(axis=2)
+
+    # Get position of current sparsity block consisting of its batch, row, and column index
+    spa_bat_o_idx = (pid_blk * s_lut_o_r_s + 0 * s_lut_o_c_s)
+    spa_bat_o_msk = (spa_bat_o_idx < s_lut_o_r * s_lut_o_r_s)
+    spa_bat_o = tl.load(s_lut_o + spa_bat_o_idx, mask=spa_bat_o_msk)
+
+    spa_row_o_idx = (pid_blk * s_lut_o_r_s + 1 * s_lut_o_c_s)
+    spa_row_o_msk = (spa_row_o_idx < s_lut_o_r * s_lut_o_r_s)
+    spa_row_o = tl.load(s_lut_o + spa_row_o_idx, mask=spa_row_o_msk)
+
+    spa_col_o_idx = (pid_blk * s_lut_o_r_s + 2 * s_lut_o_c_s)
+    spa_col_o_msk = (spa_col_o_idx < s_lut_o_r * s_lut_o_r_s)
+    spa_col_o = tl.load(s_lut_o + spa_col_o_idx, mask=spa_col_o_msk)
+
+    # Load x block
+    blk_x_idx = (spa_bat_o * x_b_s +
+                 ((spa_row_o * sparsity_block_size + pid_row * TRITON_BLOCK_SIZE +
+                   tl.arange(0, TRITON_BLOCK_SIZE)) * x_c_s)[None, :])
+    blk_x_msk = (blk_x_idx < x_b * x_b_s)
+    blk_x = tl.load(x + blk_x_idx, mask=blk_x_msk)
+
+    # Load y block
+    blk_y_idx = (spa_bat_o * y_b_s +
+                 ((spa_col_o * sparsity_block_size + pid_col * TRITON_BLOCK_SIZE +
+                   tl.arange(0, TRITON_BLOCK_SIZE)) * y_c_s)[None, :])
+    blk_y_msk = (blk_y_idx < y_b * y_b_s)
+    blk_y = tl.load(y + blk_y_idx, mask=blk_y_msk)
+
+    # Compute sum
+    blk_x, blk_y = tl.broadcast(tl.trans(blk_x), blk_y)
+    buf = blk_x + blk_y
+
+    # Store result
+    blk_o_idx = ((pid_blk * o_b_s) +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * o_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * o_c_s)[None, :])
+    blk_o_msk = (blk_o_idx < o_b * o_b_s)
+    tl.store(o + blk_o_idx, buf, mask=blk_o_msk)