blksprs 1.5__tar.gz → 1.6.1__tar.gz

{blksprs-1.5 → blksprs-1.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blksprs
-Version: 1.5
+Version: 1.6.1
 Summary: A lightweight library for operations on blocksparse matrices in PyTorch.
 Author-email: Felix Schön <schoen@kr.tuwien.ac.at>
 Project-URL: Homepage, https://github.com/FelixSchoen/blksprs
@@ -31,9 +31,10 @@ Currently supported operations (includes gradient calculation):
 - Sparse matrix multiplication (_supports any combination of sparse and dense matrices due to support
   for `sparse = sparse @ sparse` matmul_)
 - Softmax
-- Transposition
+- Transpose
 - Gather
 - Scatter (_supports either no reduction or summation, gradients are only available for summation_)
+- Splitting and merging of matrices along the last dimension
 - Conversion to and from sparse form
 - Conversion to different sparsity layouts and different sparsity block sizes
 

{blksprs-1.5 → blksprs-1.6.1}/README.md

@@ -12,9 +12,10 @@ Currently supported operations (includes gradient calculation):
 - Sparse matrix multiplication (_supports any combination of sparse and dense matrices due to support
   for `sparse = sparse @ sparse` matmul_)
 - Softmax
-- Transposition
+- Transpose
 - Gather
 - Scatter (_supports either no reduction or summation, gradients are only available for summation_)
+- Splitting and merging of matrices along the last dimension
 - Conversion to and from sparse form
 - Conversion to different sparsity layouts and different sparsity block sizes
 

{blksprs-1.5 → blksprs-1.6.1}/blksprs/__init__.py

@@ -4,10 +4,11 @@ from blksprs.ops.exp import exp
 from blksprs.ops.matmul import matmul
 from blksprs.ops.softmax import softmax
 from blksprs.ops.transpose import transpose
+from blksprs.ops.partitioning import split, merge
 
 class layout:
     from blksprs.layouting.distribution_layout import build_distribution_layout
-    from blksprs.layouting.sparsity_layout import build_sparsity_layout, build_sparsity_layout_adaption
+    from blksprs.layouting.sparsity_layout import build_sparsity_layout, build_sparsity_layout_adaption, build_sparsity_layout_matmul
 
 class misc:
     from blksprs.misc.broadcast_ops import broadcast_add, broadcast_sub
@@ -18,4 +19,4 @@ class util:
     from blksprs.utils.tools import do_shape_blocksparse, undo_shape_blocksparse, disable_validation
 
 class experimental:
-    from blksprs.experimental.distribution_mdi import gather_mdi
+    from blksprs.experimental.distribution_mdi import gather_mdi, scatter_reduce_mdi

{blksprs-1.5 → blksprs-1.6.1}/blksprs/layouting/sparsity_layout.py

@@ -188,3 +188,39 @@ def kernel_sparsity_layout_adaption(x,
                        // sparsity_block_size_to) * o_c_s))
         blk_o_msk = (blk_o_idx < o_b * o_b_s)
         tl.store(o + blk_o_idx, 1, mask=blk_o_msk)
+
+
+def build_sparsity_layout_matmul(sparsity_layout_x: Tensor, sparsity_layout_y: Tensor):
+    """Builds the precise sparsity layout of the result of a matrix multiplication between the two input tensors.
+
+    Args:
+        sparsity_layout_x (Tensor): The sparsity layout of the first block-sparse tensor.
+        sparsity_layout_y (Tensor): The sparsity layout of the second block-sparse tensor.
+
+    Returns:
+        Tensor: The precise sparsity layout of the result of a matrix multiplication between the two input tensors.
+
+    """
+    return torch.matmul(sparsity_layout_x.to(torch.float), sparsity_layout_y.to(torch.float)).to(torch.bool)
+
+
+def build_sparsity_layout_matmul_fast(sparsity_layout_x: Tensor, sparsity_layout_y: Tensor):
+    """Builds the approximate sparsity layout of the result of a matrix multiplication between the two input tensors.
+
+    Note:
+        This function is faster than the ``build_sparsity_layout_matmul`` function due to the fact that it only checks
+            whether at least one of the blocks in either of the vectors participating in the matmul is non-sparse. The
+            resulting sparsity layout may thus overestimate the actual sparsity of the result.
+
+    Args:
+        sparsity_layout_x (Tensor): The sparsity layout of the first block-sparse tensor.
+        sparsity_layout_y (Tensor): The sparsity layout of the second block-sparse tensor.
+
+    Returns:
+        Tensor: The approximate sparsity layout of the result of a matrix multiplication between the two input tensors.
+
+    """
+    sparsity_layout_x_slice = torch.max(sparsity_layout_x, dim=-1).values.unsqueeze(-1)
+    sparsity_layout_y_slice = torch.max(sparsity_layout_y, dim=-2).values.unsqueeze(1)
+
+    return torch.logical_or(sparsity_layout_x_slice, sparsity_layout_y_slice)

{blksprs-1.5 → blksprs-1.6.1}/blksprs/misc/repeat_interleave.py

@@ -35,7 +35,7 @@ def repeat_interleave(x: Tensor, sparsity_layout: Tensor, repeats: int,
     validate_sparsity_block_size(sparsity_block_size, x)
     validate_triton_block_size(triton_block_size, sparsity_block_size)
 
-    sparsity_layout_output = torch.repeat_interleave(sparsity_layout, 3, dim=0).contiguous()
+    sparsity_layout_output = torch.repeat_interleave(sparsity_layout, repeats, dim=0).contiguous()
 
     sparsity_lut = torch.nonzero(sparsity_layout).contiguous()
 

blksprs-1.6.1/blksprs/ops/partitioning.py

@@ -0,0 +1,244 @@
+import torch
+import triton
+from sympy.utilities.iterables import partitions
+from torch import Tensor
+from triton import language as tl
+
+from blksprs.utils.tools import get_triton_block_size
+
+from blksprs.utils.validation import validate_dimensions, validate_contiguous, validate_device, \
+    validate_sparsity, validate_sparsity_block_size, validate_triton_block_size
+
+
+def split(x: Tensor, sparsity_layout: Tensor, partitions: int,
+          sparsity_block_size: int, triton_block_size: int = None) -> (Tensor, Tensor):
+    """Splits a block-sparse tensor in compressed form along the last dimension into partitions.
+
+    Args:
+        x (Tensor): A block-sparse tensor in compressed form.
+        sparsity_layout (Tensor): The sparsity layout of the block-sparse tensor.
+        partitions (int): The number of partitions to split the block-sparse tensor into.
+        sparsity_block_size (int): The size of the sparsity blocks.
+        triton_block_size (int): The block size to use for the triton kernel (default ``None``).
+
+    Returns:
+        Tensor: The block-sparse tensor split into partitions in compressed form.
+        Tensor: The sparsity layout of the output tensor.
+
+    """
+    x = x.contiguous()
+
+    validate_dimensions(x)
+    validate_contiguous(x)
+    validate_device(x)
+    validate_sparsity(sparsity_block_size, (x, sparsity_layout))
+    validate_sparsity_block_size(sparsity_block_size, x)
+    validate_triton_block_size(triton_block_size, sparsity_block_size)
+
+    sparsity_layout_output = (sparsity_layout
+                              .reshape(sparsity_layout.size(0), sparsity_layout.size(1), partitions,
+                                       sparsity_layout.size(2) // partitions)
+                              .permute(0, 2, 1, 3)
+                              .reshape(sparsity_layout.size(0) * partitions, sparsity_layout.size(1),
+                                       sparsity_layout.size(2) // partitions).contiguous())
+
+    sparsity_lut = torch.nonzero(sparsity_layout_output).contiguous()
+
+    sparsity_layout_flat = sparsity_layout.reshape(-1)
+    sparsity_reverse_lut = (((torch.cumsum(sparsity_layout_flat, dim=-1) - 1) *
+                             (sparsity_layout_flat == 1) -
+                             (1 * (sparsity_layout_flat == 0)))
+                            .reshape(sparsity_layout.size())
+                            .reshape(sparsity_layout.size(0), sparsity_layout.size(1), partitions,
+                                     sparsity_layout.size(2) // partitions)
+                            .permute(0, 2, 1, 3).reshape(-1).contiguous())
+
+    n_sparse_blocks = torch.sum(sparsity_layout.to(torch.int)).item()
+
+    validate_contiguous(sparsity_layout_output, sparsity_lut, sparsity_reverse_lut)
+
+    return _BlocksparseSplit.apply(x, sparsity_layout_output, sparsity_lut, sparsity_reverse_lut, partitions,
+                                   sparsity_block_size, n_sparse_blocks, triton_block_size), sparsity_layout_output
+
+
+class _BlocksparseSplit(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, x: Tensor, sparsity_layout_o: Tensor, sparsity_lut: Tensor, sparsity_reverse_lut: Tensor,
+                num_partitions: int, sparsity_block_size: int, n_sparse_blocks: int, triton_block_size: int) -> Tensor:
+        ctx.num_partitions = num_partitions
+
+        return forward_reorder(ctx, x, sparsity_layout_o, sparsity_lut, sparsity_reverse_lut, sparsity_block_size,
+                               n_sparse_blocks, triton_block_size)
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        sparsity_layout = ctx.saved_tensors[0]
+        num_partitions = ctx.num_partitions
+        sparsity_block_size = ctx.sparsity_block_size
+        triton_block_size = ctx.triton_block_size
+
+        return merge(grad_output, sparsity_layout, num_partitions,
+                     sparsity_block_size, triton_block_size)[0], None, None, None, None, None, None, None
+
+
+def merge(x: Tensor, sparsity_layout: Tensor, partitions: int,
+          sparsity_block_size: int, triton_block_size: int = None) -> (Tensor, Tensor):
+    """Merges the specified partitions of a block-sparse tensor in compressed form along the last dimension.
+
+    Args:
+        x (Tensor): A block-sparse tensor in compressed form.
+        sparsity_layout (Tensor): The sparsity layout of the block-sparse tensor.
+        partitions (int): The number of partitions to be merged.
+        sparsity_block_size (int): The size of the sparsity blocks.
+        triton_block_size (int): The block size to use for the triton kernel (default ``None``).
+
+    Returns:
+        Tensor: The merged block-sparse tensor in compressed form.
+        Tensor: The sparsity layout of the output tensor.
+
+    """
+    x = x.contiguous()
+
+    validate_dimensions(x)
+    validate_contiguous(x)
+    validate_device(x)
+    validate_sparsity(sparsity_block_size, (x, sparsity_layout))
+    validate_sparsity_block_size(sparsity_block_size, x)
+    validate_triton_block_size(triton_block_size, sparsity_block_size)
+
+    sparsity_layout_output = (sparsity_layout.reshape(sparsity_layout.size(0) // partitions, partitions,
+                                                      sparsity_layout.size(1), sparsity_layout.size(2))
+                              .permute(0, 2, 1, 3)
+                              .reshape(sparsity_layout.size(0) // partitions,
+                                       sparsity_layout.size(1), sparsity_layout.size(2) * partitions).contiguous())
+
+    sparsity_lut = torch.nonzero(sparsity_layout_output).contiguous()
+
+    sparsity_layout_flat = sparsity_layout.reshape(-1)
+    sparsity_reverse_lut = (((torch.cumsum(sparsity_layout_flat, dim=-1) - 1) *
+                             (sparsity_layout_flat == 1) -
+                             (1 * (sparsity_layout_flat == 0)))
+                            .reshape(sparsity_layout.size(0) // partitions, partitions,
+                                     sparsity_layout.size(1), sparsity_layout.size(2))
+                            .permute(0, 2, 1, 3)
+                            .reshape(sparsity_layout.size(0) // partitions,
+                                     sparsity_layout.size(1), sparsity_layout.size(2) * partitions)
+                            .reshape(-1).contiguous())
+
+    n_sparse_blocks = torch.sum(sparsity_layout.to(torch.int)).item()
+
+    validate_contiguous(sparsity_layout_output, sparsity_lut, sparsity_reverse_lut)
+
+    return _BlocksparseMerge.apply(x, sparsity_layout_output, sparsity_lut, sparsity_reverse_lut, partitions,
+                                   sparsity_block_size, n_sparse_blocks, triton_block_size), sparsity_layout_output
+
+
+class _BlocksparseMerge(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, x: Tensor, sparsity_layout_o: Tensor, sparsity_lut: Tensor, sparsity_reverse_lut: Tensor,
+                num_partitions: int, sparsity_block_size: int, n_sparse_blocks: int, triton_block_size: int) -> Tensor:
+        ctx.num_partitions = num_partitions
+
+        return forward_reorder(ctx, x, sparsity_layout_o, sparsity_lut, sparsity_reverse_lut, sparsity_block_size,
+                               n_sparse_blocks, triton_block_size)
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        sparsity_layout = ctx.saved_tensors[0]
+        num_partitions = ctx.num_partitions
+        sparsity_block_size = ctx.sparsity_block_size
+        triton_block_size = ctx.triton_block_size
+
+        return split(grad_output, sparsity_layout, num_partitions,
+                     sparsity_block_size, triton_block_size)[0], None, None, None, None, None, None, None
+
+
+def forward_reorder(ctx, x: Tensor, sparsity_layout_o: Tensor, sparsity_lut: Tensor, sparsity_reverse_lut: Tensor,
+                    sparsity_block_size: int, n_sparse_blocks: int, triton_block_size: int) -> Tensor:
+    output = torch.empty(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size),
+                         dtype=x.dtype, device=x.device)
+
+    x_b, x_r, x_c = x.size()
+    x_b_s, x_r_s, x_c_s = x.stride()
+    s_l_b, s_l_r, s_l_c = sparsity_layout_o.size()
+    s_l_b_s, s_l_r_s, s_l_c_s = sparsity_layout_o.stride()
+    s_lut_r, s_lut_c = sparsity_lut.shape
+    s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
+    o_b, o_r, o_c = output.size()
+    o_b_s, o_r_s, o_c_s = output.stride()
+
+    if triton_block_size is None:
+        triton_block_size = get_triton_block_size(sparsity_block_size)
+
+    triton_grid = lambda meta: [o_b,
+                                triton.cdiv(o_r, meta["TRITON_BLOCK_SIZE"]),
+                                triton.cdiv(o_c, meta["TRITON_BLOCK_SIZE"])]
+
+    (kernel_blocksparse_reorder[triton_grid]
+     (x,
+      x_b, x_b_s, x_r_s, x_c_s,
+      s_l_b, s_l_b_s, s_l_r_s, s_l_c_s,
+      sparsity_lut, s_lut_r, s_lut_r_s, s_lut_c_s,
+      sparsity_reverse_lut,
+      output,
+      o_b, o_b_s,
+      triton_block_size))
+
+    # Save for backward pass
+    ctx.save_for_backward(sparsity_layout_o)
+    ctx.sparsity_block_size = sparsity_block_size
+    ctx.triton_block_size = triton_block_size
+
+    return output
+
+
+@triton.jit
+def kernel_blocksparse_reorder(x,
+                               x_b, x_b_s, x_r_s, x_c_s,
+                               s_l_b, s_l_b_s, s_l_r_s, s_l_c_s,
+                               s_lut, s_lut_r, s_lut_r_s, s_lut_c_s,
+                               r_lut,
+                               o,
+                               o_b, o_b_s,
+                               TRITON_BLOCK_SIZE: tl.constexpr) -> None:
+    # Get triton block indices
+    pid_blk = tl.program_id(axis=0)
+    pid_row = tl.program_id(axis=1)
+    pid_col = tl.program_id(axis=2)
+
+    # Get sparsity index of current output block consisting of its batch, row, and column index
+    spa_bat_idx = (pid_blk * s_lut_r_s + 0 * s_lut_c_s)
+    spa_bat_msk = (spa_bat_idx < s_lut_r * s_lut_r_s)
+    spa_bat = tl.load(s_lut + spa_bat_idx, mask=spa_bat_msk)
+
+    spa_row_idx = (pid_blk * s_lut_r_s + 1 * s_lut_c_s)
+    spa_row_msk = (spa_row_idx < s_lut_r * s_lut_r_s)
+    spa_row = tl.load(s_lut + spa_row_idx, mask=spa_row_msk)
+
+    spa_col_idx = (pid_blk * s_lut_r_s + 2 * s_lut_c_s)
+    spa_col_msk = (spa_col_idx < s_lut_r * s_lut_r_s)
+    spa_col = tl.load(s_lut + spa_col_idx, mask=spa_col_msk)
+
+    # Get reverse sparsity index
+    rev_idx_spa_idx = (spa_bat * s_l_b_s +
+                       spa_row * s_l_r_s +
+                       spa_col * s_l_c_s)
+    rev_idx_spa_msk = (rev_idx_spa_idx < s_l_b * s_l_b_s)
+    rev_idx_spa = tl.load(r_lut + rev_idx_spa_idx, mask=rev_idx_spa_msk).to(tl.int32)
+
+    if rev_idx_spa == -1:
+        assert False, "Invalid sparsity block"
+
+    blk_x_idx = (rev_idx_spa * x_b_s +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_c_s)[None, :])
+    blk_x_msk = (blk_x_idx < x_b * x_b_s)
+    blk_x = tl.load(x + blk_x_idx, mask=blk_x_msk)
+
+    blk_o_idx = (pid_blk * o_b_s +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_c_s)[None, :])
+    blk_o_msk = (blk_o_idx < o_b * o_b_s)
+    tl.store(o + blk_o_idx, blk_x, mask=blk_o_msk)

{blksprs-1.5 → blksprs-1.6.1}/blksprs/ops/transpose.py

@@ -56,16 +56,16 @@ def transpose(x: Tensor, sparsity_layout: Tensor, sparsity_block_size: int, trit
 class _BlocksparseTranspose(torch.autograd.Function):
 
     @staticmethod
-    def forward(ctx, x: Tensor,
-                sparsity_layout: Tensor, sparsity_lut: Tensor, sparsity_reverse_lut: Tensor, sparsity_block_size: int,
-                n_sparse_blocks: int, triton_block_size: int) -> (Tensor, Tensor):
+    def forward(ctx, x: Tensor, sparsity_layout_o: Tensor, sparsity_lut: Tensor, sparsity_reverse_lut: Tensor,
+                sparsity_block_size: int,
+                n_sparse_blocks: int, triton_block_size: int) -> Tensor:
         output = torch.empty(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size),
                              dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.size()
         x_b_s, x_r_s, x_c_s = x.stride()
-        s_l_b, s_l_r, s_l_c = sparsity_layout.size()
-        s_l_b_s, s_l_r_s, s_l_c_s = sparsity_layout.stride()
+        s_l_b, s_l_r, s_l_c = sparsity_layout_o.size()
+        s_l_b_s, s_l_r_s, s_l_c_s = sparsity_layout_o.stride()
         s_lut_r, s_lut_c = sparsity_lut.shape
         s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
         o_b, o_r, o_c = output.size()
@@ -89,8 +89,7 @@ class _BlocksparseTranspose(torch.autograd.Function):
           triton_block_size))
 
         # Save for backward pass
-        ctx.save_for_backward(sparsity_layout)
-        ctx.sparsity_layout = sparsity_layout
+        ctx.save_for_backward(sparsity_layout_o)
         ctx.sparsity_block_size = sparsity_block_size
         ctx.triton_block_size = triton_block_size
 

{blksprs-1.5 → blksprs-1.6.1}/blksprs/utils/validation.py

@@ -63,6 +63,8 @@ def validate_sparsity(sparsity_block_size: int, *tensor_sparsity_layout_tuples:
     for (tensor, sparsity_layout) in tensor_sparsity_layout_tuples:
         _validate_sparsity_layout_values(sparsity_layout)
 
+        if not sparsity_layout.dim() == 3:
+            raise ValueError("Sparsity layout must have exactly 3 dimensions")
         if not (tensor.size(-1) == tensor.size(-2) == sparsity_block_size):
             raise ValueError("Blocks not conforming to sparsity block size")
         if not tensor.size(0) == torch.sum(sparsity_layout.reshape(-1)):

{blksprs-1.5 → blksprs-1.6.1}/blksprs.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blksprs
-Version: 1.5
+Version: 1.6.1
 Summary: A lightweight library for operations on blocksparse matrices in PyTorch.
 Author-email: Felix Schön <schoen@kr.tuwien.ac.at>
 Project-URL: Homepage, https://github.com/FelixSchoen/blksprs
@@ -31,9 +31,10 @@ Currently supported operations (includes gradient calculation):
 - Sparse matrix multiplication (_supports any combination of sparse and dense matrices due to support
   for `sparse = sparse @ sparse` matmul_)
 - Softmax
-- Transposition
+- Transpose
 - Gather
 - Scatter (_supports either no reduction or summation, gradients are only available for summation_)
+- Splitting and merging of matrices along the last dimension
 - Conversion to and from sparse form
 - Conversion to different sparsity layouts and different sparsity block sizes
 

{blksprs-1.5 → blksprs-1.6.1}/blksprs.egg-info/SOURCES.txt

@@ -16,6 +16,7 @@ blksprs/ops/conversion.py
 blksprs/ops/distribution.py
 blksprs/ops/exp.py
 blksprs/ops/matmul.py
+blksprs/ops/partitioning.py
 blksprs/ops/softmax.py
 blksprs/ops/transpose.py
 blksprs/utils/benchmarking.py

{blksprs-1.5 → blksprs-1.6.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "blksprs"
-version = "1.5"
+version = "1.6.1"
 authors = [{ name = "Felix Schön", email = "schoen@kr.tuwien.ac.at" }]
 description = "A lightweight library for operations on blocksparse matrices in PyTorch."
 readme = "README.md"