blksprs 1.4.1__py3-none-any.whl → 1.5__py3-none-any.whl

blksprs/__init__.py

@@ -15,4 +15,7 @@ class misc:
     from blksprs.misc.row_wise import row_wise_sum, row_wise_max, row_wise_add, row_wise_sub
 
 class util:
-    from blksprs.utils.tools import do_shape_blocksparse, undo_shape_blocksparse, disable_validation
+    from blksprs.utils.tools import do_shape_blocksparse, undo_shape_blocksparse, disable_validation
+
+class experimental:
+    from blksprs.experimental.distribution_mdi import gather_mdi

blksprs/experimental/distribution_mdi.py

@@ -0,0 +1,438 @@
+import torch
+import triton
+from torch import Tensor
+from triton import language as tl
+
+from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.validation import validate_contiguous, validate_dimensions, validate_device, \
+    validate_sparsity, validate_dtype_int, validate_sparsity_block_size, validate_triton_block_size
+
+
+def gather_mdi(src: Tensor, sparsity_layout_src: Tensor,
+               idx_bat: Tensor,
+               idx_row: Tensor,
+               idx_col: Tensor,
+               sparsity_layout_idx: Tensor,
+               sparsity_block_size: int, triton_block_size: int = None) -> Tensor:
+    src = src.contiguous()
+    idx_bat = idx_bat.contiguous()
+    idx_col = idx_col.contiguous()
+
+    validate_dimensions(src, idx_bat, idx_col)
+    validate_contiguous(src, idx_bat, idx_col)
+    validate_dtype_int(idx_bat, idx_col)
+    validate_device(src, idx_bat, idx_col)
+    validate_sparsity(sparsity_block_size, (src, sparsity_layout_src),
+                      (idx_bat, sparsity_layout_idx), (idx_col, sparsity_layout_idx))
+    validate_sparsity_block_size(sparsity_block_size, src, idx_bat, idx_col)
+    validate_triton_block_size(triton_block_size, sparsity_block_size)
+
+    sparsity_layout_x_flat = sparsity_layout_src.reshape(-1)
+    sparsity_reverse_lut_x = ((torch.cumsum(sparsity_layout_x_flat, dim=-1) - 1) *
+                              (sparsity_layout_x_flat == 1) -
+                              (1 * (sparsity_layout_x_flat == 0)))
+
+    sparsity_lut_i = torch.nonzero(sparsity_layout_idx).contiguous()
+
+    validate_contiguous(sparsity_layout_src, sparsity_reverse_lut_x,
+                        sparsity_layout_idx, sparsity_lut_i)
+
+    return _BlocksparseGatherMDI.apply(src, sparsity_layout_src, sparsity_reverse_lut_x,
+                                       idx_bat, idx_col, sparsity_layout_idx, sparsity_lut_i,
+                                       sparsity_block_size, triton_block_size)
+
+
+class _BlocksparseGatherMDI(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, x: Tensor, sparsity_layout_x: Tensor, sparsity_reverse_lut_x: Tensor,
+                idx_bat: Tensor, idx_col: Tensor, sparsity_layout_i: Tensor, sparsity_lut_i: Tensor,
+                sparsity_block_size: int, triton_block_size: int = None) -> Tensor:
+        output = torch.empty_like(idx_col, dtype=x.dtype)
+
+        x_b, x_r, x_c = x.size()
+        x_b_s, x_r_s, x_c_s = x.stride()
+        s_l_x_b, s_l_x_r, s_l_x_c = sparsity_layout_x.size()
+        s_l_x_b_s, s_l_x_r_s, s_l_x_c_s = sparsity_layout_x.stride()
+        i_b, i_r, i_c = idx_col.size()
+        i_b_s, i_r_s, i_c_s = idx_col.stride()
+        s_lut_i_r, s_lut_i_c = sparsity_lut_i.size()
+        s_lut_i_r_s, s_lut_i_c_s = sparsity_lut_i.stride()
+        o_b, o_r, o_c = output.size()
+        o_b_s, o_r_s, o_c_s = output.stride()
+
+        if triton_block_size is None:
+            triton_block_size = get_triton_block_size(sparsity_block_size)
+
+        triton_grid = lambda meta: [o_b,
+                                    triton.cdiv(o_r, meta["TRITON_BLOCK_SIZE"]),
+                                    triton.cdiv(o_c, meta["TRITON_BLOCK_SIZE"])]
+
+        (_BlocksparseGatherMDI.kernel_blocksparse_gather_mdi[triton_grid]
+         (x,
+          x_b, x_b_s, x_r_s, x_c_s,
+          s_l_x_b, s_l_x_b_s, s_l_x_r_s, s_l_x_c_s,
+          sparsity_reverse_lut_x,
+          idx_bat,
+          idx_col,
+          i_b, i_b_s, i_r_s, i_c_s,
+          output,
+          o_b, o_b_s, o_r_s, o_c_s,
+          sparsity_lut_i, s_lut_i_r, s_lut_i_r_s, s_lut_i_c_s,
+          sparsity_block_size,
+          triton_block_size))
+
+        ctx.save_for_backward(sparsity_layout_x, idx_bat, idx_col, sparsity_layout_i)
+        ctx.sparsity_block_size = sparsity_block_size
+        ctx.triton_block_size = triton_block_size
+
+        return output
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        sparsity_layout_x, idx_bat, idx_col, sparsity_layout_i = ctx.saved_tensors
+        sparsity_block_size = ctx.sparsity_block_size
+        triton_block_size = ctx.triton_block_size
+
+        return scatter_reduce_mdi(grad_output, sparsity_layout_i,
+                                  idx_bat,
+                                  None,
+                                  idx_col,
+                                  sparsity_layout_x,
+                                  sparsity_block_size,
+                                  reduce_op="sum",
+                                  triton_block_size=triton_block_size), None, None, None, None, None, None, None, None
+
+    @staticmethod
+    @triton.jit
+    def kernel_blocksparse_gather_mdi(x,
+                                      x_b, x_b_s, x_r_s, x_c_s,
+                                      s_l_x_b, s_l_x_b_s, s_l_x_r_s, s_l_x_c_s,
+                                      r_lut_x,
+                                      idx_bat,
+                                      idx_col,
+                                      i_b, i_b_s, i_r_s, i_c_s,
+                                      o,
+                                      o_b, o_b_s, o_r_s, o_c_s,
+                                      s_lut_o, s_lut_o_r, s_lut_o_r_s, s_lut_o_c_s,
+                                      sparsity_block_size,
+                                      TRITON_BLOCK_SIZE: tl.constexpr) -> None:
+        # Get triton block indices
+        pid_blk = tl.program_id(axis=0)
+        pid_row = tl.program_id(axis=1)
+        pid_col = tl.program_id(axis=2)
+
+        # Load batch index values
+        blk_idx_bat_idx = ((pid_blk * i_b_s) +
+                           ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_r_s)[:, None] +
+                           ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_c_s)[None, :])
+        blk_idx_bat_msk = (blk_idx_bat_idx < i_b * i_b_s)
+        blk_idx_bat = tl.load(idx_bat + blk_idx_bat_idx, mask=blk_idx_bat_msk).to(tl.int32)
+
+        # Get position of current sparsity block row
+        spa_row_o_idx = (pid_blk * s_lut_o_r_s + 1 * s_lut_o_c_s)
+        spa_row_o_msk = (spa_row_o_idx < s_lut_o_r * s_lut_o_r_s)
+        spa_row_o = tl.load(s_lut_o + spa_row_o_idx, mask=spa_row_o_msk)
+
+        # Load column index values
+        blk_idx_col_idx = ((pid_blk * i_b_s) +
+                           ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_r_s)[:, None] +
+                           ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_c_s)[None, :])
+        blk_idx_col_msk = (blk_idx_col_idx < i_b * i_b_s)
+        blk_idx_col = tl.load(idx_col + blk_idx_col_idx, mask=blk_idx_col_msk).to(tl.int32)
+
+        # Get positions of sparsity blocks
+        pos_spa_blk_x = blk_idx_col // sparsity_block_size
+        pos_spa_col_x = blk_idx_col % sparsity_block_size
+
+        # Load reverse sparsity indices for x
+        rev_idx_spa_x_idx = ((blk_idx_bat * s_l_x_b_s) +
+                             (spa_row_o * s_l_x_r_s) +
+                             (pos_spa_blk_x * s_l_x_c_s))
+        rev_idx_spa_x_msk = (rev_idx_spa_x_idx < s_l_x_b * s_l_x_b_s)
+        rev_idx_spa_x = tl.load(r_lut_x + rev_idx_spa_x_idx, mask=rev_idx_spa_x_msk).to(tl.int32)
+
+        # Load x values
+        blk_x_idx = ((rev_idx_spa_x * x_b_s) +
+                     ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +
+                     (pos_spa_col_x * x_c_s))
+        blk_x_msk = (blk_x_idx < x_b * x_b_s)
+        blk_x = tl.load(x + blk_x_idx, mask=blk_x_msk)
+
+        # Store output
+        blk_o_idx = ((pid_blk * o_b_s) +
+                     ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * o_r_s)[:, None] +
+                     ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * o_c_s)[None, :])
+        blk_o_msk = (blk_o_idx < o_b * o_b_s)
+        tl.store(o + blk_o_idx, blk_x, mask=blk_o_msk)
+
+
+def scatter_reduce_mdi(src: Tensor, sparsity_layout_src: Tensor,
+                       idx_bat: Tensor,
+                       idx_row: Tensor,
+                       idx_col: Tensor,
+                       sparsity_layout_tgt: Tensor,
+                       sparsity_block_size: int,
+                       reduce_op: str = "sum", triton_block_size: int = None) -> Tensor:
+    src = src.contiguous()
+    idx_bat = idx_bat.contiguous()
+    idx_col = idx_col.contiguous()
+
+    validate_dimensions(src, idx_bat, idx_col)
+    validate_contiguous(src, idx_bat, idx_col)
+    validate_dtype_int(idx_bat, idx_col)
+    validate_device(src, idx_bat, idx_col)
+    validate_sparsity(sparsity_block_size, (src, sparsity_layout_src),
+                      (idx_bat, sparsity_layout_src),
+                      (idx_col, sparsity_layout_src))
+    validate_sparsity_block_size(sparsity_block_size, src, idx_bat, idx_col)
+    validate_triton_block_size(triton_block_size, sparsity_block_size)
+
+    if reduce_op not in ["none", "sum"]:
+        raise ValueError(f"Reduction operation '{reduce_op}' is not supported")
+
+    sparsity_lut_x = torch.nonzero(sparsity_layout_src).contiguous()
+
+    sparsity_layout_o_flat = sparsity_layout_tgt.reshape(-1)
+    sparsity_reverse_lut_o = ((torch.cumsum(sparsity_layout_o_flat, dim=-1) - 1) *
+                              (sparsity_layout_o_flat == 1) -
+                              (1 * (sparsity_layout_o_flat == 0)))
+
+    n_sparse_blocks = torch.sum(sparsity_layout_tgt.to(torch.int)).item()
+
+    validate_contiguous(sparsity_layout_src, sparsity_lut_x,
+                        sparsity_layout_tgt, sparsity_reverse_lut_o)
+
+    return _BlocksparseScatterReduceMDI.apply(src, sparsity_layout_src, sparsity_lut_x,
+                                              idx_bat,
+                                              idx_col,
+                                              sparsity_layout_tgt, sparsity_reverse_lut_o,
+                                              sparsity_block_size, n_sparse_blocks,
+                                              reduce_op, triton_block_size)
+
+
+class _BlocksparseScatterReduceMDI(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, x: Tensor, sparsity_layout_x: Tensor, sparsity_lut_x: Tensor,
+                idx_bat: Tensor,
+                idx_col: Tensor,
+                sparsity_layout_o: Tensor, sparsity_reverse_lut_o: Tensor,
+                sparsity_block_size: int, n_sparse_blocks: int,
+                reduce_op: str, triton_block_size: int) -> Tensor:
+        output = torch.zeros(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size),
+                             dtype=x.dtype, device=x.device)
+
+        x_b, x_r, x_c = x.size()
+        x_b_s, x_r_s, x_c_s = x.stride()
+        s_lut_x_r, s_lut_x_c = sparsity_lut_x.size()
+        s_lut_x_r_s, s_lut_x_c_s = sparsity_lut_x.stride()
+        i_b, i_r, i_c = idx_col.size()
+        i_b_s, i_r_s, i_c_s = idx_col.stride()
+        o_b, o_r, o_c = output.size()
+        o_b_s, o_r_s, o_c_s = output.stride()
+        s_l_o_b, s_l_o_r, s_l_o_c = sparsity_layout_o.size()
+        s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = sparsity_layout_o.stride()
+
+        if triton_block_size is None:
+            triton_block_size = get_triton_block_size(sparsity_block_size)
+
+        triton_grid = lambda meta: [x_b,
+                                    triton.cdiv(x_r, meta["TRITON_BLOCK_SIZE"]),
+                                    triton.cdiv(x_c, meta["TRITON_BLOCK_SIZE"])]
+
+        reduce_op_ind = 0
+        if reduce_op == "sum":
+            reduce_op_ind = 1
+
+        (_BlocksparseScatterReduceMDI.kernel_blocksparse_scatter_mdi[triton_grid]
+         (x,
+          x_b, x_b_s, x_r_s, x_c_s,
+          sparsity_lut_x, s_lut_x_r, s_lut_x_r_s, s_lut_x_c_s,
+          idx_bat,
+          idx_col,
+          i_b, i_b_s, i_r_s, i_c_s,
+          output,
+          o_b, o_b_s, o_r_s, o_c_s,
+          s_l_o_b, s_l_o_b_s, s_l_o_r_s, s_l_o_c_s,
+          sparsity_reverse_lut_o,
+          reduce_op_ind,
+          sparsity_block_size,
+          triton_block_size))
+
+        ctx.save_for_backward(sparsity_layout_x, idx_bat, idx_col, sparsity_layout_o)
+        ctx.sparsity_block_size = sparsity_block_size
+        ctx.reduce_op = reduce_op
+        ctx.triton_block_size = triton_block_size
+
+        return output
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        sparsity_layout_x, idx_bat, idx_col, sparsity_layout_o = ctx.saved_tensors
+        sparsity_block_size = ctx.sparsity_block_size
+        reduce_op = ctx.reduce_op
+        triton_block_size = ctx.triton_block_size
+
+        if reduce_op == "sum":
+            return gather_mdi(grad_output, sparsity_layout_o,
+                              idx_bat,
+                              None,
+                              idx_col,
+                              sparsity_layout_x, sparsity_block_size,
+                              triton_block_size=triton_block_size), None, None, None, None, None, None, None, None, None, None
+        else:
+            raise ValueError(f"Reduction operation '{reduce_op}' does not support backward pass")
+
+    @staticmethod
+    @triton.jit
+    def kernel_blocksparse_scatter_mdi(x,
+                                       x_b, x_b_s, x_r_s, x_c_s,
+                                       s_lut_x, s_lut_x_r, s_lut_x_r_s, s_lut_x_c_s,
+                                       idx_bat,
+                                       idx_col,
+                                       i_b, i_b_s, i_r_s, i_c_s,
+                                       o,
+                                       o_b, o_b_s, o_r_s, o_c_s,
+                                       s_l_o_b, s_l_o_b_s, s_l_o_r_s, s_l_o_c_s,
+                                       r_lut_o,
+                                       reduce_op_ind,
+                                       sparsity_block_size,
+                                       TRITON_BLOCK_SIZE: tl.constexpr) -> None:
+        # Get triton block indices
+        pid_blk = tl.program_id(axis=0)
+        pid_row = tl.program_id(axis=1)
+        pid_col = tl.program_id(axis=2)
+
+        # Load batch index values
+        blk_idx_bat_idx = ((pid_blk * i_b_s) +
+                           ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_r_s)[:, None] +
+                           ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_c_s)[None, :])
+        blk_idx_bat_msk = (blk_idx_bat_idx < i_b * i_b_s)
+        blk_idx_bat = tl.load(idx_bat + blk_idx_bat_idx, mask=blk_idx_bat_msk).to(tl.int32)
+
+        # Get position of current sparsity block row
+        spa_row_x_idx = (pid_blk * s_lut_x_r_s + 1 * s_lut_x_c_s)
+        spa_row_x_msk = (spa_row_x_idx < s_lut_x_r * s_lut_x_r_s)
+        spa_row_x = tl.load(s_lut_x + spa_row_x_idx, mask=spa_row_x_msk)
+
+        # Load x values
+        blk_x_idx = ((pid_blk * x_b_s) +
+                     ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +
+                     ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_c_s)[None, :])
+        blk_x_msk = (blk_x_idx < x_b * x_b_s)
+        blk_x = tl.load(x + blk_x_idx, mask=blk_x_msk)
+
+        # Load column index values
+        blk_idx_col_idx = ((pid_blk * i_b_s) +
+                           ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_r_s)[:, None] +
+                           ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_c_s)[None, :])
+        blk_idx_col_msk = (blk_idx_col_idx < i_b * i_b_s)
+        blk_idx_col = tl.load(idx_col + blk_idx_col_idx, mask=blk_idx_col_msk).to(tl.int32)
+
+        # Get positions of sparsity blocks
+        pos_spa_blk_o = blk_idx_col // sparsity_block_size
+        pos_spa_col_o = blk_idx_col % sparsity_block_size
+
+        # Load reverse sparsity indices for o
+        rev_idx_spa_o_idx = ((blk_idx_bat * s_l_o_b_s) +
+                             (spa_row_x * s_l_o_r_s) +
+                             (pos_spa_blk_o * s_l_o_c_s))
+        rev_idx_spa_o_msk = (rev_idx_spa_o_idx < s_l_o_b * s_l_o_b_s)
+        rev_idx_spa_o = tl.load(r_lut_o + rev_idx_spa_o_idx, mask=rev_idx_spa_o_msk).to(tl.int32)
+
+        # Store output
+        blk_o_idx = ((rev_idx_spa_o * o_b_s) +
+                     ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * o_r_s)[:, None] +
+                     (pos_spa_col_o * o_c_s))
+        blk_o_msk = (blk_o_idx < o_b * o_b_s)
+
+        if reduce_op_ind == 0:
+            tl.store(o + blk_o_idx, blk_x, mask=blk_o_msk)
+        elif reduce_op_ind == 1:
+            tl.atomic_add(o + blk_o_idx, blk_x, mask=blk_o_msk)
+
+
+def build_distribution_layout_mdi(idx_bat: Tensor, idx_row: Tensor, idx_col: Tensor, sparsity_layout_idx: Tensor,
+                                  size_target: torch.Size,
+                                  sparsity_block_size: int, triton_block_size: int = None) -> Tensor:
+    validate_dimensions(idx_bat, idx_col)
+    validate_contiguous(idx_bat, idx_col)
+    validate_device(idx_bat, idx_col)
+
+    sparsity_lut_i = torch.nonzero(sparsity_layout_idx).contiguous()
+
+    output = torch.zeros(size_target[0], size_target[1] // sparsity_block_size, size_target[2] // sparsity_block_size,
+                         dtype=torch.bool, device=idx_col.device)
+
+    i_b, i_r, i_c = idx_col.size()
+    i_b_s, i_r_s, i_c_s = idx_col.stride()
+    s_lut_i_r, s_lut_i_c = sparsity_lut_i.size()
+    s_lut_i_r_s, s_lut_i_c_s = sparsity_lut_i.stride()
+    o_b, o_r, o_c = output.size()
+    o_b_s, o_r_s, o_c_s = output.stride()
+
+    if triton_block_size is None:
+        triton_block_size = get_triton_block_size(sparsity_block_size)
+
+    validate_triton_block_size(triton_block_size, sparsity_block_size)
+
+    triton_grid = lambda meta: [i_b,
+                                triton.cdiv(i_r, meta["TRITON_BLOCK_SIZE"]),
+                                triton.cdiv(i_c, meta["TRITON_BLOCK_SIZE"])]
+
+    (kernel_distribution_layout_mdi[triton_grid]
+     (idx_bat,
+      idx_col,
+      i_b, i_b_s, i_r_s, i_c_s,
+      sparsity_lut_i,
+      s_lut_i_r, s_lut_i_r_s, s_lut_i_c_s,
+      output,
+      o_b, o_b_s, o_r_s, o_c_s,
+      sparsity_block_size,
+      triton_block_size))
+
+    return output
+
+
+@triton.jit
+def kernel_distribution_layout_mdi(idx_bat,
+                                   idx_col,
+                                   i_b, i_b_s, i_r_s, i_c_s,
+                                   s_lut_i,
+                                   s_lut_i_r, s_lut_i_r_s, s_lut_i_c_s,
+                                   o,
+                                   o_b, o_b_s, o_r_s, o_c_s,
+                                   sparsity_block_size,
+                                   TRITON_BLOCK_SIZE: tl.constexpr) -> None:
+    # Get triton block indices
+    pid_blk = tl.program_id(axis=0)
+    pid_row = tl.program_id(axis=1)
+    pid_col = tl.program_id(axis=2)
+
+    # Load batch index values
+    blk_idx_bat_idx = ((pid_blk * i_b_s) +
+                       ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_r_s)[:, None] +
+                       ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_c_s)[None, :])
+    blk_idx_bat_msk = (blk_idx_bat_idx < i_b * i_b_s)
+    blk_idx_bat = tl.load(idx_bat + blk_idx_bat_idx, mask=blk_idx_bat_msk).to(tl.int32)
+
+    # Get position of current sparsity block row
+    spa_row_i_idx = (pid_blk * s_lut_i_r_s + 1 * s_lut_i_c_s)
+    spa_row_i_msk = (spa_row_i_idx < s_lut_i_r * s_lut_i_r_s)
+    spa_row_i = tl.load(s_lut_i + spa_row_i_idx, mask=spa_row_i_msk)
+
+    blk_i_idx = (pid_blk * i_b_s +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * i_c_s)[None, :])
+    blk_i_msk = (blk_i_idx < i_b * i_b_s)
+    blk_i = tl.load(idx_col + blk_i_idx, mask=blk_i_msk)
+
+    blk_i = blk_i // sparsity_block_size
+    blk_v = tl.full((TRITON_BLOCK_SIZE, TRITON_BLOCK_SIZE), 1, dtype=tl.int32)
+
+    blk_o_idx = ((blk_idx_bat * o_b_s) +
+                 (spa_row_i * o_r_s) +
+                 (blk_i * o_c_s))
+    blk_o_msk = (blk_o_idx < o_b * o_b_s)
+    tl.store(o + blk_o_idx, blk_v, mask=blk_o_msk)

blksprs/layouting/distribution_layout.py

@@ -35,8 +35,6 @@ def build_distribution_layout(indices: Tensor, sparsity_layout_indices: Tensor,
 
     i_b, i_r, i_c = indices.size()
     i_b_s, i_r_s, i_c_s = indices.stride()
-    s_l_i_b, s_l_i_r, s_l_i_c = sparsity_layout_indices.size()
-    s_l_i_b_s, s_l_i_r_s, s_l_i_c_s = sparsity_layout_indices.stride()
     s_lut_i_r, s_lut_i_c = sparsity_lut_i.size()
     s_lut_i_r_s, s_lut_i_c_s = sparsity_lut_i.stride()
     o_b, o_r, o_c = output.size()
@@ -54,12 +52,10 @@ def build_distribution_layout(indices: Tensor, sparsity_layout_indices: Tensor,
     (kernel_distribution_layout[triton_grid]
      (indices,
       i_b, i_b_s, i_r_s, i_c_s,
-      sparsity_layout_indices,
-      s_l_i_b, s_l_i_b_s, s_l_i_r, s_l_i_r_s, s_l_i_c, s_l_i_c_s,
       sparsity_lut_i,
-      s_lut_i_r, s_lut_i_r_s, s_lut_i_c, s_lut_i_c_s,
+      s_lut_i_r, s_lut_i_r_s, s_lut_i_c_s,
       output,
-      o_b, o_b_s, o_r, o_r_s, o_c, o_c_s,
+      o_b, o_b_s, o_r_s, o_c_s,
       sparsity_block_size,
       triton_block_size))
 
@@ -69,12 +65,10 @@ def build_distribution_layout(indices: Tensor, sparsity_layout_indices: Tensor,
 @triton.jit
 def kernel_distribution_layout(i,
                                i_b, i_b_s, i_r_s, i_c_s,
-                               s_l_i,
-                               s_l_i_b, s_l_i_b_s, s_l_i_r, s_l_i_r_s, s_l_i_c, s_l_i_c_s,
                                s_lut_i,
-                               s_lut_i_r, s_lut_i_r_s, s_lut_i_c, s_lut_i_c_s,
+                               s_lut_i_r, s_lut_i_r_s, s_lut_i_c_s,
                                o,
-                               o_b, o_b_s, o_r, o_r_s, o_c, o_c_s,
+                               o_b, o_b_s, o_r_s, o_c_s,
                                sparsity_block_size,
                                TRITON_BLOCK_SIZE: tl.constexpr) -> None:
     # Get triton block indices
@@ -105,10 +99,3 @@ def kernel_distribution_layout(i,
                  (blk_i * o_c_s))
     blk_o_msk = (blk_o_idx < o_b * o_b_s)
     tl.store(o + blk_o_idx, blk_v, mask=blk_o_msk)
-
-    # if tl.min(blk_x) != 0 or tl.max(blk_x) != 0:
-    #     blk_o_idx = (pid_bat * o_b_s +
-    #                  (((pid_row * TRITON_BLOCK_SIZE) // sparsity_block_size) * o_r_s +
-    #                   ((pid_col * TRITON_BLOCK_SIZE) // sparsity_block_size) * o_c_s))
-    #     blk_o_msk = (blk_o_idx < o_b * o_b_s)
-    #     tl.store(o + blk_o_idx, 1, mask=blk_o_msk)

blksprs/misc/broadcast_ops.py

@@ -41,7 +41,7 @@ def broadcast_add(x: Tensor, y: Tensor, sparsity_layout_output: Tensor,
 
     validate_contiguous(sparsity_layout_output, sparsity_lut_o)
 
-    output = torch.zeros(n_sparse_blocks, sparsity_block_size, sparsity_block_size, device=x.device)
+    output = torch.zeros(n_sparse_blocks, sparsity_block_size, sparsity_block_size, dtype=x.dtype, device=x.device)
 
     x_b, x_c = x.size()
     x_b_s, x_c_s = x.stride()

blksprs/misc/row_wise.py

@@ -56,6 +56,7 @@ def row_wise_sum(x: Tensor, sparsity_layout: Tensor, sparsity_block_size: int,
     output = torch.zeros(size=(n_sparse_blocks_output,
                                sparsity_block_size,
                                1 if flag_slice_only else sparsity_block_size),
+                         dtype=x.dtype,
                          device=x.device)
 
     x_b, x_r, x_c = x.size()

blksprs/ops/conversion.py

@@ -186,8 +186,8 @@ class _BlocksparseToSparse(torch.autograd.Function):
     def forward(ctx, x: Tensor,
                 sparsity_layout: Tensor, sparsity_lut: Tensor,
                 sparsity_block_size: int, n_sparse_blocks: int, triton_block_size: int) -> Tensor:
-        output = torch.empty(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size), dtype=x.dtype,
-                             device=x.device)
+        output = torch.empty(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size),
+                             dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.size()
         x_b_s, x_r_s, x_c_s = x.stride()

blksprs/ops/matmul.py

@@ -78,7 +78,8 @@ class _BlocksparseMatmulSSS(torch.autograd.Function):
                 sparsity_layout_y: Tensor, sparsity_reverse_lut_y: Tensor,
                 sparsity_layout_o: Tensor, sparsity_lut_o: Tensor,
                 sparsity_block_size: int, n_sparse_blocks: int, triton_block_size: int) -> Tensor:
-        output = torch.empty(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size), device=x.device)
+        output = torch.empty(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size),
+                             dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.size()
         x_b_s, x_r_s, x_c_s = x.stride()

blksprs/ops/softmax.py

@@ -127,7 +127,7 @@ class _BlocksparseSoftmax(torch.autograd.Function):
         s_l_s_b, s_l_s_r, s_l_s_c = sparsity_layout_s.size()
         s_l_s_b_s, s_l_s_r_s, s_l_s_c_s = sparsity_layout_s.stride()
 
-        grad_x = torch.empty_like(o)
+        grad_x = torch.empty_like(o, dtype=torch.float)
 
         triton_grid = lambda meta: [o_b,
                                     triton.cdiv(o_r, meta["TRITON_BLOCK_SIZE"]),

blksprs/ops/transpose.py

@@ -59,7 +59,8 @@ class _BlocksparseTranspose(torch.autograd.Function):
     def forward(ctx, x: Tensor,
                 sparsity_layout: Tensor, sparsity_lut: Tensor, sparsity_reverse_lut: Tensor, sparsity_block_size: int,
                 n_sparse_blocks: int, triton_block_size: int) -> (Tensor, Tensor):
-        output = torch.empty(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size), device=x.device)
+        output = torch.empty(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size),
+                             dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.size()
         x_b_s, x_r_s, x_c_s = x.stride()
@@ -101,7 +102,8 @@ class _BlocksparseTranspose(torch.autograd.Function):
         sparsity_block_size = ctx.sparsity_block_size
         triton_block_size = ctx.triton_block_size
 
-        return transpose(grad_output, sparsity_layout, sparsity_block_size, triton_block_size)[0], None, None, None, None, None, None
+        return transpose(grad_output, sparsity_layout, sparsity_block_size, triton_block_size)[
+            0], None, None, None, None, None, None
 
     @staticmethod
     @triton.jit

blksprs/utils/tools.py

@@ -1,4 +1,3 @@
-import torch
 from torch import Tensor, Size
 
 from blksprs.utils.validation import _set_skip_validation
@@ -8,7 +7,7 @@ def do_shape_blocksparse(x: Tensor):
     if x.dim() == 3:
         return x.contiguous(), x.size()
 
-    return x.reshape(-1, x.size(-2), x.size(-1)), x.size()
+    return x.reshape(-1, x.size(-2), x.size(-1)).contiguous(), x.size()
 
 
 def undo_shape_blocksparse(x: Tensor, shape: Size):

blksprs/utils/validation.py

@@ -3,13 +3,13 @@ from torch import Tensor
 
 VALIDATION = True
 
-def validate_dimensions(*tensors: Tensor) -> None:
+def validate_dimensions(*tensors: Tensor, dims=3) -> None:
     if _check_skip_validation():
         return
 
     for tensor in tensors:
-        if tensor.dim() != 3:
-            raise ValueError("Tensor must have 3 dimensions")
+        if tensor.dim() != dims:
+            raise ValueError(f"Tensor must have {dims} dimensions")
 
 
 def validate_contiguous(*tensors: Tensor) -> None:
@@ -91,6 +91,9 @@ def validate_triton_block_size(triton_block_size: int, sparsity_block_size: int)
     if triton_block_size is None:
         return
 
+    if not (triton_block_size & (triton_block_size - 1)) == 0:
+        raise ValueError("Triton block size must be a power of 2")
+
     if triton_block_size > sparsity_block_size:
         raise ValueError("Triton block size cannot be larger than sparsity block size")
 

{blksprs-1.4.1.dist-info → blksprs-1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blksprs
-Version: 1.4.1
+Version: 1.5
 Summary: A lightweight library for operations on blocksparse matrices in PyTorch.
 Author-email: Felix Schön <schoen@kr.tuwien.ac.at>
 Project-URL: Homepage, https://github.com/FelixSchoen/blksprs

blksprs-1.5.dist-info/RECORD

@@ -0,0 +1,20 @@
+blksprs/__init__.py,sha256=OY9ofdbzBGsvY6hx0oLCrSszlJFdMns9x7gKE0asFI0,919
+blksprs/experimental/distribution_mdi.py,sha256=shu-3Nt7nkaLIb4O2kSajC8Lh7IWFXO9rsjzP14ASYA,20088
+blksprs/layouting/distribution_layout.py,sha256=Zv-b2t5VOvW6-ejdX42kUV7X1yYsvDCY_PXFE_wKwi0,4165
+blksprs/layouting/sparsity_layout.py,sha256=vZL8r5LkMwILYYqTYPZcN_NYFJuVFIB6mmBkdtRyXmI,7893
+blksprs/misc/broadcast_ops.py,sha256=ahm7_lI12bJ6VTKRuSkwEeaEYWRY-BeMIOhtei35zpQ,5323
+blksprs/misc/repeat_interleave.py,sha256=KJeapmxbpA7zGFfa5hUhCGrk4aFmhOhlMw-hbTh9PLI,5668
+blksprs/misc/row_wise.py,sha256=1UtjLplrGx1FkxhzQ2hjSBBY11ToLQs0JiLaXKRAkL4,16893
+blksprs/ops/conversion.py,sha256=vuiNwrwyuGI6H4PKrS_UHI7OKWJwNZd2i3LSjf6RetU,21332
+blksprs/ops/distribution.py,sha256=KhtHRVcv4_woyNlldAjIWF-7021-KX-xyIcN6rE-UgE,16879
+blksprs/ops/exp.py,sha256=CVWVq_emO2CnS_xk6Unx67P7EI7IL26dwtsmBJZOLzQ,3698
+blksprs/ops/matmul.py,sha256=743XeD5M4iUv28sYf7q6mVXDd4jZpV04JAx8bF7hWkw,11254
+blksprs/ops/softmax.py,sha256=cs1utM6UCzHhdJpf-ZysBr6CwbjI-5aQG0ahYY37Zy0,11991
+blksprs/ops/transpose.py,sha256=Ru4YKyg796WT6OnDSTCYG45tMmdgvju3hMFzkwsJnO8,6801
+blksprs/utils/benchmarking.py,sha256=4pLVlnPW_2EM-NT3n4SClaRznVYEljztLbJcccz8kZE,1360
+blksprs/utils/tools.py,sha256=JAuwsLISr_hcvxIgUVvKz5ZPf9M5ycquplsBU5dVfDc,596
+blksprs/utils/validation.py,sha256=rP6yr-C2ghXfJEERry_pfvVJ0g0VyqV4sL4HkBRlJg8,3345
+blksprs-1.5.dist-info/METADATA,sha256=dql0_6s1Vfdnx6sLFusayZWSeU9uxvfAjBDdLPk43so,7607
+blksprs-1.5.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
+blksprs-1.5.dist-info/top_level.txt,sha256=qyp0IHeY3H2GQA97i4hk_To5rRBS2YcE1HRPSLy04fk,8
+blksprs-1.5.dist-info/RECORD,,

{blksprs-1.4.1.dist-info → blksprs-1.5.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.1.0)
+Generator: setuptools (75.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

blksprs-1.4.1.dist-info/RECORD

@@ -1,19 +0,0 @@
-blksprs/__init__.py,sha256=ORAVhGR91G1wyIOs9Wecv-xfmjju3bJ4Jynq_SGOVY4,833
-blksprs/layouting/distribution_layout.py,sha256=Xd8KjZwI87L9EL1Bw5SGUW9YztFD5q0Ygr99sffvdak,4939
-blksprs/layouting/sparsity_layout.py,sha256=vZL8r5LkMwILYYqTYPZcN_NYFJuVFIB6mmBkdtRyXmI,7893
-blksprs/misc/broadcast_ops.py,sha256=RTcqvx6X_THRBb55jipeEe63YSLIAh27jdpuze0aSek,5308
-blksprs/misc/repeat_interleave.py,sha256=KJeapmxbpA7zGFfa5hUhCGrk4aFmhOhlMw-hbTh9PLI,5668
-blksprs/misc/row_wise.py,sha256=KCDO5ry5TkjI88LLD_QINZwBkzfmjoQpOOvYLfpUn5I,16853
-blksprs/ops/conversion.py,sha256=h1c5T74rQjqYgY9dwWXfPTXRpgzy0dtAhCmtUp8-6uo,21332
-blksprs/ops/distribution.py,sha256=KhtHRVcv4_woyNlldAjIWF-7021-KX-xyIcN6rE-UgE,16879
-blksprs/ops/exp.py,sha256=CVWVq_emO2CnS_xk6Unx67P7EI7IL26dwtsmBJZOLzQ,3698
-blksprs/ops/matmul.py,sha256=6DaYxecJgwiW8L-UISkgyNyzQ31AAkmDL-Oq1EjHt98,11210
-blksprs/ops/softmax.py,sha256=cSTxDnNmMRlJGOlCSpdg1U5KUIFpVtHulz8fteJFeh0,11972
-blksprs/ops/transpose.py,sha256=et8R124L29TUqihci18ms_hBoYXTtPu5LXgEA8sxk_w,6744
-blksprs/utils/benchmarking.py,sha256=4pLVlnPW_2EM-NT3n4SClaRznVYEljztLbJcccz8kZE,1360
-blksprs/utils/tools.py,sha256=RKGWCGd5h1qFOIoShsdJObx4-QsS0RxCyzFie0geNxo,596
-blksprs/utils/validation.py,sha256=Gsx3aah6355bWXRPpbFuZ1p0fOrYduIqaM3ON9d5NiI,3197
-blksprs-1.4.1.dist-info/METADATA,sha256=3xRmBFHv2U2KnrW3_QX3003SHLkQ1JCaSqh4AUBsJD4,7609
-blksprs-1.4.1.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-blksprs-1.4.1.dist-info/top_level.txt,sha256=qyp0IHeY3H2GQA97i4hk_To5rRBS2YcE1HRPSLy04fk,8
-blksprs-1.4.1.dist-info/RECORD,,