blksprs 1.6.1__tar.gz → 1.7__tar.gz

{blksprs-1.6.1 → blksprs-1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blksprs
-Version: 1.6.1
+Version: 1.7
 Summary: A lightweight library for operations on blocksparse matrices in PyTorch.
 Author-email: Felix Schön <schoen@kr.tuwien.ac.at>
 Project-URL: Homepage, https://github.com/FelixSchoen/blksprs
@@ -34,6 +34,7 @@ Currently supported operations (includes gradient calculation):
 - Transpose
 - Gather
 - Scatter (_supports either no reduction or summation, gradients are only available for summation_)
+- Repeat (_supports target sparsity layout_)
 - Splitting and merging of matrices along the last dimension
 - Conversion to and from sparse form
 - Conversion to different sparsity layouts and different sparsity block sizes
@@ -64,7 +65,7 @@ We recommend installing blksprs from [PyPI](https://pypi.org/project/blksprs/) u
 
 ### Dependencies
 
-- [PyTorch](https://pytorch.org/) (built with v2.4.0)
+- [PyTorch](https://pytorch.org/) (built with v2.5.0)
 - _[Triton](https://github.com/triton-lang/triton) (included with PyTorch)_
 
 ## Changelog

{blksprs-1.6.1 → blksprs-1.7}/README.md

@@ -15,6 +15,7 @@ Currently supported operations (includes gradient calculation):
 - Transpose
 - Gather
 - Scatter (_supports either no reduction or summation, gradients are only available for summation_)
+- Repeat (_supports target sparsity layout_)
 - Splitting and merging of matrices along the last dimension
 - Conversion to and from sparse form
 - Conversion to different sparsity layouts and different sparsity block sizes
@@ -45,7 +46,7 @@ We recommend installing blksprs from [PyPI](https://pypi.org/project/blksprs/) u
 
 ### Dependencies
 
-- [PyTorch](https://pytorch.org/) (built with v2.4.0)
+- [PyTorch](https://pytorch.org/) (built with v2.5.0)
 - _[Triton](https://github.com/triton-lang/triton) (included with PyTorch)_
 
 ## Changelog

{blksprs-1.6.1 → blksprs-1.7}/blksprs/experimental/distribution_mdi.py

@@ -3,7 +3,7 @@ import triton
 from torch import Tensor
 from triton import language as tl
 
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_contiguous, validate_dimensions, validate_device, \
     validate_sparsity, validate_dtype_int, validate_sparsity_block_size, validate_triton_block_size
 
@@ -51,15 +51,15 @@ class _BlocksparseGatherMDI(torch.autograd.Function):
         output = torch.empty_like(idx_col, dtype=x.dtype)
 
         x_b, x_r, x_c = x.size()
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         s_l_x_b, s_l_x_r, s_l_x_c = sparsity_layout_x.size()
-        s_l_x_b_s, s_l_x_r_s, s_l_x_c_s = sparsity_layout_x.stride()
+        s_l_x_b_s, s_l_x_r_s, s_l_x_c_s = stride(sparsity_layout_x)
         i_b, i_r, i_c = idx_col.size()
-        i_b_s, i_r_s, i_c_s = idx_col.stride()
+        i_b_s, i_r_s, i_c_s = stride(idx_col)
         s_lut_i_r, s_lut_i_c = sparsity_lut_i.size()
-        s_lut_i_r_s, s_lut_i_c_s = sparsity_lut_i.stride()
+        s_lut_i_r_s, s_lut_i_c_s = stride(sparsity_lut_i)
         o_b, o_r, o_c = output.size()
-        o_b_s, o_r_s, o_c_s = output.stride()
+        o_b_s, o_r_s, o_c_s = stride(output)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -224,15 +224,15 @@ class _BlocksparseScatterReduceMDI(torch.autograd.Function):
                              dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.size()
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         s_lut_x_r, s_lut_x_c = sparsity_lut_x.size()
-        s_lut_x_r_s, s_lut_x_c_s = sparsity_lut_x.stride()
+        s_lut_x_r_s, s_lut_x_c_s = stride(sparsity_lut_x)
         i_b, i_r, i_c = idx_col.size()
-        i_b_s, i_r_s, i_c_s = idx_col.stride()
+        i_b_s, i_r_s, i_c_s = stride(idx_col)
         o_b, o_r, o_c = output.size()
-        o_b_s, o_r_s, o_c_s = output.stride()
+        o_b_s, o_r_s, o_c_s = stride(output)
         s_l_o_b, s_l_o_r, s_l_o_c = sparsity_layout_o.size()
-        s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = sparsity_layout_o.stride()
+        s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = stride(sparsity_layout_o)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -366,11 +366,11 @@ def build_distribution_layout_mdi(idx_bat: Tensor, idx_row: Tensor, idx_col: Ten
                          dtype=torch.bool, device=idx_col.device)
 
     i_b, i_r, i_c = idx_col.size()
-    i_b_s, i_r_s, i_c_s = idx_col.stride()
+    i_b_s, i_r_s, i_c_s = stride(idx_col)
     s_lut_i_r, s_lut_i_c = sparsity_lut_i.size()
-    s_lut_i_r_s, s_lut_i_c_s = sparsity_lut_i.stride()
+    s_lut_i_r_s, s_lut_i_c_s = stride(sparsity_lut_i)
     o_b, o_r, o_c = output.size()
-    o_b_s, o_r_s, o_c_s = output.stride()
+    o_b_s, o_r_s, o_c_s = stride(output)
 
     if triton_block_size is None:
         triton_block_size = get_triton_block_size(sparsity_block_size)

{blksprs-1.6.1 → blksprs-1.7}/blksprs/layouting/distribution_layout.py

@@ -3,7 +3,7 @@ import triton
 from torch import Tensor
 from triton import language as tl
 
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_triton_block_size, validate_dimensions, validate_device, \
     validate_contiguous
 
@@ -34,11 +34,11 @@ def build_distribution_layout(indices: Tensor, sparsity_layout_indices: Tensor,
                          dtype=torch.bool, device=indices.device)
 
     i_b, i_r, i_c = indices.size()
-    i_b_s, i_r_s, i_c_s = indices.stride()
+    i_b_s, i_r_s, i_c_s = stride(indices)
     s_lut_i_r, s_lut_i_c = sparsity_lut_i.size()
-    s_lut_i_r_s, s_lut_i_c_s = sparsity_lut_i.stride()
+    s_lut_i_r_s, s_lut_i_c_s = stride(sparsity_lut_i)
     o_b, o_r, o_c = output.size()
-    o_b_s, o_r_s, o_c_s = output.stride()
+    o_b_s, o_r_s, o_c_s = stride(output)
 
     if triton_block_size is None:
         triton_block_size = get_triton_block_size(sparsity_block_size)

{blksprs-1.6.1 → blksprs-1.7}/blksprs/layouting/sparsity_layout.py

@@ -5,7 +5,7 @@ import triton
 from torch import Tensor
 from triton import language as tl
 
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_triton_block_size, validate_dimensions, validate_device, \
     validate_contiguous, validate_sparsity, validate_sparsity_block_size
 
@@ -30,9 +30,9 @@ def build_sparsity_layout(x: Tensor, sparsity_block_size: int, triton_block_size
                          dtype=torch.bool, device=x.device)
 
     x_b, x_r, x_c = x.size()
-    x_b_s, x_r_s, x_c_s = x.stride()
+    x_b_s, x_r_s, x_c_s = stride(x)
     o_b, o_r, o_c = output.size()
-    o_b_s, o_r_s, o_c_s = output.stride()
+    o_b_s, o_r_s, o_c_s = stride(output)
 
     if triton_block_size is None:
         triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -120,10 +120,10 @@ def build_sparsity_layout_adaption(x: Tensor, sparsity_layout_from: Tensor,
     output = torch.zeros(o_b, o_r, o_c, dtype=torch.bool, device=x.device)
 
     x_b, x_r, x_c = x.size()
-    x_b_s, x_r_s, x_c_s = x.stride()
+    x_b_s, x_r_s, x_c_s = stride(x)
     s_lut_r, s_lut_c = sparsity_lut.size()
-    s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
-    o_b_s, o_r_s, o_c_s = output.stride()
+    s_lut_r_s, s_lut_c_s = stride(sparsity_lut)
+    o_b_s, o_r_s, o_c_s = stride(output)
 
     if triton_block_size is None:
         triton_block_size = get_triton_block_size(sparsity_block_size_from)

{blksprs-1.6.1 → blksprs-1.7}/blksprs/misc/broadcast_ops.py

@@ -3,7 +3,7 @@ import triton
 from torch import Tensor
 from triton import language as tl
 
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_contiguous, validate_device, \
     validate_sparsity_block_size, validate_triton_block_size
 
@@ -44,13 +44,13 @@ def broadcast_add(x: Tensor, y: Tensor, sparsity_layout_output: Tensor,
     output = torch.zeros(n_sparse_blocks, sparsity_block_size, sparsity_block_size, dtype=x.dtype, device=x.device)
 
     x_b, x_c = x.size()
-    x_b_s, x_c_s = x.stride()
+    x_b_s, x_c_s = stride(x)
     y_b, y_c = y.size()
-    y_b_s, y_c_s = y.stride()
+    y_b_s, y_c_s = stride(y)
     o_b, o_r, o_c = output.size()
-    o_b_s, o_r_s, o_c_s = output.stride()
+    o_b_s, o_r_s, o_c_s = stride(output)
     s_lut_o_r, s_lut_o_c = sparsity_lut_o.size()
-    s_lut_o_r_s, s_lut_o_c_s = sparsity_lut_o.stride()
+    s_lut_o_r_s, s_lut_o_c_s = stride(sparsity_lut_o)
 
     if triton_block_size is None:
         triton_block_size = get_triton_block_size(sparsity_block_size)

{blksprs-1.6.1 → blksprs-1.7}/blksprs/misc/repeat_interleave.py

@@ -3,7 +3,7 @@ import triton
 from torch import Tensor
 from triton import language as tl
 
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_contiguous, validate_device, \
     validate_sparsity_block_size, validate_triton_block_size, validate_dimensions
 
@@ -52,13 +52,13 @@ def repeat_interleave(x: Tensor, sparsity_layout: Tensor, repeats: int,
                          dtype=x.dtype, device=x.device)
 
     x_b, x_r, x_c = x.size()
-    x_b_s, x_r_s, x_c_s = x.stride()
+    x_b_s, x_r_s, x_c_s = stride(x)
     s_lut_r, s_lut_c = sparsity_lut.size()
-    s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
+    s_lut_r_s, s_lut_c_s = stride(sparsity_lut)
     o_b, o_r, o_c = output.size()
-    o_b_s, o_r_s, o_c_s = output.stride()
+    o_b_s, o_r_s, o_c_s = stride(output)
     s_l_o_b, s_l_o_r, s_l_o_c = sparsity_layout_output.size()
-    s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = sparsity_layout_output.stride()
+    s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = stride(sparsity_layout_output)
 
     if triton_block_size is None:
         triton_block_size = get_triton_block_size(sparsity_block_size)

{blksprs-1.6.1 → blksprs-1.7}/blksprs/misc/row_wise.py

@@ -3,7 +3,7 @@ import triton
 from torch import Tensor
 from triton import language as tl
 
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_dimensions, validate_contiguous, validate_device, validate_sparsity, \
     validate_sparsity_block_size, validate_triton_block_size
 
@@ -60,13 +60,13 @@ def row_wise_sum(x: Tensor, sparsity_layout: Tensor, sparsity_block_size: int,
                          device=x.device)
 
     x_b, x_r, x_c = x.size()
-    x_b_s, x_r_s, x_c_s = x.stride()
+    x_b_s, x_r_s, x_c_s = stride(x)
     s_lut_x_r, s_lut_x_c = sparsity_lut.size()
-    s_lut_x_r_s, s_lut_x_c_s = sparsity_lut.stride()
+    s_lut_x_r_s, s_lut_x_c_s = stride(sparsity_lut)
     o_b, o_r, o_c = output.size()
-    o_b_s, o_r_s, o_c_s = output.stride()
+    o_b_s, o_r_s, o_c_s = stride(output)
     s_l_o_b, s_l_o_r, s_l_o_c = sparsity_layout_output.size()
-    s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = sparsity_layout_output.stride()
+    s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = stride(sparsity_layout_output)
 
     if triton_block_size is None:
         triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -183,13 +183,13 @@ def row_wise_max(x: Tensor, sparsity_layout: Tensor, sparsity_block_size: int,
                         device=x.device)
 
     x_b, x_r, x_c = x.size()
-    x_b_s, x_r_s, x_c_s = x.stride()
+    x_b_s, x_r_s, x_c_s = stride(x)
     s_lut_x_r, s_lut_x_c = sparsity_lut.size()
-    s_lut_x_r_s, s_lut_x_c_s = sparsity_lut.stride()
+    s_lut_x_r_s, s_lut_x_c_s = stride(sparsity_lut)
     o_b, o_r, o_c = output.size()
-    o_b_s, o_r_s, o_c_s = output.stride()
+    o_b_s, o_r_s, o_c_s = stride(output)
     s_l_o_b, s_l_o_r, s_l_o_c = sparsity_layout_output.size()
-    s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = sparsity_layout_output.stride()
+    s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = stride(sparsity_layout_output)
 
     if triton_block_size is None:
         triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -290,15 +290,15 @@ def row_wise_add(x: Tensor, sparsity_layout_x: Tensor, y: Tensor,
     output = torch.empty_like(x)
 
     x_b, x_r, x_c = x.size()
-    x_b_s, x_r_s, x_c_s = x.stride()
+    x_b_s, x_r_s, x_c_s = stride(x)
     s_lut_r, s_lut_c = sparsity_lut.size()
-    s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
+    s_lut_r_s, s_lut_c_s = stride(sparsity_lut)
     y_b, y_r, y_c = y.size()
-    y_b_s, y_r_s, y_c_s = y.stride()
+    y_b_s, y_r_s, y_c_s = stride(y)
     s_l_y_b, s_l_y_r, s_l_y_c = sparsity_layout_rwm.size()
-    s_l_y_b_s, s_l_y_r_s, s_l_y_c_s = sparsity_layout_rwm.stride()
+    s_l_y_b_s, s_l_y_r_s, s_l_y_c_s = stride(sparsity_layout_rwm)
     o_b, o_r, o_c = output.size()
-    o_b_s, o_r_s, o_c_s = output.stride()
+    o_b_s, o_r_s, o_c_s = stride(output)
 
     if triton_block_size is None:
         triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -361,7 +361,8 @@ def kernel_blocksparse_row_wise_add(x,
     rev_idx_spa_s = tl.load(r_lut_y + rev_idx_spa_s_idx, mask=rev_idx_spa_s_msk).to(tl.int32)
 
     if rev_idx_spa_s == -1:
-        assert False, "Invalid sparsity block"
+        tl.device_assert(False)
+        return
 
     # Load x block
     blk_x_idx = ((pid_blk * x_b_s) +

{blksprs-1.6.1 → blksprs-1.7}/blksprs/ops/conversion.py

@@ -6,7 +6,7 @@ from torch import Tensor
 from triton import language as tl
 
 from blksprs.layouting.sparsity_layout import build_sparsity_layout_adaption
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_contiguous, validate_dimensions, validate_device, \
     validate_sparsity, validate_sparsity_block_size, validate_triton_block_size
 
@@ -65,11 +65,11 @@ class _BlocksparseToDense(torch.autograd.Function):
                             dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.shape
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         s_l_b, s_l_r, s_l_c = sparsity_layout.size()
-        s_l_b_s, s_l_r_s, s_l_c_s = sparsity_layout.stride()
+        s_l_b_s, s_l_r_s, s_l_c_s = stride(sparsity_layout)
         o_b, o_r, o_c = output.size()
-        o_b_s, o_r_s, o_c_s = output.stride()
+        o_b_s, o_r_s, o_c_s = stride(output)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -190,11 +190,11 @@ class _BlocksparseToSparse(torch.autograd.Function):
                              dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.size()
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         s_lut_r, s_lut_c = sparsity_lut.size()
-        s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
+        s_lut_r_s, s_lut_c_s = stride(sparsity_lut)
         o_b, o_r, o_c = output.size()
-        o_b_s, o_r_s, o_c_s = output.stride()
+        o_b_s, o_r_s, o_c_s = stride(output)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -347,13 +347,13 @@ class _BlocksparseAdaptLayout(torch.autograd.Function):
                              dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.size()
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         s_l_x_b, s_l_x_r, s_l_x_c = sparsity_layout_from.size()
-        s_l_x_b_s, s_l_x_r_s, s_l_x_c_s = sparsity_layout_from.stride()
+        s_l_x_b_s, s_l_x_r_s, s_l_x_c_s = stride(sparsity_layout_from)
         o_b, o_r, o_c = output.size()
-        o_b_s, o_r_s, o_c_s = output.stride()
+        o_b_s, o_r_s, o_c_s = stride(output)
         s_lut_o_r, s_lut_o_c = sparsity_lut_to.size()
-        s_lut_o_r_s, s_lut_o_c_s = sparsity_lut_to.stride()
+        s_lut_o_r_s, s_lut_o_c_s = stride(sparsity_lut_to)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(min_sparsity_block_size)

{blksprs-1.6.1 → blksprs-1.7}/blksprs/ops/distribution.py

@@ -3,7 +3,7 @@ import triton
 from torch import Tensor
 from triton import language as tl
 
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_contiguous, validate_dimensions, validate_device, \
     validate_sparsity, validate_dtype_int, validate_sparsity_block_size, validate_triton_block_size
 
@@ -59,15 +59,15 @@ class _BlocksparseGather(torch.autograd.Function):
         output = torch.empty_like(i, dtype=x.dtype)
 
         x_b, x_r, x_c = x.size()
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         s_l_x_b, s_l_x_r, s_l_x_c = sparsity_layout_x.size()
-        s_l_x_b_s, s_l_x_r_s, s_l_x_c_s = sparsity_layout_x.stride()
+        s_l_x_b_s, s_l_x_r_s, s_l_x_c_s = stride(sparsity_layout_x)
         i_b, i_r, i_c = i.size()
-        i_b_s, i_r_s, i_c_s = i.stride()
+        i_b_s, i_r_s, i_c_s = stride(i)
         s_lut_i_r, s_lut_i_c = sparsity_lut_i.size()
-        s_lut_i_r_s, s_lut_i_c_s = sparsity_lut_i.stride()
+        s_lut_i_r_s, s_lut_i_c_s = stride(sparsity_lut_i)
         o_b, o_r, o_c = output.size()
-        o_b_s, o_r_s, o_c_s = output.stride()
+        o_b_s, o_r_s, o_c_s = stride(output)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -248,15 +248,15 @@ class _BlocksparseScatterReduce(torch.autograd.Function):
                              dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.size()
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         s_lut_x_r, s_lut_x_c = sparsity_lut_x.size()
-        s_lut_x_r_s, s_lut_x_c_s = sparsity_lut_x.stride()
+        s_lut_x_r_s, s_lut_x_c_s = stride(sparsity_lut_x)
         i_b, i_r, i_c = i.size()
-        i_b_s, i_r_s, i_c_s = i.stride()
+        i_b_s, i_r_s, i_c_s = stride(i)
         o_b, o_r, o_c = output.size()
-        o_b_s, o_r_s, o_c_s = output.stride()
+        o_b_s, o_r_s, o_c_s = stride(output)
         s_l_o_b, s_l_o_r, s_l_o_c = sparsity_layout_o.size()
-        s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = sparsity_layout_o.stride()
+        s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = stride(sparsity_layout_o)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(sparsity_block_size)

{blksprs-1.6.1 → blksprs-1.7}/blksprs/ops/exp.py

@@ -3,7 +3,7 @@ import triton
 from torch import Tensor
 from triton import language as tl
 
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_contiguous, validate_dimensions, validate_device, \
     validate_sparsity_block_size, validate_triton_block_size
 
@@ -43,9 +43,9 @@ class _BlocksparseExp(torch.autograd.Function):
         output = torch.empty_like(x)
 
         x_b, x_r, x_c = x.shape
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         o_b, o_r, o_c = output.shape
-        o_b_s, o_r_s, o_c_s = output.stride()
+        o_b_s, o_r_s, o_c_s = stride(output)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(sparsity_block_size)

{blksprs-1.6.1 → blksprs-1.7}/blksprs/ops/matmul.py

@@ -4,7 +4,7 @@ from torch import Tensor
 from triton import language as tl
 
 from blksprs.ops.transpose import transpose
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_contiguous, validate_dimensions, validate_device, \
     validate_sparsity, validate_sparsity_block_size, validate_triton_block_size, validate_dtype_float
 
@@ -82,17 +82,17 @@ class _BlocksparseMatmulSSS(torch.autograd.Function):
                              dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.size()
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         s_l_x_b, s_l_x_r, s_l_x_c = sparsity_layout_x.size()
-        s_l_x_b_s, s_l_x_r_s, s_l_x_c_s = sparsity_layout_x.stride()
+        s_l_x_b_s, s_l_x_r_s, s_l_x_c_s = stride(sparsity_layout_x)
         y_b, y_r, y_c = y.size()
-        y_b_s, y_r_s, y_c_s = y.stride()
+        y_b_s, y_r_s, y_c_s = stride(y)
         s_l_y_b, s_l_y_r, s_l_y_c = sparsity_layout_y.size()
-        s_l_y_b_s, s_l_y_r_s, s_l_y_c_s = sparsity_layout_y.stride()
+        s_l_y_b_s, s_l_y_r_s, s_l_y_c_s = stride(sparsity_layout_y)
         o_b, o_r, o_c = output.size()
-        o_b_s, o_r_s, o_c_s = output.stride()
+        o_b_s, o_r_s, o_c_s = stride(output)
         s_lut_o_r, s_lut_o_c = sparsity_lut_o.size()
-        s_lut_o_r_s, s_lut_o_c_s = sparsity_lut_o.stride()
+        s_lut_o_r_s, s_lut_o_c_s = stride(sparsity_lut_o)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(sparsity_block_size)

{blksprs-1.6.1 → blksprs-1.7}/blksprs/ops/partitioning.py

@@ -1,10 +1,7 @@
 import torch
-import triton
-from sympy.utilities.iterables import partitions
 from torch import Tensor
-from triton import language as tl
 
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.ops.repeat import forward_flow
 
 from blksprs.utils.validation import validate_dimensions, validate_contiguous, validate_device, \
     validate_sparsity, validate_sparsity_block_size, validate_triton_block_size
@@ -48,12 +45,11 @@ def split(x: Tensor, sparsity_layout: Tensor, partitions: int,
     sparsity_reverse_lut = (((torch.cumsum(sparsity_layout_flat, dim=-1) - 1) *
                              (sparsity_layout_flat == 1) -
                              (1 * (sparsity_layout_flat == 0)))
-                            .reshape(sparsity_layout.size())
                             .reshape(sparsity_layout.size(0), sparsity_layout.size(1), partitions,
                                      sparsity_layout.size(2) // partitions)
                             .permute(0, 2, 1, 3).reshape(-1).contiguous())
 
-    n_sparse_blocks = torch.sum(sparsity_layout.to(torch.int)).item()
+    n_sparse_blocks = torch.sum(sparsity_layout_output.to(torch.int)).item()
 
     validate_contiguous(sparsity_layout_output, sparsity_lut, sparsity_reverse_lut)
 
@@ -66,10 +62,11 @@ class _BlocksparseSplit(torch.autograd.Function):
     @staticmethod
     def forward(ctx, x: Tensor, sparsity_layout_o: Tensor, sparsity_lut: Tensor, sparsity_reverse_lut: Tensor,
                 num_partitions: int, sparsity_block_size: int, n_sparse_blocks: int, triton_block_size: int) -> Tensor:
+        ctx.save_for_backward(sparsity_layout_o)
         ctx.num_partitions = num_partitions
 
-        return forward_reorder(ctx, x, sparsity_layout_o, sparsity_lut, sparsity_reverse_lut, sparsity_block_size,
-                               n_sparse_blocks, triton_block_size)
+        return forward_flow(ctx, x, sparsity_layout_o, sparsity_lut, sparsity_reverse_lut, sparsity_block_size,
+                            n_sparse_blocks, triton_block_size)
 
     @staticmethod
     def backward(ctx, grad_output):
@@ -126,7 +123,7 @@ def merge(x: Tensor, sparsity_layout: Tensor, partitions: int,
                                      sparsity_layout.size(1), sparsity_layout.size(2) * partitions)
                             .reshape(-1).contiguous())
 
-    n_sparse_blocks = torch.sum(sparsity_layout.to(torch.int)).item()
+    n_sparse_blocks = torch.sum(sparsity_layout_output.to(torch.int)).item()
 
     validate_contiguous(sparsity_layout_output, sparsity_lut, sparsity_reverse_lut)
 
@@ -139,10 +136,11 @@ class _BlocksparseMerge(torch.autograd.Function):
     @staticmethod
     def forward(ctx, x: Tensor, sparsity_layout_o: Tensor, sparsity_lut: Tensor, sparsity_reverse_lut: Tensor,
                 num_partitions: int, sparsity_block_size: int, n_sparse_blocks: int, triton_block_size: int) -> Tensor:
+        ctx.save_for_backward(sparsity_layout_o)
         ctx.num_partitions = num_partitions
 
-        return forward_reorder(ctx, x, sparsity_layout_o, sparsity_lut, sparsity_reverse_lut, sparsity_block_size,
-                               n_sparse_blocks, triton_block_size)
+        return forward_flow(ctx, x, sparsity_layout_o, sparsity_lut, sparsity_reverse_lut, sparsity_block_size,
+                            n_sparse_blocks, triton_block_size)
 
     @staticmethod
     def backward(ctx, grad_output):
@@ -155,90 +153,3 @@ class _BlocksparseMerge(torch.autograd.Function):
                      sparsity_block_size, triton_block_size)[0], None, None, None, None, None, None, None
 
 
-def forward_reorder(ctx, x: Tensor, sparsity_layout_o: Tensor, sparsity_lut: Tensor, sparsity_reverse_lut: Tensor,
-                    sparsity_block_size: int, n_sparse_blocks: int, triton_block_size: int) -> Tensor:
-    output = torch.empty(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size),
-                         dtype=x.dtype, device=x.device)
-
-    x_b, x_r, x_c = x.size()
-    x_b_s, x_r_s, x_c_s = x.stride()
-    s_l_b, s_l_r, s_l_c = sparsity_layout_o.size()
-    s_l_b_s, s_l_r_s, s_l_c_s = sparsity_layout_o.stride()
-    s_lut_r, s_lut_c = sparsity_lut.shape
-    s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
-    o_b, o_r, o_c = output.size()
-    o_b_s, o_r_s, o_c_s = output.stride()
-
-    if triton_block_size is None:
-        triton_block_size = get_triton_block_size(sparsity_block_size)
-
-    triton_grid = lambda meta: [o_b,
-                                triton.cdiv(o_r, meta["TRITON_BLOCK_SIZE"]),
-                                triton.cdiv(o_c, meta["TRITON_BLOCK_SIZE"])]
-
-    (kernel_blocksparse_reorder[triton_grid]
-     (x,
-      x_b, x_b_s, x_r_s, x_c_s,
-      s_l_b, s_l_b_s, s_l_r_s, s_l_c_s,
-      sparsity_lut, s_lut_r, s_lut_r_s, s_lut_c_s,
-      sparsity_reverse_lut,
-      output,
-      o_b, o_b_s,
-      triton_block_size))
-
-    # Save for backward pass
-    ctx.save_for_backward(sparsity_layout_o)
-    ctx.sparsity_block_size = sparsity_block_size
-    ctx.triton_block_size = triton_block_size
-
-    return output
-
-
-@triton.jit
-def kernel_blocksparse_reorder(x,
-                               x_b, x_b_s, x_r_s, x_c_s,
-                               s_l_b, s_l_b_s, s_l_r_s, s_l_c_s,
-                               s_lut, s_lut_r, s_lut_r_s, s_lut_c_s,
-                               r_lut,
-                               o,
-                               o_b, o_b_s,
-                               TRITON_BLOCK_SIZE: tl.constexpr) -> None:
-    # Get triton block indices
-    pid_blk = tl.program_id(axis=0)
-    pid_row = tl.program_id(axis=1)
-    pid_col = tl.program_id(axis=2)
-
-    # Get sparsity index of current output block consisting of its batch, row, and column index
-    spa_bat_idx = (pid_blk * s_lut_r_s + 0 * s_lut_c_s)
-    spa_bat_msk = (spa_bat_idx < s_lut_r * s_lut_r_s)
-    spa_bat = tl.load(s_lut + spa_bat_idx, mask=spa_bat_msk)
-
-    spa_row_idx = (pid_blk * s_lut_r_s + 1 * s_lut_c_s)
-    spa_row_msk = (spa_row_idx < s_lut_r * s_lut_r_s)
-    spa_row = tl.load(s_lut + spa_row_idx, mask=spa_row_msk)
-
-    spa_col_idx = (pid_blk * s_lut_r_s + 2 * s_lut_c_s)
-    spa_col_msk = (spa_col_idx < s_lut_r * s_lut_r_s)
-    spa_col = tl.load(s_lut + spa_col_idx, mask=spa_col_msk)
-
-    # Get reverse sparsity index
-    rev_idx_spa_idx = (spa_bat * s_l_b_s +
-                       spa_row * s_l_r_s +
-                       spa_col * s_l_c_s)
-    rev_idx_spa_msk = (rev_idx_spa_idx < s_l_b * s_l_b_s)
-    rev_idx_spa = tl.load(r_lut + rev_idx_spa_idx, mask=rev_idx_spa_msk).to(tl.int32)
-
-    if rev_idx_spa == -1:
-        assert False, "Invalid sparsity block"
-
-    blk_x_idx = (rev_idx_spa * x_b_s +
-                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +
-                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_c_s)[None, :])
-    blk_x_msk = (blk_x_idx < x_b * x_b_s)
-    blk_x = tl.load(x + blk_x_idx, mask=blk_x_msk)
-
-    blk_o_idx = (pid_blk * o_b_s +
-                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +
-                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_c_s)[None, :])
-    blk_o_msk = (blk_o_idx < o_b * o_b_s)
-    tl.store(o + blk_o_idx, blk_x, mask=blk_o_msk)

blksprs-1.7/blksprs/ops/repeat.py

@@ -0,0 +1,241 @@
+import torch
+import triton
+from triton import language as tl
+from torch import Tensor
+
+from blksprs.utils.tools import get_triton_block_size, stride
+from blksprs.utils.validation import validate_dimensions, validate_contiguous, validate_device, \
+    validate_sparsity, validate_sparsity_block_size, validate_triton_block_size
+
+
+def repeat(x: Tensor, sparsity_layout_x: Tensor, repeats: tuple[int, int, int],
+           sparsity_block_size: int, sparsity_layout_output: Tensor = None, triton_block_size: int = None) -> (
+        Tensor, Tensor):
+    x = x.contiguous()
+
+    validate_dimensions(x)
+    validate_contiguous(x)
+    validate_device(x)
+    validate_sparsity(sparsity_block_size, (x, sparsity_layout_x))
+    validate_sparsity_block_size(sparsity_block_size, x)
+    validate_triton_block_size(triton_block_size, sparsity_block_size)
+
+    sparsity_layout_o = sparsity_layout_x.repeat(repeats[0], repeats[1], repeats[2])
+
+    if sparsity_layout_output is not None:
+        sparsity_layout_o = torch.logical_and(sparsity_layout_o, sparsity_layout_output)
+
+    sparsity_lut = torch.nonzero(sparsity_layout_o).contiguous()
+
+    sparsity_layout_flat = sparsity_layout_x.reshape(-1)
+    sparsity_reverse_lut = (((torch.cumsum(sparsity_layout_flat, dim=-1) - 1) *
+                             (sparsity_layout_flat == 1) -
+                             (1 * (sparsity_layout_flat == 0)))
+                            .reshape(sparsity_layout_x.size())
+                            .repeat(repeats[0], repeats[1], repeats[2])
+                            .reshape(-1).contiguous())
+
+    n_sparse_blocks = torch.sum(sparsity_layout_o.to(torch.int)).item()
+
+    validate_contiguous(sparsity_layout_o, sparsity_lut, sparsity_reverse_lut)
+
+    return _BlocksparseRepeat.apply(x, sparsity_layout_x, sparsity_layout_o, sparsity_lut, sparsity_reverse_lut,
+                                    sparsity_block_size, n_sparse_blocks, triton_block_size), sparsity_layout_o
+
+
+class _BlocksparseRepeat(torch.autograd.Function):
+
+    @staticmethod
+    def forward(ctx, x: Tensor, sparsity_layout_x: Tensor, sparsity_layout_o: Tensor, sparsity_lut: Tensor,
+                sparsity_reverse_lut: Tensor,
+                sparsity_block_size: int, n_sparse_blocks: int,
+                triton_block_size: int) -> Tensor:
+        ctx.save_for_backward(sparsity_layout_x, sparsity_layout_o, sparsity_lut, sparsity_reverse_lut)
+        ctx.x_size = x.size()
+        ctx.x_stride = stride(x)
+
+        return forward_flow(ctx, x, sparsity_layout_o, sparsity_lut, sparsity_reverse_lut, sparsity_block_size,
+                            n_sparse_blocks, triton_block_size)
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        sparsity_layout_x, sparsity_layout_o, sparsity_lut, sparsity_reverse_lut = ctx.saved_tensors
+        x_size = ctx.x_size
+        x_stride = ctx.x_stride
+        sparsity_block_size = ctx.sparsity_block_size
+        triton_block_size = ctx.triton_block_size
+
+        n_sparse_blocks = torch.sum(sparsity_layout_x.to(torch.int)).item()
+
+        output = torch.zeros(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size),
+                             dtype=grad_output.dtype, device=grad_output.device)
+
+        x_b, x_r, x_c = grad_output.size()
+        x_b_s, x_r_s, x_c_s = stride(grad_output)
+        s_l_x_b, s_l_x_r, s_l_x_c = sparsity_layout_o.size()
+        s_l_x_b_s, s_l_x_r_s, s_l_x_c_s = stride(sparsity_layout_o)
+        s_lut_r, s_lut_c = sparsity_lut.size()
+        s_lut_r_s, s_lut_c_s = stride(sparsity_lut)
+        o_b, o_r, o_c = x_size
+        o_b_s, o_r_s, o_c_s = x_stride
+
+        if triton_block_size is None:
+            triton_block_size = get_triton_block_size(sparsity_block_size)
+
+        triton_grid = lambda meta: [x_b,
+                                    triton.cdiv(x_r, meta["TRITON_BLOCK_SIZE"]),
+                                    triton.cdiv(x_c, meta["TRITON_BLOCK_SIZE"])]
+
+        (kernel_blocksparse_flow_push[triton_grid]
+         (grad_output,
+          x_b, x_b_s, x_r_s, x_c_s,
+          s_l_x_b, s_l_x_b_s, s_l_x_r_s, s_l_x_c_s,
+          sparsity_lut, s_lut_r, s_lut_r_s, s_lut_c_s,
+          sparsity_reverse_lut,
+          output,
+          o_b, o_b_s, o_r_s, o_c_s,
+          triton_block_size))
+
+        return output, None, None, None, None, None, None, None
+
+
+@triton.jit
+def kernel_blocksparse_flow_pull(x,
+                                 x_b, x_b_s, x_r_s, x_c_s,
+                                 o,
+                                 o_b, o_b_s, o_r_s, o_c_s,
+                                 s_l_o_b, s_l_o_b_s, s_l_o_r_s, s_l_o_c_s,
+                                 s_lut, s_lut_r, s_lut_r_s, s_lut_c_s,
+                                 r_lut,
+                                 TRITON_BLOCK_SIZE: tl.constexpr) -> None:
+    # Get triton block indices
+    pid_blk = tl.program_id(axis=0)
+    pid_row = tl.program_id(axis=1)
+    pid_col = tl.program_id(axis=2)
+
+    # Get sparsity index of current output block consisting of its batch, row, and column index
+    spa_bat_idx = (pid_blk * s_lut_r_s + 0 * s_lut_c_s)
+    spa_bat_msk = (spa_bat_idx < s_lut_r * s_lut_r_s)
+    spa_bat = tl.load(s_lut + spa_bat_idx, mask=spa_bat_msk)
+
+    spa_row_idx = (pid_blk * s_lut_r_s + 1 * s_lut_c_s)
+    spa_row_msk = (spa_row_idx < s_lut_r * s_lut_r_s)
+    spa_row = tl.load(s_lut + spa_row_idx, mask=spa_row_msk)
+
+    spa_col_idx = (pid_blk * s_lut_r_s + 2 * s_lut_c_s)
+    spa_col_msk = (spa_col_idx < s_lut_r * s_lut_r_s)
+    spa_col = tl.load(s_lut + spa_col_idx, mask=spa_col_msk)
+
+    # Get reverse sparsity index
+    rev_idx_spa_idx = (spa_bat * s_l_o_b_s +
+                       spa_row * s_l_o_r_s +
+                       spa_col * s_l_o_c_s)
+    rev_idx_spa_msk = (rev_idx_spa_idx < s_l_o_b * s_l_o_b_s)
+    rev_idx_spa = tl.load(r_lut + rev_idx_spa_idx, mask=rev_idx_spa_msk).to(tl.int32)
+
+    if rev_idx_spa == -1:
+        tl.device_assert(False)
+        return
+
+    blk_x_idx = (rev_idx_spa * x_b_s +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_c_s)[None, :])
+    blk_x_msk = (blk_x_idx < x_b * x_b_s)
+    blk_x = tl.load(x + blk_x_idx, mask=blk_x_msk)
+
+    blk_o_idx = (pid_blk * o_b_s +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * o_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * o_c_s)[None, :])
+    blk_o_msk = (blk_o_idx < o_b * o_b_s)
+    tl.store(o + blk_o_idx, blk_x, mask=blk_o_msk)
+
+
+@triton.jit
+def kernel_blocksparse_flow_push(x,
+                                 x_b, x_b_s, x_r_s, x_c_s,
+                                 s_l_x_b, s_l_x_b_s, s_l_x_r_s, s_l_x_c_s,
+                                 s_lut, s_lut_r, s_lut_r_s, s_lut_c_s,
+                                 r_lut,
+                                 o,
+                                 o_b, o_b_s, o_r_s, o_c_s,
+                                 TRITON_BLOCK_SIZE: tl.constexpr) -> None:
+    # Get triton block indices
+    pid_blk = tl.program_id(axis=0)
+    pid_row = tl.program_id(axis=1)
+    pid_col = tl.program_id(axis=2)
+
+    # Get sparsity index of current input block consisting of its batch, row, and column index
+    spa_bat_idx = (pid_blk * s_lut_r_s + 0 * s_lut_c_s)
+    spa_bat_msk = (spa_bat_idx < s_lut_r * s_lut_r_s)
+    spa_bat = tl.load(s_lut + spa_bat_idx, mask=spa_bat_msk)
+
+    spa_row_idx = (pid_blk * s_lut_r_s + 1 * s_lut_c_s)
+    spa_row_msk = (spa_row_idx < s_lut_r * s_lut_r_s)
+    spa_row = tl.load(s_lut + spa_row_idx, mask=spa_row_msk)
+
+    spa_col_idx = (pid_blk * s_lut_r_s + 2 * s_lut_c_s)
+    spa_col_msk = (spa_col_idx < s_lut_r * s_lut_r_s)
+    spa_col = tl.load(s_lut + spa_col_idx, mask=spa_col_msk)
+
+    # Get reverse sparsity index
+    rev_idx_spa_idx = (spa_bat * s_l_x_b_s +
+                       spa_row * s_l_x_r_s +
+                       spa_col * s_l_x_c_s)
+    rev_idx_spa_msk = (rev_idx_spa_idx < s_l_x_b * s_l_x_b_s)
+    rev_idx_spa = tl.load(r_lut + rev_idx_spa_idx, mask=rev_idx_spa_msk).to(tl.int32)
+
+    if rev_idx_spa == -1:
+        tl.device_assert(False)
+        return
+
+    blk_x_idx = (pid_blk * x_b_s +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_c_s)[None, :])
+    blk_x_msk = (blk_x_idx < x_b * x_b_s)
+    blk_x = tl.load(x + blk_x_idx, mask=blk_x_msk)
+
+    blk_o_idx = (rev_idx_spa * o_b_s +
+                 ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * o_r_s)[:, None] +
+                 ((pid_col * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * o_c_s)[None, :])
+    blk_o_msk = (blk_o_idx < o_b * o_b_s)
+    tl.atomic_add(o + blk_o_idx, blk_x, mask=blk_o_msk)
+
+
+def forward_flow(ctx, x: Tensor, sparsity_layout_o: Tensor, sparsity_lut: Tensor, sparsity_reverse_lut: Tensor,
+                 sparsity_block_size: int, n_sparse_blocks: int, triton_block_size: int) -> Tensor:
+    output = torch.empty(size=(n_sparse_blocks, sparsity_block_size, sparsity_block_size),
+                         dtype=x.dtype, device=x.device)
+    output = torch.zeros_like(output)
+
+    x_b, x_r, x_c = x.size()
+    x_b_s, x_r_s, x_c_s = stride(x)
+    o_b, o_r, o_c = output.size()
+    o_b_s, o_r_s, o_c_s = stride(output)
+    s_l_o_b, s_l_o_r, s_l_o_c = sparsity_layout_o.size()
+    s_l_o_b_s, s_l_o_r_s, s_l_o_c_s = stride(sparsity_layout_o)
+    s_lut_r, s_lut_c = sparsity_lut.size()
+    s_lut_r_s, s_lut_c_s = stride(sparsity_lut)
+    asdf = torch.tensor(sparsity_lut).stride()
+
+    if triton_block_size is None:
+        triton_block_size = get_triton_block_size(sparsity_block_size)
+
+    triton_grid = lambda meta: [o_b,
+                                triton.cdiv(o_r, meta["TRITON_BLOCK_SIZE"]),
+                                triton.cdiv(o_c, meta["TRITON_BLOCK_SIZE"])]
+
+    (kernel_blocksparse_flow_pull[triton_grid]
+     (x,
+      x_b, x_b_s, x_r_s, x_c_s,
+      output,
+      o_b, o_b_s, o_r_s, o_c_s,
+      s_l_o_b, s_l_o_b_s, s_l_o_r_s, s_l_o_c_s,
+      sparsity_lut, s_lut_r, s_lut_r_s, s_lut_c_s,
+      sparsity_reverse_lut,
+      triton_block_size))
+
+    # Save for backward pass
+    ctx.sparsity_block_size = sparsity_block_size
+    ctx.triton_block_size = triton_block_size
+
+    return output

{blksprs-1.6.1 → blksprs-1.7}/blksprs/ops/softmax.py

@@ -5,7 +5,7 @@ from triton import language as tl
 
 from blksprs.ops.exp import exp
 from blksprs.misc.row_wise import row_wise_sum, row_wise_max, row_wise_sub
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_contiguous, validate_dimensions, validate_device, \
     validate_sparsity, validate_sparsity_block_size, validate_triton_block_size
 
@@ -61,9 +61,9 @@ class _BlocksparseSoftmax(torch.autograd.Function):
         output = torch.empty_like(x)
 
         x_b, x_r, x_c = x.size()
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         s_lut_r, s_lut_c = sparsity_lut.size()
-        s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
+        s_lut_r_s, s_lut_c_s = stride(sparsity_lut)
         o_b, o_r, o_c = output.size()
 
         x_row_wise_max, sparsity_layout_rwm = row_wise_max(x, sparsity_layout, sparsity_block_size,
@@ -76,9 +76,9 @@ class _BlocksparseSoftmax(torch.autograd.Function):
                                                                triton_block_size=triton_block_size)
 
         s_b, s_r, s_c = x_exp_row_wise_sum.shape
-        s_b_s, s_r_s, s_c_s = x_exp_row_wise_sum.stride()
+        s_b_s, s_r_s, s_c_s = stride(x_exp_row_wise_sum)
         s_l_s_b, s_l_s_r, s_l_s_c = sparsity_layout_rws.shape
-        s_l_s_b_s, s_l_s_r_s, s_l_s_c_s = sparsity_layout_rws.stride()
+        s_l_s_b_s, s_l_s_r_s, s_l_s_c_s = stride(sparsity_layout_rws)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -119,13 +119,13 @@ class _BlocksparseSoftmax(torch.autograd.Function):
                                   (1 * (sparsity_layout_s_flat == 0)))
 
         o_b, o_r, o_c = o.size()
-        o_b_s, o_r_s, o_c_s = o.stride()
+        o_b_s, o_r_s, o_c_s = stride(o)
         s_lut_r, s_lut_c = sparsity_lut.size()
-        s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
+        s_lut_r_s, s_lut_c_s = stride(sparsity_lut)
         s_b, s_r, s_c = s.size()
-        s_b_s, s_r_s, s_c_s = s.stride()
+        s_b_s, s_r_s, s_c_s = stride(s)
         s_l_s_b, s_l_s_r, s_l_s_c = sparsity_layout_s.size()
-        s_l_s_b_s, s_l_s_r_s, s_l_s_c_s = sparsity_layout_s.stride()
+        s_l_s_b_s, s_l_s_r_s, s_l_s_c_s = stride(sparsity_layout_s)
 
         grad_x = torch.empty_like(o, dtype=torch.float)
 
@@ -181,7 +181,8 @@ class _BlocksparseSoftmax(torch.autograd.Function):
         rev_idx_spa_s = tl.load(r_lut_s + rev_idx_spa_s_idx, mask=rev_idx_spa_s_msk).to(tl.int32)
 
         if rev_idx_spa_s == -1:
-            assert False, "Invalid sparsity block"
+            tl.device_assert(False)
+            return
 
         # Load x block
         blk_x_idx = ((pid_blk * x_b_s) +

{blksprs-1.6.1 → blksprs-1.7}/blksprs/ops/transpose.py

@@ -3,7 +3,7 @@ import triton
 from torch import Tensor
 from triton import language as tl
 
-from blksprs.utils.tools import get_triton_block_size
+from blksprs.utils.tools import get_triton_block_size, stride
 from blksprs.utils.validation import validate_dimensions, validate_contiguous, validate_device, \
     validate_sparsity, validate_sparsity_block_size, validate_triton_block_size
 
@@ -63,13 +63,13 @@ class _BlocksparseTranspose(torch.autograd.Function):
                              dtype=x.dtype, device=x.device)
 
         x_b, x_r, x_c = x.size()
-        x_b_s, x_r_s, x_c_s = x.stride()
+        x_b_s, x_r_s, x_c_s = stride(x)
         s_l_b, s_l_r, s_l_c = sparsity_layout_o.size()
-        s_l_b_s, s_l_r_s, s_l_c_s = sparsity_layout_o.stride()
+        s_l_b_s, s_l_r_s, s_l_c_s = stride(sparsity_layout_o)
         s_lut_r, s_lut_c = sparsity_lut.shape
-        s_lut_r_s, s_lut_c_s = sparsity_lut.stride()
+        s_lut_r_s, s_lut_c_s = stride(sparsity_lut)
         o_b, o_r, o_c = output.size()
-        o_b_s, o_r_s, o_c_s = output.stride()
+        o_b_s, o_r_s, o_c_s = stride(output)
 
         if triton_block_size is None:
             triton_block_size = get_triton_block_size(sparsity_block_size)
@@ -140,7 +140,8 @@ class _BlocksparseTranspose(torch.autograd.Function):
         rev_idx_spa = tl.load(r_lut + rev_idx_spa_idx, mask=rev_idx_spa_msk).to(tl.int32)
 
         if rev_idx_spa == -1:
-            assert False, "Invalid sparsity block"
+            tl.device_assert(False)
+            return
 
         blk_x_idx = (rev_idx_spa * x_b_s +
                      ((pid_row * TRITON_BLOCK_SIZE + tl.arange(0, TRITON_BLOCK_SIZE)) * x_r_s)[:, None] +

{blksprs-1.6.1 → blksprs-1.7}/blksprs/utils/tools.py

@@ -23,3 +23,6 @@ def get_triton_block_size(sparsity_block_size: int, limit: int = 128):
 
 def disable_validation():
     _set_skip_validation(True)
+
+def stride(x: Tensor):
+    return x.view(x.shape).stride()

{blksprs-1.6.1 → blksprs-1.7}/blksprs.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: blksprs
-Version: 1.6.1
+Version: 1.7
 Summary: A lightweight library for operations on blocksparse matrices in PyTorch.
 Author-email: Felix Schön <schoen@kr.tuwien.ac.at>
 Project-URL: Homepage, https://github.com/FelixSchoen/blksprs
@@ -34,6 +34,7 @@ Currently supported operations (includes gradient calculation):
 - Transpose
 - Gather
 - Scatter (_supports either no reduction or summation, gradients are only available for summation_)
+- Repeat (_supports target sparsity layout_)
 - Splitting and merging of matrices along the last dimension
 - Conversion to and from sparse form
 - Conversion to different sparsity layouts and different sparsity block sizes
@@ -64,7 +65,7 @@ We recommend installing blksprs from [PyPI](https://pypi.org/project/blksprs/) u
 
 ### Dependencies
 
-- [PyTorch](https://pytorch.org/) (built with v2.4.0)
+- [PyTorch](https://pytorch.org/) (built with v2.5.0)
 - _[Triton](https://github.com/triton-lang/triton) (included with PyTorch)_
 
 ## Changelog

{blksprs-1.6.1 → blksprs-1.7}/blksprs.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ blksprs/ops/distribution.py
 blksprs/ops/exp.py
 blksprs/ops/matmul.py
 blksprs/ops/partitioning.py
+blksprs/ops/repeat.py
 blksprs/ops/softmax.py
 blksprs/ops/transpose.py
 blksprs/utils/benchmarking.py

{blksprs-1.6.1 → blksprs-1.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "blksprs"
-version = "1.6.1"
+version = "1.7"
 authors = [{ name = "Felix Schön", email = "schoen@kr.tuwien.ac.at" }]
 description = "A lightweight library for operations on blocksparse matrices in PyTorch."
 readme = "README.md"