xformers 0.0.34.dev1098__tar.gz → 0.0.34.dev1102__tar.gz

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xformers
-Version: 0.0.34.dev1098
+Version: 0.0.34.dev1102
 Summary: XFormers: A collection of composable Transformer building blocks.
 Home-page: https://facebookresearch.github.io/xformers/
 Author: Facebook AI Research

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/setup.py

@@ -449,8 +449,6 @@ def get_extensions():
     source_cuda = list(set(source_cuda) - set(source_hip_generated))
     sources = list(set(sources) - set(source_hip))
 
-    sputnik_dir = os.path.join(this_dir, "third_party", "sputnik")
-
     xformers_pt_cutlass_attn = os.getenv("XFORMERS_PT_CUTLASS_ATTN")
     # By default, we try to link to torch internal CUTLASS attention implementation
     # and silently switch to local CUTLASS attention build if no compatibility
@@ -519,7 +517,6 @@ def get_extensions():
             # CUDA 12.5
             sources.remove(os.path.join(extensions_dir, "swiglu_fairinternal.cu"))
         include_dirs += [
-            sputnik_dir,
             cutlass_dir,
             cutlass_util_dir,
             cutlass_examples_dir,

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/tests/test_core_attention.py

@@ -8,7 +8,6 @@ import functools
 import pytest
 import torch
 
-from xformers.components.attention._sputnik_sparse import SparseCS
 from xformers.components.attention.core import scaled_dot_product_attention
 
 
@@ -120,19 +119,3 @@ def test_amp_attention_sparse(device):
 
     expected_device = torch.float32
     assert r.dtype == expected_device
-
-
-@pytest.mark.parametrize("device", _devices)
-def test_amp_attention_sparsecs(device):
-    b, s, d = 8, 64, 32
-    prob = 0.9
-
-    a = torch.rand(b, s, d, device=device)
-    m = torch.rand(s, s, device=device) > prob
-    m = SparseCS(m, device)
-
-    with torch.amp.autocast("cuda"):
-        r = scaled_dot_product_attention(a, a, a, m)
-
-    expected_device = torch.float32
-    assert r.dtype == expected_device

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/tests/test_sparse_tensors.py

@@ -9,7 +9,7 @@ import torch
 # needed to register custom ops
 import xformers  # noqa: F401
 from xformers.ops import masked_matmul
-from xformers.sparse import BlockSparseTensor, SparseCSRTensor
+from xformers.sparse import BlockSparseTensor
 
 from .utils import disable_tf32
 
@@ -17,7 +17,7 @@ cuda_only = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires C
 _devices = (
     ["cpu", "cuda:0"] if torch.cuda.is_available() and torch.version.cuda else ["cpu"]
 )
-_tensor_types = [BlockSparseTensor, SparseCSRTensor]
+_tensor_types = [BlockSparseTensor]
 
 
 def _create_blocksparse_tensor(
@@ -33,31 +33,12 @@ def _create_blocksparse_tensor(
     return BlockSparseTensor(values, layout)
 
 
-def _create_csr_tensor(device, dtype, shape, sparsity, divisible_by=4):
-    matrix = torch.rand(shape, dtype=torch.float32, device=device).to(dtype)
-    assert matrix.ndim == 3
-    keep = torch.rand_like(matrix[0], dtype=torch.float32) > sparsity
-    nonzero = torch.nonzero(keep)
-    nnz = nonzero.shape[0]
-    # NOTE: need to make it a multiple of 4 for sputnik
-    nonzero = nonzero[: (nnz - nnz % divisible_by)]
-    i, j = nonzero.unbind(1)
-    output = torch.zeros_like(matrix)
-    bdim = torch.arange(matrix.shape[0], device=matrix.device)[:, None]
-    output[bdim, i, j] = matrix[bdim, i, j]
-    return SparseCSRTensor.from_dense(output)
-
-
 def _create_tensor(tensor_type, device, dtype, shape, sparsity):
     if tensor_type == BlockSparseTensor:
         block_size = 16
         return _create_blocksparse_tensor(
             device=device, dtype=dtype, block_size=block_size
         )
-    elif tensor_type == SparseCSRTensor:
-        return _create_csr_tensor(
-            device=device, dtype=dtype, shape=shape, sparsity=sparsity
-        )
 
 
 def _seed():
@@ -83,28 +64,6 @@ def _get_dtype_atol(tensor_type, device: str):
     return torch.float32, 1e-5
 
 
-@pytest.mark.parametrize("device", _devices)
-@pytest.mark.parametrize("func", [torch.add, torch.mul])
-def test_sparse_binary_ops(func, device):
-    # TODO: add for BlockSparseTensor as well
-    N, H, W = 8, 64, 64
-    sparsity = 0.5
-    shape = (N, H, W)
-
-    a_sparse = _create_tensor(
-        SparseCSRTensor, device, dtype=torch.float32, shape=shape, sparsity=sparsity
-    )
-    a = a_sparse.to_dense()
-
-    b = a
-    b_sparse = a_sparse
-
-    res = func(a_sparse, b_sparse).to_dense()
-    res_gt = func(a, b)
-
-    assert torch.allclose(res, res_gt)
-
-
 @disable_tf32
 @pytest.mark.parametrize("tensor_type", _tensor_types)
 @pytest.mark.parametrize("device", _devices)

xformers-0.0.34.dev1102/version.txt

@@ -0,0 +1 @@
+0.0.34.dev1102

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/xformers/components/attention/__init__.py

@@ -15,7 +15,6 @@ from xformers.utils import (
     import_all_modules,
 )
 
-from ._sputnik_sparse import SparseCS
 from .attention_mask import AttentionMask
 from .base import Attention, AttentionConfig  # noqa
 
@@ -30,7 +29,6 @@ ATTENTION_CLASS_NAMES: Set[str] = set()
 # Arbitrary threshold for now,
 # in between dense and sparse matrix algorithms for the attention mechanism
 _DENSITY_THRESHOLD = 0.30  # noqa # from the sputnik paper, vs.
-_USE_SPUTNIK = True
 
 
 def build_attention(config: Union[Dict[str, Any], AttentionConfig]):
@@ -91,8 +89,6 @@ def maybe_sparsify(matrix) -> Any:
 
 
 def sparsify(matrix):
-    if _USE_SPUTNIK:
-        return SparseCS(matrix)
     return matrix.to_sparse()
 
 

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/xformers/components/attention/core.py

@@ -11,12 +11,8 @@ from typing import Optional, Union
 
 import torch
 
-from xformers import _has_cpp_library
 from xformers.components.attention.attention_mask import AttentionMask
 
-if _has_cpp_library:
-    from ._sputnik_sparse import SparseCS
-
 logger = logging.getLogger("xformers")
 
 
@@ -63,30 +59,16 @@ def _broadcast_batch(mask, batch_size):
 def _matmul_with_mask(
     a: torch.Tensor,
     b: torch.Tensor,
-    mask: Optional[Union[torch.Tensor, "SparseCS"]],
+    mask: Optional[torch.Tensor],
 ) -> torch.Tensor:
     if mask is None:
         return a @ b
 
-    if _has_cpp_library and mask.dtype == torch.bool:
-        if isinstance(mask, SparseCS):
-            return mask.matmul_with_mask(a, b)
-        if mask.is_sparse:
-            # perform broadcasting if needed
-            mask = _broadcast_batch(mask, a.shape[0])
-
-            # coalesced is not implemented for bool tensors, so need to cast
-            mask = mask.to(dtype=a.dtype)  # type: ignore  # mypy is missing the catch above
-
-        return torch.ops.xformers.matmul_with_mask(a, b, mask)
-
     # Non optimized codepath
-    if _has_cpp_library:
-        assert not isinstance(mask, SparseCS)
-
     att = a @ b
     if mask.dtype == torch.bool:
-        assert not isinstance(mask, SparseCS)
+        if mask.is_sparse:
+            mask = mask.to_dense()
         if mask.ndim == 2:
             mask = mask.unsqueeze(0).expand(att.shape[0], -1, -1)
         # mask is presumed false == ignore
@@ -95,8 +77,7 @@ def _matmul_with_mask(
         # mask is presumed additive
         # repeat if batch sizes don't match
         if (
-            not isinstance(mask, SparseCS)
-            and mask.ndim == 3
+            mask.ndim == 3
             and mask.shape[0] != att.shape[0]
             and (att.shape[0] % mask.shape[0]) == 0
         ):
@@ -108,57 +89,13 @@ def _matmul_with_mask(
 
 
 def _softmax(a: torch.Tensor, causal: bool = False) -> torch.Tensor:
-    if _has_cpp_library and isinstance(a, SparseCS):
-        return a.softmax()
-
     if a.is_sparse:
         return torch.sparse.softmax(a, dim=a.ndim - 1)
 
     return torch.softmax(a, dim=a.ndim - 1)
 
 
-if _has_cpp_library:
-
-    class SparseBMM(torch.autograd.Function):
-        @staticmethod
-        def forward(ctx, a, b):
-            a = a.coalesce()
-            r = torch.bmm(a, b)
-            ctx.save_for_backward(a, b)
-            return r
-
-        @staticmethod
-        def backward(ctx, grad):
-            a, b = ctx.saved_tensors
-
-            # gradients w.r.t. a
-            ga = None
-            if ctx.needs_input_grad[0]:
-                ga = torch.ops.xformers.matmul_with_mask(grad, b.transpose(-2, -1), a)
-
-            # gradients w.r.t. b
-            gb = None
-            if ctx.needs_input_grad[1]:
-                gb = a.transpose(1, 2).bmm(grad)
-
-            return ga, gb
-
-    def _sparse_bmm(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
-        """
-        Batch matrix multiply between a sparse matrix and a dense matrix
-        """
-        assert a.ndim == b.ndim == 3
-        assert a.shape[0] == b.shape[0]
-        assert a.shape[2] == b.shape[1]
-        return SparseBMM.apply(a, b)
-
-
 def bmm(a: torch.Tensor, b: torch.Tensor) -> torch.Tensor:
-    if _has_cpp_library:
-        if isinstance(a, SparseCS):
-            return a.spmm(b)
-        if a.is_sparse:
-            return _sparse_bmm(a, b)
     return a @ b
 
 
@@ -166,30 +103,6 @@ def _apply_dropout(att, dropout):
     if dropout is None:
         return att
 
-    # Dropout chokes on sparse tensors
-    if _has_cpp_library:
-        if isinstance(att, SparseCS):
-            values = att.values.clone()
-            values = dropout(values)
-            att = SparseCS.wrap(
-                att.shape,
-                values,
-                att.row_indices,
-                att.row_offsets,
-                att.column_indices,
-                att._transp_info,
-            )
-        elif att.is_sparse:
-            att = att.coalesce()
-            values = att.values().clone()  # protect against in-place dropout
-            values = dropout(values)
-            att = torch.sparse_coo_tensor(att.indices(), values, att.shape)
-        else:
-            # Simple dense case
-            att = dropout(att)
-
-        return att
-
     # Non optimized vanilla dropout
     att = dropout(att)
     return att
@@ -198,7 +111,7 @@ def _apply_dropout(att, dropout):
 def scaled_query_key_softmax(
     q: torch.Tensor,
     k: torch.Tensor,
-    att_mask: Optional[Union[AttentionMask, "SparseCS", torch.Tensor]],
+    att_mask: Optional[Union[AttentionMask, torch.Tensor]],
 ) -> torch.Tensor:
     # TODO assume we have (N, S, hs) instead of (B, nh, S, hs), with N = B x nh
     # this is needed due to limitations in sparse_bmm for now
@@ -209,7 +122,7 @@ def scaled_query_key_softmax(
     # Matmul with mask
     if att_mask is not None and isinstance(att_mask, AttentionMask):
         # Additive mask
-        mask: Optional[Union[SparseCS, torch.Tensor]] = att_mask.values
+        mask: Optional[torch.Tensor] = att_mask.values
     else:
         mask = att_mask
 
@@ -225,14 +138,10 @@ def scaled_dot_product_attention(
     q: torch.Tensor,
     k: torch.Tensor,
     v: torch.Tensor,
-    att_mask: Optional[Union[AttentionMask, "SparseCS", torch.Tensor]],
+    att_mask: Optional[Union[AttentionMask, torch.Tensor]],
     dropout: Optional[torch.nn.Module] = None,
 ) -> torch.Tensor:
-    autocast_disabled = (
-        _has_cpp_library
-        and isinstance(att_mask, SparseCS)
-        or (att_mask is not None and att_mask.is_sparse)
-    )
+    autocast_disabled = att_mask is not None and att_mask.is_sparse
     with torch.amp.autocast("cuda", enabled=False) if autocast_disabled else nullcontext():  # type: ignore
         if autocast_disabled:
             q, k, v = q.float(), k.float(), v.float()

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/xformers/csrc/sparse24/sparse24_metadata.h

@@ -83,6 +83,7 @@ struct MetadataCuSparseLtSm80 {
     // TODO: Cast metadata to Short
     static_assert(kBytesPerScalar == 2, "or modify the last dim below");
     metadata = metadata.view({rows / 128, cols / 32, 256});
+    storage = storage.view({rows, -1});
     return std::make_tuple(storage, packed, metadata);
   }
   MetadataCuSparseLtSm80(

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/xformers/info.py

@@ -8,7 +8,7 @@ from typing import Dict
 
 import torch
 
-from . import __version__, _cpp_lib, _is_opensource, _is_triton_available, ops
+from . import __version__, _cpp_lib, _is_opensource, _is_triton_available
 from .ops.common import OPERATORS_REGISTRY
 from .profiler.profiler_dcgm import DCGM_PROFILER_AVAILABLE
 
@@ -18,8 +18,6 @@ def get_features_status() -> Dict[str, str]:
     for op in OPERATORS_REGISTRY:
         status_str = "available" if op.is_available() else "unavailable"
         features[f"{op.OPERATOR_CATEGORY}.{op.NAME}"] = status_str
-    for k, v in ops.swiglu_op._info().items():
-        features[f"swiglu.{k}"] = v
     features["is_triton_available"] = str(_is_triton_available())
     return features
 

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/xformers/ops/__init__.py

@@ -33,15 +33,7 @@ from .sequence_parallel_fused_ops import (
     fused_linear_and_reducescatter,
 )
 from .sp24 import Sparse24Tensor, sparsify24, sparsify24_like
-from .swiglu_op import (
-    SwiGLU,
-    swiglu,
-    SwiGLUEagerOp,
-    SwiGLUFusedOp,
-    SwiGLUOp,
-    SwiGLUOpDispatch,
-    SwiGLUPackedFusedOp,
-)
+from .swiglu_op import SwiGLU, swiglu, SwiGLUEagerOp, SwiGLUOp, SwiGLUOpDispatch
 from .tiled_matmul import tiled_matmul
 from .unbind import get_stack_strides, stack_or_none, unbind
 
@@ -108,10 +100,8 @@ __all__ = [
     # swiglu_op
     "SwiGLU",
     "SwiGLUEagerOp",
-    "SwiGLUFusedOp",
     "SwiGLUOp",
     "SwiGLUOpDispatch",
-    "SwiGLUPackedFusedOp",
     "swiglu",
     # tiled_matmul
     "tiled_matmul",

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/xformers/ops/differentiable_collectives.py

@@ -97,8 +97,10 @@ class _CopyToModelParallelRegion(torch.autograd.Function):
 
 
 def copy_to_model_parallel_region(
-    x: torch.Tensor, process_group: torch.distributed.ProcessGroup
+    x: torch.Tensor, process_group: Optional[torch.distributed.ProcessGroup]
 ) -> torch.Tensor:
+    if process_group is None:
+        return x
     return _CopyToModelParallelRegion.apply(x, process_group)
 
 
@@ -119,8 +121,10 @@ class _ReduceFromModelParallelRegion(torch.autograd.Function):
 
 
 def reduce_from_model_parallel_region(
-    x: torch.Tensor, process_group: torch.distributed.ProcessGroup
+    x: torch.Tensor, process_group: Optional[torch.distributed.ProcessGroup]
 ) -> torch.Tensor:
+    if process_group is None:
+        return x
     return _ReduceFromModelParallelRegion.apply(x, process_group)
 
 
@@ -145,8 +149,10 @@ class _GatherFromSequenceParallelRegion(torch.autograd.Function):
 
 
 def gather_from_sequence_parallel_region(
-    x: torch.Tensor, process_group: torch.distributed.ProcessGroup
+    x: torch.Tensor, process_group: Optional[torch.distributed.ProcessGroup]
 ) -> torch.Tensor:
+    if process_group is None:
+        return x
     return _GatherFromSequenceParallelRegion.apply(x, process_group)
 
 
@@ -169,6 +175,8 @@ class _ScatterToSequenceParallelRegion(torch.autograd.Function):
 
 
 def scatter_to_sequence_parallel_region(
-    x: torch.Tensor, process_group: torch.distributed.ProcessGroup
+    x: torch.Tensor, process_group: Optional[torch.distributed.ProcessGroup]
 ) -> torch.Tensor:
+    if process_group is None:
+        return x
     return _ScatterToSequenceParallelRegion.apply(x, process_group)

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/xformers/ops/modpar_layers.py

@@ -3,7 +3,7 @@
 # This source code is licensed under the BSD license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Callable, List
+from typing import Callable, List, Optional
 
 import torch
 import torch.distributed
@@ -18,15 +18,15 @@ from .seqpar import sequence_parallel_leading_matmul, sequence_parallel_trailing
 def _init_2d_weight(
     weight: torch.Tensor,
     init_method: Callable[[torch.Tensor], torch.Tensor],
-    process_group: torch.distributed.ProcessGroup,
+    process_group: Optional[torch.distributed.ProcessGroup],
     partition_dim: int,
 ) -> None:
     # Mimick FairScale's _initialize_affine_weight, for backwards compatibility.
     # The reason we initialize the full unpartitioned/gathered weight is so that
     # different ranks get different initial values and thus "break the symmetry"
     # and in order to achieve the same init for any value of model parallelism.
-    rank = process_group.rank()
-    world_size = process_group.size()
+    rank = process_group.rank() if process_group is not None else 0
+    world_size = process_group.size() if process_group is not None else 1
 
     nrows, ncols = weight.shape
     if partition_dim == 0:

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/xformers/ops/swiglu_op.py

@@ -4,51 +4,14 @@
 # LICENSE file in the root directory of this source tree.
 
 from dataclasses import dataclass
-from typing import Dict, Optional, Sequence, Tuple, Union
+from typing import Optional, Tuple, Union
 
 import torch
 import torch.nn.functional as F
 from torch import nn
-from torch.amp import custom_bwd, custom_fwd
 
-from .common import BaseOperator, get_xformers_operator, register_operator
 from .unbind import stack_or_none, unbind
 
-if torch.version.hip:
-
-    @torch.library.register_kernel("xformers::dual_gemm_silu_identity_mul", "cuda")  # type: ignore
-    def dual_gemm_silu_identity_mul_cuda(
-        x: torch.Tensor,
-        w1: torch.Tensor,
-        b1: Optional[torch.Tensor],
-        w2: torch.Tensor,
-        b2: Optional[torch.Tensor],
-    ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
-        x1 = x @ w1.T
-        if b1 is not None:
-            x1 += b1
-
-        x2 = x @ w2.T
-        if b2 is not None:
-            x2 += b2
-
-        x4 = F.silu(x1) * x2
-        return x1, x2, x4
-
-
-@register_operator
-class DualGemmSiluOp(BaseOperator):
-    OPERATOR = get_xformers_operator("dual_gemm_silu_identity_mul")
-    OPERATOR_CATEGORY = "swiglu"
-    NAME = "dual_gemm_silu"
-
-
-@register_operator
-class GemmFusedSumOp(BaseOperator):
-    OPERATOR = get_xformers_operator("gemm_fused_operand_sum")
-    OPERATOR_CATEGORY = "swiglu"
-    NAME = "gemm_fused_operand_sum"
-
 
 class _SwiGLUDecomposedFunc(torch.autograd.Function):
     """
@@ -106,68 +69,6 @@ class _SwiGLUDecomposedFunc(torch.autograd.Function):
         return (dx, dw1, db1, dw2, db2, dw3, db3)
 
 
-class _SwiGLUFusedFunc(torch.autograd.Function):
-    NAME = "fused.py"
-
-    @classmethod
-    @custom_fwd(device_type="cuda")
-    def forward(cls, ctx, x, w1, b1, w2, b2, w3, b3):
-        x1, x2, x4 = DualGemmSiluOp.OPERATOR(x, w1, b1, w2, b2)
-
-        x5 = F.linear(x4, w3, b3)
-        ctx.save_for_backward(x, w1, w2, w3, x1, x2)
-        ctx.bias = [b1 is not None, b2 is not None, b3 is not None]
-        return x5
-
-    @staticmethod
-    def _linear_bw(
-        dy: torch.Tensor, x: torch.Tensor, bias: bool
-    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
-        if not bias:
-            return (dy.transpose(-2, -1) @ x), None
-        dw, db = GemmFusedSumOp.OPERATOR(dy.transpose(-2, -1), x)
-        return dw, db
-
-    @classmethod
-    @custom_bwd(device_type="cuda")
-    def backward(cls, ctx, dx5):
-        x, w1, w2, w3, x1, x2 = ctx.saved_tensors
-        w1w2 = stack_or_none([w1, w2], dim=0)
-
-        dx4 = dx5 @ w3  # 255us (nn)
-        dx1dx2, x4 = torch.ops.xformers.silu_bw_fused(x1, x2, dx4)
-        dx1, dx2 = dx1dx2.unbind(1)
-        del x1, x2, dx4
-
-        dw3, db3 = cls._linear_bw(dx5, x4, bias=ctx.bias[2])
-        del x4, dx5
-        if w1w2 is not None:
-            assert dx1dx2.is_contiguous()
-            assert w1w2.is_contiguous()
-            w1w2 = w1w2.view([w1.shape[0] * 2, w1.shape[1]])
-            dx = dx1dx2.view([dx1.shape[0], 2 * dx1.shape[1]]) @ w1w2
-
-            # backward of linear1 + linear2 - packed
-            dw1dw2 = dx1dx2.view([dx1.shape[0], 2 * dx1.shape[1]]).transpose(-2, -1) @ x
-            dw1dw2, db1db2 = cls._linear_bw(
-                dx1dx2.view([dx1.shape[0], 2 * dx1.shape[1]]), x, bias=ctx.bias[0]
-            )
-            dw1, dw2 = dw1dw2.view([2, *w1.shape]).unbind(0)
-            if ctx.bias[0]:
-                db1db2 = db1db2.view([2, dx1.shape[1]])
-                db1, db2 = torch.unbind(db1db2, dim=0)
-            else:
-                db1 = db2 = None
-        else:
-            dx = dx2 @ w2  # 260us (nn)
-            torch.addmm(
-                dx, dx1, w1.to(dx1.dtype), beta=1, alpha=1, out=dx
-            )  # dx += dx1 @ w1
-            dw2, db2 = cls._linear_bw(dx2, x, bias=ctx.bias[1])
-            dw1, db1 = cls._linear_bw(dx1, x, bias=ctx.bias[0])
-        return (dx, dw1, db1, dw2, db2, dw3, db3)
-
-
 class SwiGLUOp:
     """Base class for any swiglu operator in :attr:`xformers.ops.swiglu`"""
 
@@ -241,13 +142,6 @@ class SwiGLUOpDispatch:
         Returns:
             SwiGLUOp: The best operator for the configuration
         """
-        priorities: Sequence[SwiGLUOp] = [
-            SwiGLUPackedFusedOp,
-            SwiGLUFusedOp,
-        ]
-        for op in priorities:
-            if op.supports(self):
-                return op
         return SwiGLUEagerOp
 
     @staticmethod
@@ -273,18 +167,6 @@ class SwiGLUOpDispatch:
         )
 
 
-def _only_sm80(op: SwiGLUOpDispatch) -> bool:
-    device_type = op.device if isinstance(op.device, str) else op.device.type
-    return device_type == "cuda" and torch.cuda.get_device_capability(op.device)[0] >= 8
-
-
-def _only_half_or_autocast(op: SwiGLUOpDispatch) -> bool:
-    HALF_DTYPES = [torch.half, torch.bfloat16]
-    return op.dtype in HALF_DTYPES or (
-        op.dtype_autocast_gpu is not None and op.dtype_autocast_gpu in HALF_DTYPES
-    )
-
-
 def _bias_enabled(op: SwiGLUOpDispatch) -> bool:
     return op.bias_enabled
 
@@ -292,15 +174,6 @@ def _bias_enabled(op: SwiGLUOpDispatch) -> bool:
 _SwiGLUDecomposedOp = _ForwardToPythonAutogradFunc(
     _SwiGLUDecomposedFunc, False, "decomposed", constraints=[_bias_enabled]
 )
-SwiGLUFusedOp = _ForwardToPythonAutogradFunc(
-    _SwiGLUFusedFunc, False, "fused", constraints=[_only_sm80, _only_half_or_autocast]
-)
-SwiGLUPackedFusedOp = _ForwardToFunc(
-    get_xformers_operator("swiglu_packedw"),
-    True,
-    "fused.p.cpp",
-    constraints=[_only_sm80, _only_half_or_autocast],
-)
 SwiGLUEagerOp = _ForwardToFunc(
     _eager_functional_swiglu,
     False,
@@ -309,10 +182,6 @@ SwiGLUEagerOp = _ForwardToFunc(
 )
 
 
-def _info() -> Dict[str, str]:
-    return {op.NAME: op.info() for op in [SwiGLUPackedFusedOp]}
-
-
 def swiglu(
     x: torch.Tensor,
     w1: torch.Tensor,

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/xformers/sparse/__init__.py

@@ -4,4 +4,3 @@
 # LICENSE file in the root directory of this source tree.
 
 from .blocksparse_tensor import BlockSparseTensor  # noqa: F401
-from .csr_tensor import SparseCSRTensor  # noqa: F401

{xformers-0.0.34.dev1098 → xformers-0.0.34.dev1102}/xformers.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: xformers
-Version: 0.0.34.dev1098
+Version: 0.0.34.dev1102
 Summary: XFormers: A collection of composable Transformer building blocks.
 Home-page: https://facebookresearch.github.io/xformers/
 Author: Facebook AI Research