xformers 0.0.35.dev1146__tar.gz → 0.0.35.dev1148__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/PKG-INFO +1 -1
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_sequence_parallel_fused_ops.py +27 -0
- xformers-0.0.35.dev1148/version.txt +1 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/sequence_parallel_fused_ops.py +1 -1
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers.egg-info/PKG-INFO +1 -1
- xformers-0.0.35.dev1146/version.txt +0 -1
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/LICENSE +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/MANIFEST.in +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/README.md +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/pyproject.toml +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/requirements.txt +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/setup.cfg +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/setup.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_attention_patterns.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_checkpoint.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_fmha_flop_formula.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_fmha_merge_attentions.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_fwbw_overlap.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_indexing.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_mem_eff_attention.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_multiprocessing_utils.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_profiler.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_rmsnorm.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_rope_padded.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_seqpar.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_sparse_tensors.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_splitk_reference.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_tiled_matmul.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_tree_attention.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_unbind.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/__init__.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/_cpp_lib.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/_deprecation_warning.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/attn_bias_utils.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/__init__.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/benchmark_attn_decoding.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/benchmark_indexing.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/benchmark_mem_eff_attention.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/benchmark_merge_attentions.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/benchmark_sequence_parallel_fused.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/benchmark_tiled_matmul.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/utils.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/checkpoint.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/csrc/nvcc_info.cu +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/csrc/pt_stable_utils.cu +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/csrc/pt_stable_utils.h +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/flash_attn_3/__init__.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/fwbw_overlap.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/info.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/__init__.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/__init__.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/k_index_select_cat.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/k_scaled_index_add.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/matmul_perf_model.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/rmsnorm_kernels.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/rope_padded_kernels.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/tiled_matmul_kernels.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/common.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/differentiable_collectives.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/__init__.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/_triton/__init__.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/_triton/splitk_kernels.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/attn_bias.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/ck.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/ck_splitk.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/common.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/cutlass.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/cutlass_blackwell.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/dispatch.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/flash.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/flash3.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/merge_training.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/torch_attention_compat.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/triton_splitk.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/indexing.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/modpar_layers.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/rmsnorm.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/rope_padded.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/seqpar.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/swiglu_op.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/tiled_matmul.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/tree_attention.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/unbind.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/profiler/__init__.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/profiler/api.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/profiler/device_limits.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/profiler/find_slowest.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/profiler/profile_analyzer.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/profiler/profiler.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/profiler/profiler_dcgm.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/profiler/profiler_dcgm_impl.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/sparse/__init__.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/sparse/blocksparse_tensor.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/sparse/utils.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/test.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/triton/__init__.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/triton/importing.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/utils.py +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers.egg-info/SOURCES.txt +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers.egg-info/dependency_links.txt +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers.egg-info/not-zip-safe +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers.egg-info/requires.txt +0 -0
- {xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers.egg-info/top_level.txt +0 -0
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/tests/test_sequence_parallel_fused_ops.py
RENAMED
|
@@ -194,6 +194,33 @@ def test_sequence_parallel_fused(
|
|
|
194
194
|
)
|
|
195
195
|
|
|
196
196
|
|
|
197
|
+
def inner_sequence_parallel_fused_out(seed: int) -> None:
|
|
198
|
+
subgroup = torch.distributed.new_group()
|
|
199
|
+
# new_group can return NON_GROUP_MEMBER, but not for a group of all ranks
|
|
200
|
+
assert isinstance(subgroup, torch.distributed.ProcessGroup)
|
|
201
|
+
|
|
202
|
+
torch.random.manual_seed(seed)
|
|
203
|
+
gathered_input = torch.randn((4, 8), dtype=torch.float32, device="cuda")
|
|
204
|
+
weight = torch.randn((6, 8), dtype=torch.float32, device="cuda")
|
|
205
|
+
out = torch.empty((4, 6), dtype=torch.float32, device="cuda")
|
|
206
|
+
|
|
207
|
+
returned = fused_linear_and_reducescatter(
|
|
208
|
+
gathered_input, weight, group=subgroup, out=out
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
assert returned is out
|
|
212
|
+
torch.testing.assert_close(out, torch.nn.functional.linear(gathered_input, weight))
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@cuda_sm80_only
|
|
216
|
+
def test_sequence_parallel_fused_reducescatter_out() -> None:
|
|
217
|
+
launch_subprocesses(
|
|
218
|
+
1,
|
|
219
|
+
inner_sequence_parallel_fused_out,
|
|
220
|
+
seed=random.getrandbits(32),
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
|
|
197
224
|
def inner_sequence_parallel_fused_handle_all_dtypes(
|
|
198
225
|
seed: int,
|
|
199
226
|
step: str,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
0.0.35.dev1148
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/sequence_parallel_fused_ops.py
RENAMED
|
@@ -701,7 +701,7 @@ def fused_linear_and_reducescatter(
|
|
|
701
701
|
if out is not None:
|
|
702
702
|
assert isinstance(out, list) == isinstance(weight, list)
|
|
703
703
|
scattered_outputs = out if isinstance(out, list) else [out]
|
|
704
|
-
assert len(scattered_outputs) == scattered_output_shapes
|
|
704
|
+
assert len(scattered_outputs) == len(scattered_output_shapes)
|
|
705
705
|
assert all(so.device == gathered_input.device for so in scattered_outputs)
|
|
706
706
|
assert all(so.dtype == gathered_input.dtype for so in scattered_outputs)
|
|
707
707
|
assert all(
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
0.0.35.dev1146
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/benchmark_attn_decoding.py
RENAMED
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/benchmark_indexing.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/benchmarks/benchmark_tiled_matmul.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/k_index_select_cat.py
RENAMED
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/k_scaled_index_add.py
RENAMED
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/matmul_perf_model.py
RENAMED
|
File without changes
|
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/rope_padded_kernels.py
RENAMED
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/_triton/tiled_matmul_kernels.py
RENAMED
|
File without changes
|
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/differentiable_collectives.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/_triton/splitk_kernels.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{xformers-0.0.35.dev1146 → xformers-0.0.35.dev1148}/xformers/ops/fmha/torch_attention_compat.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|