quack-kernels 0.2.5__py3-none-any.whl → 0.2.6__py3-none-any.whl

quack/topk.py

@@ -105,7 +105,7 @@ class TopK:
 
         if tXcX[0][0] < shape[0]:
             copy(tXgX, tXrX)
-        tXrX_f32 = cute.make_fragment(tXrX.shape, Float32)
+        tXrX_f32 = cute.make_rmem_tensor(tXrX.shape, Float32)
         tXrX_f32.store(tXrX.load().to(Float32))
 
         # Encode the indices into the bottom bits of values.
@@ -138,7 +138,7 @@ class TopK:
         # 1 -> 0b11111, 2 -> 0b11110, 4 -> 0b11100, 8 -> 0b11000, 16 -> 0b10000, 32 -> 0b00000
         mask = cute.arch.WARP_SIZE - threads_per_row
         mask_and_clamp = mask << 8 | (cute.arch.WARP_SIZE - 1)
-        topk_vals_split = cute.make_fragment((vecsize_out, nvec_per_thread), Float32)
+        topk_vals_split = cute.make_rmem_tensor((vecsize_out, nvec_per_thread), Float32)
         for i in cutlass.range(cute.ceil_div(self.k, vecsize_out), unroll_full=True):
             should_receive = tidx % threads_per_row == i % threads_per_row
             for v in cutlass.range(vecsize_out, unroll_full=True):
@@ -154,7 +154,7 @@ class TopK:
 
         # Extract indices and clean values
         topk_vals_i32 = cute.recast_tensor(topk_vals_split, Int32)
-        topk_indices = cute.make_fragment(topk_vals_i32.shape, Int32)
+        topk_indices = cute.make_rmem_tensor(topk_vals_i32.shape, Int32)
         for i in cutlass.range(cute.size(topk_vals_i32), unroll_full=True):
             # Extract the encoded index from the last log_N bits
             encoded_idx = topk_vals_i32[i] & idx_mask
@@ -420,7 +420,7 @@ class TopKBackward(ReductionBase):
             grads = vals_f32 * (dvals_f32 - dot)
         else:
             grads = dvals_f32
-        grad_cvt = cute.make_fragment(tXrdV.shape, mdX.element_type)
+        grad_cvt = cute.make_rmem_tensor(tXrdV.shape, mdX.element_type)
         grad_cvt.store(grads.to(mdX.element_type))
 
         # Scatter values to smem

quack/utils.py

@@ -1,7 +1,6 @@
 # Copyright (c) 2025, Wentao Guo, Ted Zadouri, Tri Dao.
 
 import math
-from functools import partial
 from typing import Optional, Tuple, Union
 
 import cutlass
@@ -12,18 +11,6 @@ from cutlass.cutlass_dsl import T, dsl_user_op
 from cutlass._mlir.dialects import llvm, nvvm, vector
 
 
-# cute.arch.{fma,mul,add}_packed_f32x2 uses RZ rounding mode by default
-fma_packed_f32x2 = partial(cute.arch.fma_packed_f32x2, rnd=nvvm.RoundingModeKind.RN)
-mul_packed_f32x2 = partial(cute.arch.mul_packed_f32x2, rnd=nvvm.RoundingModeKind.RN)
-add_packed_f32x2 = partial(cute.arch.add_packed_f32x2, rnd=nvvm.RoundingModeKind.RN)
-sub_packed_f32x2 = partial(
-    cute.arch.calc_packed_f32x2_op,
-    src_c=None,
-    calc_func=nvvm.sub_packed_f32x2,
-    rnd=nvvm.RoundingModeKind.RN,
-)
-
-
 @dsl_user_op
 def elem_pointer(x: cute.Tensor, coord: cute.Coord, *, loc=None, ip=None) -> cute.Pointer:
     return x.iterator + cute.crd2idx(coord, x.layout, loc=loc, ip=ip)
@@ -89,6 +76,54 @@ def store_shared_remote(
     )
 
 
+@dsl_user_op
+def store_shared_remote_x4(
+    val0: Float32 | Int32,
+    val1: Float32 | Int32,
+    val2: Float32 | Int32,
+    val3: Float32 | Int32,
+    smem_ptr: cute.Pointer,
+    mbar_ptr: cute.Pointer,
+    peer_cta_rank_in_cluster: cute.typing.Int,
+    *,
+    loc=None,
+    ip=None,
+) -> None:
+    remote_smem_ptr_i32 = set_block_rank(
+        smem_ptr, peer_cta_rank_in_cluster, loc=loc, ip=ip
+    ).ir_value()
+    remote_mbar_ptr_i32 = set_block_rank(
+        mbar_ptr, peer_cta_rank_in_cluster, loc=loc, ip=ip
+    ).ir_value()
+    assert isinstance(val0, (Float32, Int32)), "val must be Float32, or Int32"
+    dtype = Float32 if isinstance(val0, Float32) else Int32
+    suffix = {Float32: "f32", Int32: "s32"}[dtype]
+    constraint = {Float32: "f", Int32: "r"}[dtype]
+    llvm.inline_asm(
+        None,
+        [
+            remote_smem_ptr_i32,
+            remote_mbar_ptr_i32,
+            dtype(val0).ir_value(loc=loc, ip=ip),
+            dtype(val1).ir_value(loc=loc, ip=ip),
+            dtype(val2).ir_value(loc=loc, ip=ip),
+            dtype(val3).ir_value(loc=loc, ip=ip),
+        ],
+        "{\n\t"
+        f".reg .v4 .{suffix} abcd;\n\t"
+        f"mov.{suffix} abcd.x, $2;\n\t"
+        f"mov.{suffix} abcd.y, $3;\n\t"
+        f"mov.{suffix} abcd.z, $4;\n\t"
+        f"mov.{suffix} abcd.w, $5;\n\t"
+        f"st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.{suffix} [$0], abcd, [$1];\n\t"
+        "}\n",
+        f"r,r,{constraint},{constraint},{constraint},{constraint}",
+        has_side_effects=True,
+        is_align_stack=False,
+        asm_dialect=llvm.AsmDialect.AD_ATT,
+    )
+
+
 @dsl_user_op
 def fmin(a: Union[float, Float32], b: Union[float, Float32], *, loc=None, ip=None) -> Float32:
     return Float32(
@@ -132,25 +167,6 @@ def ceil(a: float | Float32, *, loc=None, ip=None) -> Int32:
     )
 
 
-@dsl_user_op
-def prmt(a: int | Int32, b: int | Int32, c: int | Int32, *, loc=None, ip=None) -> Int32:
-    return Int32(
-        llvm.inline_asm(
-            T.i32(),
-            [
-                Int32(a).ir_value(loc=loc, ip=ip),
-                Int32(b).ir_value(loc=loc, ip=ip),
-                Int32(c).ir_value(loc=loc, ip=ip),
-            ],
-            "prmt.b32 $0, $1, $2, $3;",
-            "=r,r,r,r",
-            has_side_effects=False,
-            is_align_stack=False,
-            asm_dialect=llvm.AsmDialect.AD_ATT,
-        )
-    )
-
-
 @cute.jit
 def fill_oob(tXsX: cute.Tensor, tXpX: Optional[cute.Tensor], fill_value: cute.Numeric) -> None:
     """Fill out-of-bounds values in shared memory tensor.
@@ -210,14 +226,34 @@ def warp_prefix_sum(val: Int32, lane: Optional[Int32] = None) -> Int32:
 
 
 @dsl_user_op
-def atomic_add_i32(a: int | Int32, gmem_ptr: cute.Pointer, *, loc=None, ip=None) -> Int32:
-    return nvvm.atomicrmw(
-        res=T.i32(), op=nvvm.AtomicOpKind.ADD, ptr=gmem_ptr.llvm_ptr, a=Int32(a).ir_value()
-    )
+def atomic_inc_i32(a: int | Int32, gmem_ptr: cute.Pointer, *, loc=None, ip=None) -> Int32:
+    from cutlass import CUDA_VERSION
+
+    # * NVVM call based on nvvm version
+    if CUDA_VERSION.major == 12 and CUDA_VERSION.minor == 9:
+        # Old API: requires explicit result type as first positional argument
+        return nvvm.atomicrmw(
+            res=T.i32(), op=nvvm.AtomicOpKind.INC, ptr=gmem_ptr.llvm_ptr, a=Int32(a).ir_value()
+        )
+    else:
+        # New API: infers result type automatically
+        return nvvm.atomicrmw(
+            op=nvvm.AtomicOpKind.INC, ptr=gmem_ptr.llvm_ptr, a=Int32(a).ir_value()
+        )
 
 
 @dsl_user_op
-def atomic_inc_i32(a: int | Int32, gmem_ptr: cute.Pointer, *, loc=None, ip=None) -> Int32:
-    return nvvm.atomicrmw(
-        res=T.i32(), op=nvvm.AtomicOpKind.INC, ptr=gmem_ptr.llvm_ptr, a=Int32(a).ir_value()
-    )
+def atomic_add_i32(a: int | Int32, gmem_ptr: cute.Pointer, *, loc=None, ip=None) -> Int32:
+    from cutlass import CUDA_VERSION
+
+    # * NVVM call based on nvvm version
+    if CUDA_VERSION.major == 12 and CUDA_VERSION.minor == 9:
+        # Old API: requires explicit result type as first positional argument
+        return nvvm.atomicrmw(
+            res=T.i32(), op=nvvm.AtomicOpKind.ADD, ptr=gmem_ptr.llvm_ptr, a=Int32(a).ir_value()
+        )
+    else:
+        # New API: infers result type automatically
+        return nvvm.atomicrmw(
+            op=nvvm.AtomicOpKind.ADD, ptr=gmem_ptr.llvm_ptr, a=Int32(a).ir_value()
+        )

{quack_kernels-0.2.5.dist-info → quack_kernels-0.2.6.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.4
 Name: quack-kernels
-Version: 0.2.5
+Version: 0.2.6
 Requires-Python: >=3.10
 License-File: LICENSE
-Requires-Dist: nvidia-cutlass-dsl>=4.4.0.dev0
+Requires-Dist: nvidia-cutlass-dsl>=4.4.0.dev1
 Requires-Dist: torch
 Requires-Dist: apache-tvm-ffi<0.2,>=0.1.6
 Requires-Dist: torch-c-dlpack-ext

quack_kernels-0.2.6.dist-info/RECORD

@@ -0,0 +1,45 @@
+quack/__init__.py,sha256=GrpcEMDzBp43ePLzVo6TroIs5GcJv2dbypfFYpLGHmE,427
+quack/activation.py,sha256=T_ypcXoz6a4wPPNZW2gKZuEj8JeucaKtKxQiQl5XrXc,21243
+quack/autotuner.py,sha256=atw0ntedi22RPwSdjWOoge4S56S8VFvRocJQcYhpAlo,13454
+quack/broadcast_utils.py,sha256=TwDdyR9BeIdTgCNJKSGIgN9N_mkQoCwtEnCmocPWSbw,1286
+quack/compile_utils.py,sha256=qJ3oTsDlbAiddrJHtEO7LPYVqn_s-neNfiw-_KvfXZU,591
+quack/copy_utils.py,sha256=GuO2DMXEzy0e3irfT51yIvUt4klWhhmlttgIJCC4tTY,23518
+quack/cross_entropy.py,sha256=w6fjHC_vXt5ji2KfoLrSOdAvpLrQszrYU9rmRij2yY8,24899
+quack/cute_dsl_ptxas.py,sha256=IfBnTJ9amdfDOQkuSdWCLTh7CkZziIvs_xrAc8taxhk,5122
+quack/cute_dsl_utils.py,sha256=4uQx5aYDG9UvVzbWwJTjjJLrnoympz70_CD8b37FQWo,3854
+quack/fast_math.py,sha256=4vrW_HB65rBlCyg0d6h6Y8Q4Gvjti5g74yqCv7acsxA,1020
+quack/gemm.py,sha256=8V23MPq49QbV3csv-_AxjfE9qf8R3NIqFK9Q9db6t2c,7417
+quack/gemm_act.py,sha256=_2NrcI2y0GM1MKwHjRc-6BZq8kthNXqSGxww1ci5Xac,31951
+quack/gemm_config.py,sha256=94o3g9x7H0wi7aBbsb7H67H8nSzTurwL2zgvKDtQUas,3575
+quack/gemm_dact.py,sha256=u2sLwNmaFZH8cetpwS-N0jQSSHpzc2-MNj01FhMpwR0,30116
+quack/gemm_default_epi.py,sha256=1GdsILB1rrP7hrQDexeAn21R2qd_pBrdiDOOBi4r8Yk,11928
+quack/gemm_interface.py,sha256=eoBtqqT7TQFb2gIQS8jTe7w1uFhXAMXqEcxU1RhAgng,60275
+quack/gemm_sm100.py,sha256=lcFK5yKTvwc4RS28PbBHZBQeYCBIw9pi6s1SDZrifqs,120751
+quack/gemm_sm90.py,sha256=ppru1dyrC4Ph0tGO-G_bbIBGqH-zxZ9r-IJqqQ0PTnM,88687
+quack/gemm_symmetric.py,sha256=ljZtEuc_q3aABHlq4ukEM0vxBzkMqWVv2cIHPm1N17A,13161
+quack/gemm_wrapper_utils.py,sha256=EaPyR3Lq19z_RkdB2_xxRj0IPSJMgyfpkrTXyvY3B6M,12775
+quack/layout_utils.py,sha256=q2YkyAHcvT_DsEPc-UnJjGoSRWRxXEhYyDCT9uvz928,12187
+quack/linear.py,sha256=xw2aGwIAAZUictUFHwRgefaT_9-mFfHZmYjFDL5_iMY,11311
+quack/linear_cross_entropy.py,sha256=Zhy_gdMsKHOie-jntBaqIuiDJtkiq6qEBwnyuWwIRw4,10092
+quack/mlp.py,sha256=YjdwQRwEePA9KyidFXp5H1-lxiJc8dZ41vl8Fv8pgss,2259
+quack/pipeline.py,sha256=wQWT7h20qzQNlPj5Aub2uuQwzhDFSDuJzMFQawWdTHg,10504
+quack/reduce.py,sha256=Yx-aB1gQOrKR2t3-joM5PCU0mcrPJLC6Jgs0FEZ6b3Q,12704
+quack/reduction_base.py,sha256=QqlPs5L2VCxwDrO4CHPq-KY6f_BAYRbvsR6k81LPzTU,3180
+quack/rmsnorm.py,sha256=TmDuLOSADfbTbHIRdF9Log7nCrEBGTWw6AiI2v12MFU,44580
+quack/sm100_utils.py,sha256=-p5qj3Wi9n4WDLy2sl-fApYpGp5rH3JvZQb712OTxPs,1901
+quack/sm90_utils.py,sha256=y1GPyzIm9Gxxobr9bNaKuH5LwiIJn5q9Q51P30iHi1w,5576
+quack/softmax.py,sha256=ZqeVbnGfzwkro1LfWBHagbS7B7ug7b9SLZWuGx_Y3Kc,14367
+quack/tensormap_manager.py,sha256=Ts3Mxp0_es2RNA0ffvUjWMXN79lsfWEBZ0DQYhtbcnw,5338
+quack/tile_scheduler.py,sha256=H9Mhyup-8b-S4u3cS1NWIZrBpETporaif9U7ibA2-_A,44343
+quack/topk.py,sha256=XnKvJD7sWdPg_1etExVPQif7I29EcY1fjqcY2QM9BWc,22534
+quack/utils.py,sha256=DeABbD_2H63MGegcJFHR5qOiML2ujVh-PAo7qV3eZ6s,8731
+quack/varlen_utils.py,sha256=SOYkomxX2FoqjYlybg99CqNhS9IARM6F9ba2AkIVvT4,15811
+quack/sort/bitonic_sort.py,sha256=C4i9VfZOBGJbHNCOoXOwrHkCgLBr3c64Ueymc64d31U,4840
+quack/sort/generate_sorting_networks.py,sha256=vkJBOjTVEinQkWT4OtFqOWxFVdTIPoNAQocneKc9-rM,14477
+quack/sort/sorting_networks.py,sha256=l_26zi3gXD_z-tnm2eAczRrmE-mbaz00KmqH6ONivL8,9686
+quack/sort/utils.py,sha256=RbubEY1GcEpsjiz_6o5o2WB47IeMOzaajW6Jis0s444,1059
+quack_kernels-0.2.6.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+quack_kernels-0.2.6.dist-info/METADATA,sha256=OHPfmEcfEGXK_0e10oPBZ-ZCcG8-hdfY58K_OIrTFds,366
+quack_kernels-0.2.6.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+quack_kernels-0.2.6.dist-info/top_level.txt,sha256=6e4Jr_vNJbZTYwlO_Ahf_sDeHDE0zcqcf7Le11FKxxo,6
+quack_kernels-0.2.6.dist-info/RECORD,,

quack_kernels-0.2.5.dist-info/RECORD

@@ -1,45 +0,0 @@
-quack/__init__.py,sha256=0MnyCHBHxG4k12KHNzd-JlElf1M0qRrzhs89PJZJUHA,427
-quack/activation.py,sha256=-lZgojraqdyLjOzgOXBehoVeRBhBq30UX7kOkXsCpGI,20855
-quack/autotuner.py,sha256=atw0ntedi22RPwSdjWOoge4S56S8VFvRocJQcYhpAlo,13454
-quack/broadcast_utils.py,sha256=X5vWg2RtIIWU9Z7nEUW6m0EP0Cfd9XtCKxp4tSyp4Mg,1283
-quack/compile_utils.py,sha256=qJ3oTsDlbAiddrJHtEO7LPYVqn_s-neNfiw-_KvfXZU,591
-quack/copy_utils.py,sha256=IIXtLJv0wQSKfinjIJwG10xQScRvAZvKw1yBV2MXckw,23682
-quack/cross_entropy.py,sha256=w6fjHC_vXt5ji2KfoLrSOdAvpLrQszrYU9rmRij2yY8,24899
-quack/cute_dsl_ptxas.py,sha256=IfBnTJ9amdfDOQkuSdWCLTh7CkZziIvs_xrAc8taxhk,5122
-quack/cute_dsl_utils.py,sha256=4uQx5aYDG9UvVzbWwJTjjJLrnoympz70_CD8b37FQWo,3854
-quack/fast_math.py,sha256=E1XUqfUt0_n9BPZNggF-UDzZ6anso9bYUrwqafemWvQ,2297
-quack/gemm.py,sha256=8V23MPq49QbV3csv-_AxjfE9qf8R3NIqFK9Q9db6t2c,7417
-quack/gemm_act.py,sha256=Y8HJKfw3tCoFKecwhwhd5xpXd9jCQCGZT_V2xXf-CnU,20823
-quack/gemm_config.py,sha256=94o3g9x7H0wi7aBbsb7H67H8nSzTurwL2zgvKDtQUas,3575
-quack/gemm_dact.py,sha256=l__UhCrFbPjD9a1TAVgP7_C7p5lLfX5DkRcM6z0ofOw,7789
-quack/gemm_default_epi.py,sha256=6qO8Ovtcw8sQQ_kXTBTTQ5IHh1lS6RBCGZG0lgLHNrs,11916
-quack/gemm_interface.py,sha256=AF5PYTNgEHjb3MNXcNvvEpOcShAHtak0Xu12l1zrOAw,44804
-quack/gemm_sm100.py,sha256=U9jmzpST_d1W6CBFf1ZHhTtr0K8hENCsUz7dXvHaMZc,122344
-quack/gemm_sm90.py,sha256=u-Q3fN6DPm1fEdz0LcMecMbGTBcRunUCWopufwO8cHU,92015
-quack/gemm_symmetric.py,sha256=mqx7wgOCY6Dh9hjL6gR9PBstMD476GhpA_NkGeaEtik,13349
-quack/gemm_wrapper_utils.py,sha256=EaPyR3Lq19z_RkdB2_xxRj0IPSJMgyfpkrTXyvY3B6M,12775
-quack/layout_utils.py,sha256=qar8x_6VPKOdrz_lAGH_c4W_HKfYLk3Lhtd3Rv1OBBE,12197
-quack/linear.py,sha256=mhN2A98w7H7X4MS63XCCK3gpOm1eS8H7a4WO9ovkt5U,9791
-quack/linear_cross_entropy.py,sha256=Zhy_gdMsKHOie-jntBaqIuiDJtkiq6qEBwnyuWwIRw4,10092
-quack/mlp.py,sha256=YjdwQRwEePA9KyidFXp5H1-lxiJc8dZ41vl8Fv8pgss,2259
-quack/pipeline.py,sha256=3d4D8CPHw7ytZfdH9HFkfDng12YTnGf3pAe2DYxHjK4,11993
-quack/reduce.py,sha256=ySKT2xh1_pIlbJX29BPmwH6yJ7MxIrRZyxHIPPYVpm0,12698
-quack/reduction_base.py,sha256=QqlPs5L2VCxwDrO4CHPq-KY6f_BAYRbvsR6k81LPzTU,3180
-quack/rmsnorm.py,sha256=esy18s5JtT7KBPRPhWf_anLRTrtromwqeJmg2yzOm60,44678
-quack/sm100_utils.py,sha256=-p5qj3Wi9n4WDLy2sl-fApYpGp5rH3JvZQb712OTxPs,1901
-quack/sm90_utils.py,sha256=RLfIZFPhx7Mb9gXwilJ-QSULaj_Q4unaQJA2tFjGIJ4,5545
-quack/softmax.py,sha256=ZqeVbnGfzwkro1LfWBHagbS7B7ug7b9SLZWuGx_Y3Kc,14367
-quack/tensormap_manager.py,sha256=Ts3Mxp0_es2RNA0ffvUjWMXN79lsfWEBZ0DQYhtbcnw,5338
-quack/tile_scheduler.py,sha256=vbKq0xp94eII0uJ63yY_3sgvJkQI7Irc8y1OttO6cRA,42514
-quack/topk.py,sha256=43xHpRGbwZCSRsulmfrG4WA_r2eLHc3sniaUFU7wn-o,22522
-quack/utils.py,sha256=WIttE1iiwyPIwR1NpaeO26Pn9YkZb361TDxFTUDH-IE,7354
-quack/varlen_utils.py,sha256=SOYkomxX2FoqjYlybg99CqNhS9IARM6F9ba2AkIVvT4,15811
-quack/sort/bitonic_sort.py,sha256=-4VmHGmnqRLaVF-IrNhbJqNEJcz-FJT5GuzSWTFeIfI,4831
-quack/sort/generate_sorting_networks.py,sha256=vkJBOjTVEinQkWT4OtFqOWxFVdTIPoNAQocneKc9-rM,14477
-quack/sort/sorting_networks.py,sha256=l_26zi3gXD_z-tnm2eAczRrmE-mbaz00KmqH6ONivL8,9686
-quack/sort/utils.py,sha256=RbubEY1GcEpsjiz_6o5o2WB47IeMOzaajW6Jis0s444,1059
-quack_kernels-0.2.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-quack_kernels-0.2.5.dist-info/METADATA,sha256=5FnKfn7JrhBVjFUILnccs-OL2I8UN6Lqo7QR0i4tAlA,366
-quack_kernels-0.2.5.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-quack_kernels-0.2.5.dist-info/top_level.txt,sha256=6e4Jr_vNJbZTYwlO_Ahf_sDeHDE0zcqcf7Le11FKxxo,6
-quack_kernels-0.2.5.dist-info/RECORD,,