cuda-cccl 0.3.2__cp313-cp313-manylinux_2_24_aarch64.whl → 0.3.4__cp313-cp313-manylinux_2_24_aarch64.whl

cuda/cccl/headers/include/cub/block/block_radix_rank.cuh

@@ -1,30 +1,6 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
+// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
+// SPDX-FileCopyrightText: Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
 
 //! @file
 //! cub::BlockRadixRank provides operations for ranking unsigned integer types within a CUDA thread block
@@ -50,7 +26,9 @@
 
 #include <cuda/__ptx/instructions/get_sreg.h>
 #include <cuda/std/__algorithm/max.h>
+#include <cuda/std/__bit/countl.h>
 #include <cuda/std/__bit/integral.h>
+#include <cuda/std/__bit/popcount.h>
 #include <cuda/std/__functional/operations.h>
 #include <cuda/std/__type_traits/conditional.h>
 #include <cuda/std/__type_traits/is_same.h>
@@ -65,12 +43,12 @@ CUB_NAMESPACE_BEGIN
 //!        initial arrangements of keys to function properly.
 enum RadixRankAlgorithm
 {
-  //! Ranking using the BlockRadixRank algorithm with `MEMOIZE_OUTER_SCAN == false`.
+  //! Ranking using the BlockRadixRank algorithm with `MemoizeOuterScan == false`.
   //! It uses thread-private histograms, and thus uses more shared memory.
   //! Requires blocked arrangement of keys. Does not support count callbacks.
   RADIX_RANK_BASIC,
 
-  //! Ranking using the BlockRadixRank algorithm with `MEMOIZE_OUTER_SCAN == true`.
+  //! Ranking using the BlockRadixRank algorithm with `MemoizeOuterScan == true`.
   //! Similar to RADIX_RANK BASIC, it requires blocked arrangement of keys and does not support count callbacks.
   RADIX_RANK_MEMOIZE,
 
@@ -102,7 +80,6 @@ struct BlockRadixRankEmptyCallback
 #ifndef _CCCL_DOXYGEN_INVOKED // Do not document
 namespace detail
 {
-
 template <int Bits, int PartialWarpThreads, int PartialWarpId>
 struct warp_in_block_matcher_t
 {
@@ -125,7 +102,6 @@ struct warp_in_block_matcher_t<Bits, 0, PartialWarpId>
     return MatchAny<Bits>(label);
   }
 };
-
 } // namespace detail
 #endif // _CCCL_DOXYGEN_INVOKED
 
@@ -136,6 +112,8 @@ struct warp_in_block_matcher_t<Bits, 0, PartialWarpId>
 //! +++++++++++++++++++++++++++++++++++++++++++++
 //!
 //! - Keys must be in a form suitable for radix ranking (i.e., unsigned bits).
+//! - **Important**: BlockRadixRank ranks only ``RadixBits`` bits at a time from the keys, not the entire key.
+//!   The digit extractor determines which bits are ranked.
 //! - @blocked
 //!
 //! Performance Considerations
@@ -153,25 +131,26 @@ struct warp_in_block_matcher_t<Bits, 0, PartialWarpId>
 //!      constexpr int radix_bits = 5;
 //!
 //!      // Specialize BlockRadixRank for a 1D block of 2 threads
-//!      // Specialize BlockRadixRank for a 1D block of 2 threads
 //!      using block_radix_rank = cub::BlockRadixRank<block_threads, radix_bits, false>;
 //!      using storage_t = typename block_radix_rank::TempStorage;
 //!
-//!      // Allocate shared memory for BlockRadixSort
+//!      // Allocate shared memory for BlockRadixRank
 //!      __shared__ storage_t temp_storage;
 //!
 //!      // Obtain a segment of consecutive items that are blocked across threads
-//!      int keys[2];
+//!      unsigned int keys[2];
 //!      int ranks[2];
 //!      ...
 //!
-//!      cub::BFEDigitExtractor<int> extractor(0, radix_bits);
+//!      // Extract the lowest radix_bits from each key
+//!      cub::BFEDigitExtractor<unsigned> extractor(0, radix_bits);
 //!      block_radix_rank(temp_storage).RankKeys(keys, ranks, extractor);
 //!
 //!      ...
 //!    }
 //!
 //! Suppose the set of input ``keys`` across the block of threads is ``{ [16,10], [9,11] }``.
+//! The extractor will rank only the lowest 5 bits: ``{ [16,10], [9,11] }`` (bits 0-4).
 //! The corresponding output ``ranks`` in those threads will be ``{ [3,1], [0,2] }``.
 //!
 //! Re-using dynamically allocating shared memory
@@ -183,41 +162,41 @@ struct warp_in_block_matcher_t<Bits, 0, PartialWarpId>
 //!
 //! @endrst
 //!
-//! @tparam BLOCK_DIM_X
+//! @tparam BlockDimX
 //!   The thread block length in threads along the X dimension
 //!
-//! @tparam RADIX_BITS
+//! @tparam RadixBits
 //!   The number of radix bits per digit place
 //!
-//! @tparam IS_DESCENDING
+//! @tparam IsDescending
 //!   Whether or not the sorted-order is high-to-low
 //!
-//! @tparam MEMOIZE_OUTER_SCAN
+//! @tparam MemoizeOuterScan
 //!   **[optional]** Whether or not to buffer outer raking scan
 //!   partials to incur fewer shared memory reads at the expense of higher register pressure
 //!   (default: true for architectures SM35 and newer, false otherwise).
 //!   See `BlockScanAlgorithm::BLOCK_SCAN_RAKING_MEMOIZE` for more details.
 //!
-//! @tparam INNER_SCAN_ALGORITHM
+//! @tparam InnerScanAlgorithm
 //!   **[optional]** The cub::BlockScanAlgorithm algorithm to use (default: cub::BLOCK_SCAN_WARP_SCANS)
 //!
-//! @tparam SMEM_CONFIG
+//! @tparam SMemConfig
 //!   **[optional]** Shared memory bank mode (default: `cudaSharedMemBankSizeFourByte`)
 //!
-//! @tparam BLOCK_DIM_Y
+//! @tparam BlockDimY
 //!   **[optional]** The thread block length in threads along the Y dimension (default: 1)
 //!
-//! @tparam BLOCK_DIM_Z
+//! @tparam BlockDimZ
 //!   **[optional]** The thread block length in threads along the Z dimension (default: 1)
 //!
-template <int BLOCK_DIM_X,
-          int RADIX_BITS,
-          bool IS_DESCENDING,
-          bool MEMOIZE_OUTER_SCAN                 = true,
-          BlockScanAlgorithm INNER_SCAN_ALGORITHM = BLOCK_SCAN_WARP_SCANS,
-          cudaSharedMemConfig SMEM_CONFIG         = cudaSharedMemBankSizeFourByte,
-          int BLOCK_DIM_Y                         = 1,
-          int BLOCK_DIM_Z                         = 1>
+template <int BlockDimX,
+          int RadixBits,
+          bool IsDescending,
+          bool MemoizeOuterScan                 = true,
+          BlockScanAlgorithm InnerScanAlgorithm = BLOCK_SCAN_WARP_SCANS,
+          cudaSharedMemConfig SMemConfig        = cudaSharedMemBankSizeFourByte,
+          int BlockDimY                         = 1,
+          int BlockDimZ                         = 1>
 class BlockRadixRank
 {
 private:
@@ -226,16 +205,16 @@ private:
 
   // Integer type for packing DigitCounters into columns of shared memory banks
   using PackedCounter =
-    ::cuda::std::_If<SMEM_CONFIG == cudaSharedMemBankSizeEightByte, unsigned long long, unsigned int>;
+    ::cuda::std::_If<SMemConfig == cudaSharedMemBankSizeEightByte, unsigned long long, unsigned int>;
 
   static constexpr DigitCounter max_tile_size = ::cuda::std::numeric_limits<DigitCounter>::max();
 
   enum
   {
     // The thread block size in threads
-    BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    BLOCK_THREADS = BlockDimX * BlockDimY * BlockDimZ,
 
-    RADIX_DIGITS = 1 << RADIX_BITS,
+    RADIX_DIGITS = 1 << RadixBits,
 
     LOG_WARP_THREADS = detail::log2_warp_threads,
     WARP_THREADS     = 1 << LOG_WARP_THREADS,
@@ -248,7 +227,7 @@ private:
     LOG_PACKING_RATIO = Log2<PACKING_RATIO>::VALUE,
 
     // Always at least one lane
-    LOG_COUNTER_LANES = ::cuda::std::max(RADIX_BITS - LOG_PACKING_RATIO, 0),
+    LOG_COUNTER_LANES = ::cuda::std::max(RadixBits - LOG_PACKING_RATIO, 0),
     COUNTER_LANES     = 1 << LOG_COUNTER_LANES,
 
     // The number of packed counters per thread (plus one for padding)
@@ -265,7 +244,7 @@ public:
 
 private:
   /// BlockScan type
-  using BlockScan = BlockScan<PackedCounter, BLOCK_DIM_X, INNER_SCAN_ALGORITHM, BLOCK_DIM_Y, BLOCK_DIM_Z>;
+  using BlockScan = BlockScan<PackedCounter, BlockDimX, InnerScanAlgorithm, BlockDimY, BlockDimZ>;
 
 #ifndef _CCCL_DOXYGEN_INVOKED // Do not document
   struct __align__(16) _TempStorage
@@ -306,7 +285,7 @@ private:
   _CCCL_DEVICE _CCCL_FORCEINLINE PackedCounter Upsweep()
   {
     auto& smem_raking_ptr = temp_storage.aliasable.raking_grid[linear_tid];
-    if constexpr (MEMOIZE_OUTER_SCAN)
+    if constexpr (MemoizeOuterScan)
     {
       // Copy data into registers
       _CCCL_PRAGMA_UNROLL_FULL()
@@ -327,12 +306,12 @@ private:
   {
     PackedCounter* smem_raking_ptr = temp_storage.aliasable.raking_grid[linear_tid];
 
-    PackedCounter* raking_ptr = (MEMOIZE_OUTER_SCAN) ? cached_segment : smem_raking_ptr;
+    PackedCounter* raking_ptr = (MemoizeOuterScan) ? cached_segment : smem_raking_ptr;
 
     // Exclusive raking downsweep scan
     detail::ThreadScanExclusive<RAKING_SEGMENT>(raking_ptr, raking_ptr, ::cuda::std::plus<>{}, raking_partial);
 
-    if (MEMOIZE_OUTER_SCAN)
+    if (MemoizeOuterScan)
     {
       // Copy data back to smem
       _CCCL_PRAGMA_UNROLL_FULL()
@@ -404,7 +383,7 @@ public:
   //! @brief Collective constructor using a private static allocation of shared memory as temporary storage.
   _CCCL_DEVICE _CCCL_FORCEINLINE BlockRadixRank()
       : temp_storage(PrivateStorage())
-      , linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+      , linear_tid(RowMajorTid(BlockDimX, BlockDimY, BlockDimZ))
   {}
 
   /**
@@ -415,7 +394,7 @@ public:
    */
   _CCCL_DEVICE _CCCL_FORCEINLINE BlockRadixRank(TempStorage& temp_storage)
       : temp_storage(temp_storage.Alias())
-      , linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+      , linear_tid(RowMajorTid(BlockDimX, BlockDimY, BlockDimZ))
   {}
 
   //! @} end member group
@@ -461,7 +440,7 @@ public:
       // Get counter lane
       ::cuda::std::uint32_t counter_lane = digit & (COUNTER_LANES - 1);
 
-      if (IS_DESCENDING)
+      if (IsDescending)
       {
         sub_counter  = PACKING_RATIO - 1 - sub_counter;
         counter_lane = COUNTER_LANES - 1 - counter_lane;
@@ -533,7 +512,7 @@ public:
 
       if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
       {
-        if (IS_DESCENDING)
+        if (IsDescending)
         {
           bin_idx = RADIX_DIGITS - bin_idx - 1;
         }
@@ -554,12 +533,12 @@ public:
 /**
  * Radix-rank using match.any
  */
-template <int BLOCK_DIM_X,
-          int RADIX_BITS,
-          bool IS_DESCENDING,
-          BlockScanAlgorithm INNER_SCAN_ALGORITHM = BLOCK_SCAN_WARP_SCANS,
-          int BLOCK_DIM_Y                         = 1,
-          int BLOCK_DIM_Z                         = 1>
+template <int BlockDimX,
+          int RadixBits,
+          bool IsDescending,
+          BlockScanAlgorithm InnerScanAlgorithm = BLOCK_SCAN_WARP_SCANS,
+          int BlockDimY                         = 1,
+          int BlockDimZ                         = 1>
 class BlockRadixRankMatch
 {
 private:
@@ -569,9 +548,9 @@ private:
   enum
   {
     // The thread block size in threads
-    BLOCK_THREADS = BLOCK_DIM_X * BLOCK_DIM_Y * BLOCK_DIM_Z,
+    BLOCK_THREADS = BlockDimX * BlockDimY * BlockDimZ,
 
-    RADIX_DIGITS = 1 << RADIX_BITS,
+    RADIX_DIGITS = 1 << RadixBits,
 
     LOG_WARP_THREADS     = detail::log2_warp_threads,
     WARP_THREADS         = 1 << LOG_WARP_THREADS,
@@ -594,7 +573,7 @@ public:
 
 private:
   /// BlockScan type
-  using BlockScanT = BlockScan<DigitCounterT, BLOCK_THREADS, INNER_SCAN_ALGORITHM, BLOCK_DIM_Y, BLOCK_DIM_Z>;
+  using BlockScanT = BlockScan<DigitCounterT, BLOCK_THREADS, InnerScanAlgorithm, BlockDimY, BlockDimZ>;
 
 #ifndef _CCCL_DOXYGEN_INVOKED // Do not document
   struct __align__(16) _TempStorage
@@ -631,7 +610,7 @@ public:
    */
   _CCCL_DEVICE _CCCL_FORCEINLINE BlockRadixRankMatch(TempStorage& temp_storage)
       : temp_storage(temp_storage.Alias())
-      , linear_tid(RowMajorTid(BLOCK_DIM_X, BLOCK_DIM_Y, BLOCK_DIM_Z))
+      , linear_tid(RowMajorTid(BlockDimX, BlockDimY, BlockDimZ))
   {}
 
   //! @}  end member group
@@ -666,7 +645,7 @@ public:
 
       if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
       {
-        if (IS_DESCENDING)
+        if (IsDescending)
         {
           bin_idx     = RADIX_DIGITS - bin_idx - 1;
           bins[track] = (bin_idx > 0 ? temp_storage.aliasable.warp_digit_counters[bin_idx - 1][0] : TILE_ITEMS)
@@ -724,14 +703,14 @@ public:
       // My digit
       ::cuda::std::uint32_t digit = digit_extractor.Digit(keys[ITEM]);
 
-      if (IS_DESCENDING)
+      if (IsDescending)
       {
         digit = RADIX_DIGITS - digit - 1;
       }
 
       // Mask of peers who have same digit as me
       uint32_t peer_mask =
-        detail::warp_in_block_matcher_t<RADIX_BITS, PARTIAL_WARP_THREADS, WARPS - 1>::match_any(digit, warp_id);
+        detail::warp_in_block_matcher_t<RadixBits, PARTIAL_WARP_THREADS, WARPS - 1>::match_any(digit, warp_id);
 
       // Pointer to smem digit counter for this key
       digit_counters[ITEM] = &temp_storage.aliasable.warp_digit_counters[digit][warp_id];
@@ -743,10 +722,10 @@ public:
       __syncwarp(0xFFFFFFFF);
 
       // Number of peers having same digit as me
-      int32_t digit_count = __popc(peer_mask);
+      int32_t digit_count = ::cuda::std::popcount(peer_mask);
 
       // Number of lower-ranked peers having same digit seen so far
-      int32_t peer_digit_prefix = __popc(peer_mask & lane_mask_lt);
+      int32_t peer_digit_prefix = ::cuda::std::popcount(peer_mask & lane_mask_lt);
 
       if (peer_digit_prefix == 0)
       {
@@ -839,7 +818,7 @@ public:
 
       if ((BLOCK_THREADS == RADIX_DIGITS) || (bin_idx < RADIX_DIGITS))
       {
-        if (IS_DESCENDING)
+        if (IsDescending)
         {
           bin_idx = RADIX_DIGITS - bin_idx - 1;
         }
@@ -888,19 +867,19 @@ enum WarpMatchAlgorithm
  * decoupled look-back, where it reduces the time other thread blocks need to
  * wait for digit counts to become available.
  */
-template <int BLOCK_DIM_X,
-          int RADIX_BITS,
-          bool IS_DESCENDING,
-          BlockScanAlgorithm INNER_SCAN_ALGORITHM = BLOCK_SCAN_WARP_SCANS,
-          WarpMatchAlgorithm MATCH_ALGORITHM      = WARP_MATCH_ANY,
-          int NUM_PARTS                           = 1>
+template <int BlockDimX,
+          int RadixBits,
+          bool IsDescending,
+          BlockScanAlgorithm InnerScanAlgorithm = BLOCK_SCAN_WARP_SCANS,
+          WarpMatchAlgorithm MATCH_ALGORITHM    = WARP_MATCH_ANY,
+          int NUM_PARTS                         = 1>
 struct BlockRadixRankMatchEarlyCounts
 {
   // constants
   enum
   {
-    BLOCK_THREADS           = BLOCK_DIM_X,
-    RADIX_DIGITS            = 1 << RADIX_BITS,
+    BLOCK_THREADS           = BlockDimX,
+    RADIX_DIGITS            = 1 << RadixBits,
     BINS_PER_THREAD         = (RADIX_DIGITS + BLOCK_THREADS - 1) / BLOCK_THREADS,
     BINS_TRACKED_PER_THREAD = BINS_PER_THREAD,
     FULL_BINS               = BINS_PER_THREAD * BLOCK_THREADS == RADIX_DIGITS,
@@ -915,7 +894,7 @@ struct BlockRadixRankMatchEarlyCounts
   };
 
   // types
-  using BlockScan = cub::BlockScan<int, BLOCK_THREADS, INNER_SCAN_ALGORITHM>;
+  using BlockScan = cub::BlockScan<int, BLOCK_THREADS, InnerScanAlgorithm>;
 
   struct TempStorage
   {
@@ -945,13 +924,13 @@ struct BlockRadixRankMatchEarlyCounts
     _CCCL_DEVICE _CCCL_FORCEINLINE ::cuda::std::uint32_t Digit(UnsignedBits key)
     {
       ::cuda::std::uint32_t digit = digit_extractor.Digit(key);
-      return IS_DESCENDING ? RADIX_DIGITS - 1 - digit : digit;
+      return IsDescending ? RADIX_DIGITS - 1 - digit : digit;
     }
 
     _CCCL_DEVICE _CCCL_FORCEINLINE int ThreadBin(int u)
     {
       int bin = threadIdx.x * BINS_PER_THREAD + u;
-      return IS_DESCENDING ? RADIX_DIGITS - 1 - bin : bin;
+      return IsDescending ? RADIX_DIGITS - 1 - bin : bin;
     }
 
     _CCCL_DEVICE _CCCL_FORCEINLINE void ComputeHistogramsWarp(UnsignedBits (&keys)[KEYS_PER_THREAD])
@@ -969,7 +948,7 @@ struct BlockRadixRankMatchEarlyCounts
           warp_histograms[bin][part] = 0;
         }
       }
-      if (MATCH_ALGORITHM == WARP_MATCH_ATOMIC_OR)
+      if constexpr (MATCH_ALGORITHM == WARP_MATCH_ATOMIC_OR)
       {
         int* match_masks = &s.match_masks[warp][0];
 
@@ -992,7 +971,7 @@ struct BlockRadixRankMatchEarlyCounts
 
       // sum different parts;
       // no extra work is necessary if NUM_PARTS == 1
-      if (NUM_PARTS > 1)
+      if constexpr (NUM_PARTS > 1)
       {
         __syncwarp(WARP_MASK);
         // TODO: handle RADIX_DIGITS % WARP_THREADS != 0 if it becomes necessary
@@ -1073,10 +1052,12 @@ struct BlockRadixRankMatchEarlyCounts
         int* p_match_mask         = &match_masks[bin];
         atomicOr(p_match_mask, lane_mask);
         __syncwarp(WARP_MASK);
-        int bin_mask    = *p_match_mask;
-        int leader      = ::cuda::std::__bit_log2(static_cast<unsigned>(bin_mask));
+        int bin_mask = *p_match_mask;
+        // TODO(bgruber): __bit_log2 regresses cub.bench.radix_sort.keys.base up to 30% on H200, see cccl_private/#586
+        // int leader      = ::cuda::std::__bit_log2(static_cast<unsigned>(bin_mask));
+        int leader      = (WARP_THREADS - 1) - ::cuda::std::countl_zero(static_cast<unsigned>(bin_mask));
         int warp_offset = 0;
-        int popc        = __popc(bin_mask & ::cuda::ptx::get_sreg_lanemask_le());
+        int popc        = ::cuda::std::popcount(bin_mask & ::cuda::ptx::get_sreg_lanemask_le());
         if (lane == leader)
         {
           // atomic is a bit faster
@@ -1103,10 +1084,12 @@ struct BlockRadixRankMatchEarlyCounts
       {
         ::cuda::std::uint32_t bin = Digit(keys[u]);
         int bin_mask =
-          detail::warp_in_block_matcher_t<RADIX_BITS, PARTIAL_WARP_THREADS, BLOCK_WARPS - 1>::match_any(bin, warp);
-        int leader      = ::cuda::std::__bit_log2(static_cast<unsigned>(bin_mask));
+          detail::warp_in_block_matcher_t<RadixBits, PARTIAL_WARP_THREADS, BLOCK_WARPS - 1>::match_any(bin, warp);
+        // TODO(bgruber): __bit_log2 regresses cub.bench.radix_sort.keys.base up to 30% on H200, see cccl_private/#586
+        // int leader      = ::cuda::std::__bit_log2(static_cast<unsigned>(bin_mask));
+        int leader      = (WARP_THREADS - 1) - ::cuda::std::countl_zero(static_cast<unsigned>(bin_mask));
         int warp_offset = 0;
-        int popc        = __popc(bin_mask & ::cuda::ptx::get_sreg_lanemask_le());
+        int popc        = ::cuda::std::popcount(bin_mask & ::cuda::ptx::get_sreg_lanemask_le());
         if (lane == leader)
         {
           // atomic is a bit faster
@@ -1192,7 +1175,6 @@ struct BlockRadixRankMatchEarlyCounts
 #ifndef _CCCL_DOXYGEN_INVOKED // Do not document
 namespace detail
 {
-
 // `BlockRadixRank` doesn't conform to the typical pattern, not exposing the algorithm
 // template parameter. Other algorithms don't provide the same template parameters, not allowing
 // multi-dimensional thread block specializations.
@@ -1215,7 +1197,6 @@ using block_radix_rank_t = ::cuda::std::_If<
         RankAlgorithm == RADIX_RANK_MATCH_EARLY_COUNTS_ANY,
         BlockRadixRankMatchEarlyCounts<BlockDimX, RadixBits, IsDescending, ScanAlgorithm, WARP_MATCH_ANY>,
         BlockRadixRankMatchEarlyCounts<BlockDimX, RadixBits, IsDescending, ScanAlgorithm, WARP_MATCH_ATOMIC_OR>>>>>;
-
 } // namespace detail
 #endif // _CCCL_DOXYGEN_INVOKED