cuda-cccl 0.3.2__cp313-cp313-manylinux_2_24_aarch64.whl → 0.3.4__cp313-cp313-manylinux_2_24_aarch64.whl

cuda/cccl/headers/include/cub/device/dispatch/kernels/{histogram.cuh → kernel_histogram.cuh}

@@ -16,18 +16,16 @@
 #include <cub/agent/agent_histogram.cuh>
 #include <cub/grid/grid_queue.cuh>
 
+#include <cuda/std/__numeric/reduce.h>
+
 CUB_NAMESPACE_BEGIN
 namespace detail::histogram
 {
-
 template <typename LevelT, typename OffsetT, typename SampleT>
 struct Transforms
 {
-  enum
-  {
-    // Maximum number of bins per channel for which we will use a privatized smem strategy
-    MAX_PRIVATIZED_SMEM_BINS = 256
-  };
+  // Maximum number of bins per channel for which we will use a privatized smem strategy
+  static constexpr int MAX_PRIVATIZED_SMEM_BINS = 256;
 
   //---------------------------------------------------------------------
   // Transform functors for converting samples to bin-ids
@@ -40,12 +38,10 @@ struct Transforms
     LevelIteratorT d_levels; // Pointer to levels array
     int num_output_levels; // Number of levels in array
 
-    /**
-     * @brief Initializer
-     *
-     * @param d_levels_ Pointer to levels array
-     * @param num_output_levels_ Number of levels in array
-     */
+    //! @brief Initializer
+    //!
+    //! @param d_levels_ Pointer to levels array
+    //! @param num_output_levels_ Number of levels in array
     _CCCL_HOST_DEVICE _CCCL_FORCEINLINE void Init(LevelIteratorT d_levels_, int num_output_levels_)
     {
       this->d_levels          = d_levels_;
@@ -66,10 +62,10 @@ struct Transforms
 
       WrappedLevelIteratorT wrapped_levels(d_levels);
 
-      int num_bins = num_output_levels - 1;
+      const int num_bins = num_output_levels - 1;
       if (valid)
       {
-        bin = UpperBound(wrapped_levels, num_output_levels, (LevelT) sample) - 1;
+        bin = UpperBound(wrapped_levels, num_output_levels, static_cast<LevelT>(sample)) - 1;
         if (bin >= num_bins)
         {
           bin = -1;
@@ -202,9 +198,7 @@ struct Transforms
     }
 #endif // _CCCL_HAS_NVFP16()
 
-    /**
-     * @brief Bin computation for floating point (and extended floating point) types
-     */
+    //! @brief Bin computation for floating point (and extended floating point) types
     template <typename T>
     _CCCL_HOST_DEVICE _CCCL_FORCEINLINE int
     ComputeBin(T sample, T min_level, ScaleT scale, ::cuda::std::true_type /* is_fp */)
@@ -212,9 +206,7 @@ struct Transforms
       return static_cast<int>((sample - min_level) * scale.reciprocal);
     }
 
-    /**
-     * @brief Bin computation for custom types and __[u]int128
-     */
+    //! @brief Bin computation for custom types and __[u]int128
     template <typename T>
     _CCCL_HOST_DEVICE _CCCL_FORCEINLINE int
     ComputeBin(T sample, T min_level, ScaleT scale, ::cuda::std::false_type /* is_fp */)
@@ -222,9 +214,7 @@ struct Transforms
       return static_cast<int>(((sample - min_level) * scale.fraction.bins) / scale.fraction.range);
     }
 
-    /**
-     * @brief Bin computation for integral types of up to 64-bit types
-     */
+    //! @brief Bin computation for integral types of up to 64-bit types
     template <typename T, ::cuda::std::enable_if_t<is_integral_excl_int128<T>::value, int> = 0>
     _CCCL_HOST_DEVICE _CCCL_FORCEINLINE int ComputeBin(T sample, T min_level, ScaleT scale)
     {
@@ -249,35 +239,32 @@ struct Transforms
     }
 #endif // _CCCL_HAS_NVFP16()
 
-    _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool
-    MayOverflow(CommonT /* num_bins */, ::cuda::std::false_type /* is_integral */)
-    {
-      return false;
-    }
-
-    /**
-     * @brief Returns true if the bin computation for a given combination of range `(max_level -
-     * min_level)` and number of bins may overflow.
-     */
-    _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool MayOverflow(CommonT num_bins, ::cuda::std::true_type /* is_integral */)
+    //! @brief Returns true if the bin computation for a given combination of range `(max_level - min_level)` and number
+    //! of bins may overflow.
+    _CCCL_HOST_DEVICE _CCCL_FORCEINLINE bool MayOverflow([[maybe_unused]] CommonT num_bins)
     {
-      return static_cast<IntArithmeticT>(m_max - m_min)
-           > (::cuda::std::numeric_limits<IntArithmeticT>::max() / static_cast<IntArithmeticT>(num_bins));
+      if constexpr (::cuda::std::is_integral_v<CommonT>)
+      {
+        return static_cast<IntArithmeticT>(m_max - m_min)
+             > (::cuda::std::numeric_limits<IntArithmeticT>::max() / static_cast<IntArithmeticT>(num_bins));
+      }
+      else
+      {
+        return false;
+      }
     }
 
   public:
-    /**
-     * @brief Initializes the ScaleTransform for the given parameters
-     * @return cudaErrorInvalidValue if the ScaleTransform for the given values may overflow,
-     * cudaSuccess otherwise
-     */
+    //! @brief Initializes the ScaleTransform for the given parameters
+    //! @return cudaErrorInvalidValue if the ScaleTransform for the given values may overflow,
+    //! cudaSuccess otherwise
     _CCCL_HOST_DEVICE _CCCL_FORCEINLINE cudaError_t Init(int num_levels, LevelT max_level, LevelT min_level)
     {
       m_max = static_cast<CommonT>(max_level);
       m_min = static_cast<CommonT>(min_level);
 
       // Check whether accurate bin computation for an integral sample type may overflow
-      if (MayOverflow(static_cast<CommonT>(num_levels - 1), ::cuda::std::is_integral<CommonT>{}))
+      if (MayOverflow(static_cast<CommonT>(num_levels - 1)))
       {
         return cudaErrorInvalidValue;
       }
@@ -314,7 +301,7 @@ struct Transforms
     {
       if (valid)
       {
-        bin = (int) sample;
+        bin = static_cast<int>(sample);
       }
     }
   };
@@ -324,128 +311,133 @@ struct Transforms
  * Histogram kernel entry points
  *****************************************************************************/
 
-/**
- * Histogram initialization kernel entry point
- *
- * @tparam NUM_ACTIVE_CHANNELS
- *   Number of channels actively being histogrammed
- *
- * @tparam CounterT
- *   Integer type for counting sample occurrences per histogram bin
- *
- * @tparam OffsetT
- *   Signed integer type for global offsets
- *
- * @param num_output_bins_wrapper
- *   Number of output histogram bins per channel
- *
- * @param d_output_histograms_wrapper
- *   Histogram counter data having logical dimensions
- *   `CounterT[NUM_ACTIVE_CHANNELS][num_bins.array[CHANNEL]]`
- *
- * @param tile_queue
- *   Drain queue descriptor for dynamically mapping tile data onto thread blocks
- */
-template <int NUM_ACTIVE_CHANNELS, typename CounterT, typename OffsetT>
+//! Histogram initialization kernel entry point
+//!
+//! @tparam NumActiveChannels
+//!   Number of channels actively being histogrammed
+//!
+//! @tparam CounterT
+//!   Integer type for counting sample occurrences per histogram bin
+//!
+//! @tparam OffsetT
+//!   Signed integer type for global offsets
+//!
+//! @param num_output_bins_wrapper
+//!   Number of output histogram bins per channel
+//!
+//! @param d_output_histograms_wrapper
+//!   Histogram counter data having logical dimensions `CounterT[NUM_ACTIVE_CHANNELS][num_bins.array[CHANNEL]]`
+//!
+//! @param tile_queue
+//!   Drain queue descriptor for dynamically mapping tile data onto thread blocks
+template <typename ChainedPolicyT, int NumActiveChannels, typename CounterT, typename OffsetT>
 CUB_DETAIL_KERNEL_ATTRIBUTES void DeviceHistogramInitKernel(
-  ::cuda::std::array<int, NUM_ACTIVE_CHANNELS> num_output_bins_wrapper,
-  ::cuda::std::array<CounterT*, NUM_ACTIVE_CHANNELS> d_output_histograms_wrapper,
+  ::cuda::std::array<int, NumActiveChannels> num_output_bins_wrapper,
+  ::cuda::std::array<CounterT*, NumActiveChannels> d_output_histograms_wrapper,
   GridQueue<int> tile_queue)
 {
+  _CCCL_PDL_GRID_DEPENDENCY_SYNC(); // TODO(bgruber): if we had the guarantee that there would be no pending
+                                    // writes/reads to the temp storage, we could omit the sync here
+
+  // we trigger the sweep kernel only if we have a small number of remaining writes in this kernel
+  NV_IF_TARGET(NV_PROVIDES_SM_90,
+               (if (::cuda::std::reduce(num_output_bins_wrapper.begin(), num_output_bins_wrapper.end())
+                    <= ChainedPolicyT::ActivePolicy::pdl_trigger_next_launch_in_init_kernel_max_bin_count) {
+                 _CCCL_PDL_TRIGGER_NEXT_LAUNCH();
+               }));
+
   if ((threadIdx.x == 0) && (blockIdx.x == 0))
   {
     tile_queue.ResetDrain();
   }
 
-  int output_bin = (blockIdx.x * blockDim.x) + threadIdx.x;
+  const int output_bin = (blockIdx.x * blockDim.x) + threadIdx.x;
 
   _CCCL_PRAGMA_UNROLL_FULL()
-  for (int CHANNEL = 0; CHANNEL < NUM_ACTIVE_CHANNELS; ++CHANNEL)
+  for (int ch = 0; ch < NumActiveChannels; ++ch)
   {
-    if (output_bin < num_output_bins_wrapper[CHANNEL])
+    if (output_bin < num_output_bins_wrapper[ch])
     {
-      d_output_histograms_wrapper[CHANNEL][output_bin] = 0;
+      d_output_histograms_wrapper[ch][output_bin] = 0;
     }
   }
 }
 
-/**
- * Histogram privatized sweep kernel entry point (multi-block).
- * Computes privatized histograms, one per thread block.
- *
- *
- * @tparam AgentHistogramPolicyT
- *   Parameterized AgentHistogramPolicy tuning policy type
- *
- * @tparam PRIVATIZED_SMEM_BINS
- *   Maximum number of histogram bins per channel (e.g., up to 256)
- *
- * @tparam NUM_CHANNELS
- *   Number of channels interleaved in the input data (may be greater than the number of channels
- *   being actively histogrammed)
- *
- * @tparam NUM_ACTIVE_CHANNELS
- *   Number of channels actively being histogrammed
- *
- * @tparam SampleIteratorT
- *   The input iterator type. @iterator.
- *
- * @tparam CounterT
- *   Integer type for counting sample occurrences per histogram bin
- *
- * @tparam PrivatizedDecodeOpT
- *   The transform operator type for determining privatized counter indices from samples,
- *   one for each channel
- *
- * @tparam OutputDecodeOpT
- *   The transform operator type for determining output bin-ids from privatized counter indices,
- *   one for each channel
- *
- * @tparam OffsetT
- *   integer type for global offsets
- *
- * @param d_samples
- *   Input data to reduce
- *
- * @param num_output_bins_wrapper
- *   The number bins per final output histogram
- *
- * @param num_privatized_bins_wrapper
- *   The number bins per privatized histogram
- *
- * @param d_output_histograms_wrapper
- *   Reference to final output histograms
- *
- * @param d_privatized_histograms_wrapper
- *   Reference to privatized histograms
- *
- * @param output_decode_op_wrapper
- *   The transform operator for determining output bin-ids from privatized counter indices,
- *   one for each channel
- *
- * @param privatized_decode_op_wrapper
- *   The transform operator for determining privatized counter indices from samples,
- *   one for each channel
- *
- * @param num_row_pixels
- *   The number of multi-channel pixels per row in the region of interest
- *
- * @param num_rows
- *   The number of rows in the region of interest
- *
- * @param row_stride_samples
- *   The number of samples between starts of consecutive rows in the region of interest
- *
- * @param tiles_per_row
- *   Number of image tiles per row
- *
- * @param tile_queue
- *   Drain queue descriptor for dynamically mapping tile data onto thread blocks
- */
+//! Histogram privatized sweep kernel entry point (multi-block).
+//! Computes privatized histograms, one per thread block.
+//!
+//!
+//! @tparam ChainedPolicyT
+//!   Max policy from a policy hub containing the AgentHistogramPolicy policy
+//!
+//! @tparam PrivatizedSmemBins
+//!   Maximum number of histogram bins per channel (e.g., up to 256)
+//!
+//! @tparam NumChannels
+//!   Number of channels interleaved in the input data (may be greater than the number of channels
+//!   being actively histogrammed)
+//!
+//! @tparam NumActiveChannels
+//!   Number of channels actively being histogrammed
+//!
+//! @tparam SampleIteratorT
+//!   The input iterator type. @iterator.
+//!
+//! @tparam CounterT
+//!   Integer type for counting sample occurrences per histogram bin
+//!
+//! @tparam PrivatizedDecodeOpT
+//!   The transform operator type for determining privatized counter indices from samples,
+//!   one for each channel
+//!
+//! @tparam OutputDecodeOpT
+//!   The transform operator type for determining output bin-ids from privatized counter indices,
+//!   one for each channel
+//!
+//! @tparam OffsetT
+//!   integer type for global offsets
+//!
+//! @param d_samples
+//!   Input data to reduce
+//!
+//! @param num_output_bins_wrapper
+//!   The number bins per final output histogram
+//!
+//! @param num_privatized_bins_wrapper
+//!   The number bins per privatized histogram
+//!
+//! @param d_output_histograms_wrapper
+//!   Reference to final output histograms
+//!
+//! @param d_privatized_histograms_wrapper
+//!   Reference to privatized histograms
+//!
+//! @param output_decode_op_wrapper
+//!   The transform operator for determining output bin-ids from privatized counter indices,
+//!   one for each channel
+//!
+//! @param privatized_decode_op_wrapper
+//!   The transform operator for determining privatized counter indices from samples,
+//!   one for each channel
+//!
+//! @param num_row_pixels
+//!   The number of multi-channel pixels per row in the region of interest
+//!
+//! @param num_rows
+//!   The number of rows in the region of interest
+//!
+//! @param row_stride_samples
+//!   The number of samples between starts of consecutive rows in the region of interest
+//!
+//! @param tiles_per_row
+//!   Number of image tiles per row
+//!
+//! @param tile_queue
+//!   Drain queue descriptor for dynamically mapping tile data onto thread blocks
 template <typename ChainedPolicyT,
-          int PRIVATIZED_SMEM_BINS,
-          int NUM_CHANNELS,
-          int NUM_ACTIVE_CHANNELS,
+          int PrivatizedSmemBins,
+          int NumChannels,
+          int NumActiveChannels,
           typename SampleIteratorT,
           typename CounterT,
           typename PrivatizedDecodeOpT,
@@ -454,12 +446,12 @@ template <typename ChainedPolicyT,
 __launch_bounds__(int(ChainedPolicyT::ActivePolicy::AgentHistogramPolicyT::BLOCK_THREADS))
   CUB_DETAIL_KERNEL_ATTRIBUTES void DeviceHistogramSweepKernel(
     SampleIteratorT d_samples,
-    ::cuda::std::array<int, NUM_ACTIVE_CHANNELS> num_output_bins_wrapper,
-    ::cuda::std::array<int, NUM_ACTIVE_CHANNELS> num_privatized_bins_wrapper,
-    ::cuda::std::array<CounterT*, NUM_ACTIVE_CHANNELS> d_output_histograms_wrapper,
-    ::cuda::std::array<CounterT*, NUM_ACTIVE_CHANNELS> d_privatized_histograms_wrapper,
-    ::cuda::std::array<OutputDecodeOpT, NUM_ACTIVE_CHANNELS> output_decode_op_wrapper,
-    ::cuda::std::array<PrivatizedDecodeOpT, NUM_ACTIVE_CHANNELS> privatized_decode_op_wrapper,
+    ::cuda::std::array<int, NumActiveChannels> num_output_bins_wrapper,
+    ::cuda::std::array<int, NumActiveChannels> num_privatized_bins_wrapper,
+    ::cuda::std::array<CounterT*, NumActiveChannels> d_output_histograms_wrapper,
+    ::cuda::std::array<CounterT*, NumActiveChannels> d_privatized_histograms_wrapper,
+    ::cuda::std::array<OutputDecodeOpT, NumActiveChannels> output_decode_op_wrapper,
+    ::cuda::std::array<PrivatizedDecodeOpT, NumActiveChannels> privatized_decode_op_wrapper,
     OffsetT num_row_pixels,
     OffsetT num_rows,
     OffsetT row_stride_samples,
@@ -470,9 +462,9 @@ __launch_bounds__(int(ChainedPolicyT::ActivePolicy::AgentHistogramPolicyT::BLOCK
   using AgentHistogramPolicyT = typename ChainedPolicyT::ActivePolicy::AgentHistogramPolicyT;
   using AgentHistogramT =
     AgentHistogram<AgentHistogramPolicyT,
-                   PRIVATIZED_SMEM_BINS,
-                   NUM_CHANNELS,
-                   NUM_ACTIVE_CHANNELS,
+                   PrivatizedSmemBins,
+                   NumChannels,
+                   NumActiveChannels,
                    SampleIteratorT,
                    CounterT,
                    PrivatizedDecodeOpT,

cuda/cccl/headers/include/cub/device/dispatch/kernels/{merge_sort.cuh → kernel_merge_sort.cuh}

@@ -22,7 +22,6 @@ CUB_NAMESPACE_BEGIN
 
 namespace detail::merge_sort
 {
-
 /**
  * @brief Helper class template that provides two agent template instantiations: one instantiated with the default
  * policy and one with the fallback policy. This helps to avoid having to enlist all the agent's template parameters
@@ -328,7 +327,6 @@ __launch_bounds__(
   // If applicable, hints to discard modified cache lines for vsmem
   VSmemHelperT::discard_temp_storage(temp_storage);
 }
-
 } // namespace detail::merge_sort
 
 CUB_NAMESPACE_END

cuda/cccl/headers/include/cub/device/dispatch/kernels/{radix_sort.cuh → kernel_radix_sort.cuh}

@@ -36,7 +36,6 @@ CUB_NAMESPACE_BEGIN
 
 namespace detail::radix_sort
 {
-
 /**
  * @brief Upsweep digit-counting kernel entry point (multi-block).
  *        Computes privatized digit histograms, one per block.
@@ -797,7 +796,6 @@ CUB_DETAIL_KERNEL_ATTRIBUTES void DeviceRadixSortExclusiveSumKernel(OffsetT* d_b
     d_bins[bin_start + bin] = bins[u];
   }
 }
-
 } // namespace detail::radix_sort
 
 CUB_NAMESPACE_END

cuda/cccl/headers/include/cub/device/dispatch/kernels/{reduce.cuh → kernel_reduce.cuh}

@@ -1,29 +1,5 @@
-/******************************************************************************
- * Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
 
 #pragma once
 
@@ -49,7 +25,6 @@ CUB_NAMESPACE_BEGIN
 
 namespace detail::reduce
 {
-
 /**
  * All cub::DeviceReduce::* algorithms are using the same implementation. Some of them, however,
  * should use initial value only for empty problems. If this struct is used as initial value with
@@ -577,7 +552,6 @@ CUB_DETAIL_KERNEL_ATTRIBUTES __launch_bounds__(int(
       (atomicAdd(&d_out[0], blockIdx.x == 0 ? reduction_op(init, block_aggregate) : block_aggregate);));
   }
 }
-
 } // namespace detail::reduce
 
 CUB_NAMESPACE_END

cuda/cccl/headers/include/cub/device/dispatch/kernels/{scan.cuh → kernel_scan.cuh}

@@ -1,29 +1,5 @@
-/******************************************************************************
- * Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
 
 #pragma once
 
@@ -44,7 +20,6 @@ CUB_NAMESPACE_BEGIN
 
 namespace detail::scan
 {
-
 /******************************************************************************
  * Kernel entry points
  *****************************************************************************/
@@ -183,7 +158,6 @@ __launch_bounds__(int(ChainedPolicyT::ActivePolicy::ScanPolicyT::BLOCK_THREADS))
   // Process tiles
   AgentScanT(temp_storage, d_in, d_out, scan_op, real_init_value).ConsumeRange(num_items, tile_state, start_tile);
 }
-
 } // namespace detail::scan
 
 CUB_NAMESPACE_END

cuda/cccl/headers/include/cub/device/dispatch/kernels/{segmented_reduce.cuh → kernel_segmented_reduce.cuh}

@@ -1,29 +1,5 @@
-/******************************************************************************
- * Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
- * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
 
 #pragma once
 
@@ -38,14 +14,13 @@
 #endif // no system header
 
 #include <cub/agent/agent_reduce.cuh>
-#include <cub/device/dispatch/kernels/reduce.cuh> // finalize_and_store_aggregate
+#include <cub/device/dispatch/kernels/kernel_reduce.cuh> // finalize_and_store_aggregate
 #include <cub/iterator/arg_index_input_iterator.cuh>
 
 CUB_NAMESPACE_BEGIN
 
 namespace detail::reduce
 {
-
 /// Normalize input iterator to segment offset
 template <typename T, typename OffsetT, typename IteratorT>
 _CCCL_DEVICE _CCCL_FORCEINLINE void NormalizeReductionOutput(T& /*val*/, OffsetT /*base_offset*/, IteratorT /*itr*/)
@@ -315,7 +290,6 @@ __launch_bounds__(int(ChainedPolicyT::ActivePolicy::ReducePolicy::BLOCK_THREADS)
     }
   }
 }
-
 } // namespace detail::reduce
 
 CUB_NAMESPACE_END

cuda/cccl/headers/include/cub/device/dispatch/kernels/{segmented_sort.cuh → kernel_segmented_sort.cuh}

@@ -23,7 +23,6 @@
 CUB_NAMESPACE_BEGIN
 namespace detail::segmented_sort
 {
-
 // Type used to index within segments within a single invocation
 using local_segment_index_t = ::cuda::std::uint32_t;
 // Type used for total number of segments and to index within segments globally

cuda/cccl/headers/include/cub/device/dispatch/kernels/{three_way_partition.cuh → kernel_three_way_partition.cuh}

@@ -19,7 +19,6 @@ CUB_NAMESPACE_BEGIN
 
 namespace detail::three_way_partition
 {
-
 // Offset type used to instantiate the stream three-way-partition-kernel and agent to index the items within one
 // partition
 using per_partition_offset_t = ::cuda::std::int32_t;