cuda-cccl 0.3.2__cp313-cp313-manylinux_2_24_aarch64.whl → 0.3.4__cp313-cp313-manylinux_2_24_aarch64.whl

cuda/cccl/headers/include/cub/device/dispatch/dispatch_segmented_sort.cuh

@@ -1,29 +1,5 @@
-/******************************************************************************
- * Copyright (c) 2011-2021, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
+// SPDX-FileCopyrightText: Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
 
 #pragma once
 
@@ -40,7 +16,7 @@
 #include <cub/detail/device_double_buffer.cuh>
 #include <cub/detail/temporary_storage.cuh>
 #include <cub/device/device_partition.cuh>
-#include <cub/device/dispatch/kernels/segmented_sort.cuh>
+#include <cub/device/dispatch/kernels/kernel_segmented_sort.cuh>
 #include <cub/device/dispatch/tuning/tuning_segmented_sort.cuh>
 #include <cub/util_debug.cuh>
 #include <cub/util_device.cuh>

cuda/cccl/headers/include/cub/device/dispatch/dispatch_select_if.cuh

@@ -1,30 +1,6 @@
-/******************************************************************************
- * Copyright (c) 2011, Duane Merrill.  All rights reserved.
- * Copyright (c) 2011-2018, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
+// SPDX-FileCopyrightText: Copyright (c) 2011, Duane Merrill. All rights reserved.
+// SPDX-FileCopyrightText: Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
 
 /**
  * @file

cuda/cccl/headers/include/cub/device/dispatch/dispatch_streaming_reduce.cuh

@@ -30,7 +30,6 @@ CUB_NAMESPACE_BEGIN
 
 namespace detail::reduce
 {
-
 template <typename GlobalAccumT, typename PromoteToGlobalOpT, typename GlobalReductionOpT, typename FinalResultOutIteratorT>
 struct accumulating_transform_output_op
 {
@@ -334,7 +333,6 @@ struct dispatch_streaming_arg_reduce_t
     return cudaSuccess;
   }
 };
-
 } // namespace detail::reduce
 CUB_NAMESPACE_END
 

cuda/cccl/headers/include/cub/device/dispatch/dispatch_streaming_reduce_by_key.cuh

@@ -34,7 +34,6 @@ CUB_NAMESPACE_BEGIN
 
 namespace detail::reduce
 {
-
 /******************************************************************************
  * Dispatch
  ******************************************************************************/
@@ -220,26 +219,6 @@ struct DispatchStreamingReduceByKey
           ? (num_items - current_partition_offset)
           : capped_num_items_per_invocation;
 
-      streaming_context_t streaming_context{};
-      if constexpr (use_streaming_invocation)
-      {
-        auto tmp_num_uniques = static_cast<global_offset_t*>(allocations[1]);
-        auto tmp_prefix      = static_cast<AccumT*>(allocations[2]);
-
-        const bool is_first_partition = (partition_idx == 0);
-        const bool is_last_partition  = (partition_idx + 1 == num_partitions);
-        const int buffer_selector     = partition_idx % 2;
-
-        streaming_context = streaming_context_t{
-          is_first_partition,
-          is_last_partition,
-          is_first_partition ? d_keys_in : d_keys_in + current_partition_offset - 1,
-          &tmp_prefix[buffer_selector],
-          &tmp_prefix[buffer_selector ^ 0x01],
-          &tmp_num_uniques[buffer_selector],
-          &tmp_num_uniques[buffer_selector ^ 0x01]};
-      }
-
       // Construct the tile status interface
       const auto num_current_tiles = static_cast<int>(::cuda::ceil_div(current_num_items, tile_size));
 
@@ -293,20 +272,57 @@ struct DispatchStreamingReduceByKey
 #endif // CUB_DEBUG_LOG
 
       // Invoke reduce_by_key_kernel
-      THRUST_NS_QUALIFIER::cuda_cub::detail::triple_chevron(num_current_tiles, block_threads, 0, stream)
-        .doit(reduce_by_key_kernel,
-              d_keys_in + current_partition_offset,
-              d_unique_out,
-              d_values_in + current_partition_offset,
-              d_aggregates_out,
-              d_num_runs_out,
-              tile_state,
-              0,
-              equality_op,
-              reduction_op,
-              static_cast<local_offset_t>(current_num_items),
-              streaming_context,
-              cub::detail::vsmem_t{nullptr});
+
+      if constexpr (use_streaming_invocation)
+      {
+        auto tmp_num_uniques = static_cast<global_offset_t*>(allocations[1]);
+        auto tmp_prefix      = static_cast<AccumT*>(allocations[2]);
+
+        const bool is_first_partition = (partition_idx == 0);
+        const bool is_last_partition  = (partition_idx + 1 == num_partitions);
+        const int buffer_selector     = partition_idx % 2;
+
+        streaming_context_t streaming_context{
+          is_first_partition,
+          is_last_partition,
+          is_first_partition ? d_keys_in : d_keys_in + current_partition_offset - 1,
+          &tmp_prefix[buffer_selector],
+          &tmp_prefix[buffer_selector ^ 0x01],
+          &tmp_num_uniques[buffer_selector],
+          &tmp_num_uniques[buffer_selector ^ 0x01]};
+
+        THRUST_NS_QUALIFIER::cuda_cub::detail::triple_chevron(num_current_tiles, block_threads, 0, stream)
+          .doit(reduce_by_key_kernel,
+                d_keys_in + current_partition_offset,
+                d_unique_out,
+                d_values_in + current_partition_offset,
+                d_aggregates_out,
+                d_num_runs_out,
+                tile_state,
+                0,
+                equality_op,
+                reduction_op,
+                static_cast<local_offset_t>(current_num_items),
+                streaming_context,
+                cub::detail::vsmem_t{nullptr});
+      }
+      else
+      {
+        THRUST_NS_QUALIFIER::cuda_cub::detail::triple_chevron(num_current_tiles, block_threads, 0, stream)
+          .doit(reduce_by_key_kernel,
+                d_keys_in + current_partition_offset,
+                d_unique_out,
+                d_values_in + current_partition_offset,
+                d_aggregates_out,
+                d_num_runs_out,
+                tile_state,
+                0,
+                equality_op,
+                reduction_op,
+                static_cast<local_offset_t>(current_num_items),
+                NullType{},
+                cub::detail::vsmem_t{nullptr});
+      }
 
       // Check for failure to launch
       error = CubDebug(cudaPeekAtLastError());
@@ -434,7 +450,6 @@ struct DispatchStreamingReduceByKey
     return error;
   }
 };
-
 } // namespace detail::reduce
 
 CUB_NAMESPACE_END

cuda/cccl/headers/include/cub/device/dispatch/dispatch_three_way_partition.cuh

@@ -1,29 +1,5 @@
-/******************************************************************************
- * Copyright (c) 2011-2021, NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
+// SPDX-FileCopyrightText: Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
 
 #pragma once
 
@@ -39,7 +15,7 @@
 
 #include <cub/agent/agent_three_way_partition.cuh>
 #include <cub/device/dispatch/dispatch_scan.cuh>
-#include <cub/device/dispatch/kernels/three_way_partition.cuh>
+#include <cub/device/dispatch/kernels/kernel_three_way_partition.cuh>
 #include <cub/device/dispatch/tuning/tuning_three_way_partition.cuh>
 #include <cub/util_device.cuh>
 #include <cub/util_math.cuh>
@@ -59,7 +35,6 @@ CUB_NAMESPACE_BEGIN
 
 namespace detail::three_way_partition
 {
-
 template <typename MaxPolicyT,
           typename InputIteratorT,
           typename FirstOutputIteratorT,

cuda/cccl/headers/include/cub/device/dispatch/dispatch_topk.cuh

@@ -621,7 +621,6 @@ struct DispatchTopK
     return CubDebug(max_policy_t::Invoke(ptx_version, dispatch));
   }
 };
-
 } // namespace detail::topk
 
 CUB_NAMESPACE_END

cuda/cccl/headers/include/cub/device/dispatch/dispatch_transform.cuh

@@ -16,7 +16,7 @@
 #include <cub/detail/detect_cuda_runtime.cuh>
 #include <cub/detail/launcher/cuda_runtime.cuh>
 #include <cub/detail/uninitialized_copy.cuh>
-#include <cub/device/dispatch/kernels/transform.cuh>
+#include <cub/device/dispatch/kernels/kernel_transform.cuh>
 #include <cub/util_arch.cuh>
 #include <cub/util_device.cuh>
 #include <cub/util_math.cuh>
@@ -51,6 +51,21 @@ CUB_NAMESPACE_BEGIN
 
 namespace detail::transform
 {
+template <typename T>
+using cuda_expected = ::cuda::std::expected<T, cudaError_t>;
+
+struct async_config
+{
+  int items_per_thread;
+  int max_occupancy;
+  int sm_count;
+};
+
+struct prefetch_config
+{
+  int max_occupancy;
+  int sm_count;
+};
 
 template <typename Offset,
           typename RandomAccessIteratorsIn,
@@ -82,9 +97,16 @@ struct TransformKernelSource<Offset,
                      THRUST_NS_QUALIFIER::try_unwrap_contiguous_iterator_t<RandomAccessIteratorOut>,
                      THRUST_NS_QUALIFIER::try_unwrap_contiguous_iterator_t<RandomAccessIteratorsIn>...>);
 
-  CUB_RUNTIME_FUNCTION static constexpr bool CanCacheConfiguration()
+  template <class ActionT>
+  CUB_RUNTIME_FUNCTION cuda_expected<async_config> CacheAsyncConfiguration(const ActionT& action)
+  {
+    NV_IF_TARGET(NV_IS_HOST, (static auto cached_config = action(); return cached_config;), (return action();))
+  }
+
+  template <class ActionT>
+  CUB_RUNTIME_FUNCTION cuda_expected<prefetch_config> CachePrefetchConfiguration(const ActionT& action)
   {
-    return true;
+    NV_IF_TARGET(NV_IS_HOST, (static auto cached_config = action(); return cached_config;), (return action();))
   }
 
   CUB_RUNTIME_FUNCTION static constexpr int LoadedBytesPerIteration()
@@ -138,22 +160,6 @@ enum class requires_stable_address
   yes
 };
 
-template <typename T>
-using cuda_expected = ::cuda::std::expected<T, cudaError_t>;
-
-struct async_config
-{
-  int items_per_thread;
-  int max_occupancy;
-  int sm_count;
-};
-
-struct prefetch_config
-{
-  int max_occupancy;
-  int sm_count;
-};
-
 template <
   requires_stable_address StableAddress,
   typename Offset,
@@ -282,27 +288,12 @@ struct dispatch_t<StableAddress,
       }
       return last_config;
     };
-    cuda_expected<async_config> config = [&]() {
-      NV_IF_TARGET(
-        NV_IS_HOST,
-        (
-          if constexpr (KernelSource::CanCacheConfiguration()) {
-            static auto cached_config = determine_element_counts();
-            return cached_config;
-          } else {
-            // we cannot cache the determined element count when not allowed
-            return determine_element_counts();
-          }),
-        (
-          // we cannot cache the determined element count in device code
-          return determine_element_counts();))
-    }();
+    cuda_expected<async_config> config = kernel_source.CacheAsyncConfiguration(determine_element_counts);
     if (!config)
     {
       return ::cuda::std::unexpected<cudaError_t /* nvcc 12.0 fails CTAD here */>(config.error());
     }
     _CCCL_ASSERT(config->items_per_thread > 0, "");
-    _CCCL_ASSERT(config->items_per_thread > 0, "");
     _CCCL_ASSERT((config->items_per_thread * block_threads) % alignment == 0, "");
 
     const int ipt =
@@ -422,25 +413,7 @@ struct dispatch_t<StableAddress,
       return prefetch_config{max_occupancy, sm_count};
     };
 
-    cuda_expected<prefetch_config> config = [&]() {
-      NV_IF_TARGET(
-        NV_IS_HOST,
-        (
-          // this static variable exists for each template instantiation of the surrounding function and
-          // class, on which the chosen element count solely depends (assuming max SMEM is constant during a
-          // program execution)
-          if constexpr (KernelSource::CanCacheConfiguration()) {
-            // we can cache the determined element count in host code if allowed
-            static auto cached_config = determine_config();
-            return cached_config;
-          } else {
-            // we cannot cache the determined element count when not allowed
-            return determine_config();
-          }),
-        (
-          // we cannot cache the determined element count in device code
-          return determine_config();))
-    }();
+    cuda_expected<prefetch_config> config = kernel_source.CachePrefetchConfiguration(determine_config);
     if (!config)
     {
       return config.error();
@@ -562,7 +535,6 @@ struct dispatch_t<StableAddress,
     return CubDebug(max_policy.Invoke(ptx_version, dispatch));
   }
 };
-
 } // namespace detail::transform
 CUB_NAMESPACE_END
 

cuda/cccl/headers/include/cub/device/dispatch/dispatch_unique_by_key.cuh

@@ -1,30 +1,6 @@
 
-/******************************************************************************
- * Copyright (c) NVIDIA CORPORATION.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of the NVIDIA CORPORATION nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- ******************************************************************************/
+// SPDX-FileCopyrightText: Copyright (c), NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
 
 /**
  * @file
@@ -45,8 +21,8 @@
 #endif // no system header
 
 #include <cub/device/dispatch/dispatch_scan.cuh>
-#include <cub/device/dispatch/kernels/scan.cuh>
-#include <cub/device/dispatch/kernels/unique_by_key.cuh>
+#include <cub/device/dispatch/kernels/kernel_scan.cuh>
+#include <cub/device/dispatch/kernels/kernel_unique_by_key.cuh>
 #include <cub/device/dispatch/tuning/tuning_unique_by_key.cuh>
 #include <cub/util_device.cuh>
 #include <cub/util_math.cuh>

cuda/cccl/headers/include/cub/device/dispatch/kernels/{for_each.cuh → kernel_for_each.cuh}

@@ -33,7 +33,6 @@ CUB_NAMESPACE_BEGIN
 
 namespace detail::for_each
 {
-
 template <class Fn>
 struct first_parameter
 {
@@ -255,7 +254,6 @@ struct op_wrapper_extents_t
     impl(i, ::cuda::std::make_index_sequence<ExtentsType::rank()>{});
   }
 };
-
 } // namespace detail::for_each
 
 CUB_NAMESPACE_END