cuda-cccl 0.3.2__cp313-cp313-manylinux_2_24_aarch64.whl → 0.3.4__cp313-cp313-manylinux_2_24_aarch64.whl

cuda/cccl/headers/include/cuda/__numeric/sub_overflow.h

@@ -0,0 +1,344 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___NUMERIC_SUB_OVERFLOW_H
+#define _CUDA___NUMERIC_SUB_OVERFLOW_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__numeric/overflow_cast.h>
+#include <cuda/__numeric/overflow_result.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__limits/numeric_limits.h>
+#include <cuda/std/__type_traits/common_type.h>
+#include <cuda/std/__type_traits/conditional.h>
+#include <cuda/std/__type_traits/is_constant_evaluated.h>
+#include <cuda/std/__type_traits/is_integer.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/__type_traits/is_unsigned.h>
+#include <cuda/std/__type_traits/is_void.h>
+#include <cuda/std/__type_traits/make_nbit_int.h>
+#include <cuda/std/__type_traits/make_signed.h>
+#include <cuda/std/__type_traits/make_unsigned.h>
+
+#include <nv/target>
+
+#if _CCCL_COMPILER(MSVC) && _CCCL_ARCH(X86_64)
+#  include <intrin.h>
+#endif // _CCCL_COMPILER(MSVC) && _CCCL_ARCH(X86_64)
+
+#include <cuda/std/__cccl/prologue.h>
+
+#if _CCCL_CHECK_BUILTIN(__builtin_sub_overflow) || _CCCL_COMPILER(GCC)
+#  define _CCCL_BUILTIN_SUB_OVERFLOW(...) __builtin_sub_overflow(__VA_ARGS__)
+#endif // _CCCL_CHECK_BUILTIN(__builtin_sub_overflow)
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+// Subtracts using unsigned intermediates to avoid signed overflow (UB) before casting to the requested result type.
+template <typename _Result, typename _Lhs, typename _Rhs>
+[[nodiscard]] _CCCL_API constexpr _Result __sub_as_unsigned(_Lhs __lhs, _Rhs __rhs) noexcept
+{
+  using _UnsignedResult = ::cuda::std::make_unsigned_t<_Result>;
+  const auto __lhs1     = static_cast<_UnsignedResult>(__lhs);
+  const auto __rhs1     = static_cast<_UnsignedResult>(__rhs);
+  return static_cast<_Result>(__lhs1 - __rhs1);
+}
+
+// addition with unsigned types to avoid UB, return an unsigned type
+template <typename _Result, typename _Lhs, typename _Rhs>
+[[nodiscard]] _CCCL_API constexpr _Result __add_as_unsigned(_Lhs __lhs, _Rhs __rhs) noexcept
+{
+  using _UnsignedResult = ::cuda::std::make_unsigned_t<_Result>;
+  const auto __lhs1     = static_cast<_UnsignedResult>(__lhs);
+  const auto __rhs1     = static_cast<_UnsignedResult>(__rhs);
+  return static_cast<_Result>(__lhs1 + __rhs1);
+}
+
+template <typename _Tp>
+[[nodiscard]] _CCCL_API constexpr overflow_result<_Tp> __sub_overflow_generic_impl(_Tp __lhs, _Tp __rhs) noexcept
+{
+  const auto __sub = ::cuda::__sub_as_unsigned<_Tp>(__lhs, __rhs);
+  if constexpr (::cuda::std::is_signed_v<_Tp>)
+  {
+    return overflow_result<_Tp>{__sub, (__sub > __lhs) == (__rhs >= _Tp{0})};
+  }
+  else
+  {
+    return overflow_result<_Tp>{__sub, __sub > __lhs};
+  }
+}
+
+#if _CCCL_DEVICE_COMPILATION()
+
+template <class _Tp>
+[[nodiscard]] _CCCL_DEVICE_API overflow_result<_Tp> __sub_overflow_device(_Tp __lhs, _Tp __rhs) noexcept
+{
+  // signed version of sub.cc behave the same way of unsigned integers
+  if constexpr ((sizeof(_Tp) == 4 || sizeof(_Tp) == 8) && ::cuda::std::is_unsigned_v<_Tp>)
+  {
+    _Tp __result;
+    int __overflow;
+    if constexpr (sizeof(_Tp) == 4)
+    {
+      asm("sub.cc.u32 %0, %2, %3;"
+          "subc.u32 %1, 0, 0;"
+          : "=r"(__result), "=r"(__overflow)
+          : "r"(__lhs), "r"(__rhs));
+    }
+    else if constexpr (sizeof(_Tp) == 8)
+    {
+      asm("sub.cc.u64 %0, %2, %3;"
+          "subc.u32 %1, 0, 0;"
+          : "=l"(__result), "=r"(__overflow)
+          : "l"(__lhs), "l"(__rhs));
+    }
+    return overflow_result<_Tp>{__result, static_cast<bool>(__overflow)};
+  }
+  else
+  {
+    return ::cuda::__sub_overflow_generic_impl(__lhs, __rhs); // do not use builtin functions
+  }
+}
+
+#endif // _CCCL_DEVICE_COMPILATION()
+
+#if _CCCL_HOST_COMPILATION()
+
+template <class _Tp>
+[[nodiscard]] _CCCL_HOST_API overflow_result<_Tp> __sub_overflow_host(_Tp __lhs, _Tp __rhs) noexcept
+{
+#  if _CCCL_COMPILER(MSVC) && _CCCL_ARCH(X86_64)
+  if constexpr (sizeof(_Tp) <= 8)
+  {
+#    if _CCCL_COMPILER(MSVC, >=, 19, 37)
+    if constexpr (::cuda::std::is_signed_v<_Tp>)
+    {
+      overflow_result<_Tp> __result;
+      if constexpr (sizeof(_Tp) == 1)
+      {
+        __result.overflow = ::_sub_overflow_i8(0, __lhs, __rhs, reinterpret_cast<signed char*>(&__result.value));
+      }
+      else if constexpr (sizeof(_Tp) == 2)
+      {
+        __result.overflow = ::_sub_overflow_i16(0, __lhs, __rhs, reinterpret_cast<short*>(&__result.value));
+      }
+      else if constexpr (sizeof(_Tp) == 4)
+      {
+        __result.overflow = ::_sub_overflow_i32(0, __lhs, __rhs, reinterpret_cast<int*>(&__result.value));
+      }
+      else if constexpr (sizeof(_Tp) == 8)
+      {
+        __result.overflow = ::_sub_overflow_i64(0, __lhs, __rhs, reinterpret_cast<long long*>(&__result.value));
+      }
+      return __result;
+    }
+    else
+#    endif // _CCCL_COMPILER(MSVC, >=, 19, 37)
+      if constexpr (::cuda::std::is_unsigned_v<_Tp>)
+      { // unsigned
+        overflow_result<_Tp> __result;
+        if constexpr (sizeof(_Tp) == 1)
+        {
+          __result.overflow = ::_subborrow_u8(0, __lhs, __rhs, reinterpret_cast<unsigned char*>(&__result.value));
+        }
+        else if constexpr (sizeof(_Tp) == 2)
+        {
+          __result.overflow = ::_subborrow_u16(0, __lhs, __rhs, reinterpret_cast<unsigned short*>(&__result.value));
+        }
+        else if constexpr (sizeof(_Tp) == 4)
+        {
+          __result.overflow = ::_subborrow_u32(0, __lhs, __rhs, reinterpret_cast<unsigned int*>(&__result.value));
+        }
+        else if constexpr (sizeof(_Tp) == 8)
+        {
+          __result.overflow = ::_subborrow_u64(0, __lhs, __rhs, reinterpret_cast<unsigned long long*>(&__result.value));
+        }
+        return __result;
+      }
+      else
+      {
+        return ::cuda::__sub_overflow_generic_impl(__lhs, __rhs);
+      }
+  }
+  else
+#  endif // ^^^ _CCCL_COMPILER(MSVC) || _CCCL_ARCH(X86_64) ^^^
+  {
+    return ::cuda::__sub_overflow_generic_impl(__lhs, __rhs);
+  }
+}
+
+#endif // _CCCL_HOST_COMPILATION()
+
+template <typename _Tp>
+[[nodiscard]] _CCCL_API constexpr overflow_result<_Tp> __sub_overflow_uniform_type(_Tp __lhs, _Tp __rhs) noexcept
+{
+  if (!::cuda::std::__cccl_default_is_constant_evaluated())
+  {
+    NV_IF_TARGET(NV_IS_DEVICE,
+                 (return ::cuda::__sub_overflow_device(__lhs, __rhs);),
+                 (return ::cuda::__sub_overflow_host(__lhs, __rhs);))
+  }
+  return ::cuda::__sub_overflow_generic_impl(__lhs, __rhs);
+}
+
+template <typename _Result, typename _Lhs, typename _Rhs>
+inline constexpr bool __is_sub_representable_v = sizeof(_Result) > sizeof(_Lhs) && sizeof(_Result) > sizeof(_Rhs)
+                                              && ::cuda::std::is_signed_v<_Result>;
+
+/***********************************************************************************************************************
+ * Public interface
+ **********************************************************************************************************************/
+
+_CCCL_TEMPLATE(typename _Result = void,
+               typename _Lhs,
+               typename _Rhs,
+               typename _Common       = ::cuda::std::common_type_t<_Lhs, _Rhs>,
+               typename _ActualResult = ::cuda::std::conditional_t<::cuda::std::is_void_v<_Result>, _Common, _Result>)
+_CCCL_REQUIRES((::cuda::std::is_void_v<_Result> || ::cuda::std::__cccl_is_integer_v<_Result>)
+                 _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Lhs> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Rhs>)
+[[nodiscard]]
+_CCCL_API constexpr overflow_result<_ActualResult> sub_overflow(const _Lhs __lhs, const _Rhs __rhs) noexcept
+{
+// (1) __builtin_sub_overflow is not available in a constant expression with gcc + nvcc
+// (2) __builtin_sub_overflow generates suboptimal code with nvc++ and clang-cuda for device code
+#if defined(_CCCL_BUILTIN_SUB_OVERFLOW) && _CCCL_HOST_COMPILATION() \
+  && !(_CCCL_COMPILER(GCC) && _CCCL_CUDA_COMPILER(NVCC))
+  overflow_result<_ActualResult> __result;
+  __result.overflow = _CCCL_BUILTIN_SUB_OVERFLOW(__lhs, __rhs, &__result.value);
+  return __result;
+#else
+  using ::cuda::std::common_type_t;
+  using ::cuda::std::is_signed_v;
+  using ::cuda::std::is_unsigned_v;
+  using ::cuda::std::make_signed_t;
+  using ::cuda::std::make_unsigned_t;
+  using ::cuda::std::numeric_limits;
+  // shortcut for the case where inputs are representable with the result type
+  if constexpr (__is_sub_representable_v<_ActualResult, _Lhs, _Rhs>)
+  {
+    constexpr auto __max_lhs_rhs = sizeof(_Lhs) > sizeof(_Rhs) ? sizeof(_Lhs) : sizeof(_Rhs);
+    constexpr auto __max_bits = __max_lhs_rhs * 2 <= sizeof(_ActualResult) ? __max_lhs_rhs * 2 : sizeof(_ActualResult);
+    using _ComputeType        = ::cuda::std::__make_nbit_int_t<__max_bits * 8>;
+    const auto __lhs1         = static_cast<_ComputeType>(__lhs);
+    const auto __rhs1         = static_cast<_ComputeType>(__rhs);
+    const auto __sub          = static_cast<_ComputeType>(__lhs1 - __rhs1);
+    return overflow_result<_ActualResult>{static_cast<_ActualResult>(__sub), false};
+  }
+  // all types have the same sign
+  else if constexpr (is_signed_v<_Lhs> == is_signed_v<_Rhs> && is_signed_v<_Lhs> == is_signed_v<_ActualResult>)
+  {
+    using _CommonAll  = common_type_t<_Common, _ActualResult>;
+    const auto __lhs1 = static_cast<_CommonAll>(__lhs);
+    const auto __rhs1 = static_cast<_CommonAll>(__rhs);
+    const auto __sub  = ::cuda::__sub_overflow_uniform_type(__lhs1, __rhs1);
+    const auto __ret  = ::cuda::overflow_cast<_ActualResult>(__sub.value);
+    return overflow_result<_ActualResult>{__ret.value, __ret.overflow || __sub.overflow};
+  }
+  else if (::cuda::std::cmp_less(__lhs, __rhs)) // lhs < rhs -> negative result
+  {
+    if constexpr (is_unsigned_v<_ActualResult>) // if _ActualResult is unsigned, any negative result is an underflow
+    {
+      const auto __lhs1 = static_cast<_ActualResult>(__lhs);
+      const auto __rhs1 = static_cast<_ActualResult>(__rhs);
+      return overflow_result<_ActualResult>{static_cast<_ActualResult>(__lhs1 - __rhs1), true};
+    }
+    else
+    {
+      // perform the subtraction as signed (negative result) and check if the result is out of range
+      // Then, there are two cases depending on the sign of the rhs
+      using _SignedCommonAll          = make_signed_t<common_type_t<_Common, _ActualResult>>;
+      const auto __sub                = ::cuda::__sub_as_unsigned<_SignedCommonAll>(__lhs, __rhs);
+      constexpr auto __result_min     = numeric_limits<_ActualResult>::min();
+      const auto __is_out_of_range    = ::cuda::std::cmp_less(__sub, __result_min);
+      const auto __sub_ret            = static_cast<_ActualResult>(__sub);
+      const bool __rhs_less_than_zero = !is_unsigned_v<_Rhs> && __rhs < _Rhs{0};
+      if (__rhs_less_than_zero || __is_out_of_range) // if rhs < 0, lhs - rhs > lhs -> no overflow
+      {
+        return overflow_result<_ActualResult>{__sub_ret, __is_out_of_range};
+      }
+      else // rhs >= 0 -> lhs - rhs < result_min? -> lhs < result_min + rhs
+      {
+        // Now, the problem is to compute 'result_min + rhs' correctly
+        //      note: rhs >= 0, result_min < 0
+        // * if sizeof(_ActualResult) >= sizeof(_Rhs) or _Rhs is signed, we can use the signed common type
+        //   because the sum is always representable
+        // * if sizeof(_ActualResult) < sizeof(_Rhs) and _Rhs is unsigned, we can still use the signed common type
+        //   if rhs <= result_max because the sum is always representable, e.g. INT_MIN + uint64_t{UINT_MAX} = INT_MAX
+        constexpr auto __signed_min = numeric_limits<_ActualResult>::min();
+        constexpr auto __result_max = numeric_limits<make_signed_t<_ActualResult>>::max();
+        if (sizeof(_ActualResult) >= sizeof(_Rhs)
+            || is_signed_v<_Rhs> || ::cuda::std::cmp_less_equal(__rhs, __result_max))
+        {
+          // use __add_as_unsigned to avoid UB with INT_MIN
+          using _SumType            = make_signed_t<common_type_t<_Rhs, _ActualResult>>;
+          const auto __usum         = ::cuda::__add_as_unsigned<_SumType>(__signed_min, __rhs);
+          const bool __is_underflow = ::cuda::std::cmp_less(__lhs, __usum);
+          return overflow_result<_ActualResult>{__sub_ret, __is_underflow};
+        }
+        else // * otherwise, rhs > result_max and we need to use the unsigned common type
+        {
+          using _SumType            = make_unsigned_t<common_type_t<_Rhs, _ActualResult>>;
+          const auto __usum         = ::cuda::__add_as_unsigned<_SumType>(__signed_min, __rhs);
+          const bool __is_underflow = ::cuda::std::cmp_less(__lhs, __usum);
+          return overflow_result<_ActualResult>{__sub_ret, __is_underflow};
+        }
+      }
+    }
+  }
+  else // lhs >= rhs -> positive result
+  {
+    // perform the subtraction as unsigned (positive result) and check if the result is out of range
+    // Then, there are two cases depending on the sign of the rhs
+    using _UnsignedCommonAll     = make_unsigned_t<common_type_t<_Common, _ActualResult>>;
+    const auto __sub             = ::cuda::__sub_as_unsigned<_UnsignedCommonAll>(__lhs, __rhs);
+    const auto __sub_ret         = static_cast<_ActualResult>(__sub);
+    constexpr auto __result_max  = numeric_limits<_ActualResult>::max();
+    const auto __is_out_of_range = ::cuda::std::cmp_greater(__sub, __result_max);
+    const bool __is_rhs_ge_zero  = is_unsigned_v<_Rhs> || __rhs >= 0;
+    if (__is_rhs_ge_zero || __is_out_of_range) // rhs >= 0 -> lhs - rhs < lhs -> no overflow
+    {
+      return overflow_result<_ActualResult>{__sub_ret, __is_out_of_range};
+    }
+    else // lhs >= 0 && rhs < 0 -> lhs - rhs > result_max? -> lhs > result_max + rhs
+    {
+      using _Up                     = make_unsigned_t<common_type_t<_Rhs, _ActualResult>>;
+      constexpr auto __unsigned_max = numeric_limits<_ActualResult>::max();
+      const auto __sum              = ::cuda::__add_as_unsigned<_Up>(__unsigned_max, __rhs);
+      const bool __is_overflow      = ::cuda::std::cmp_greater(__lhs, __sum);
+      return overflow_result<_ActualResult>{__sub_ret, __is_overflow};
+    }
+  }
+#endif // defined(_CCCL_BUILTIN_SUB_OVERFLOW) && !_CCCL_CUDA_COMPILER(NVCC)
+}
+
+//! @brief Subtracts two numbers \p __lhs and \p __rhs with overflow detection
+_CCCL_TEMPLATE(typename _Result, typename _Lhs, typename _Rhs)
+_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Result> _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Lhs>
+                 _CCCL_AND ::cuda::std::__cccl_is_integer_v<_Rhs>)
+[[nodiscard]] _CCCL_API constexpr bool sub_overflow(_Result& __result, const _Lhs __lhs, const _Rhs __rhs) noexcept
+{
+  const auto __res = ::cuda::sub_overflow<_Result>(__lhs, __rhs);
+  __result         = __res.value;
+  return __res.overflow;
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___NUMERIC_SUB_OVERFLOW_H

cuda/cccl/headers/include/cuda/__nvtx/nvtx3.h

@@ -664,12 +664,10 @@
 
 namespace nvtx3
 {
-
 NVTX3_INLINE_IF_REQUESTED namespace NVTX3_VERSION_NAMESPACE
 {
   namespace detail
   {
-
   template <typename Unused>
   struct always_false : std::false_type
   {};
@@ -706,7 +704,6 @@ NVTX3_INLINE_IF_REQUESTED namespace NVTX3_VERSION_NAMESPACE
 
   template <typename T>
   using is_uint32 = std::is_same<typename std::decay<T>::type, uint32_t>;
-
   } // namespace detail
 
   /**
@@ -2217,7 +2214,6 @@ NVTX3_INLINE_IF_REQUESTED namespace NVTX3_VERSION_NAMESPACE
 
   namespace detail
   {
-
   /// @cond internal
   template <typename D = domain::global>
   class NVTX3_MAYBE_UNUSED optional_scoped_range_in
@@ -2264,7 +2260,6 @@ NVTX3_INLINE_IF_REQUESTED namespace NVTX3_VERSION_NAMESPACE
 #  endif
   };
   /// @endcond
-
   } // namespace detail
 
   /**
@@ -2811,7 +2806,6 @@ NVTX3_INLINE_IF_REQUESTED namespace NVTX3_VERSION_NAMESPACE
   }
 
 } // namespace NVTX3_VERSION_NAMESPACE
-
 } // namespace nvtx3
 
 #  ifndef NVTX_DISABLE

cuda/cccl/headers/include/cuda/__ptx/instructions/shfl_sync.h

@@ -83,11 +83,11 @@ _CCCL_DEVICE static inline void __shfl_sync_checks(
   [[maybe_unused]] uint32_t __lane_mask)
 {
   static_assert(sizeof(_Tp) == 4, "shfl.sync only accepts 4-byte data types");
+  _CCCL_ASSERT(__lane_mask & (1u << ::cuda::ptx::get_sreg_laneid()), "lane_mask must contain the current lane");
   if (__shfl_mode != __dot_shfl_mode::__idx)
   {
     _CCCL_ASSERT(__lane_idx_offset < 32, "the lane index or offset must be less than the warp size");
   }
-  _CCCL_ASSERT(__lane_mask != 0, "lane_mask must be non-zero");
   _CCCL_ASSERT((__clamp_segmask | 0b1111100011111) == 0b1111100011111,
                "clamp value + segmentation mask must use the bit positions [0:4] and [8:12]");
   _CCCL_ASSERT(::cuda::ptx::__shfl_sync_dst_lane(__shfl_mode, __lane_idx_offset, __clamp_segmask) & __lane_mask,

cuda/cccl/headers/include/cuda/__ptx/pragmas/enable_smem_spilling.h

@@ -0,0 +1,47 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___PTX_ENABLE_SMEM_SPILLING_H
+#define _CUDA___PTX_ENABLE_SMEM_SPILLING_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_PTX
+
+#if __cccl_ptx_isa >= 900
+
+extern "C" _CCCL_DEVICE void __cuda_ptx_pragma_enable_smem_spilling_is_not_supported_before_SM_75__();
+_CCCL_DEVICE static _CCCL_FORCEINLINE void enable_smem_spilling() noexcept
+{
+#  if _CCCL_CUDA_COMPILER(NVHPC) || __CUDA_ARCH__ >= 750
+  asm volatile(".pragma \"enable_smem_spilling\";");
+#  else
+  // Unsupported architectures will have a linker error with a semi-decent error message
+  ::cuda::ptx::__cuda_ptx_pragma_enable_smem_spilling_is_not_supported_before_SM_75__();
+#  endif
+}
+
+#endif // __cccl_ptx_isa >= 900
+
+_CCCL_END_NAMESPACE_CUDA_PTX
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___PTX_ENABLE_SMEM_SPILLING_H

cuda/cccl/headers/include/cuda/{std/__cuda → __runtime}/api_wrapper.h

@@ -8,8 +8,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _CUDA__STD__CUDA_API_WRAPPER_H
-#define _CUDA__STD__CUDA_API_WRAPPER_H
+#ifndef _CUDA___RUNTIME_API_WRAPPER_H
+#define _CUDA___RUNTIME_API_WRAPPER_H
 
 #include <cuda/std/detail/__config>
 
@@ -59,4 +59,4 @@
     return __status;                                                               \
   }()
 
-#endif //_CUDA__STD__CUDA_API_WRAPPER_H
+#endif //_CUDA___RUNTIME_API_WRAPPER_H

cuda/cccl/headers/include/cuda/__stream/get_stream.h

@@ -26,7 +26,6 @@
 #  include <cuda/__stream/stream_ref.h>
 #  include <cuda/std/__concepts/concept_macros.h>
 #  include <cuda/std/__concepts/convertible_to.h>
-#  include <cuda/std/__cuda/api_wrapper.h>
 #  include <cuda/std/__execution/env.h>
 #  include <cuda/std/__type_traits/is_convertible.h>
 

cuda/cccl/headers/include/cuda/{__fwd/barrier_native_handle.h → __stream/internal_streams.h}

@@ -1,15 +1,15 @@
 //===----------------------------------------------------------------------===//
 //
-// Part of libcu++, the C++ Standard Library for your entire system,
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
 // under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef _CUDA___FWD_BARRIER_NATIVE_HANDLE_H
-#define _CUDA___FWD_BARRIER_NATIVE_HANDLE_H
+#ifndef _CUDA___STREAM_INTERNAL_STREAMS_H
+#define _CUDA___STREAM_INTERNAL_STREAMS_H
 
 #include <cuda/std/detail/__config>
 
@@ -21,22 +21,24 @@
 #  pragma system_header
 #endif // no system header
 
-#include <cuda/__fwd/barrier.h>
-#include <cuda/std/__atomic/scopes.h>
-#include <cuda/std/cstdint>
+#include <cuda/__stream/stream.h>
 
-#include <cuda/std/__cccl/prologue.h>
-
-_CCCL_BEGIN_NV_DIAG_SUPPRESS(821) // extern inline function was referenced but not defined
+#include <cuda_runtime_api.h>
 
-_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE
+#include <cuda/std/__cccl/prologue.h>
 
-_CCCL_DEVICE inline ::cuda::std::uint64_t* barrier_native_handle(barrier<thread_scope_block>& __b);
+_CCCL_BEGIN_NAMESPACE_CUDA
 
-_CCCL_END_NAMESPACE_CUDA_DEVICE
+//! @brief internal stream used for memory allocations, no real blocking work
+//! should ever be pushed into it
+inline ::cuda::stream_ref __cccl_allocation_stream()
+{
+  static ::cuda::stream __stream{device_ref{0}};
+  return __stream;
+}
 
-_CCCL_END_NV_DIAG_SUPPRESS()
+_CCCL_END_NAMESPACE_CUDA
 
 #include <cuda/std/__cccl/epilogue.h>
 
-#endif // _CUDA___FWD_BARRIER_NATIVE_HANDLE_H
+#endif // _CUDA___STREAM_INTERNAL_STREAMS_H

cuda/cccl/headers/include/cuda/__utility/__basic_any/basic_any_ptr.h

@@ -289,12 +289,12 @@ private:
 
 _CCCL_TEMPLATE(template <class...> class _Interface, class _Super)
 _CCCL_REQUIRES(__is_interface<_Interface<_Super>>)
-_CCCL_PUBLIC_API __basic_any(_Interface<_Super>*) //
+_CCCL_HOST_DEVICE __basic_any(_Interface<_Super>*) //
   -> __basic_any<__normalized_interface_of<__basic_any<_Super>*>>;
 
 _CCCL_TEMPLATE(template <class...> class _Interface, class _Super)
 _CCCL_REQUIRES(__is_interface<_Interface<_Super>>)
-_CCCL_PUBLIC_API __basic_any(_Interface<_Super> const*) //
+_CCCL_HOST_DEVICE __basic_any(_Interface<_Super> const*) //
   -> __basic_any<__normalized_interface_of<__basic_any<_Super> const*>>;
 
 _CCCL_END_NAMESPACE_CUDA

cuda/cccl/headers/include/cuda/__utility/__basic_any/basic_any_value.h

@@ -32,6 +32,7 @@
 #include <cuda/__utility/__basic_any/virtual_tables.h>
 #include <cuda/std/__concepts/constructible.h>
 #include <cuda/std/__concepts/same_as.h>
+#include <cuda/std/__new/device_new.h>
 #include <cuda/std/__new/launder.h>
 #include <cuda/std/__type_traits/decay.h>
 #include <cuda/std/__type_traits/is_callable.h>

cuda/cccl/headers/include/cuda/__utility/__basic_any/semiregular.h

@@ -35,6 +35,7 @@
 #include <cuda/std/__concepts/equality_comparable.h>
 #include <cuda/std/__concepts/movable.h>
 #include <cuda/std/__memory/addressof.h>
+#include <cuda/std/__new/device_new.h>
 #include <cuda/std/__type_traits/always_false.h>
 #include <cuda/std/__type_traits/decay.h>
 #include <cuda/std/__type_traits/is_same.h>

cuda/cccl/headers/include/cuda/__utility/__basic_any/virtcall.h

@@ -32,7 +32,7 @@
 #include <cuda/std/__cccl/prologue.h>
 
 _CCCL_DIAG_PUSH
-_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedef")
+_CCCL_DIAG_SUPPRESS_CLANG("-Wunused-local-typedefs")
 
 _CCCL_BEGIN_NAMESPACE_CUDA
 
@@ -115,6 +115,7 @@ _CCCL_API auto __virtcall(_Self* __self, _Args&&... __args) //
   // map the member function pointer to the correct one if necessary
   using __virtual_fn_t = __virtual_fn_for<_Mbr, _Interface, _Super>;
   return __vptr->__virtual_fn_t::__fn_(__obj, static_cast<_Args&&>(__args)...);
+  _CCCL_UNREACHABLE();
 }
 
 _CCCL_TEMPLATE(auto _Mbr, template <class...> class _Interface, class _Super, class... _Args)