cuda-cccl 0.3.2__cp313-cp313-manylinux_2_24_aarch64.whl → 0.3.4__cp313-cp313-manylinux_2_24_aarch64.whl

cuda/cccl/headers/include/cuda/barrier

@@ -29,7 +29,6 @@
 #include <cuda/__barrier/barrier_arrive_tx.h>
 #include <cuda/__barrier/barrier_block_scope.h>
 #include <cuda/__barrier/barrier_expect_tx.h>
-#include <cuda/__barrier/barrier_native_handle.h>
 #include <cuda/__barrier/barrier_thread_scope.h>
 #include <cuda/__memcpy_async/memcpy_async.h>
 #include <cuda/__memcpy_async/memcpy_async_tx.h>
@@ -69,8 +68,9 @@ _CCCL_BEGIN_NAMESPACE_CUDA_DEVICE_EXPERIMENTAL
 // device compilation begins.
 #ifdef __cccl_lib_experimental_ctk12_cp_async_exposure
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk
-inline _CCCL_DEVICE void cp_async_bulk_global_to_shared(
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk instead") inline _CCCL_DEVICE void cp_async_bulk_global_to_shared(
   void* __dest, const void* __src, ::cuda::std::uint32_t __size, ::cuda::barrier<::cuda::thread_scope_block>& __bar)
 {
   _CCCL_ASSERT(__size % 16 == 0, "Size must be multiple of 16.");
@@ -88,8 +88,10 @@ inline _CCCL_DEVICE void cp_async_bulk_global_to_shared(
     ::cuda::device::barrier_native_handle(__bar));
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk
-inline _CCCL_DEVICE void cp_async_bulk_shared_to_global(void* __dest, const void* __src, ::cuda::std::uint32_t __size)
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk instead") inline _CCCL_DEVICE void
+cp_async_bulk_shared_to_global(void* __dest, const void* __src, ::cuda::std::uint32_t __size)
 {
   _CCCL_ASSERT(__size % 16 == 0, "Size must be multiple of 16.");
   _CCCL_ASSERT(::cuda::device::is_address_from(__dest, ::cuda::device::address_space::global),
@@ -100,8 +102,10 @@ inline _CCCL_DEVICE void cp_async_bulk_shared_to_global(void* __dest, const void
   ::cuda::ptx::cp_async_bulk(::cuda::ptx::space_global, ::cuda::ptx::space_shared, __dest, __src, __size);
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
-inline _CCCL_DEVICE void cp_async_bulk_tensor_1d_global_to_shared(
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_tensor instead") inline _CCCL_DEVICE void
+cp_async_bulk_tensor_1d_global_to_shared(
   void* __dest, const CUtensorMap* __tensor_map, int __c0, ::cuda::barrier<::cuda::thread_scope_block>& __bar)
 {
   const ::cuda::std::int32_t __coords[]{__c0};
@@ -115,8 +119,10 @@ inline _CCCL_DEVICE void cp_async_bulk_tensor_1d_global_to_shared(
     ::cuda::device::barrier_native_handle(__bar));
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
-inline _CCCL_DEVICE void cp_async_bulk_tensor_2d_global_to_shared(
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_tensor instead") inline _CCCL_DEVICE void
+cp_async_bulk_tensor_2d_global_to_shared(
   void* __dest, const CUtensorMap* __tensor_map, int __c0, int __c1, ::cuda::barrier<::cuda::thread_scope_block>& __bar)
 {
   const ::cuda::std::int32_t __coords[]{__c0, __c1};
@@ -130,8 +136,10 @@ inline _CCCL_DEVICE void cp_async_bulk_tensor_2d_global_to_shared(
     ::cuda::device::barrier_native_handle(__bar));
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
-inline _CCCL_DEVICE void cp_async_bulk_tensor_3d_global_to_shared(
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_tensor instead") inline _CCCL_DEVICE void
+cp_async_bulk_tensor_3d_global_to_shared(
   void* __dest,
   const CUtensorMap* __tensor_map,
   int __c0,
@@ -150,8 +158,10 @@ inline _CCCL_DEVICE void cp_async_bulk_tensor_3d_global_to_shared(
     ::cuda::device::barrier_native_handle(__bar));
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
-inline _CCCL_DEVICE void cp_async_bulk_tensor_4d_global_to_shared(
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_tensor instead") inline _CCCL_DEVICE void
+cp_async_bulk_tensor_4d_global_to_shared(
   void* __dest,
   const CUtensorMap* __tensor_map,
   int __c0,
@@ -171,8 +181,10 @@ inline _CCCL_DEVICE void cp_async_bulk_tensor_4d_global_to_shared(
     ::cuda::device::barrier_native_handle(__bar));
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
-inline _CCCL_DEVICE void cp_async_bulk_tensor_5d_global_to_shared(
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_tensor instead") inline _CCCL_DEVICE void
+cp_async_bulk_tensor_5d_global_to_shared(
   void* __dest,
   const CUtensorMap* __tensor_map,
   int __c0,
@@ -193,8 +205,9 @@ inline _CCCL_DEVICE void cp_async_bulk_tensor_5d_global_to_shared(
     ::cuda::device::barrier_native_handle(__bar));
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
-inline _CCCL_DEVICE void
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_tensor instead") inline _CCCL_DEVICE void
 cp_async_bulk_tensor_1d_shared_to_global(const CUtensorMap* __tensor_map, int __c0, const void* __src)
 {
   const ::cuda::std::int32_t __coords[]{__c0};
@@ -202,8 +215,9 @@ cp_async_bulk_tensor_1d_shared_to_global(const CUtensorMap* __tensor_map, int __
   ::cuda::ptx::cp_async_bulk_tensor(::cuda::ptx::space_global, ::cuda::ptx::space_shared, __tensor_map, __coords, __src);
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
-inline _CCCL_DEVICE void
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_tensor instead") inline _CCCL_DEVICE void
 cp_async_bulk_tensor_2d_shared_to_global(const CUtensorMap* __tensor_map, int __c0, int __c1, const void* __src)
 {
   const ::cuda::std::int32_t __coords[]{__c0, __c1};
@@ -211,8 +225,10 @@ cp_async_bulk_tensor_2d_shared_to_global(const CUtensorMap* __tensor_map, int __
   ::cuda::ptx::cp_async_bulk_tensor(::cuda::ptx::space_global, ::cuda::ptx::space_shared, __tensor_map, __coords, __src);
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
-inline _CCCL_DEVICE void cp_async_bulk_tensor_3d_shared_to_global(
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_tensor instead") inline _CCCL_DEVICE void
+cp_async_bulk_tensor_3d_shared_to_global(
   const CUtensorMap* __tensor_map, int __c0, int __c1, int __c2, const void* __src)
 {
   const ::cuda::std::int32_t __coords[]{__c0, __c1, __c2};
@@ -220,8 +236,10 @@ inline _CCCL_DEVICE void cp_async_bulk_tensor_3d_shared_to_global(
   ::cuda::ptx::cp_async_bulk_tensor(::cuda::ptx::space_global, ::cuda::ptx::space_shared, __tensor_map, __coords, __src);
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
-inline _CCCL_DEVICE void cp_async_bulk_tensor_4d_shared_to_global(
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_tensor instead") inline _CCCL_DEVICE void
+cp_async_bulk_tensor_4d_shared_to_global(
   const CUtensorMap* __tensor_map, int __c0, int __c1, int __c2, int __c3, const void* __src)
 {
   const ::cuda::std::int32_t __coords[]{__c0, __c1, __c2, __c3};
@@ -229,8 +247,10 @@ inline _CCCL_DEVICE void cp_async_bulk_tensor_4d_shared_to_global(
   ::cuda::ptx::cp_async_bulk_tensor(::cuda::ptx::space_global, ::cuda::ptx::space_shared, __tensor_map, __coords, __src);
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor
-inline _CCCL_DEVICE void cp_async_bulk_tensor_5d_shared_to_global(
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_tensor instead") inline _CCCL_DEVICE void
+cp_async_bulk_tensor_5d_shared_to_global(
   const CUtensorMap* __tensor_map, int __c0, int __c1, int __c2, int __c3, int __c4, const void* __src)
 {
   const ::cuda::std::int32_t __coords[]{__c0, __c1, __c2, __c3, __c4};
@@ -238,21 +258,27 @@ inline _CCCL_DEVICE void cp_async_bulk_tensor_5d_shared_to_global(
   ::cuda::ptx::cp_async_bulk_tensor(::cuda::ptx::space_global, ::cuda::ptx::space_shared, __tensor_map, __coords, __src);
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar
-inline _CCCL_DEVICE void fence_proxy_async_shared_cta()
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::fence_proxy_async instead") inline _CCCL_DEVICE void
+fence_proxy_async_shared_cta()
 {
   ::cuda::ptx::fence_proxy_async(::cuda::ptx::space_shared);
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group
-inline _CCCL_DEVICE void cp_async_bulk_commit_group()
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_commit_group instead") inline _CCCL_DEVICE void
+cp_async_bulk_commit_group()
 {
   ::cuda::ptx::cp_async_bulk_commit_group();
 }
 
+//! Deprecated [Since 3.2]
 // https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group
 template <int __n_prior>
-inline _CCCL_DEVICE void cp_async_bulk_wait_group_read()
+CCCL_DEPRECATED_BECAUSE("Use cuda::ptx::cp_async_bulk_wait_group_read instead") inline _CCCL_DEVICE void
+cp_async_bulk_wait_group_read()
 {
   static_assert(__n_prior <= 63, "cp_async_bulk_wait_group_read: waiting for more than 63 groups is not supported.");
   ::cuda::ptx::cp_async_bulk_wait_group_read(::cuda::ptx::n32_t<__n_prior>{});

cuda/cccl/headers/include/cuda/memory

@@ -30,6 +30,7 @@
 #include <cuda/__memory/is_aligned.h>
 #include <cuda/__memory/ptr_in_range.h>
 #include <cuda/__memory/ptr_rebind.h>
+#include <cuda/__memory/ranges_overlap.h>
 #include <cuda/std/memory>
 
 #endif // _CUDA_MEMORY

cuda/cccl/headers/include/cuda/memory_resource

@@ -26,10 +26,15 @@
 //!
 //!@endrst
 
+#include <cuda/__memory_resource/any_resource.h>
+#include <cuda/__memory_resource/device_memory_pool.h>
 #include <cuda/__memory_resource/get_memory_resource.h>
 #include <cuda/__memory_resource/get_property.h>
+#include <cuda/__memory_resource/legacy_managed_memory_resource.h>
+#include <cuda/__memory_resource/legacy_pinned_memory_resource.h>
+#include <cuda/__memory_resource/managed_memory_pool.h>
+#include <cuda/__memory_resource/pinned_memory_pool.h>
 #include <cuda/__memory_resource/properties.h>
 #include <cuda/__memory_resource/resource.h>
-#include <cuda/__memory_resource/resource_ref.h>
 
 #endif //_CCCL_BEGIN_NAMESPACE_CUDA

cuda/cccl/headers/include/cuda/numeric

@@ -21,9 +21,11 @@
 #endif // no system header
 
 #include <cuda/__numeric/add_overflow.h>
+#include <cuda/__numeric/div_overflow.h>
 #include <cuda/__numeric/narrow.h>
 #include <cuda/__numeric/overflow_cast.h>
 #include <cuda/__numeric/overflow_result.h>
+#include <cuda/__numeric/sub_overflow.h>
 #include <cuda/std/numeric>
 
 #endif // _CUDA_NUMERIC

cuda/cccl/headers/include/cuda/pipeline

@@ -29,6 +29,7 @@
 #include <cuda/atomic>
 #include <cuda/barrier>
 #include <cuda/std/__algorithm/max.h>
+#include <cuda/std/__bit/popcount.h>
 #include <cuda/std/__chrono/duration.h>
 #include <cuda/std/__chrono/time_point.h>
 #include <cuda/std/cstdint>
@@ -112,7 +113,7 @@ public:
         ::__match_any_sync(::__activemask(), reinterpret_cast<uintptr_t>(__shared_state_get_refcount()));
       const uint32_t __elected_id = ::__ffs(__match_mask) - 1;
       __elected                   = (::cuda::ptx::get_sreg_laneid() == __elected_id);
-      __sub_count                 = ::__popc(__match_mask);
+      __sub_count                 = ::cuda::std::popcount(__match_mask);
       , __elected = true;
       __sub_count = 1;)
     bool __released = false;
@@ -301,7 +302,7 @@ make_pipeline(const _Group& __group, pipeline_shared_state<_Scope, _Stages_count
         ::__match_any_sync(::__activemask(), reinterpret_cast<uintptr_t>(&__shared_state->__refcount));
       const uint32_t __elected_id = ::__ffs(__match_mask) - 1;
       __elected                   = (::cuda::ptx::get_sreg_laneid() == __elected_id);
-      __add_count                 = ::__popc(__match_mask);
+      __add_count                 = ::cuda::std::popcount(__match_mask);
       , __elected = true;
       __add_count = 1;)
     if (__elected)

cuda/cccl/headers/include/cuda/ptx

@@ -114,6 +114,7 @@
 #include <cuda/__ptx/instructions/tensormap_cp_fenceproxy.h>
 #include <cuda/__ptx/instructions/tensormap_replace.h>
 #include <cuda/__ptx/instructions/trap.h>
+#include <cuda/__ptx/pragmas/enable_smem_spilling.h>
 
 #include <cuda/std/__cccl/prologue.h>
 

cuda/cccl/headers/include/cuda/std/__algorithm/unique_copy.h

@@ -35,14 +35,12 @@ _CCCL_BEGIN_NAMESPACE_CUDA_STD
 
 namespace __unique_copy_tags
 {
-
 struct __reread_from_input_tag
 {};
 struct __reread_from_output_tag
 {};
 struct __read_from_tmp_value_tag
 {};
-
 } // namespace __unique_copy_tags
 
 _CCCL_EXEC_CHECK_DISABLE

cuda/cccl/headers/include/cuda/std/__atomic/api/reference.h

@@ -24,7 +24,7 @@
 #include <cuda/std/__atomic/api/common.h>
 #include <cuda/std/__atomic/order.h>
 #include <cuda/std/__atomic/scopes.h>
-#include <cuda/std/__atomic/types/reference.h>
+#include <cuda/std/__atomic/types.h>
 #include <cuda/std/__atomic/wait/notify_wait.h>
 #include <cuda/std/__atomic/wait/polling.h>
 #include <cuda/std/__type_traits/conditional.h>

cuda/cccl/headers/include/cuda/std/__atomic/functions/cuda_ptx_derived.h

@@ -22,6 +22,7 @@
 #endif // no system header
 
 #include <cuda/std/__atomic/functions/cuda_ptx_generated.h>
+#include <cuda/std/__functional/operations.h>
 #include <cuda/std/__type_traits/conditional.h>
 #include <cuda/std/__type_traits/enable_if.h>
 #include <cuda/std/__type_traits/is_scalar.h>
@@ -147,6 +148,45 @@ _CCCL_DEVICE _Type __cuda_atomic_fetch_update(_Type* __ptr, const _Fn& __op, _Or
   }
 }
 
+template <class _Type, template <class> class _Op>
+struct __cuda_atomic_op_bind
+{
+  _Type __val;
+
+  [[nodiscard]] _CCCL_DEVICE _Type operator()(_Type __old) const
+  {
+    return _Op<_Type>{}(__val, __old);
+  }
+};
+
+template <class _Type>
+struct __cuda_atomic_op_store
+{
+  // Just return first value
+  [[nodiscard]] _CCCL_DEVICE _Type operator()(_Type __val, _Type) const
+  {
+    return __val;
+  }
+};
+
+template <class _Type>
+struct __cuda_atomic_op_fetch_min
+{
+  [[nodiscard]] _CCCL_DEVICE _Type operator()(_Type __op, _Type __old) const
+  {
+    return __op < __old ? __op : __old;
+  }
+};
+
+template <class _Type>
+struct __cuda_atomic_op_fetch_max
+{
+  [[nodiscard]] _CCCL_DEVICE _Type operator()(_Type __op, _Type __old) const
+  {
+    return __old < __op ? __op : __old;
+  }
+};
+
 // Optimized fetch_update CAS loop with op determined after first load reducing waste.
 template <class _Type,
           class _Fn,
@@ -179,9 +219,7 @@ __cuda_atomic_store(_Type* __ptr, _Type __val, _Order, _Operand, _Sco, __atomic_
   // Store requires cas on 8/16b types
   __cuda_atomic_fetch_update(
     __ptr,
-    [__val](_Type __old) {
-      return __val;
-    },
+    __cuda_atomic_op_bind<_Type, ::cuda::std::__cuda_atomic_op_store>{__val},
     _Order{},
     __atomic_cuda_operand_tag<__atomic_cuda_operand::_b, _Operand::__size>{},
     _Sco{});
@@ -192,9 +230,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_add(_Type* __ptr, _Type& __d
 {
   __dst = __cuda_atomic_fetch_update(
     __ptr,
-    [__op](_Type __old) {
-      return __old + __op;
-    },
+    __cuda_atomic_op_bind<_Type, ::cuda::std::plus>{__op},
     _Order{},
     __atomic_cuda_operand_tag<__atomic_cuda_operand::_b, _Operand::__size>{},
     _Sco{});
@@ -205,9 +241,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_and(_Type* __ptr, _Type& __d
 {
   __dst = __cuda_atomic_fetch_update(
     __ptr,
-    [__op](_Type __old) {
-      return __old & __op;
-    },
+    __cuda_atomic_op_bind<_Type, ::cuda::std::bit_and>{__op},
     _Order{},
     __atomic_cuda_operand_tag<__atomic_cuda_operand::_b, _Operand::__size>{},
     _Sco{});
@@ -218,9 +252,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_xor(_Type* __ptr, _Type& __d
 {
   __dst = __cuda_atomic_fetch_update(
     __ptr,
-    [__op](_Type __old) {
-      return __old ^ __op;
-    },
+    __cuda_atomic_op_bind<_Type, ::cuda::std::bit_xor>{__op},
     _Order{},
     __atomic_cuda_operand_tag<__atomic_cuda_operand::_b, _Operand::__size>{},
     _Sco{});
@@ -231,9 +263,7 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_or(_Type* __ptr, _Type& __ds
 {
   __dst = __cuda_atomic_fetch_update(
     __ptr,
-    [__op](_Type __old) {
-      return __old | __op;
-    },
+    __cuda_atomic_op_bind<_Type, ::cuda::std::bit_or>{__op},
     _Order{},
     __atomic_cuda_operand_tag<__atomic_cuda_operand::_b, _Operand::__size>{},
     _Sco{});
@@ -244,21 +274,18 @@ static inline _CCCL_DEVICE void __cuda_atomic_fetch_min(_Type* __ptr, _Type& __d
 {
   __dst = __cuda_atomic_fetch_update(
     __ptr,
-    [__op](_Type __old) {
-      return __op < __old ? __op : __old;
-    },
+    __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_min>{__op},
     _Order{},
     __atomic_cuda_operand_tag<__atomic_cuda_operand::_b, _Operand::__size>{},
     _Sco{});
 }
+
 template <class _Type, class _Order, class _Operand, class _Sco, __cuda_atomic_enable_non_native_arithmetic<_Operand> = 0>
 static inline _CCCL_DEVICE void __cuda_atomic_fetch_max(_Type* __ptr, _Type& __dst, _Type __op, _Order, _Operand, _Sco)
 {
   __dst = __cuda_atomic_fetch_update(
     __ptr,
-    [__op](_Type __old) {
-      return __old < __op ? __op : __old;
-    },
+    __cuda_atomic_op_bind<_Type, __cuda_atomic_op_fetch_max>{__op},
     _Order{},
     __atomic_cuda_operand_tag<__atomic_cuda_operand::_b, _Operand::__size>{},
     _Sco{});
@@ -269,16 +296,15 @@ static inline _CCCL_DEVICE void __cuda_atomic_exchange(_Type* __ptr, _Type& __ds
 {
   __dst = __cuda_atomic_fetch_update(
     __ptr,
-    [__op](_Type __old) {
-      return __op;
-    },
+    __cuda_atomic_op_bind<_Type, ::cuda::std::__cuda_atomic_op_store>{__op},
     _Order{},
     __atomic_cuda_operand_tag<__atomic_cuda_operand::_b, _Operand::__size>{},
     _Sco{});
 }
 
 template <typename _Tp, typename _Fn, typename _Sco>
-_CCCL_DEVICE _Tp __atomic_fetch_update_cuda(_Tp* __ptr, const _Fn& __op, int __memorder, _Sco)
+[[nodiscard]] static inline _CCCL_DEVICE _Tp
+__atomic_fetch_update_cuda(_Tp* __ptr, const _Fn& __op, int __memorder, _Sco)
 {
   _Tp __expected = __atomic_load_n_cuda(__ptr, __ATOMIC_RELAXED, _Sco{});
   _Tp __desired  = __op(__expected);
@@ -289,7 +315,8 @@ _CCCL_DEVICE _Tp __atomic_fetch_update_cuda(_Tp* __ptr, const _Fn& __op, int __m
   return __expected;
 }
 template <typename _Tp, typename _Fn, typename _Sco>
-_CCCL_DEVICE _Tp __atomic_fetch_update_cuda(_Tp volatile* __ptr, const _Fn& __op, int __memorder, _Sco)
+[[nodiscard]] static inline _CCCL_DEVICE _Tp
+__atomic_fetch_update_cuda(_Tp volatile* __ptr, const _Fn& __op, int __memorder, _Sco)
 {
   _Tp __expected = __atomic_load_n_cuda(__ptr, __ATOMIC_RELAXED, _Sco{});
   _Tp __desired  = __op(__expected);
@@ -301,14 +328,14 @@ _CCCL_DEVICE _Tp __atomic_fetch_update_cuda(_Tp volatile* __ptr, const _Fn& __op
 }
 
 template <typename _Tp, typename _Sco>
-_CCCL_DEVICE _Tp __atomic_load_n_cuda(const _Tp* __ptr, int __memorder, _Sco)
+[[nodiscard]] static inline _CCCL_DEVICE _Tp __atomic_load_n_cuda(const _Tp* __ptr, int __memorder, _Sco)
 {
   _Tp __ret;
   __atomic_load_cuda(__ptr, __ret, __memorder, _Sco{});
   return __ret;
 }
 template <typename _Tp, typename _Sco>
-_CCCL_DEVICE _Tp __atomic_load_n_cuda(const _Tp volatile* __ptr, int __memorder, _Sco)
+[[nodiscard]] static inline _CCCL_DEVICE _Tp __atomic_load_n_cuda(const _Tp volatile* __ptr, int __memorder, _Sco)
 {
   _Tp __ret;
   __atomic_load_cuda(__ptr, __ret, __memorder, _Sco{});
@@ -327,64 +354,94 @@ _CCCL_DEVICE void __atomic_store_n_cuda(_Tp volatile* __ptr, _Tp __val, int __me
 }
 
 template <typename _Tp, typename _Sco>
-_CCCL_DEVICE _Tp __atomic_exchange_n_cuda(_Tp* __ptr, _Tp __val, int __memorder, _Sco)
+[[nodiscard]] static inline _CCCL_DEVICE _Tp __atomic_exchange_n_cuda(_Tp* __ptr, _Tp __val, int __memorder, _Sco)
 {
   _Tp __ret;
   __atomic_exchange_cuda(__ptr, __ret, __val, __memorder, _Sco{});
   return __ret;
 }
 template <typename _Tp, typename _Sco>
-_CCCL_DEVICE _Tp __atomic_exchange_n_cuda(_Tp volatile* __ptr, _Tp __val, int __memorder, _Sco)
+[[nodiscard]] static inline _CCCL_DEVICE _Tp
+__atomic_exchange_n_cuda(_Tp volatile* __ptr, _Tp __val, int __memorder, _Sco)
 {
   _Tp __ret;
   __atomic_exchange_cuda(__ptr, __ret, __val, __memorder, _Sco{});
   return __ret;
 }
 
+template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_arithmetic<_Tp> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Tp __atomic_fetch_add_cuda(_Tp* __ptr, _Up __val, int __memorder, _Sco)
+{
+  return __atomic_fetch_update_cuda(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::plus>{__val}, __memorder, _Sco{});
+}
+template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_arithmetic<_Tp> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Tp
+__atomic_fetch_add_cuda(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco)
+{
+  return __atomic_fetch_update_cuda(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::plus>{__val}, __memorder, _Sco{});
+}
+
+template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_bitwise<_Tp> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Tp __atomic_fetch_and_cuda(_Tp* __ptr, _Up __val, int __memorder, _Sco)
+{
+  return __atomic_fetch_update_cuda(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_and>{__val}, __memorder, _Sco{});
+}
+template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_bitwise<_Tp> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Tp
+__atomic_fetch_and_cuda(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco)
+{
+  return __atomic_fetch_update_cuda(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_and>{__val}, __memorder, _Sco{});
+}
+
+template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_bitwise<_Tp> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Tp __atomic_fetch_or_cuda(_Tp* __ptr, _Up __val, int __memorder, _Sco)
+{
+  return __atomic_fetch_update_cuda(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_or>{__val}, __memorder, _Sco{});
+}
+template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_bitwise<_Tp> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Tp __atomic_fetch_or_cuda(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco)
+{
+  return __atomic_fetch_update_cuda(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_or>{__val}, __memorder, _Sco{});
+}
+
+template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_bitwise<_Tp> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Tp __atomic_fetch_xor_cuda(_Tp* __ptr, _Up __val, int __memorder, _Sco)
+{
+  return __atomic_fetch_update_cuda(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_xor>{__val}, __memorder, _Sco{});
+}
+template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_bitwise<_Tp> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Tp
+__atomic_fetch_xor_cuda(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco)
+{
+  return __atomic_fetch_update_cuda(__ptr, __cuda_atomic_op_bind<_Tp, ::cuda::std::bit_xor>{__val}, __memorder, _Sco{});
+}
+
 template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_minmax<_Tp> = 0>
-_CCCL_DEVICE _Tp __atomic_fetch_min_cuda(_Tp* __ptr, _Up __val, int __memorder, _Sco)
+[[nodiscard]] static inline _CCCL_DEVICE _Tp __atomic_fetch_min_cuda(_Tp* __ptr, _Up __val, int __memorder, _Sco)
 {
   return __atomic_fetch_update_cuda(
-    __ptr,
-    [__val](_Tp __old) {
-      return __val < __old ? __val : __old;
-    },
-    __memorder,
-    _Sco{});
+    __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_min>{__val}, __memorder, _Sco{});
 }
 template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_minmax<_Tp> = 0>
-_CCCL_DEVICE _Tp __atomic_fetch_min_cuda(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco)
+[[nodiscard]] static inline _CCCL_DEVICE _Tp
+__atomic_fetch_min_cuda(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco)
 {
   return __atomic_fetch_update_cuda(
-    __ptr,
-    [__val](_Tp __old) {
-      return __val < __old ? __val : __old;
-    },
-    __memorder,
-    _Sco{});
+    __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_min>{__val}, __memorder, _Sco{});
 }
 
 template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_minmax<_Tp> = 0>
-_CCCL_DEVICE _Tp __atomic_fetch_max_cuda(_Tp* __ptr, _Up __val, int __memorder, _Sco)
+[[nodiscard]] static inline _CCCL_DEVICE _Tp __atomic_fetch_max_cuda(_Tp* __ptr, _Up __val, int __memorder, _Sco)
 {
   return __atomic_fetch_update_cuda(
-    __ptr,
-    [__val](_Tp __old) {
-      return __old < __val ? __val : __old;
-    },
-    __memorder,
-    _Sco{});
+    __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_max>{__val}, __memorder, _Sco{});
 }
 template <typename _Tp, typename _Up, typename _Sco, __atomic_enable_if_not_native_minmax<_Tp> = 0>
-_CCCL_DEVICE _Tp __atomic_fetch_max_cuda(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco)
+[[nodiscard]] static inline _CCCL_DEVICE _Tp
+__atomic_fetch_max_cuda(volatile _Tp* __ptr, _Up __val, int __memorder, _Sco)
 {
   return __atomic_fetch_update_cuda(
-    __ptr,
-    [__val](_Tp __old) {
-      return __old < __val ? __val : __old;
-    },
-    __memorder,
-    _Sco{});
+    __ptr, __cuda_atomic_op_bind<_Tp, __cuda_atomic_op_fetch_max>{__val}, __memorder, _Sco{});
 }
 
 _CCCL_DEVICE static inline void __atomic_signal_fence_cuda(int)