cuda-cccl 0.3.1__cp311-cp311-manylinux_2_24_aarch64.whl → 0.3.2__cp311-cp311-manylinux_2_24_aarch64.whl

cuda/cccl/headers/include/thrust/detail/allocator/fill_construct_range.h

@@ -26,14 +26,56 @@
 #  pragma system_header
 #endif // no system header
 
+#include <thrust/detail/allocator/allocator_traits.h>
+#include <thrust/detail/type_traits.h>
+#include <thrust/detail/type_traits/pointer_traits.h>
+#include <thrust/for_each.h>
+#include <thrust/uninitialized_fill.h>
+
+#include <cuda/std/__cccl/memory_wrapper.h>
+
 THRUST_NAMESPACE_BEGIN
 namespace detail
 {
 
+// fill_construct_range has 2 cases:
+// if Allocator has an effectful member function construct:
+//   1. construct via the allocator
+// else
+//   2. construct via uninitialized_fill
+
+template <typename Allocator, typename T, typename Arg1>
+inline constexpr bool has_effectful_member_construct2 =
+  allocator_traits_detail::has_member_construct2<Allocator, T, Arg1>::value;
+
+// std::allocator::construct's only effect is to invoke placement new
+template <typename U, typename T, typename Arg1>
+inline constexpr bool has_effectful_member_construct2<std::allocator<U>, T, Arg1> = false;
+
+template <typename Allocator, typename Arg1>
+struct construct2_via_allocator
+{
+  Allocator& a;
+  Arg1 arg;
+
+  template <typename T>
+  inline _CCCL_HOST_DEVICE void operator()(T& x)
+  {
+    allocator_traits<Allocator>::construct(a, &x, arg);
+  }
+};
 template <typename Allocator, typename Pointer, typename Size, typename T>
-_CCCL_HOST_DEVICE inline void fill_construct_range(Allocator& a, Pointer p, Size n, const T& value);
+_CCCL_HOST_DEVICE void fill_construct_range(Allocator& a, Pointer p, Size n, const T& value)
+{
+  if constexpr (has_effectful_member_construct2<Allocator, typename pointer_element<Pointer>::type, T>)
+  {
+    thrust::for_each_n(allocator_system<Allocator>::get(a), p, n, construct2_via_allocator<Allocator, T>{a, value});
+  }
+  else
+  {
+    thrust::uninitialized_fill_n(allocator_system<Allocator>::get(a), p, n, value);
+  }
+}
 
 } // namespace detail
 THRUST_NAMESPACE_END
-
-#include <thrust/detail/allocator/fill_construct_range.inl>

cuda/cccl/headers/include/thrust/detail/allocator/malloc_allocator.h

@@ -25,29 +25,54 @@
 #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
 #  pragma system_header
 #endif // no system header
+
+#include <thrust/detail/allocator/malloc_allocator.h>
 #include <thrust/detail/allocator/tagged_allocator.h>
+#include <thrust/detail/malloc_and_free.h>
+#include <thrust/detail/raw_pointer_cast.h>
 #include <thrust/detail/type_traits/pointer_traits.h>
+#include <thrust/system/detail/bad_alloc.h>
+#include <thrust/system/detail/generic/select_system.h>
 
 THRUST_NAMESPACE_BEGIN
 namespace detail
 {
 
 template <typename T, typename System, typename Pointer>
-class malloc_allocator : public thrust::detail::tagged_allocator<T, System, Pointer>
+class malloc_allocator : public tagged_allocator<T, System, Pointer>
 {
 private:
-  using super_t = thrust::detail::tagged_allocator<T, System, Pointer>;
+  using super_t = tagged_allocator<T, System, Pointer>;
 
 public:
   using pointer   = typename super_t::pointer;
   using size_type = typename super_t::size_type;
 
-  pointer allocate(size_type cnt);
+  pointer allocate(size_type cnt)
+  {
+    using thrust::system::detail::generic::select_system;
+
+    // XXX should use a hypothetical thrust::static_pointer_cast here
+    System system;
+
+    pointer result = thrust::malloc<T>(select_system(system), cnt);
+
+    if (result.get() == 0)
+    {
+      throw thrust::system::detail::bad_alloc("malloc_allocator::allocate: malloc failed");
+    } // end if
 
-  void deallocate(pointer p, size_type n) noexcept;
+    return result;
+  }
+
+  void deallocate(pointer p, size_type n) noexcept
+  {
+    using thrust::system::detail::generic::select_system;
+
+    System system;
+    thrust::free(select_system(system), p);
+  }
 };
 
 } // namespace detail
 THRUST_NAMESPACE_END
-
-#include <thrust/detail/allocator/malloc_allocator.inl>

cuda/cccl/headers/include/thrust/detail/allocator/tagged_allocator.h

@@ -25,9 +25,12 @@
 #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
 #  pragma system_header
 #endif // no system header
+#include <thrust/detail/allocator/tagged_allocator.h>
 #include <thrust/detail/type_traits/pointer_traits.h>
 #include <thrust/iterator/iterator_traits.h>
 
+#include <cuda/std/limits>
+
 THRUST_NAMESPACE_BEGIN
 namespace detail
 {
@@ -72,31 +75,41 @@ public:
     using other = tagged_allocator<U, Tag, Pointer>;
   }; // end rebind
 
-  _CCCL_HOST_DEVICE inline tagged_allocator();
+  tagged_allocator() = default;
 
-  _CCCL_HOST_DEVICE inline tagged_allocator(const tagged_allocator&);
+  tagged_allocator(const tagged_allocator&) = default;
 
   template <typename U, typename OtherPointer>
-  _CCCL_HOST_DEVICE inline tagged_allocator(const tagged_allocator<U, Tag, OtherPointer>&);
+  _CCCL_HOST_DEVICE tagged_allocator(const tagged_allocator<U, Tag, OtherPointer>&)
+  {}
 
-  _CCCL_HOST_DEVICE inline ~tagged_allocator();
+  ~tagged_allocator() = default;
 
-  _CCCL_HOST_DEVICE pointer address(reference x) const;
+  _CCCL_HOST_DEVICE pointer address(reference x) const
+  {
+    return &x;
+  }
 
-  _CCCL_HOST_DEVICE const_pointer address(const_reference x) const;
+  _CCCL_HOST_DEVICE const_pointer address(const_reference x) const
+  {
+    return &x;
+  }
 
-  size_type max_size() const;
-};
+  size_type max_size() const
+  {
+    return (::cuda::std::numeric_limits<size_type>::max)() / sizeof(T);
+  }
 
-template <typename T1, typename Pointer1, typename T2, typename Pointer2, typename Tag>
-_CCCL_HOST_DEVICE bool
-operator==(const tagged_allocator<T1, Pointer1, Tag>&, const tagged_allocator<T2, Pointer2, Tag>&);
+  _CCCL_HOST_DEVICE friend bool operator==(const tagged_allocator&, const tagged_allocator&)
+  {
+    return true;
+  }
 
-template <typename T1, typename Pointer1, typename T2, typename Pointer2, typename Tag>
-_CCCL_HOST_DEVICE bool
-operator!=(const tagged_allocator<T1, Pointer1, Tag>&, const tagged_allocator<T2, Pointer2, Tag>&);
+  _CCCL_HOST_DEVICE friend bool operator!=(const tagged_allocator&, const tagged_allocator&)
+  {
+    return false;
+  }
+};
 
 } // namespace detail
 THRUST_NAMESPACE_END
-
-#include <thrust/detail/allocator/tagged_allocator.inl>

cuda/cccl/headers/include/thrust/detail/allocator/temporary_allocator.h

@@ -25,11 +25,23 @@
 #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
 #  pragma system_header
 #endif // no system header
+
 #include <thrust/detail/allocator/allocator_traits.h>
 #include <thrust/detail/allocator/tagged_allocator.h>
+#include <thrust/detail/allocator/temporary_allocator.h>
 #include <thrust/detail/execution_policy.h>
+#include <thrust/detail/temporary_buffer.h>
 #include <thrust/memory.h>
 #include <thrust/pair.h>
+#include <thrust/system/detail/bad_alloc.h>
+
+#include <cuda/std/cassert>
+
+#include <nv/target>
+
+#if _CCCL_CUDA_COMPILATION() && _CCCL_DEVICE_COMPILATION()
+#  include <thrust/system/cuda/detail/terminate.h>
+#endif // _CCCL_CUDA_COMPILATION() && _CCCL_DEVICE_COMPILATION()
 
 THRUST_NAMESPACE_BEGIN
 namespace detail
@@ -60,9 +72,36 @@ public:
       , m_system(thrust::detail::derived_cast(system))
   {}
 
-  _CCCL_HOST_DEVICE pointer allocate(size_type cnt);
+  _CCCL_HOST_DEVICE pointer allocate(size_type cnt)
+  {
+    pointer_and_size result = thrust::get_temporary_buffer<T>(system(), cnt);
+
+    // handle failure
+    if (result.second < cnt)
+    {
+      // deallocate and throw
+      // note that we pass cnt to deallocate, not a value derived from result.second
+      deallocate(result.first, cnt);
+
+#if _CCCL_CUDA_COMPILATION()
+      NV_IF_TARGET(
+        NV_IS_HOST,
+        (throw thrust::system::detail::bad_alloc("temporary_buffer::allocate: get_temporary_buffer failed");),
+        ( // NV_IS_DEVICE
+          thrust::system::cuda::detail::terminate_with_message("temporary_buffer::allocate: "
+                                                               "get_temporary_buffer failed");));
+#else
+      throw thrust::system::detail::bad_alloc("temporary_buffer::allocate: get_temporary_buffer failed");
+#endif
+    } // end if
 
-  _CCCL_HOST_DEVICE void deallocate(pointer p, size_type n) noexcept;
+    return result.first;
+  }
+
+  _CCCL_HOST_DEVICE void deallocate(pointer p, size_type n) noexcept
+  {
+    return thrust::return_temporary_buffer(system(), p, n);
+  }
 
   _CCCL_HOST_DEVICE inline System& system()
   {
@@ -75,5 +114,3 @@ private:
 
 } // namespace detail
 THRUST_NAMESPACE_END
-
-#include <thrust/detail/allocator/temporary_allocator.inl>

cuda/cccl/headers/include/thrust/detail/allocator/value_initialize_range.h

@@ -26,14 +26,52 @@
 #  pragma system_header
 #endif // no system header
 
+#include <thrust/detail/allocator/allocator_traits.h>
+#include <thrust/detail/type_traits.h>
+#include <thrust/detail/type_traits/pointer_traits.h>
+#include <thrust/for_each.h>
+#include <thrust/uninitialized_fill.h>
+
 THRUST_NAMESPACE_BEGIN
 namespace detail
 {
+template <typename Allocator>
+struct construct1_via_allocator
+{
+  Allocator& a;
 
-template <typename Allocator, typename Pointer, typename Size>
-_CCCL_HOST_DEVICE inline void value_initialize_range(Allocator& a, Pointer p, Size n);
+  template <typename T>
+  inline _CCCL_HOST_DEVICE void operator()(T& x)
+  {
+    allocator_traits<Allocator>::construct(a, &x);
+  }
+};
+
+// we need to construct T via the allocator if the Allocator does something interesting or if T's default constructor
+// does something interesting
+template <typename Allocator, typename T>
+inline constexpr bool needs_default_construct_via_allocator =
+  allocator_traits_detail::has_member_construct1<Allocator, T>::value
+  || !::cuda::std::is_trivially_default_constructible_v<T>;
 
+// we know that std::allocator::construct's only effect is to call T's
+// default constructor, so we needn't use it for default construction
+// unless T's constructor does something interesting
+template <typename U, typename T>
+inline constexpr bool needs_default_construct_via_allocator<std::allocator<U>, T> =
+  !::cuda::std::is_trivially_default_constructible_v<T>;
+
+template <typename Allocator, typename Pointer, typename Size>
+_CCCL_HOST_DEVICE void value_initialize_range(Allocator& a, Pointer p, Size n)
+{
+  if constexpr (needs_default_construct_via_allocator<Allocator, typename pointer_element<Pointer>::type>)
+  {
+    thrust::for_each_n(allocator_system<Allocator>::get(a), p, n, construct1_via_allocator<Allocator>{a});
+  }
+  else
+  {
+    thrust::uninitialized_fill_n(allocator_system<Allocator>::get(a), p, n, typename pointer_element<Pointer>::type());
+  }
+}
 } // namespace detail
 THRUST_NAMESPACE_END
-
-#include <thrust/detail/allocator/value_initialize_range.inl>

cuda/cccl/headers/include/thrust/detail/complex/ccosh.h

@@ -106,9 +106,9 @@ _CCCL_HOST_DEVICE inline thrust::complex<double> ccosh(const thrust::complex<dou
     else if (ix < 0x4096bbaa)
     {
       /* x < 1455: scale to avoid overflow */
-      thrust::complex<double> z_;
-      z_ = ldexp_cexp(thrust::complex<double>(fabs(x), y), -1);
-      return (thrust::complex<double>(z_.real(), z_.imag() * copysign(1.0, x)));
+      thrust::complex<double> z_ = ldexp_cexp(thrust::complex<double>(fabs(x), y), -1);
+      z_.imag(copysign(z_.imag(), x));
+      return z_;
     }
     else
     {

cuda/cccl/headers/include/thrust/detail/internal_functional.h

@@ -30,7 +30,6 @@
 #  pragma system_header
 #endif // no system header
 
-#include <thrust/detail/memory_wrapper.h> // for ::new
 #include <thrust/detail/raw_reference_cast.h>
 #include <thrust/detail/static_assert.h>
 #include <thrust/detail/type_traits.h>
@@ -43,6 +42,7 @@
 #include <cuda/__iterator/tabulate_output_iterator.h>
 #include <cuda/__iterator/transform_input_output_iterator.h>
 #include <cuda/__iterator/transform_output_iterator.h>
+#include <cuda/std/__cccl/memory_wrapper.h> // for ::new
 #include <cuda/std/type_traits>
 
 THRUST_NAMESPACE_BEGIN

cuda/cccl/headers/include/thrust/detail/memory_algorithms.h

@@ -18,10 +18,10 @@
 #  pragma system_header
 #endif // no system header
 #include <thrust/detail/allocator/allocator_traits.h>
-#include <thrust/detail/memory_wrapper.h>
 #include <thrust/detail/type_traits.h>
 #include <thrust/iterator/iterator_traits.h>
 
+#include <cuda/std/__cccl/memory_wrapper.h>
 #include <cuda/std/__memory/addressof.h>
 #include <cuda/std/utility>
 

cuda/cccl/headers/include/thrust/detail/temporary_array.h

@@ -45,10 +45,10 @@ THRUST_NAMESPACE_END
 #include <thrust/detail/allocator/no_throw_allocator.h>
 #include <thrust/detail/allocator/temporary_allocator.h>
 #include <thrust/detail/contiguous_storage.h>
-#include <thrust/detail/memory_wrapper.h>
 #include <thrust/iterator/detail/tagged_iterator.h>
 #include <thrust/iterator/iterator_traits.h>
 
+#include <cuda/std/__cccl/memory_wrapper.h>
 #include <cuda/std/type_traits>
 
 THRUST_NAMESPACE_BEGIN

cuda/cccl/headers/include/thrust/detail/type_traits.h

@@ -123,7 +123,7 @@ struct larger_type
 {};
 
 template <class F, class... Us>
-using invoke_result = ::cuda::std::__invoke_of<F, Us...>;
+using invoke_result = ::cuda::std::invoke_result<F, Us...>;
 
 template <class F, class... Us>
 using invoke_result_t = typename invoke_result<F, Us...>::type;

cuda/cccl/headers/include/thrust/device_delete.h

@@ -29,10 +29,22 @@
 #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
 #  pragma system_header
 #endif // no system header
+
+#include <thrust/detail/allocator/destroy_range.h>
+#include <thrust/device_delete.h>
+#include <thrust/device_free.h>
 #include <thrust/device_ptr.h>
+#include <thrust/execution_policy.h>
 
 THRUST_NAMESPACE_BEGIN
 
+namespace detail
+{
+// define an empty allocator class to use below
+struct device_delete_allocator
+{};
+} // namespace detail
+
 /*! \addtogroup memory_management Memory Management
  *  \{
  */
@@ -49,11 +61,14 @@ THRUST_NAMESPACE_BEGIN
  *  \see device_new
  */
 template <typename T>
-inline void device_delete(thrust::device_ptr<T> ptr, const size_t n = 1);
+inline void device_delete(thrust::device_ptr<T> ptr, const size_t n = 1)
+{
+  // we don't have an allocator, so there is no need to go through thrust::detail::destroy_range
+  thrust::for_each_n(device, ptr, n, detail::gozer{});
+  thrust::device_free(ptr);
+}
 
 /*! \} // memory_management
  */
 
 THRUST_NAMESPACE_END
-
-#include <thrust/detail/device_delete.inl>

cuda/cccl/headers/include/thrust/device_free.h

@@ -29,7 +29,12 @@
 #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
 #  pragma system_header
 #endif // no system header
+
+#include <thrust/detail/malloc_and_free.h>
+#include <thrust/device_free.h>
 #include <thrust/device_ptr.h>
+#include <thrust/iterator/iterator_traits.h>
+#include <thrust/system/detail/generic/select_system.h>
 
 THRUST_NAMESPACE_BEGIN
 
@@ -62,11 +67,19 @@ THRUST_NAMESPACE_BEGIN
  *  \see device_ptr
  *  \see device_malloc
  */
-inline void device_free(thrust::device_ptr<void> ptr);
+inline void device_free(thrust::device_ptr<void> ptr)
+{
+  using thrust::system::detail::generic::select_system;
+
+  using system = thrust::iterator_system<thrust::device_ptr<void>>::type;
+
+  // XXX lower to select_system(system) here
+  system s;
+
+  thrust::free(s, ptr);
+}
 
 /*! \} // memory_management
  */
 
 THRUST_NAMESPACE_END
-
-#include <thrust/detail/device_free.inl>

cuda/cccl/headers/include/thrust/device_new.h

@@ -30,10 +30,13 @@
 #  pragma system_header
 #endif // no system header
 
-// #include this for size_t
+#include <thrust/detail/allocator/value_initialize_range.h>
+#include <thrust/device_allocator.h>
+#include <thrust/device_malloc.h>
+#include <thrust/device_new.h>
 #include <thrust/device_ptr.h>
-
-#include <cuda/std/cstddef>
+#include <thrust/execution_policy.h>
+#include <thrust/uninitialized_fill.h>
 
 THRUST_NAMESPACE_BEGIN
 
@@ -55,7 +58,15 @@ THRUST_NAMESPACE_BEGIN
  *  \see device_ptr
  */
 template <typename T>
-device_ptr<T> device_new(device_ptr<void> p, const size_t n = 1);
+device_ptr<T> device_new(device_ptr<void> p, const size_t n = 1)
+{
+  auto* dev_ptr = static_cast<T*>(p.get());
+  // TODO(bgruber): ideally, we would have an thrust::uninitialized_default_construct. Until then, use vector's
+  // infrastructure
+  device_allocator<T> alloc; // not needed for allocation, just for construct() called in value_initialize_range()
+  detail::value_initialize_range(alloc, dev_ptr, n);
+  return device_ptr<T>{dev_ptr};
+}
 
 /*! \p device_new implements the placement new operator for types
  *  resident in device memory. \p device_new calls <tt>T</tt>'s copy
@@ -72,7 +83,15 @@ device_ptr<T> device_new(device_ptr<void> p, const size_t n = 1);
  *  \see fill
  */
 template <typename T>
-device_ptr<T> device_new(device_ptr<void> p, const T& exemplar, const size_t n = 1);
+device_ptr<T> device_new(device_ptr<void> p, const T& exemplar, const size_t n = 1)
+{
+  device_ptr<T> result(static_cast<T*>(p.get()));
+
+  // run copy constructors at p here
+  thrust::uninitialized_fill(device, result, result + n, exemplar);
+
+  return result;
+}
 
 /*! \p device_new implements the new operator for types resident in device memory.
  *  It allocates device memory large enough to hold \p n new objects of type \c T.
@@ -81,11 +100,13 @@ device_ptr<T> device_new(device_ptr<void> p, const T& exemplar, const size_t n =
  *  \return A \p device_ptr to the newly allocated region of device memory.
  */
 template <typename T>
-device_ptr<T> device_new(const size_t n = 1);
+device_ptr<T> device_new(const size_t n = 1)
+{
+  // call placement new version of device_new
+  return device_new<T>(thrust::device_malloc<T>(n));
+}
 
 /*! \} // memory_management
  */
 
 THRUST_NAMESPACE_END
-
-#include <thrust/detail/device_new.inl>

cuda/cccl/headers/include/thrust/host_vector.h

@@ -30,9 +30,9 @@
 #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
 #  pragma system_header
 #endif // no system header
-#include <thrust/detail/memory_wrapper.h>
 #include <thrust/detail/vector_base.h>
 
+#include <cuda/std/__cccl/memory_wrapper.h>
 #include <cuda/std/initializer_list>
 #include <cuda/std/utility>
 

cuda/cccl/headers/include/thrust/iterator/tabulate_output_iterator.h

@@ -104,7 +104,8 @@ inline constexpr bool is_proxy_reference_v<tabulate_output_iterator_proxy<Binary
 //!
 //! \see make_tabulate_output_iterator
 template <typename BinaryFunction, typename System = use_default, typename DifferenceT = ptrdiff_t>
-class tabulate_output_iterator : public detail::make_tabulate_output_iterator_base<BinaryFunction, System, DifferenceT>
+class CCCL_DEPRECATED_BECAUSE("Use cuda::tabulate_output_iterator instead") tabulate_output_iterator
+    : public detail::make_tabulate_output_iterator_base<BinaryFunction, System, DifferenceT>
 {
 public:
   //! \cond
@@ -138,7 +139,9 @@ private:
 //! \param fun The \c BinaryFunction invoked whenever assigning to a dereferenced \p tabulate_output_iterator
 //! \see tabulate_output_iterator
 template <typename BinaryFunction>
-tabulate_output_iterator<BinaryFunction> _CCCL_HOST_DEVICE make_tabulate_output_iterator(BinaryFunction fun)
+CCCL_DEPRECATED_BECAUSE("Use cuda::make_tabulate_output_iterator instead")
+_CCCL_HOST_DEVICE tabulate_output_iterator<BinaryFunction>
+make_tabulate_output_iterator(BinaryFunction fun)
 {
   return tabulate_output_iterator<BinaryFunction>(fun);
 }

cuda/cccl/headers/include/thrust/mr/disjoint_pool.h

@@ -34,7 +34,6 @@
 #include <thrust/detail/config.h>
 
 #include <thrust/binary_search.h>
-#include <thrust/detail/algorithm_wrapper.h>
 #include <thrust/detail/seq.h>
 #include <thrust/find.h>
 #include <thrust/host_vector.h>
@@ -44,6 +43,7 @@
 
 #include <cuda/std/__algorithm/max.h>
 #include <cuda/std/__algorithm/min.h>
+#include <cuda/std/__cccl/algorithm_wrapper.h>
 #include <cuda/std/cassert>
 #include <cuda/std/cstdint>
 

cuda/cccl/headers/include/thrust/mr/pool.h

@@ -32,12 +32,12 @@
 #  pragma system_header
 #endif // no system header
 
-#include <thrust/detail/algorithm_wrapper.h>
 #include <thrust/host_vector.h>
 #include <thrust/mr/allocator.h>
 #include <thrust/mr/memory_resource.h>
 #include <thrust/mr/pool_options.h>
 
+#include <cuda/std/__cccl/algorithm_wrapper.h>
 #include <cuda/std/cassert>
 #include <cuda/std/cstdint>