cuda-cccl 0.3.2__cp313-cp313-manylinux_2_24_aarch64.whl → 0.3.4__cp313-cp313-manylinux_2_24_aarch64.whl

cuda/cccl/headers/include/thrust/system/tbb/detail/malloc_and_free.h

@@ -1,18 +1,5 @@
-/*
- *  Copyright 2008-2013 NVIDIA Corporation
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
+// SPDX-FileCopyrightText: Copyright (c) 2008-2013, NVIDIA Corporation. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 #pragma once
 

cuda/cccl/headers/include/thrust/system/tbb/detail/merge.h

@@ -1,18 +1,5 @@
-/*
- *  Copyright 2008-2013 NVIDIA Corporation
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
+// SPDX-FileCopyrightText: Copyright (c) 2008-2021, NVIDIA Corporation. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 #pragma once
 
@@ -25,31 +12,250 @@
 #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
 #  pragma system_header
 #endif // no system header
+#include <thrust/binary_search.h>
+#include <thrust/detail/seq.h>
+#include <thrust/detail/temporary_array.h>
+#include <thrust/iterator/iterator_traits.h>
+#include <thrust/merge.h>
 #include <thrust/system/tbb/detail/execution_policy.h>
 
+#include <tbb/parallel_for.h>
+
 THRUST_NAMESPACE_BEGIN
-namespace system
+namespace system::tbb::detail
+{
+namespace merge_detail
+{
+template <typename InputIterator1, typename InputIterator2, typename OutputIterator, typename StrictWeakOrdering>
+struct range
+{
+  InputIterator1 first1, last1;
+  InputIterator2 first2, last2;
+  OutputIterator result;
+  StrictWeakOrdering comp;
+  size_t grain_size;
+
+  range(InputIterator1 first1,
+        InputIterator1 last1,
+        InputIterator2 first2,
+        InputIterator2 last2,
+        OutputIterator result,
+        StrictWeakOrdering comp,
+        size_t grain_size = 1024)
+      : first1(first1)
+      , last1(last1)
+      , first2(first2)
+      , last2(last2)
+      , result(result)
+      , comp(comp)
+      , grain_size(grain_size)
+  {}
+
+  range(range& r, ::tbb::split)
+      : first1(r.first1)
+      , last1(r.last1)
+      , first2(r.first2)
+      , last2(r.last2)
+      , result(r.result)
+      , comp(r.comp)
+      , grain_size(r.grain_size)
+  {
+    // we can assume n1 and n2 are not both 0
+    size_t n1 = ::cuda::std::distance(first1, last1);
+    size_t n2 = ::cuda::std::distance(first2, last2);
+
+    InputIterator1 mid1 = first1;
+    InputIterator2 mid2 = first2;
+
+    if (n1 > n2)
+    {
+      mid1 += n1 / 2;
+      mid2 = thrust::lower_bound(thrust::seq, first2, last2, raw_reference_cast(*mid1), comp);
+    }
+    else
+    {
+      mid2 += n2 / 2;
+      mid1 = thrust::upper_bound(thrust::seq, first1, last1, raw_reference_cast(*mid2), comp);
+    }
+
+    // set first range to [first1, mid1), [first2, mid2), result
+    r.last1 = mid1;
+    r.last2 = mid2;
+
+    // set second range to [mid1, last1), [mid2, last2), result + (mid1 - first1) + (mid2 - first2)
+    first1 = mid1;
+    first2 = mid2;
+    result += ::cuda::std::distance(r.first1, mid1) + ::cuda::std::distance(r.first2, mid2);
+  }
+
+  bool empty() const
+  {
+    return (first1 == last1) && (first2 == last2);
+  }
+
+  bool is_divisible() const
+  {
+    return static_cast<size_t>(::cuda::std::distance(first1, last1) + ::cuda::std::distance(first2, last2))
+         > grain_size;
+  }
+};
+
+struct body
 {
-namespace tbb
+  template <typename Range>
+  void operator()(Range& r) const
+  {
+    thrust::merge(thrust::seq, r.first1, r.last1, r.first2, r.last2, r.result, r.comp);
+  }
+};
+} // end namespace merge_detail
+
+namespace merge_by_key_detail
 {
-namespace detail
+template <typename InputIterator1,
+          typename InputIterator2,
+          typename InputIterator3,
+          typename InputIterator4,
+          typename OutputIterator1,
+          typename OutputIterator2,
+          typename StrictWeakOrdering>
+struct range
 {
+  InputIterator1 keys_first1, keys_last1;
+  InputIterator2 keys_first2, keys_last2;
+  InputIterator3 values_first1;
+  InputIterator4 values_first2;
+  OutputIterator1 keys_result;
+  OutputIterator2 values_result;
+  StrictWeakOrdering comp;
+  size_t grain_size;
 
-template <typename ExecutionPolicy,
+  range(InputIterator1 keys_first1,
+        InputIterator1 keys_last1,
+        InputIterator2 keys_first2,
+        InputIterator2 keys_last2,
+        InputIterator3 values_first1,
+        InputIterator4 values_first2,
+        OutputIterator1 keys_result,
+        OutputIterator2 values_result,
+        StrictWeakOrdering comp,
+        size_t grain_size = 1024)
+      : keys_first1(keys_first1)
+      , keys_last1(keys_last1)
+      , keys_first2(keys_first2)
+      , keys_last2(keys_last2)
+      , values_first1(values_first1)
+      , values_first2(values_first2)
+      , keys_result(keys_result)
+      , values_result(values_result)
+      , comp(comp)
+      , grain_size(grain_size)
+  {}
+
+  range(range& r, ::tbb::split)
+      : keys_first1(r.keys_first1)
+      , keys_last1(r.keys_last1)
+      , keys_first2(r.keys_first2)
+      , keys_last2(r.keys_last2)
+      , values_first1(r.values_first1)
+      , values_first2(r.values_first2)
+      , keys_result(r.keys_result)
+      , values_result(r.values_result)
+      , comp(r.comp)
+      , grain_size(r.grain_size)
+  {
+    // we can assume n1 and n2 are not both 0
+    size_t n1 = ::cuda::std::distance(keys_first1, keys_last1);
+    size_t n2 = ::cuda::std::distance(keys_first2, keys_last2);
+
+    InputIterator1 mid1 = keys_first1;
+    InputIterator2 mid2 = keys_first2;
+
+    if (n1 > n2)
+    {
+      mid1 += n1 / 2;
+      mid2 = thrust::lower_bound(thrust::seq, keys_first2, keys_last2, raw_reference_cast(*mid1), comp);
+    }
+    else
+    {
+      mid2 += n2 / 2;
+      mid1 = thrust::upper_bound(thrust::seq, keys_first1, keys_last1, raw_reference_cast(*mid2), comp);
+    }
+
+    // set first range to [keys_first1, mid1), [keys_first2, mid2), keys_result, values_result
+    r.keys_last1 = mid1;
+    r.keys_last2 = mid2;
+
+    // set second range to [mid1, keys_last1), [mid2, keys_last2), keys_result + (mid1 - keys_first1) + (mid2 -
+    // keys_first2), values_result + (mid1 - keys_first1) + (mid2 - keys_first2)
+    keys_first1 = mid1;
+    keys_first2 = mid2;
+    values_first1 += ::cuda::std::distance(r.keys_first1, mid1);
+    values_first2 += ::cuda::std::distance(r.keys_first2, mid2);
+    keys_result += ::cuda::std::distance(r.keys_first1, mid1) + ::cuda::std::distance(r.keys_first2, mid2);
+    values_result += ::cuda::std::distance(r.keys_first1, mid1) + ::cuda::std::distance(r.keys_first2, mid2);
+  }
+
+  bool empty() const
+  {
+    return (keys_first1 == keys_last1) && (keys_first2 == keys_last2);
+  }
+
+  bool is_divisible() const
+  {
+    return static_cast<size_t>(
+             ::cuda::std::distance(keys_first1, keys_last1) + ::cuda::std::distance(keys_first2, keys_last2))
+         > grain_size;
+  }
+};
+
+struct body
+{
+  template <typename Range>
+  void operator()(Range& r) const
+  {
+    thrust::merge_by_key(
+      thrust::seq,
+      r.keys_first1,
+      r.keys_last1,
+      r.keys_first2,
+      r.keys_last2,
+      r.values_first1,
+      r.values_first2,
+      r.keys_result,
+      r.values_result,
+      r.comp);
+  }
+};
+} // end namespace merge_by_key_detail
+
+template <typename DerivedPolicy,
           typename InputIterator1,
           typename InputIterator2,
           typename OutputIterator,
           typename StrictWeakOrdering>
 OutputIterator
-merge(execution_policy<ExecutionPolicy>& exec,
+merge(execution_policy<DerivedPolicy>&,
       InputIterator1 first1,
       InputIterator1 last1,
       InputIterator2 first2,
       InputIterator2 last2,
       OutputIterator result,
-      StrictWeakOrdering comp);
+      StrictWeakOrdering comp)
+{
+  using Range = typename merge_detail::range<InputIterator1, InputIterator2, OutputIterator, StrictWeakOrdering>;
+  using Body  = merge_detail::body;
+  Range range(first1, last1, first2, last2, result, comp);
+  Body body;
+
+  ::tbb::parallel_for(range, body);
+
+  ::cuda::std::advance(result, ::cuda::std::distance(first1, last1) + ::cuda::std::distance(first2, last2));
 
-template <typename ExecutionPolicy,
+  return result;
+} // end merge()
+
+template <typename DerivedPolicy,
           typename InputIterator1,
           typename InputIterator2,
           typename InputIterator3,
@@ -58,7 +264,7 @@ template <typename ExecutionPolicy,
           typename OutputIterator2,
           typename StrictWeakOrdering>
 thrust::pair<OutputIterator1, OutputIterator2> merge_by_key(
-  execution_policy<ExecutionPolicy>& exec,
+  execution_policy<DerivedPolicy>&,
   InputIterator1 keys_first1,
   InputIterator1 keys_last1,
   InputIterator2 keys_first2,
@@ -67,11 +273,30 @@ thrust::pair<OutputIterator1, OutputIterator2> merge_by_key(
   InputIterator4 values_first4,
   OutputIterator1 keys_result,
   OutputIterator2 values_result,
-  StrictWeakOrdering comp);
+  StrictWeakOrdering comp)
+{
+  using Range = typename merge_by_key_detail::range<
+    InputIterator1,
+    InputIterator2,
+    InputIterator3,
+    InputIterator4,
+    OutputIterator1,
+    OutputIterator2,
+    StrictWeakOrdering>;
+  using Body = merge_by_key_detail::body;
 
-} // namespace detail
-} // namespace tbb
-} // namespace system
-THRUST_NAMESPACE_END
+  Range range(
+    keys_first1, keys_last1, keys_first2, keys_last2, values_first3, values_first4, keys_result, values_result, comp);
+  Body body;
+
+  ::tbb::parallel_for(range, body);
 
-#include <thrust/system/tbb/detail/merge.inl>
+  ::cuda::std::advance(keys_result,
+                       ::cuda::std::distance(keys_first1, keys_last1) + ::cuda::std::distance(keys_first2, keys_last2));
+  ::cuda::std::advance(values_result,
+                       ::cuda::std::distance(keys_first1, keys_last1) + ::cuda::std::distance(keys_first2, keys_last2));
+
+  return thrust::make_pair(keys_result, values_result);
+}
+} // end namespace system::tbb::detail
+THRUST_NAMESPACE_END

cuda/cccl/headers/include/thrust/system/tbb/detail/mismatch.h

@@ -1,18 +1,5 @@
-/*
- *  Copyright 2008-2013 NVIDIA Corporation
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
+// SPDX-FileCopyrightText: Copyright (c) 2008-2013, NVIDIA Corporation. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 #pragma once
 

cuda/cccl/headers/include/thrust/system/tbb/detail/partition.h

@@ -1,18 +1,5 @@
-/*
- *  Copyright 2008-2013 NVIDIA Corporation
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
+// SPDX-FileCopyrightText: Copyright (c) 2008-2013, NVIDIA Corporation. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 #pragma once
 
@@ -26,19 +13,19 @@
 #  pragma system_header
 #endif // no system header
 #include <thrust/pair.h>
+#include <thrust/system/detail/generic/partition.h>
 #include <thrust/system/tbb/detail/execution_policy.h>
 
 THRUST_NAMESPACE_BEGIN
-namespace system
+namespace system::tbb::detail
 {
-namespace tbb
-{
-namespace detail
-{
-
 template <typename DerivedPolicy, typename ForwardIterator, typename Predicate>
 ForwardIterator
-stable_partition(execution_policy<DerivedPolicy>& exec, ForwardIterator first, ForwardIterator last, Predicate pred);
+stable_partition(execution_policy<DerivedPolicy>& exec, ForwardIterator first, ForwardIterator last, Predicate pred)
+{
+  // tbb prefers generic::stable_partition to cpp::stable_partition
+  return thrust::system::detail::generic::stable_partition(exec, first, last, pred);
+} // end stable_partition()
 
 template <typename DerivedPolicy, typename ForwardIterator, typename InputIterator, typename Predicate>
 ForwardIterator stable_partition(
@@ -46,7 +33,11 @@ ForwardIterator stable_partition(
   ForwardIterator first,
   ForwardIterator last,
   InputIterator stencil,
-  Predicate pred);
+  Predicate pred)
+{
+  // tbb prefers generic::stable_partition to cpp::stable_partition
+  return thrust::system::detail::generic::stable_partition(exec, first, last, stencil, pred);
+} // end stable_partition()
 
 template <typename DerivedPolicy,
           typename InputIterator,
@@ -59,7 +50,11 @@ thrust::pair<OutputIterator1, OutputIterator2> stable_partition_copy(
   InputIterator last,
   OutputIterator1 out_true,
   OutputIterator2 out_false,
-  Predicate pred);
+  Predicate pred)
+{
+  // tbb prefers generic::stable_partition_copy to cpp::stable_partition_copy
+  return thrust::system::detail::generic::stable_partition_copy(exec, first, last, out_true, out_false, pred);
+} // end stable_partition_copy()
 
 template <typename DerivedPolicy,
           typename InputIterator1,
@@ -74,11 +69,10 @@ thrust::pair<OutputIterator1, OutputIterator2> stable_partition_copy(
   InputIterator2 stencil,
   OutputIterator1 out_true,
   OutputIterator2 out_false,
-  Predicate pred);
-
-} // end namespace detail
-} // end namespace tbb
-} // end namespace system
+  Predicate pred)
+{
+  // tbb prefers generic::stable_partition_copy to cpp::stable_partition_copy
+  return thrust::system::detail::generic::stable_partition_copy(exec, first, last, stencil, out_true, out_false, pred);
+} // end stable_partition_copy()
+} // end namespace system::tbb::detail
 THRUST_NAMESPACE_END
-
-#include <thrust/system/tbb/detail/partition.inl>

cuda/cccl/headers/include/thrust/system/tbb/detail/per_device_resource.h

@@ -1,18 +1,5 @@
-/*
- *  Copyright 2018 NVIDIA Corporation
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
+// SPDX-FileCopyrightText: Copyright (c) 2018, NVIDIA Corporation. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 #pragma once
 

cuda/cccl/headers/include/thrust/system/tbb/detail/reduce.h

@@ -1,18 +1,5 @@
-/*
- *  Copyright 2008-2013 NVIDIA Corporation
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
+// SPDX-FileCopyrightText: Copyright (c) 2008-2013, NVIDIA Corporation. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
 
 /*! \file reduce.h
  *  \brief TBB implementation of reduce.
@@ -29,26 +16,105 @@
 #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
 #  pragma system_header
 #endif // no system header
+#include <thrust/detail/function.h>
+#include <thrust/detail/static_assert.h>
+#include <thrust/distance.h>
+#include <thrust/iterator/iterator_traits.h>
+#include <thrust/reduce.h>
 #include <thrust/system/tbb/detail/execution_policy.h>
 
+#include <tbb/blocked_range.h>
+#include <tbb/parallel_reduce.h>
+
 THRUST_NAMESPACE_BEGIN
-namespace system
+namespace system::tbb::detail
 {
-namespace tbb
+namespace reduce_detail
 {
-namespace detail
+template <typename RandomAccessIterator, typename OutputType, typename BinaryFunction>
+struct body
 {
+  RandomAccessIterator first;
+  OutputType sum;
+  bool first_call; // TBB can invoke operator() multiple times on the same body
+  thrust::detail::wrapped_function<BinaryFunction, OutputType> binary_op;
+
+  // note: we only initialize sum with init to avoid calling OutputType's default constructor
+  body(RandomAccessIterator first, OutputType init, BinaryFunction binary_op)
+      : first(first)
+      , sum(init)
+      , first_call(true)
+      , binary_op{binary_op}
+  {}
+
+  // note: we only initialize sum with b.sum to avoid calling OutputType's default constructor
+  body(body& b, ::tbb::split)
+      : first(b.first)
+      , sum(b.sum)
+      , first_call(true)
+      , binary_op{b.binary_op}
+  {}
+
+  template <typename Size>
+  void operator()(const ::tbb::blocked_range<Size>& r)
+  {
+    // we assume that blocked_range specifies a contiguous range of integers
+
+    if (r.empty())
+    {
+      return; // nothing to do
+    }
+
+    RandomAccessIterator iter = first + r.begin();
+
+    OutputType temp = thrust::raw_reference_cast(*iter);
+
+    ++iter;
+
+    for (Size i = r.begin() + 1; i != r.end(); ++i, ++iter)
+    {
+      temp = binary_op(temp, *iter);
+    }
+
+    if (first_call)
+    {
+      // first time body has been invoked
+      first_call = false;
+      sum        = temp;
+    }
+    else
+    {
+      // body has been previously invoked, accumulate temp into sum
+      sum = binary_op(sum, temp);
+    }
+  } // end operator()()
+
+  void join(body& b)
+  {
+    sum = binary_op(sum, b.sum);
+  }
+}; // end body
+} // namespace reduce_detail
 
 template <typename DerivedPolicy, typename InputIterator, typename OutputType, typename BinaryFunction>
-OutputType reduce(execution_policy<DerivedPolicy>& exec,
-                  InputIterator begin,
-                  InputIterator end,
-                  OutputType init,
-                  BinaryFunction binary_op);
-
-} // end namespace detail
-} // end namespace tbb
-} // end namespace system
-THRUST_NAMESPACE_END
+OutputType reduce(
+  execution_policy<DerivedPolicy>&, InputIterator begin, InputIterator end, OutputType init, BinaryFunction binary_op)
+{
+  using Size = thrust::detail::it_difference_t<InputIterator>;
 
-#include <thrust/system/tbb/detail/reduce.inl>
+  Size n = ::cuda::std::distance(begin, end);
+
+  if (n == 0)
+  {
+    return init;
+  }
+  else
+  {
+    using Body = typename reduce_detail::body<InputIterator, OutputType, BinaryFunction>;
+    Body reduce_body(begin, init, binary_op);
+    ::tbb::parallel_reduce(::tbb::blocked_range<Size>(0, n), reduce_body);
+    return binary_op(init, reduce_body.sum);
+  }
+}
+} // end namespace system::tbb::detail
+THRUST_NAMESPACE_END