warp-lang 1.7.0__py3-none-manylinux_2_34_aarch64.whl

warp/native/sort.cu

@@ -0,0 +1,277 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "warp.h"
+#include "cuda_util.h"
+#include "sort.h"
+
+#define THRUST_IGNORE_CUB_VERSION_CHECK
+
+#include <cub/cub.cuh>
+
+#include <map>
+
+// temporary buffer for radix sort
+struct RadixSortTemp
+{
+    void* mem = NULL;
+    size_t size = 0;
+};
+
+// map temp buffers to CUDA contexts
+static std::map<void*, RadixSortTemp> g_radix_sort_temp_map;
+
+
+template <typename KeyType>
+void radix_sort_reserve_internal(void* context, int n, void** mem_out, size_t* size_out)
+{
+    ContextGuard guard(context);
+
+    cub::DoubleBuffer<KeyType> d_keys;
+	cub::DoubleBuffer<int> d_values;
+
+    // compute temporary memory required
+	size_t sort_temp_size;
+    check_cuda(cub::DeviceRadixSort::SortPairs(
+        NULL,
+        sort_temp_size,
+        d_keys,
+        d_values,
+        n, 0, sizeof(KeyType)*8,
+        (cudaStream_t)cuda_stream_get_current()));
+
+    if (!context)
+        context = cuda_context_get_current();
+
+    RadixSortTemp& temp = g_radix_sort_temp_map[context];
+
+    if (sort_temp_size > temp.size)
+    {
+	    free_device(WP_CURRENT_CONTEXT, temp.mem);
+        temp.mem = alloc_device(WP_CURRENT_CONTEXT, sort_temp_size);
+        temp.size = sort_temp_size;
+    }
+    
+    if (mem_out)
+        *mem_out = temp.mem;
+    if (size_out)
+        *size_out = temp.size;
+}
+
+void radix_sort_reserve(void* context, int n, void** mem_out, size_t* size_out)
+{
+    radix_sort_reserve_internal<int>(context, n, mem_out, size_out);
+}
+
+template <typename KeyType>
+void radix_sort_pairs_device(void* context, KeyType* keys, int* values, int n)
+{
+    ContextGuard guard(context);
+
+    cub::DoubleBuffer<KeyType> d_keys(keys, keys + n);
+	cub::DoubleBuffer<int> d_values(values, values + n);
+
+    RadixSortTemp temp;
+    radix_sort_reserve_internal<KeyType>(WP_CURRENT_CONTEXT, n, &temp.mem, &temp.size);
+
+    // sort
+    check_cuda(cub::DeviceRadixSort::SortPairs(
+        temp.mem,
+        temp.size,
+        d_keys, 
+        d_values, 
+        n, 0, sizeof(KeyType)*8, 
+        (cudaStream_t)cuda_stream_get_current()));
+
+	if (d_keys.Current() != keys)
+		memcpy_d2d(WP_CURRENT_CONTEXT, keys, d_keys.Current(), sizeof(KeyType)*n);
+
+	if (d_values.Current() != values)
+		memcpy_d2d(WP_CURRENT_CONTEXT, values, d_values.Current(), sizeof(int)*n);
+}
+
+void radix_sort_pairs_device(void* context, int* keys, int* values, int n)
+{
+    radix_sort_pairs_device<int>(context, keys, values, n);
+}
+
+void radix_sort_pairs_device(void* context, float* keys, int* values, int n)
+{
+    radix_sort_pairs_device<float>(context, keys, values, n);
+}
+
+void radix_sort_pairs_device(void* context, int64_t* keys, int* values, int n)
+{
+    radix_sort_pairs_device<int64_t>(context, keys, values, n);
+}
+
+void radix_sort_pairs_int_device(uint64_t keys, uint64_t values, int n)
+{
+    radix_sort_pairs_device(
+        WP_CURRENT_CONTEXT,
+        reinterpret_cast<int *>(keys),
+        reinterpret_cast<int *>(values), n);
+}
+
+void radix_sort_pairs_float_device(uint64_t keys, uint64_t values, int n)
+{
+    radix_sort_pairs_device(
+        WP_CURRENT_CONTEXT,
+        reinterpret_cast<float *>(keys),
+        reinterpret_cast<int *>(values), n);
+}
+
+void radix_sort_pairs_int64_device(uint64_t keys, uint64_t values, int n)
+{
+    radix_sort_pairs_device(
+        WP_CURRENT_CONTEXT,
+        reinterpret_cast<int64_t *>(keys),
+        reinterpret_cast<int *>(values), n);
+}
+
+void segmented_sort_reserve(void* context, int n, int num_segments, void** mem_out, size_t* size_out)
+{
+    ContextGuard guard(context);
+
+    cub::DoubleBuffer<int> d_keys;
+	cub::DoubleBuffer<int> d_values;
+
+    int* start_indices = NULL;
+    int* end_indices = NULL;
+
+    // compute temporary memory required
+	size_t sort_temp_size;
+    check_cuda(cub::DeviceSegmentedRadixSort::SortPairs(
+        NULL,
+        sort_temp_size,
+        d_keys,
+        d_values,
+        n, 
+        num_segments,
+        start_indices,
+        end_indices,
+        0,
+        32,
+        (cudaStream_t)cuda_stream_get_current()));
+
+    if (!context)
+        context = cuda_context_get_current();
+
+    RadixSortTemp& temp = g_radix_sort_temp_map[context];
+
+    if (sort_temp_size > temp.size)
+    {
+	    free_device(WP_CURRENT_CONTEXT, temp.mem);
+        temp.mem = alloc_device(WP_CURRENT_CONTEXT, sort_temp_size);
+        temp.size = sort_temp_size;
+    }
+    
+    if (mem_out)
+        *mem_out = temp.mem;
+    if (size_out)
+        *size_out = temp.size;
+}
+
+// segment_start_indices and segment_end_indices are arrays of length num_segments, where segment_start_indices[i] is the index of the first element 
+// in the i-th segment and segment_end_indices[i] is the index after the last element in the i-th segment
+// https://nvidia.github.io/cccl/cub/api/structcub_1_1DeviceSegmentedRadixSort.html
+void segmented_sort_pairs_device(void* context, float* keys, int* values, int n, int* segment_start_indices, int* segment_end_indices, int num_segments)
+{
+    ContextGuard guard(context);
+
+    cub::DoubleBuffer<float> d_keys(keys, keys + n);
+	cub::DoubleBuffer<int> d_values(values, values + n);
+
+    RadixSortTemp temp;
+    segmented_sort_reserve(WP_CURRENT_CONTEXT, n, num_segments, &temp.mem, &temp.size);
+
+    // sort
+    check_cuda(cub::DeviceSegmentedRadixSort::SortPairs(
+        temp.mem,
+        temp.size,
+        d_keys, 
+        d_values, 
+        n,
+        num_segments,
+        segment_start_indices,
+        segment_end_indices,
+        0,
+        32,
+        (cudaStream_t)cuda_stream_get_current()));
+
+	if (d_keys.Current() != keys)
+		memcpy_d2d(WP_CURRENT_CONTEXT, keys, d_keys.Current(), sizeof(float)*n);
+
+	if (d_values.Current() != values)
+		memcpy_d2d(WP_CURRENT_CONTEXT, values, d_values.Current(), sizeof(int)*n);
+}
+
+void segmented_sort_pairs_float_device(uint64_t keys, uint64_t values, int n, uint64_t segment_start_indices, uint64_t segment_end_indices, int num_segments)
+{
+    segmented_sort_pairs_device(
+        WP_CURRENT_CONTEXT,
+        reinterpret_cast<float *>(keys),
+        reinterpret_cast<int *>(values), n,
+        reinterpret_cast<int *>(segment_start_indices),
+        reinterpret_cast<int *>(segment_end_indices),
+        num_segments);
+}
+
+// segment_indices is an array of length num_segments + 1, where segment_indices[i] is the index of the first element in the i-th segment
+// The end of a segment is given by segment_indices[i+1]
+// https://nvidia.github.io/cccl/cub/api/structcub_1_1DeviceSegmentedSort.html#a-simple-example
+void segmented_sort_pairs_device(void* context, int* keys, int* values, int n, int* segment_start_indices, int* segment_end_indices, int num_segments)
+{
+    ContextGuard guard(context);
+
+    cub::DoubleBuffer<int> d_keys(keys, keys + n);
+	cub::DoubleBuffer<int> d_values(values, values + n);
+
+    RadixSortTemp temp;
+    segmented_sort_reserve(WP_CURRENT_CONTEXT, n, num_segments, &temp.mem, &temp.size);
+
+    // sort
+    check_cuda(cub::DeviceSegmentedRadixSort::SortPairs(
+        temp.mem,
+        temp.size,
+        d_keys, 
+        d_values, 
+        n,
+        num_segments,
+        segment_start_indices,
+        segment_end_indices,
+        0,
+        32,
+        (cudaStream_t)cuda_stream_get_current()));
+
+	if (d_keys.Current() != keys)
+		memcpy_d2d(WP_CURRENT_CONTEXT, keys, d_keys.Current(), sizeof(float)*n);
+
+	if (d_values.Current() != values)
+		memcpy_d2d(WP_CURRENT_CONTEXT, values, d_values.Current(), sizeof(int)*n);
+}
+
+void segmented_sort_pairs_int_device(uint64_t keys, uint64_t values, int n, uint64_t segment_start_indices, uint64_t segment_end_indices, int num_segments)
+{
+    segmented_sort_pairs_device(
+        WP_CURRENT_CONTEXT,
+        reinterpret_cast<int *>(keys),
+        reinterpret_cast<int *>(values), n,
+        reinterpret_cast<int *>(segment_start_indices),
+        reinterpret_cast<int *>(segment_end_indices),
+        num_segments);
+}

warp/native/sort.h

@@ -0,0 +1,33 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <stddef.h>
+
+void radix_sort_reserve(void* context, int n, void** mem_out=NULL, size_t* size_out=NULL);
+void radix_sort_pairs_host(int* keys, int* values, int n);
+void radix_sort_pairs_host(float* keys, int* values, int n);
+void radix_sort_pairs_host(int64_t* keys, int* values, int n);
+void radix_sort_pairs_device(void* context, int* keys, int* values, int n);
+void radix_sort_pairs_device(void* context, float* keys, int* values, int n);
+void radix_sort_pairs_device(void* context, int64_t* keys, int* values, int n);
+
+void segmented_sort_pairs_host(float* keys, int* values, int n, int* segment_start_indices, int* segment_end_indices, int num_segments);
+void segmented_sort_pairs_device(void* context, float* keys, int* values, int n, int* segment_start_indices, int* segment_end_indices, int num_segments);
+void segmented_sort_pairs_host(void* context, int* keys, int* values, int n, int* segment_start_indices, int* segment_end_indices, int num_segments);
+void segmented_sort_pairs_device(void* context, int* keys, int* values, int n, int* segment_start_indices, int* segment_end_indices, int num_segments);

warp/native/sparse.cpp

@@ -0,0 +1,378 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "warp.h"
+
+#include <algorithm>
+#include <numeric>
+#include <vector>
+
+namespace
+{
+
+// Specialized is_zero and accumulation function for common block sizes
+// Rely on compiler to unroll loops when block size is known
+
+template <int N, typename T> bool bsr_fixed_block_is_zero(const T* val, int value_size)
+{
+    return std::all_of(val, val + N, [](float v) { return v == T(0); });
+}
+
+template <typename T> bool bsr_dyn_block_is_zero(const T* val, int value_size)
+{
+    return std::all_of(val, val + value_size, [](float v) { return v == T(0); });
+}
+
+template <int N, typename T> void bsr_fixed_block_accumulate(const T* val, T* sum, int value_size)
+{
+    for (int i = 0; i < N; ++i, ++val, ++sum)
+    {
+        *sum += *val;
+    }
+}
+
+template <typename T> void bsr_dyn_block_accumulate(const T* val, T* sum, int value_size)
+{
+    for (int i = 0; i < value_size; ++i, ++val, ++sum)
+    {
+        *sum += *val;
+    }
+}
+
+template <int Rows, int Cols, typename T>
+void bsr_fixed_block_transpose(const T* src, T* dest, int row_count, int col_count)
+{
+    for (int r = 0; r < Rows; ++r)
+    {
+        for (int c = 0; c < Cols; ++c)
+        {
+            dest[c * Rows + r] = src[r * Cols + c];
+        }
+    }
+}
+
+template <typename T> void bsr_dyn_block_transpose(const T* src, T* dest, int row_count, int col_count)
+{
+    for (int r = 0; r < row_count; ++r)
+    {
+        for (int c = 0; c < col_count; ++c)
+        {
+            dest[c * row_count + r] = src[r * col_count + c];
+        }
+    }
+}
+
+} // namespace
+
+template <typename T>
+int bsr_matrix_from_triplets_host(const int rows_per_block, const int cols_per_block, const int row_count,
+                                  const int nnz, const int* tpl_rows, const int* tpl_columns, const T* tpl_values,
+                                  const bool prune_numerical_zeros, const bool masked, int* bsr_offsets,
+                                  int* bsr_columns, T* bsr_values)
+{
+
+    // get specialized accumulator for common block sizes (1,1), (1,2), (1,3),
+    // (2,2), (2,3), (3,3)
+    const int block_size = rows_per_block * cols_per_block;
+    void (*block_accumulate_func)(const T*, T*, int);
+    bool (*block_is_zero_func)(const T*, int);
+    switch (block_size)
+    {
+    case 1:
+        block_accumulate_func = bsr_fixed_block_accumulate<1, T>;
+        block_is_zero_func = bsr_fixed_block_is_zero<1, T>;
+        break;
+    case 2:
+        block_accumulate_func = bsr_fixed_block_accumulate<2, T>;
+        block_is_zero_func = bsr_fixed_block_is_zero<2, T>;
+        break;
+    case 3:
+        block_accumulate_func = bsr_fixed_block_accumulate<3, T>;
+        block_is_zero_func = bsr_fixed_block_is_zero<3, T>;
+        break;
+    case 4:
+        block_accumulate_func = bsr_fixed_block_accumulate<4, T>;
+        block_is_zero_func = bsr_fixed_block_is_zero<4, T>;
+        break;
+    case 6:
+        block_accumulate_func = bsr_fixed_block_accumulate<6, T>;
+        block_is_zero_func = bsr_fixed_block_is_zero<6, T>;
+        break;
+    case 9:
+        block_accumulate_func = bsr_fixed_block_accumulate<9, T>;
+        block_is_zero_func = bsr_fixed_block_is_zero<9, T>;
+        break;
+    default:
+        block_accumulate_func = bsr_dyn_block_accumulate<T>;
+        block_is_zero_func = bsr_dyn_block_is_zero<T>;
+    }
+
+    std::vector<int> block_indices(nnz);
+    std::iota(block_indices.begin(), block_indices.end(), 0);
+
+    // remove zero blocks  and invalid row indices
+
+    auto discard_block = [&](int i)
+    {
+        const int row = tpl_rows[i];
+        if (row < 0 || row >= row_count)
+        {
+            return true;
+        }
+
+        if (prune_numerical_zeros && tpl_values && block_is_zero_func(tpl_values + i * block_size, block_size))
+        {
+            return true;
+        }
+
+        if (!masked)
+        {
+            return false;
+        }
+
+        const int* beg = bsr_columns + bsr_offsets[row];
+        const int* end = bsr_columns + bsr_offsets[row + 1];
+        const int col = tpl_columns[i];
+        const int* block = std::lower_bound(beg, end, col);
+        return block == end || *block != col;
+    };
+
+    block_indices.erase(std::remove_if(block_indices.begin(), block_indices.end(), discard_block), block_indices.end());
+
+    // sort block indices according to lexico order
+    std::sort(block_indices.begin(), block_indices.end(), [tpl_rows, tpl_columns](int i, int j) -> bool
+              { return tpl_rows[i] < tpl_rows[j] || (tpl_rows[i] == tpl_rows[j] && tpl_columns[i] < tpl_columns[j]); });
+
+    // accumulate blocks at same locations, count blocks per row
+    std::fill_n(bsr_offsets, row_count + 1, 0);
+
+    int current_row = -1;
+    int current_col = -1;
+
+    // so that we get back to the start for the first block
+    if (bsr_values)
+    {
+        bsr_values -= block_size;
+    }
+
+    for (int i = 0; i < block_indices.size(); ++i)
+    {
+        int idx = block_indices[i];
+        int row = tpl_rows[idx];
+        int col = tpl_columns[idx];
+        const T* val = tpl_values + idx * block_size;
+
+        if (row == current_row && col == current_col)
+        {
+            if (bsr_values)
+            {
+                block_accumulate_func(val, bsr_values, block_size);
+            }
+        }
+        else
+        {
+            *(bsr_columns++) = col;
+
+            if (bsr_values)
+            {
+                bsr_values += block_size;
+                std::copy_n(val, block_size, bsr_values);
+            }
+
+            bsr_offsets[row + 1]++;
+
+            current_row = row;
+            current_col = col;
+        }
+    }
+
+    // build postfix sum of row counts
+    std::partial_sum(bsr_offsets, bsr_offsets + row_count + 1, bsr_offsets);
+
+    return bsr_offsets[row_count];
+}
+
+template <typename T>
+void bsr_transpose_host(int rows_per_block, int cols_per_block, int row_count, int col_count, int nnz_up,
+                        const int* bsr_offsets, const int* bsr_columns, const T* bsr_values,
+                        int* transposed_bsr_offsets, int* transposed_bsr_columns, T* transposed_bsr_values)
+{
+    const int nnz = bsr_offsets[row_count];
+    const int block_size = rows_per_block * cols_per_block;
+
+    void (*block_transpose_func)(const T*, T*, int, int) = bsr_dyn_block_transpose<T>;
+    switch (rows_per_block)
+    {
+    case 1:
+        switch (cols_per_block)
+        {
+        case 1:
+            block_transpose_func = bsr_fixed_block_transpose<1, 1, T>;
+            break;
+        case 2:
+            block_transpose_func = bsr_fixed_block_transpose<1, 2, T>;
+            break;
+        case 3:
+            block_transpose_func = bsr_fixed_block_transpose<1, 3, T>;
+            break;
+        }
+        break;
+    case 2:
+        switch (cols_per_block)
+        {
+        case 1:
+            block_transpose_func = bsr_fixed_block_transpose<2, 1, T>;
+            break;
+        case 2:
+            block_transpose_func = bsr_fixed_block_transpose<2, 2, T>;
+            break;
+        case 3:
+            block_transpose_func = bsr_fixed_block_transpose<2, 3, T>;
+            break;
+        }
+        break;
+    case 3:
+        switch (cols_per_block)
+        {
+        case 1:
+            block_transpose_func = bsr_fixed_block_transpose<3, 1, T>;
+            break;
+        case 2:
+            block_transpose_func = bsr_fixed_block_transpose<3, 2, T>;
+            break;
+        case 3:
+            block_transpose_func = bsr_fixed_block_transpose<3, 3, T>;
+            break;
+        }
+        break;
+    }
+
+    std::vector<int> block_indices(nnz), bsr_rows(nnz);
+    std::iota(block_indices.begin(), block_indices.end(), 0);
+
+    // Fill row indices from offsets
+    for (int row = 0; row < row_count; ++row)
+    {
+        std::fill(bsr_rows.begin() + bsr_offsets[row], bsr_rows.begin() + bsr_offsets[row + 1], row);
+    }
+
+    // sort block indices according to (transposed) lexico order
+    std::sort(
+        block_indices.begin(), block_indices.end(), [&bsr_rows, bsr_columns](int i, int j) -> bool
+        { return bsr_columns[i] < bsr_columns[j] || (bsr_columns[i] == bsr_columns[j] && bsr_rows[i] < bsr_rows[j]); });
+
+    // Count blocks per column and transpose blocks
+    std::fill_n(transposed_bsr_offsets, col_count + 1, 0);
+
+    for (int i = 0; i < nnz; ++i)
+    {
+        int idx = block_indices[i];
+        int row = bsr_rows[idx];
+        int col = bsr_columns[idx];
+
+        ++transposed_bsr_offsets[col + 1];
+        transposed_bsr_columns[i] = row;
+
+        if (transposed_bsr_values != nullptr)
+        {
+            const T* src_block = bsr_values + idx * block_size;
+            T* dst_block = transposed_bsr_values + i * block_size;
+            block_transpose_func(src_block, dst_block, rows_per_block, cols_per_block);
+        }
+    }
+
+    // build postfix sum of column counts
+    std::partial_sum(transposed_bsr_offsets, transposed_bsr_offsets + col_count + 1, transposed_bsr_offsets);
+}
+
+WP_API void bsr_matrix_from_triplets_float_host(int rows_per_block, int cols_per_block, int row_count, int nnz,
+                                                int* tpl_rows, int* tpl_columns, void* tpl_values,
+                                                bool prune_numerical_zeros, bool masked, int* bsr_offsets,
+                                                int* bsr_columns, void* bsr_values, int* bsr_nnz, void* bsr_nnz_event)
+{
+    bsr_matrix_from_triplets_host<float>(rows_per_block, cols_per_block, row_count, nnz, tpl_rows, tpl_columns,
+                                         static_cast<const float*>(tpl_values), prune_numerical_zeros, masked,
+                                         bsr_offsets, bsr_columns, static_cast<float*>(bsr_values));
+    if (bsr_nnz)
+    {
+        *bsr_nnz = bsr_offsets[row_count];
+    }
+}
+
+WP_API void bsr_matrix_from_triplets_double_host(int rows_per_block, int cols_per_block, int row_count, int nnz,
+                                                 int* tpl_rows, int* tpl_columns, void* tpl_values,
+                                                 bool prune_numerical_zeros, bool masked, int* bsr_offsets,
+                                                 int* bsr_columns, void* bsr_values, int* bsr_nnz, void* bsr_nnz_event)
+{
+    bsr_matrix_from_triplets_host<double>(rows_per_block, cols_per_block, row_count, nnz, tpl_rows, tpl_columns,
+                                          static_cast<const double*>(tpl_values), prune_numerical_zeros, masked,
+                                          bsr_offsets, bsr_columns, static_cast<double*>(bsr_values));
+    if (bsr_nnz)
+    {
+        *bsr_nnz = bsr_offsets[row_count];
+    }
+}
+
+WP_API void bsr_transpose_float_host(int rows_per_block, int cols_per_block, int row_count, int col_count, int nnz,
+                                     int* bsr_offsets, int* bsr_columns, void* bsr_values, int* transposed_bsr_offsets,
+                                     int* transposed_bsr_columns, void* transposed_bsr_values)
+{
+    bsr_transpose_host(rows_per_block, cols_per_block, row_count, col_count, nnz, bsr_offsets, bsr_columns,
+                       static_cast<const float*>(bsr_values), transposed_bsr_offsets, transposed_bsr_columns,
+                       static_cast<float*>(transposed_bsr_values));
+}
+
+WP_API void bsr_transpose_double_host(int rows_per_block, int cols_per_block, int row_count, int col_count, int nnz,
+                                      int* bsr_offsets, int* bsr_columns, void* bsr_values, int* transposed_bsr_offsets,
+                                      int* transposed_bsr_columns, void* transposed_bsr_values)
+{
+    bsr_transpose_host(rows_per_block, cols_per_block, row_count, col_count, nnz, bsr_offsets, bsr_columns,
+                       static_cast<const double*>(bsr_values), transposed_bsr_offsets, transposed_bsr_columns,
+                       static_cast<double*>(transposed_bsr_values));
+}
+
+#if !WP_ENABLE_CUDA
+WP_API void bsr_matrix_from_triplets_float_device(int rows_per_block, int cols_per_block, int row_count, int nnz,
+                                                  int* tpl_rows, int* tpl_columns, void* tpl_values,
+                                                  bool prune_numerical_zeros, bool masked, int* bsr_offsets,
+                                                  int* bsr_columns, void* bsr_values, int* bsr_nnz, void* bsr_nnz_event)
+{
+}
+
+WP_API void bsr_matrix_from_triplets_double_device(int rows_per_block, int cols_per_block, int row_count, int nnz,
+                                                   int* tpl_rows, int* tpl_columns, void* tpl_values,
+                                                   bool prune_numerical_zeros, bool masked, int* bsr_offsets,
+                                                   int* bsr_columns, void* bsr_values, int* bsr_nnz,
+                                                   void* bsr_nnz_event)
+{
+}
+
+WP_API void bsr_transpose_float_device(int rows_per_block, int cols_per_block, int row_count, int col_count, int nnz,
+                                       int* bsr_offsets, int* bsr_columns, void* bsr_values,
+                                       int* transposed_bsr_offsets, int* transposed_bsr_columns,
+                                       void* transposed_bsr_values)
+{
+}
+
+WP_API void bsr_transpose_double_device(int rows_per_block, int cols_per_block, int row_count, int col_count, int nnz,
+                                        int* bsr_offsets, int* bsr_columns, void* bsr_values,
+                                        int* transposed_bsr_offsets, int* transposed_bsr_columns,
+                                        void* transposed_bsr_values)
+{
+}
+
+#endif