numba-cuda 0.22.0__cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl

numba_cuda/numba/cuda/mathimpl.py

@@ -0,0 +1,374 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+import sys
+import math
+import operator
+from llvmlite import ir
+from numba.cuda import types, typing
+from numba.cuda import cgutils
+from numba.cuda.core.imputils import Registry
+from numba.cuda.types import float32, float64, int64, uint64
+from numba.cuda import libdevice
+from numba.cuda.core import targetconfig
+
+registry = Registry("mathimpl")
+lower = registry.lower
+
+
+booleans = []
+booleans += [("isnand", "isnanf", math.isnan)]
+booleans += [("isinfd", "isinff", math.isinf)]
+booleans += [("isfinited", "finitef", math.isfinite)]
+
+unarys = []
+unarys += [("ceil", "ceilf", math.ceil)]
+unarys += [("floor", "floorf", math.floor)]
+unarys += [("fabs", "fabsf", math.fabs)]
+unarys += [("exp", "expf", math.exp)]
+if sys.version_info >= (3, 11):
+    unarys += [("exp2", "exp2f", math.exp2)]
+unarys += [("expm1", "expm1f", math.expm1)]
+unarys += [("erf", "erff", math.erf)]
+unarys += [("erfc", "erfcf", math.erfc)]
+unarys += [("tgamma", "tgammaf", math.gamma)]
+unarys += [("lgamma", "lgammaf", math.lgamma)]
+unarys += [("sqrt", "sqrtf", math.sqrt)]
+unarys += [("log", "logf", math.log)]
+unarys += [("log2", "log2f", math.log2)]
+unarys += [("log10", "log10f", math.log10)]
+unarys += [("log1p", "log1pf", math.log1p)]
+unarys += [("acosh", "acoshf", math.acosh)]
+unarys += [("acos", "acosf", math.acos)]
+unarys += [("cos", "cosf", math.cos)]
+unarys += [("cosh", "coshf", math.cosh)]
+unarys += [("asinh", "asinhf", math.asinh)]
+unarys += [("asin", "asinf", math.asin)]
+unarys += [("sin", "sinf", math.sin)]
+unarys += [("sinh", "sinhf", math.sinh)]
+unarys += [("atan", "atanf", math.atan)]
+unarys += [("atanh", "atanhf", math.atanh)]
+unarys += [("tan", "tanf", math.tan)]
+unarys += [("trunc", "truncf", math.trunc)]
+
+unarys_fastmath = {}
+unarys_fastmath["cosf"] = "fast_cosf"
+unarys_fastmath["sinf"] = "fast_sinf"
+unarys_fastmath["tanf"] = "fast_tanf"
+unarys_fastmath["expf"] = "fast_expf"
+unarys_fastmath["log2f"] = "fast_log2f"
+unarys_fastmath["log10f"] = "fast_log10f"
+unarys_fastmath["logf"] = "fast_logf"
+
+binarys = []
+binarys += [("copysign", "copysignf", math.copysign)]
+binarys += [("atan2", "atan2f", math.atan2)]
+binarys += [("pow", "powf", math.pow)]
+binarys += [("fmod", "fmodf", math.fmod)]
+binarys += [("hypot", "hypotf", math.hypot)]
+binarys += [("remainder", "remainderf", math.remainder)]
+binarys += [("nextafter", "nextafterf", math.nextafter)]
+
+binarys_fastmath = {}
+binarys_fastmath["powf"] = "fast_powf"
+
+
+@lower(math.isinf, types.Integer)
+@lower(math.isnan, types.Integer)
+def math_isinf_isnan_int(context, builder, sig, args):
+    return context.get_constant(types.boolean, 0)
+
+
+@lower(operator.truediv, types.float32, types.float32)
+def maybe_fast_truediv(context, builder, sig, args):
+    if context.fastmath:
+        sig = typing.signature(float32, float32, float32)
+        impl = context.get_function(libdevice.fast_fdividef, sig)
+        return impl(builder, args)
+    else:
+        with cgutils.if_zero(builder, args[1]):
+            context.error_model.fp_zero_division(builder, ("division by zero",))
+        res = builder.fdiv(*args)
+        return res
+
+
+@lower(math.isfinite, types.Integer)
+def math_isfinite_int(context, builder, sig, args):
+    return context.get_constant(types.boolean, 1)
+
+
+def impl_boolean(key, ty, libfunc):
+    def lower_boolean_impl(context, builder, sig, args):
+        libfunc_impl = context.get_function(
+            libfunc, typing.signature(types.int32, ty)
+        )
+        result = libfunc_impl(builder, args)
+        return context.cast(builder, result, types.int32, types.boolean)
+
+    lower(key, ty)(lower_boolean_impl)
+
+
+def get_lower_unary_impl(key, ty, libfunc):
+    def lower_unary_impl(context, builder, sig, args):
+        actual_libfunc = libfunc
+        fast_replacement = None
+        if ty == float32 and context.fastmath:
+            fast_replacement = unarys_fastmath.get(libfunc.__name__)
+
+        if fast_replacement is not None:
+            actual_libfunc = getattr(libdevice, fast_replacement)
+
+        libfunc_impl = context.get_function(
+            actual_libfunc, typing.signature(ty, ty)
+        )
+        return libfunc_impl(builder, args)
+
+    return lower_unary_impl
+
+
+def get_unary_impl_for_fn_and_ty(fn, ty):
+    # tanh is a special case - because it is not registered like the other
+    # unary implementations, it does not appear in the unarys list. However,
+    # its implementation can be looked up by key like the other
+    # implementations, so we add it to the list we search here.
+    tanh_impls = ("tanh", "tanhf", math.tanh)
+    for fname64, fname32, key in unarys + [tanh_impls]:
+        if fn == key:
+            if ty == float32:
+                impl = getattr(libdevice, fname32)
+            elif ty == float64:
+                impl = getattr(libdevice, fname64)
+
+            return get_lower_unary_impl(key, ty, impl)
+
+    raise RuntimeError(f"Implementation of {fn} for {ty} not found")
+
+
+def impl_unary(key, ty, libfunc):
+    lower_unary_impl = get_lower_unary_impl(key, ty, libfunc)
+    lower(key, ty)(lower_unary_impl)
+
+
+def impl_unary_int(key, ty, libfunc):
+    def lower_unary_int_impl(context, builder, sig, args):
+        if sig.args[0] == int64:
+            convert = builder.sitofp
+        elif sig.args[0] == uint64:
+            convert = builder.uitofp
+        else:
+            m = "Only 64-bit integers are supported for generic unary int ops"
+            raise TypeError(m)
+
+        arg = convert(args[0], ir.DoubleType())
+        sig = typing.signature(float64, float64)
+        libfunc_impl = context.get_function(libfunc, sig)
+        return libfunc_impl(builder, [arg])
+
+    lower(key, ty)(lower_unary_int_impl)
+
+
+def get_lower_binary_impl(key, ty, libfunc):
+    def lower_binary_impl(context, builder, sig, args):
+        actual_libfunc = libfunc
+        fast_replacement = None
+        if ty == float32 and context.fastmath:
+            fast_replacement = binarys_fastmath.get(libfunc.__name__)
+
+        if fast_replacement is not None:
+            actual_libfunc = getattr(libdevice, fast_replacement)
+
+        libfunc_impl = context.get_function(
+            actual_libfunc, typing.signature(ty, ty, ty)
+        )
+        return libfunc_impl(builder, args)
+
+    return lower_binary_impl
+
+
+def get_binary_impl_for_fn_and_ty(fn, ty):
+    for fname64, fname32, key in binarys:
+        if fn == key:
+            if ty == float32:
+                impl = getattr(libdevice, fname32)
+            elif ty == float64:
+                impl = getattr(libdevice, fname64)
+
+            return get_lower_binary_impl(key, ty, impl)
+
+    raise RuntimeError(f"Implementation of {fn} for {ty} not found")
+
+
+def impl_binary(key, ty, libfunc):
+    lower_binary_impl = get_lower_binary_impl(key, ty, libfunc)
+    lower(key, ty, ty)(lower_binary_impl)
+
+
+def impl_binary_int(key, ty, libfunc):
+    def lower_binary_int_impl(context, builder, sig, args):
+        if sig.args[0] == int64:
+            convert = builder.sitofp
+        elif sig.args[0] == uint64:
+            convert = builder.uitofp
+        else:
+            m = "Only 64-bit integers are supported for generic binary int ops"
+            raise TypeError(m)
+
+        args = [convert(arg, ir.DoubleType()) for arg in args]
+        sig = typing.signature(float64, float64, float64)
+        libfunc_impl = context.get_function(libfunc, sig)
+        return libfunc_impl(builder, args)
+
+    lower(key, ty, ty)(lower_binary_int_impl)
+
+
+for fname64, fname32, key in booleans:
+    impl32 = getattr(libdevice, fname32)
+    impl64 = getattr(libdevice, fname64)
+    impl_boolean(key, float32, impl32)
+    impl_boolean(key, float64, impl64)
+
+
+for fname64, fname32, key in unarys:
+    impl32 = getattr(libdevice, fname32)
+    impl64 = getattr(libdevice, fname64)
+    impl_unary(key, float32, impl32)
+    impl_unary(key, float64, impl64)
+    impl_unary_int(key, int64, impl64)
+    impl_unary_int(key, uint64, impl64)
+
+
+for fname64, fname32, key in binarys:
+    impl32 = getattr(libdevice, fname32)
+    impl64 = getattr(libdevice, fname64)
+    impl_binary(key, float32, impl32)
+    impl_binary(key, float64, impl64)
+    impl_binary_int(key, int64, impl64)
+    impl_binary_int(key, uint64, impl64)
+
+
+def impl_pow_int(ty, libfunc):
+    def lower_pow_impl_int(context, builder, sig, args):
+        powi_sig = typing.signature(ty, ty, types.int32)
+        libfunc_impl = context.get_function(libfunc, powi_sig)
+        return libfunc_impl(builder, args)
+
+    lower(math.pow, ty, types.int32)(lower_pow_impl_int)
+
+
+impl_pow_int(types.float32, libdevice.powif)
+impl_pow_int(types.float64, libdevice.powi)
+
+
+def impl_modf(ty, libfunc):
+    retty = types.UniTuple(ty, 2)
+
+    def lower_modf_impl(context, builder, sig, args):
+        modf_sig = typing.signature(retty, ty)
+        libfunc_impl = context.get_function(libfunc, modf_sig)
+        return libfunc_impl(builder, args)
+
+    lower(math.modf, ty)(lower_modf_impl)
+
+
+impl_modf(types.float32, libdevice.modff)
+impl_modf(types.float64, libdevice.modf)
+
+
+def impl_frexp(ty, libfunc):
+    retty = types.Tuple((ty, types.int32))
+
+    def lower_frexp_impl(context, builder, sig, args):
+        frexp_sig = typing.signature(retty, ty)
+        libfunc_impl = context.get_function(libfunc, frexp_sig)
+        return libfunc_impl(builder, args)
+
+    lower(math.frexp, ty)(lower_frexp_impl)
+
+
+impl_frexp(types.float32, libdevice.frexpf)
+impl_frexp(types.float64, libdevice.frexp)
+
+
+def impl_ldexp(ty, libfunc):
+    def lower_ldexp_impl(context, builder, sig, args):
+        ldexp_sig = typing.signature(ty, ty, types.int32)
+        libfunc_impl = context.get_function(libfunc, ldexp_sig)
+        return libfunc_impl(builder, args)
+
+    lower(math.ldexp, ty, types.int32)(lower_ldexp_impl)
+
+
+impl_ldexp(types.float32, libdevice.ldexpf)
+impl_ldexp(types.float64, libdevice.ldexp)
+
+
+def impl_tanh(ty, libfunc):
+    def lower_tanh_impl(context, builder, sig, args):
+        def get_compute_capability():
+            flags = targetconfig.ConfigStack().top()
+            return flags.compute_capability
+
+        def tanh_impl_libdevice():
+            tanh_sig = typing.signature(ty, ty)
+            libfunc_impl = context.get_function(libfunc, tanh_sig)
+            return libfunc_impl(builder, args)
+
+        def tanhf_impl_fastmath():
+            fnty = ir.FunctionType(ir.FloatType(), [ir.FloatType()])
+            asm = ir.InlineAsm(fnty, "tanh.approx.f32 $0, $1;", "=f,f")
+            return builder.call(asm, args)
+
+        if ty == float32 and context.fastmath:
+            cc = get_compute_capability()
+            if cc >= (7, 5):
+                return tanhf_impl_fastmath()
+
+        return tanh_impl_libdevice()
+
+    lower(math.tanh, ty)(lower_tanh_impl)
+
+
+impl_tanh(types.float32, libdevice.tanhf)
+impl_tanh(types.float64, libdevice.tanh)
+
+impl_unary_int(math.tanh, int64, libdevice.tanh)
+impl_unary_int(math.tanh, uint64, libdevice.tanh)
+
+
+# Complex power implementations - translations of _Py_c_pow from CPython
+# https://github.com/python/cpython/blob/a755410e054e1e2390de5830befc08fe80706c66/Objects/complexobject.c#L123-L151
+#
+# The complex64 variant casts all constants and some variables to ensure that
+# as much computation is done in single precision as possible. A small number
+# of operations are still done in 64-bit, but these come from libdevice code.
+
+
+def cpow_implement(fty, cty):
+    def core(context, builder, sig, args):
+        def cpow_internal(a, b):
+            if b.real == fty(0.0) and b.imag == fty(0.0):
+                return cty(1.0) + cty(0.0j)
+            elif a.real == fty(0.0) and b.real == fty(0.0):
+                return cty(0.0) + cty(0.0j)
+
+            vabs = math.hypot(a.real, a.imag)
+            len = math.pow(vabs, b.real)
+            at = math.atan2(a.imag, a.real)
+            phase = at * b.real
+            if b.imag != fty(0.0):
+                len /= math.exp(at * b.imag)
+                phase += b.imag * math.log(vabs)
+
+            return len * (
+                cty(math.cos(phase)) + cty(math.sin(phase) * cty(1.0j))
+            )
+
+        return context.compile_internal(builder, cpow_internal, sig, args)
+
+    lower(operator.pow, cty, cty)(core)
+    lower(operator.ipow, cty, cty)(core)
+    lower(pow, cty, cty)(core)
+
+
+cpow_implement(types.float32, types.complex64)
+cpow_implement(types.float64, types.complex128)

numba_cuda/numba/cuda/memory_management/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: BSD-2-Clause
+
+from numba.cuda.memory_management.nrt import rtsys  # noqa: F401

numba_cuda/numba/cuda/memory_management/memsys.cu

@@ -0,0 +1,99 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include "memsys.cuh"
+
+__device__ size_t memsys_size = sizeof(NRT_MemSys);
+
+namespace detail
+{
+  void __device__ check_memsys()
+  {
+    if (TheMSys == nullptr)
+    {
+      assert(false && "TheMSys pointer is null. Please use NRT_MemSys_set to set pointer first.");
+    }
+  }
+}
+
+extern "C" __global__ void NRT_MemSys_set(NRT_MemSys *memsys_ptr)
+{
+  TheMSys = memsys_ptr;
+}
+
+extern "C" __global__ void NRT_MemSys_read(uint64_t *managed_memsys)
+{
+  detail::check_memsys();
+  managed_memsys[0] = TheMSys->stats.alloc;
+  managed_memsys[1] = TheMSys->stats.free;
+  managed_memsys[2] = TheMSys->stats.mi_alloc;
+  managed_memsys[3] = TheMSys->stats.mi_free;
+}
+
+extern "C" __global__ void NRT_MemSys_read_alloc(uint64_t *managed_result)
+{
+  detail::check_memsys();
+  managed_result[0] = TheMSys->stats.alloc;
+}
+
+extern "C" __global__ void NRT_MemSys_read_free(uint64_t *managed_result)
+{
+  detail::check_memsys();
+  managed_result[0] = TheMSys->stats.free;
+}
+
+extern "C" __global__ void NRT_MemSys_read_mi_alloc(uint64_t *managed_result)
+{
+  detail::check_memsys();
+  managed_result[0] = TheMSys->stats.mi_alloc;
+}
+
+extern "C" __global__ void NRT_MemSys_read_mi_free(uint64_t *managed_result)
+{
+  detail::check_memsys();
+  managed_result[0] = TheMSys->stats.mi_free;
+}
+
+extern "C" __global__ void NRT_MemSys_init(void)
+{
+  detail::check_memsys();
+  TheMSys->stats.enabled = false;
+  TheMSys->stats.alloc = 0;
+  TheMSys->stats.free = 0;
+  TheMSys->stats.mi_alloc = 0;
+  TheMSys->stats.mi_free = 0;
+}
+
+extern "C" __global__ void NRT_MemSys_enable_stats(void)
+{
+  detail::check_memsys();
+  TheMSys->stats.enabled = true;
+}
+
+extern "C" __global__ void NRT_MemSys_disable_stats(void)
+{
+  detail::check_memsys();
+  TheMSys->stats.enabled = false;
+}
+
+extern "C" __global__ void NRT_MemSys_stats_enabled(uint8_t *enabled)
+{
+  detail::check_memsys();
+  *enabled = static_cast<uint8_t>(TheMSys->stats.enabled);
+}
+
+extern "C" __global__ void NRT_MemSys_print(void)
+{
+  if (TheMSys != nullptr)
+  {
+    printf("TheMSys->stats.enabled %d\n", TheMSys->stats.enabled);
+    printf("TheMSys->stats.alloc %zu\n", TheMSys->stats.alloc.load());
+    printf("TheMSys->stats.free %zu\n", TheMSys->stats.free.load());
+    printf("TheMSys->stats.mi_alloc %zu\n", TheMSys->stats.mi_alloc.load());
+    printf("TheMSys->stats.mi_free %zu\n", TheMSys->stats.mi_free.load());
+  } else {
+    printf("TheMsys is null.\n");
+  }
+}

numba_cuda/numba/cuda/memory_management/memsys.cuh

@@ -0,0 +1,22 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <cuda/atomic>
+
+// Globally needed variables
+struct NRT_MemSys {
+    struct {
+      bool enabled;
+      cuda::atomic<size_t, cuda::thread_scope_device> alloc;
+      cuda::atomic<size_t, cuda::thread_scope_device> free;
+      cuda::atomic<size_t, cuda::thread_scope_device> mi_alloc;
+      cuda::atomic<size_t, cuda::thread_scope_device> mi_free;
+    } stats;
+  };
+
+/* The Memory System object */
+__device__ NRT_MemSys* TheMSys;
+
+extern "C" __global__ void NRT_MemSys_set(NRT_MemSys *memsys_ptr);

numba_cuda/numba/cuda/memory_management/nrt.cu

@@ -0,0 +1,212 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef _NRT_H
+#define _NRT_H
+
+#include <cuda/atomic>
+
+#include "memsys.cuh"
+#include "nrt.cuh"
+
+
+extern "C" __global__ void NRT_MemSys_set(NRT_MemSys *memsys_ptr)
+{
+  TheMSys = memsys_ptr;
+}
+
+
+extern "C" __device__ void* NRT_Allocate(size_t size)
+{
+  void* ptr = NULL;
+  ptr       = malloc(size);
+  if (TheMSys && TheMSys->stats.enabled) {
+    TheMSys->stats.alloc.fetch_add(1, cuda::memory_order_relaxed); }
+  return ptr;
+}
+
+extern "C" __device__ void NRT_MemInfo_init(NRT_MemInfo* mi,
+                                            void* data,
+                                            size_t size,
+                                            NRT_dtor_function dtor,
+                                            void* dtor_info)
+{
+  mi->refct     = 1; /* starts with 1 refct */
+  mi->dtor      = dtor;
+  mi->dtor_info = dtor_info;
+  mi->data      = data;
+  mi->size      = size;
+ if (TheMSys && TheMSys->stats.enabled) {
+  TheMSys->stats.mi_alloc.fetch_add(1, cuda::memory_order_relaxed); }
+}
+
+extern "C"
+__device__ NRT_MemInfo* NRT_MemInfo_new(
+  void* data, size_t size, NRT_dtor_function dtor, void* dtor_info)
+{
+  NRT_MemInfo* mi = (NRT_MemInfo*)NRT_Allocate(sizeof(NRT_MemInfo));
+  if (mi != NULL) { NRT_MemInfo_init(mi, data, size, dtor, dtor_info); }
+  return mi;
+}
+
+extern "C" __device__ void NRT_Free(void* ptr)
+{
+  free(ptr);
+  if (TheMSys && TheMSys->stats.enabled) {
+    TheMSys->stats.free.fetch_add(1, cuda::memory_order_relaxed); }
+}
+
+extern "C" __device__ void NRT_dealloc(NRT_MemInfo* mi)
+{
+  NRT_Free(mi);
+}
+
+extern "C" __device__ void NRT_MemInfo_destroy(NRT_MemInfo* mi)
+{
+  NRT_dealloc(mi);
+  if (TheMSys && TheMSys->stats.enabled) {
+    TheMSys->stats.mi_free.fetch_add(1, cuda::memory_order_relaxed); }
+}
+
+extern "C" __device__ void NRT_MemInfo_call_dtor(NRT_MemInfo* mi)
+{
+  if (mi->dtor) /* We have a destructor */
+    mi->dtor(mi->data, mi->size, NULL);
+  /* Clear and release MemInfo */
+  NRT_MemInfo_destroy(mi);
+}
+
+static void __device__
+nrt_varsize_dtor(void *ptr, size_t size, void *info) {
+    if (info) {
+        /* call element dtor */
+        typedef void dtor_fn_t(void *ptr);
+        dtor_fn_t *dtor = (dtor_fn_t *)info;
+        dtor(ptr);
+    }
+    NRT_Free(ptr);
+}
+
+extern "C"
+__device__ NRT_MemInfo* NRT_MemInfo_new_varsize(size_t size)
+{
+    void *data = NRT_Allocate(size);
+    if (data == NULL) {
+        return NULL; /* return early as allocation failed */
+    }
+
+    NRT_MemInfo *mi = NRT_MemInfo_new(data, size, nrt_varsize_dtor, NULL);
+
+    return mi;
+}
+
+extern "C"
+__device__ NRT_MemInfo* NRT_MemInfo_new_varsize_dtor(size_t size, NRT_dtor_function dtor) {
+    NRT_MemInfo *mi = NRT_MemInfo_new_varsize(size);
+    if (mi) {
+        mi->dtor_info = (void*)dtor;
+    }
+    return mi;
+}
+
+extern "C" __device__ void* NRT_MemInfo_data_fast(NRT_MemInfo *mi)
+{
+  return mi->data;
+}
+
+extern "C" __device__ NRT_MemInfo *NRT_MemInfo_alloc_aligned(size_t size, unsigned align) {
+    NRT_MemInfo *mi = NULL;
+    void *data = nrt_allocate_meminfo_and_data_align(size, align, &mi);
+    if (data == NULL) {
+        return NULL; /* return early as allocation failed */
+    }
+    //NRT_Debug(nrt_debug_print("NRT_MemInfo_alloc_aligned %p\n", data));
+    NRT_MemInfo_init(mi, data, size, NULL, NULL);
+    return mi;
+}
+
+static
+__device__ void *nrt_allocate_meminfo_and_data_align(size_t size, unsigned align,
+                                          NRT_MemInfo **mi)
+{
+    size_t offset = 0, intptr = 0, remainder = 0;
+    //NRT_Debug(nrt_debug_print("nrt_allocate_meminfo_and_data_align %p\n", allocator));
+    char *base = (char *)nrt_allocate_meminfo_and_data(size + 2 * align, mi);
+    if (base == NULL) {
+        return NULL; /* return early as allocation failed */
+    }
+    intptr = (size_t) base;
+    /*
+     * See if the allocation is aligned already...
+     * Check if align is a power of 2, if so the modulo can be avoided.
+     */
+    if((align & (align - 1)) == 0)
+    {
+        remainder = intptr & (align - 1);
+    }
+    else
+    {
+        remainder = intptr % align;
+    }
+    if (remainder == 0){ /* Yes */
+        offset = 0;
+    } else { /* No, move forward `offset` bytes */
+        offset = align - remainder;
+    }
+    return (void*)((char *)base + offset);
+}
+
+static
+__device__ void *nrt_allocate_meminfo_and_data(size_t size, NRT_MemInfo **mi_out) {
+    NRT_MemInfo *mi = NULL;
+    //NRT_Debug(nrt_debug_print("nrt_allocate_meminfo_and_data %p\n", allocator));
+    char *base = (char *)NRT_Allocate_External(sizeof(NRT_MemInfo) + size);
+    if (base == NULL) {
+        *mi_out = NULL; /* set meminfo to NULL as allocation failed */
+        return NULL; /* return early as allocation failed */
+    }
+    mi = (NRT_MemInfo *) base;
+    *mi_out = mi;
+    return (void*)((char *)base + sizeof(NRT_MemInfo));
+}
+
+extern "C" __device__ void* NRT_Allocate_External(size_t size) {
+    void *ptr = NULL;
+    ptr = malloc(size);
+    //NRT_Debug(nrt_debug_print("NRT_Allocate_External bytes=%zu ptr=%p\n", size, ptr));
+
+    if (TheMSys && TheMSys->stats.enabled)
+    {
+       TheMSys->stats.alloc.fetch_add(1, cuda::memory_order_relaxed);
+    }
+    return ptr;
+}
+
+
+/*
+  c++ version of the NRT_decref function that usually is added to
+  the final kernel link in PTX form by numba. This version may be
+  used by c++ APIs that accept ownership of live objects and must
+  manage them going forward.
+*/
+extern "C" __device__ void NRT_decref(NRT_MemInfo* mi)
+{
+  if (mi != NULL) {
+    mi->refct--;
+    if (mi->refct == 0) { NRT_MemInfo_call_dtor(mi); }
+  }
+}
+
+
+
+
+extern "C" __device__ void NRT_incref(NRT_MemInfo* mi)
+{
+  if (mi != NULL) {
+    mi->refct++;
+  }
+}
+
+#endif