nvmath-python 1.0.0__cp314-cp314t-win_amd64.whl

nvmath/device/types.py

@@ -0,0 +1,138 @@
+# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+__all__ = [
+    "Complex",
+    "Vector",
+    "complex32",
+    "complex64",
+    "complex128",
+    "half2",
+    "half4",
+    "np_float16x2",
+    "np_float16x4",
+    "REAL_NP_TYPES",
+    "INT_NP_TYPES",
+]
+
+import warnings
+
+import numpy as np
+
+np_float16x2 = np.dtype([("x", np.float16), ("y", np.float16)], align=True)
+np_float16x4 = np.dtype([("x", np.float16), ("y", np.float16), ("z", np.float16), ("w", np.float16)], align=True)
+
+REAL_NP_TYPES: list = [np.float16, np.float32, np.float64]
+INT_NP_TYPES: list = [np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64]
+
+_alignment_warning_msg = (
+    "You are using the host counterpart of a dtype that is under-aligned "
+    "compared to what the device function expects. In most cases this will "
+    "work, since device memory is typically allocated with at least 256-byte "
+    "alignment. For details on device memory access and built-in type "
+    "alignment requirements, see "
+    "https://docs.nvidia.com/cuda/cuda-c-programming-guide/#device-memory-accesses "
+    "and "
+    "https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/cpp-language-extensions.html#built-in-types"
+)
+
+
+class Complex:
+    """
+    Complex type that can be used to represent complex numbers both
+    on host and device side. Numpy does not provide a built-in complex type with
+    16-bit real and imaginary parts, so we define our own dtype for that case.
+    For 32-bit and 64-bit complex numbers, we can use the built-in numpy dtypes.
+    However on device side we expect those types to be aligned to the full size
+    of the complex type, so the array defined on host and device side will have
+    different type and alignment. :py:const:`np_float16x2`,
+    ``numpy.dtype(numpy.complex64)`` and
+    ``numpy.dtype(numpy.complex128)`` are the host side dtypes and
+    :py:const:`float16x2_type`, :py:const:`float32x2_type` and
+    :py:const:`float64x2_type` are the device side types.
+    """
+
+    def __init__(self, real_dtype):
+        self._real_dtype = real_dtype
+
+    @property
+    def real_dtype(self):
+        return self._real_dtype
+
+    @property
+    def dtype(self):
+        warnings.warn(_alignment_warning_msg, UserWarning, stacklevel=2)
+        if self._real_dtype == np.float16:
+            return np_float16x2
+        elif self._real_dtype == np.float32:
+            return np.dtype(np.complex64)
+        assert self._real_dtype == np.float64
+        return np.dtype(np.complex128)
+
+    @property
+    def _numba_type(self):
+        from .vector_types_numba import float16x2_type, float32x2_type, float64x2_type
+
+        if self.real_dtype == np.float16:
+            return float16x2_type
+        if self.real_dtype == np.float32:
+            return float32x2_type
+        assert self.real_dtype == np.float64
+        return float64x2_type
+
+
+complex32 = Complex(np.float16)
+complex64 = Complex(np.float32)
+complex128 = Complex(np.float64)
+
+
+class Vector:
+    """
+    Vector type that can be used to represent vector values both
+    on host and device side. Host side representation uses numpy structured
+    dtypes to represent the vector components, while device side representation
+    uses custom numba types. This difference is necessary because device
+    functions expect alignment of the vector types to be the same as the size of
+    the vector, which is not the case for numpy structured dtypes.
+    :py:const:`np_float16x2` and :py:const:`np_float16x4` are the host side
+    dtypes and :py:const:`float16x2_type` and :py:const:`float16x4_type` are the
+    device side types.
+    """
+
+    def __init__(self, real_dtype, size):
+        if size not in (2, 4):
+            raise ValueError(f"Unsupported vector size {size}, only 2 and 4 are supported")
+        if real_dtype != np.float16:
+            raise ValueError(f"Unsupported vector real dtype {real_dtype}, only float16 is supported")
+        self._real_dtype = real_dtype
+        self._size = size
+
+    @property
+    def real_dtype(self):
+        return self._real_dtype
+
+    @property
+    def size(self):
+        return self._size
+
+    @property
+    def dtype(self):
+        warnings.warn(_alignment_warning_msg, UserWarning, stacklevel=2)
+        if self._size == 2:
+            return np_float16x2
+        assert self._size == 4
+        return np_float16x4
+
+    @property
+    def _numba_type(self):
+        from .vector_types_numba import float16x2_type, float16x4_type
+
+        if self._size == 2:
+            return float16x2_type
+        assert self._size == 4
+        return float16x4_type
+
+
+half2 = Vector(np.float16, 2)
+half4 = Vector(np.float16, 4)

nvmath/device/vector_types_numba.py

@@ -0,0 +1,259 @@
+# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+__all__ = [
+    "float16x2",
+    "float16x4",
+    "float32x2",
+    "float32x4",
+    "float64x2",
+    "float64x4",
+    "uint32x4",
+    "float16x2_type",
+    "float16x4_type",
+    "float32x2_type",
+    "float32x4_type",
+    "float64x2_type",
+    "float64x4_type",
+    "uint32x4_type",
+]
+
+import typing
+
+from llvmlite import ir
+from numba.core import types
+from numba.core.typeconv import Conversion
+from numba.core.typing.templates import AttributeTemplate
+from numba.cuda.cudadecl import registry as cuda_registry
+from numba.cuda.cudaimpl import lower_attr as cuda_lower_attr
+from numba.cuda.extending import as_numba_type, lower_builtin, lower_cast, overload, register_model, type_callable
+from numba.extending import models
+
+
+def make_vector(
+    scalar_type: types.Number,
+    llvm_scalar_type,
+    vector_length: typing.Literal[2, 4],
+):
+    assert vector_length == 2 or vector_length == 4
+    assert isinstance(scalar_type, (types.Float, types.Integer))
+    vector_name = f"{scalar_type.name}x{vector_length}"
+    vector_bitwidth = vector_length * scalar_type.bitwidth
+    if isinstance(scalar_type, types.Float):
+        scalar_class = types.Float
+    else:
+        scalar_class = types.Integer
+
+    # User visible types and functions
+    class vector:
+        def __init__(self, *args):
+            pass
+
+    def create(input):
+        pass
+
+    # FE types
+    class vectorType(types.Number):
+        def __init__(self):
+            super().__init__(name=f"vector({scalar_type.name}x{vector_length})")
+            self.dtype = scalar_type
+            self.count = vector_length
+            self.bitwidth = vector_bitwidth
+            self.make = vector
+
+        def can_convert_from(self, typingctx, other):
+            if scalar_class is types.Float and vector_length == 2 and isinstance(other, types.Complex):
+                if other.bitwidth == vector_bitwidth:
+                    return Conversion.exact
+                elif other.bitwidth < vector_bitwidth:
+                    return Conversion.safe
+                elif other.bitwidth > vector_bitwidth:
+                    return Conversion.unsafe
+
+        def can_convert_to(self, typingctx, other):
+            if scalar_class is types.Float and vector_length == 2 and isinstance(other, types.Complex):
+                if other.bitwidth == vector_bitwidth:
+                    return Conversion.exact
+                elif other.bitwidth < vector_bitwidth:
+                    return Conversion.unsafe
+                elif other.bitwidth > vector_bitwidth:
+                    return Conversion.safe
+
+        def __name__(self):
+            return f"vector({scalar_type.name}x{vector_length})"
+
+    vector_type = vectorType()
+
+    types.number_domain = types.number_domain | frozenset([vector_type])
+
+    # Type inference (Python --> FE type)
+    as_numba_type.register(vector, vector_type)
+
+    # How to build a vectorType from individual elements
+    @type_callable(vector)
+    def type(context):
+        def typer(x, y=None, z=None, w=None):
+            if (
+                (
+                    scalar_class is types.Float
+                    and vector_length == 2
+                    and isinstance(x, types.Complex)
+                    and y is None
+                    and z is None
+                    and w is None
+                )
+                or (vector_length == 2 and all(isinstance(v, scalar_class) for v in [x, y]) and z is None and w is None)
+                or (vector_length == 4 and all(isinstance(v, scalar_class) for v in [x, y, z, w]))
+            ):
+                return vector_type
+
+        return typer
+
+    # How to represent vectorType in Memory?
+    @register_model(vectorType)
+    class vectorTypeModel(models.PrimitiveModel):
+        def __init__(self, dmm, fe_type):
+            be_type = ir.IntType(vector_bitwidth)
+            super().__init__(dmm, fe_type, be_type)
+
+    # How to build a vectorType from individual float values?
+    @lower_builtin(vector, *(vector_length * [scalar_class]))  # FIXME
+    def vector_ctor(context, builder, sig, args):
+        typ = sig.return_type
+        args = list(args)
+        assert len(args) == vector_length
+        for i in range(vector_length):
+            args[i] = context.cast(builder, args[i], sig.args[i], scalar_type)
+
+        vt = ir.VectorType(llvm_scalar_type, vector_length)
+        val = ir.Constant(vt, ir.Undefined)
+
+        for i in range(vector_length):
+            val = builder.insert_element(val, args[i], context.get_constant(types.int32, i))
+
+        return builder.bitcast(val, context.get_value_type(typ))
+
+    # How to access .x, .y, .z, .w
+    @cuda_registry.register_attr
+    class complex_attrs(AttributeTemplate):
+        key = vectorType
+
+        def resolve_x(self, mod):
+            return scalar_type
+
+        def resolve_y(self, mod):
+            return scalar_type
+
+        def resolve_z(self, mod):
+            return scalar_type
+
+        def resolve_w(self, mod):
+            return scalar_type
+
+    def complex_get(context, builder, typ, val, i):
+        vt = ir.VectorType(llvm_scalar_type, vector_length)
+        vec = builder.bitcast(val, vt)
+        index = context.get_constant(types.int32, i)
+        return builder.extract_element(vec, index)
+
+    # This cannot be done with a loop
+    @cuda_lower_attr(vectorType, "x")
+    def complex_get_x(context, builder, typ, val):
+        return complex_get(context, builder, typ, val, 0)
+
+    @cuda_lower_attr(vectorType, "y")
+    def complex_get_y(context, builder, typ, val):
+        return complex_get(context, builder, typ, val, 1)
+
+    if vector_length == 4:
+
+        @cuda_lower_attr(vectorType, "z")
+        def complex_get_z(context, builder, typ, val):
+            return complex_get(context, builder, typ, val, 2)
+
+        @cuda_lower_attr(vectorType, "w")
+        def complex_get_w(context, builder, typ, val):
+            return complex_get(context, builder, typ, val, 3)
+
+    # Conversions from types.Complex
+    if scalar_class is types.Float and vector_length == 2:
+
+        @overload(create)
+        def make_complex(cplx):
+            if isinstance(cplx, types.Complex):
+
+                def impl(cplx):
+                    return vector(cplx.real, cplx.imag)
+
+                return impl
+
+        @lower_cast(types.Complex, vector_type)
+        def np_complex_to_complex(context, builder, fromty, toty, val):
+            src = context.make_complex(builder, fromty, value=val)
+            src_float_ty = fromty.underlying_float
+            ctor_sig = toty(src_float_ty, src_float_ty)
+            ctor_args = (src.real, src.imag)
+            return vector_ctor(context, builder, ctor_sig, ctor_args)
+
+        @lower_cast(vector_type, types.Complex)
+        def complex_to_np_complex(context, builder, fromty, toty, val):
+            dst_float_ty = toty.underlying_float
+            x = complex_get(context, builder, fromty, val, 0)
+            y = complex_get(context, builder, fromty, val, 1)
+            x = context.cast(builder, x, scalar_type, dst_float_ty)
+            y = context.cast(builder, y, scalar_type, dst_float_ty)
+            vt = context.get_value_type(toty)
+            val = ir.Constant(vt, ir.Undefined)
+            val = builder.insert_value(val, x, [0])
+            val = builder.insert_value(val, y, [1])
+            return val
+
+    vector.__name__ = vector_name
+    attrs = "``x`` and ``y``" if vector_length == 2 else "``x``, ``y``, ``z``, and ``w``"
+    vector.__doc__ = f"""
+    {vector_name}({"x, y" if vector_length == 2 else "x, y, z, w"})
+
+    A Numba-compliant vector object for {scalar_type.name} with vector length {vector_length}.
+    Elements can be accessed with the {attrs} attributes.
+    """
+    return vector, vector_type
+
+
+class VectorTypeClass(types.NumberClass):
+    """
+    NumberClass for the vector FE types.
+    Reuses NumberClass for the DTypeSpec behavior but routes cls(x, y, ...)
+    to the vector constructor instead of NumberClass's single-value scalar cast.
+    """
+
+    def get_impl_key(self, sig):
+        return self.instance_type.make
+
+
+register_model(VectorTypeClass)(models.OpaqueModel)
+
+
+@cuda_registry.register_attr
+class _VectorTypeClassCallAttribute(AttributeTemplate):
+    """
+    Overrides NumberClass's scalar-cast __call__.
+    Resolves cls(x, y, ...) to the vector constructor,
+    so calls build a vector instead of attempting a single-value cast.
+    """
+
+    key = VectorTypeClass
+
+    def resolve___call__(self, classty):
+        return self.context.resolve_value_type(classty.instance_type.make)
+
+
+# IMPORTANT: take care to update symbol list in docs/sphinx/device-apis/utils.rst
+# and docstring processing in docs/sphinx/conf.py if names are changed or types are added
+float16x2, float16x2_type = make_vector(types.float16, ir.IntType(16), 2)
+float16x4, float16x4_type = make_vector(types.float16, ir.IntType(16), 4)
+float32x2, float32x2_type = make_vector(types.float32, ir.FloatType(), 2)
+float32x4, float32x4_type = make_vector(types.float32, ir.FloatType(), 4)
+float64x2, float64x2_type = make_vector(types.float64, ir.DoubleType(), 2)
+float64x4, float64x4_type = make_vector(types.float64, ir.DoubleType(), 4)
+uint32x4, uint32x4_type = make_vector(types.uint32, ir.IntType(32), 4)

nvmath/distributed/__init__.py

@@ -0,0 +1,200 @@
+# Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import sys
+
+if sys.platform != "linux":
+    raise ImportError("nvmath.distributed is only supported on Linux.")
+
+import atexit
+from collections.abc import Sequence
+from dataclasses import dataclass
+from threading import Lock
+from typing import TYPE_CHECKING, Any, Literal
+
+import numpy as np
+from cuda.core import Device
+
+if TYPE_CHECKING:
+    import mpi4py
+
+from nvmath.internal.utils import device_ctx
+
+from . import distribution, fft, linalg
+from ._internal import nvshmem
+from ._utils import allocate_symmetric_memory, free_symmetric_memory
+from .process_group import MPIProcessGroup, ProcessGroup, TorchProcessGroup
+
+__all__ = [
+    "initialize",
+    "finalize",
+    "get_context",
+    "ProcessGroup",
+    "MPIProcessGroup",
+    "TorchProcessGroup",
+    "allocate_symmetric_memory",
+    "free_symmetric_memory",
+    "distribution",
+    "fft",
+    "linalg",
+]
+
+_initialize_mutex = Lock()
+_atexit_registered = False
+_ctx = None
+
+
+@dataclass(frozen=True)
+class DistributedContext:
+    """
+    Context of initialized ``nvmath.distributed`` runtime.
+
+    Attributes:
+        device_id: CUDA device ID associated with the distributed runtime
+            on this process.
+
+        process_group: ``nvmath.distributed`` participating processes.
+
+        nvshmem_available: True if NVSHMEM backend was selected at initialization.
+
+        nccl_comm: nccl4py communicator if NCCL backend was selected at
+            initialization, None otherwise.
+    """
+
+    device_id: int
+    process_group: ProcessGroup
+    nvshmem_available: bool
+    nccl_comm: Any | None
+
+
+def initialize(
+    device_id: int,
+    process_group: ProcessGroup | mpi4py.MPI.Comm,
+    backends: Sequence[Literal["nvshmem", "nccl"]],
+) -> None:
+    """Initialize ``nvmath.distributed`` runtime. This is required before any distributed
+    operations can be performed. **Note that this is a collective operation and must be
+    called by all processes.**
+
+    If the runtime is already initialized this function will raise an error. If you need
+    to reinitialize the runtime (for example with different backends) you have to finalize
+    it first.
+
+    Note: NCCL doesn't allow assigning more than one process to the same GPU.
+
+    Args:
+        device_id: CUDA device ID to associate with the ``nvmath.distributed`` runtime
+            on this process.
+
+        process_group: ProcessGroup (or mpi4py communicator) specifying the participating
+            processes. This is used for setup and not for communication during compute.
+
+        backends: Communication backends to use in distributed computations. Valid values
+            are "nvshmem" and "nccl". Note that specific libraries (cuFFTMp, cuBLASMp, ...)
+            have specific required backends.
+    """
+    if not isinstance(device_id, int):
+        raise TypeError(
+            "The device ID used to initialize the nvmath.distributed module "
+            f"must be an integer. The provided device ID is {device_id}."
+        )
+
+    valid_backends = ("nvshmem", "nccl")
+    for backend in backends:
+        if backend not in valid_backends:
+            raise ValueError(f"backend must be one of {valid_backends}, got {backend}")
+
+    if len(backends) == 0:
+        raise ValueError(f"Need to specify at least one of {valid_backends} communication backends")
+
+    if process_group is None:
+        raise ValueError("process_group cannot be None")
+
+    with _initialize_mutex:
+        global _atexit_registered, _ctx
+
+        if _ctx is not None:
+            raise RuntimeError("nvmath.distributed has already been initialized")
+
+        if not _atexit_registered:
+            atexit.register(finalize)
+            _atexit_registered = True
+
+        if not isinstance(process_group, ProcessGroup):
+            valid_process_group = False
+            if "mpi4py" in sys.modules:
+                import mpi4py.MPI
+
+                if isinstance(process_group, mpi4py.MPI.Comm):
+                    process_group = MPIProcessGroup(process_group)
+                    valid_process_group = True
+            if not valid_process_group:
+                raise TypeError(
+                    f"Unrecognized process group type ({process_group}). "
+                    "Need nvmath.distributed.ProcessGroup or mpi4py communicator."
+                )
+
+        assert isinstance(process_group, ProcessGroup)  # for type-checker
+        rank = process_group.rank
+        nranks = process_group.nranks
+
+        # Set the device for NVSHMEM and NCCL initialization, but also need to make sure
+        # that a CUDA context has been created. We can't rely on `device_ctx` to do it
+        # since it's not guaranteed to make a runtime API call.
+        old_device = Device()
+        device = Device(device_id)
+        device.set_current()
+
+        try:
+            nvshmem_available = False
+            if "nvshmem" in backends:
+                nvshmem.initialize(device_id, process_group)
+                nvshmem_available = True
+
+            nccl_comm = None
+            if "nccl" in backends:
+                # Don't want to import this at the global scope yet because
+                # `import nccl.core` imports cupy and torch (with nccl4py 0.1.1)
+                import nccl.core as nccl  # type: ignore
+
+                # Create NCCL communicator.
+                unique_id = nccl.get_unique_id()
+                process_group.broadcast_buffer(unique_id.as_ndarray.view(np.int8), root=0)
+                nccl_comm = nccl.Communicator.init(nranks=nranks, rank=rank, unique_id=unique_id)
+
+            _ctx = DistributedContext(
+                device_id=device_id, process_group=process_group, nvshmem_available=nvshmem_available, nccl_comm=nccl_comm
+            )
+        finally:
+            old_device.set_current()
+
+
+def finalize() -> None:
+    """Finalize ``nvmath.distributed`` runtime (this is called automatically at exit
+    if the runtime is initialized). **Note that this is a collective operation and
+    must be called by all processes.**"""
+    global _ctx
+    with _initialize_mutex:
+        if _ctx is None:
+            return
+
+        linalg.advanced.matmulmod._grid_cache.clear()
+        linalg.generic._caching.clear_handles()
+
+        if _ctx.nccl_comm is not None:
+            with device_ctx(_ctx.device_id):
+                _ctx.nccl_comm.finalize()
+                _ctx.nccl_comm.destroy()
+
+        if _ctx.nvshmem_available:
+            nvshmem.finalize(_ctx.device_id)
+
+        _ctx = None
+
+
+def get_context() -> DistributedContext | None:
+    """Return the distributed runtime's context or None if not initialized."""
+    return _ctx

nvmath/distributed/_internal/nccl.py

@@ -0,0 +1,86 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+__all__ = [
+    "nccl_empty_dlpack",
+    "NcclNDBufferAllocator",
+]
+
+import logging
+from typing import TYPE_CHECKING
+
+from .symmetric_memory import calculate_symmetric_size
+
+if TYPE_CHECKING:
+    from cuda.core import Buffer, Stream
+
+    from nvmath.distributed import DistributedContext
+    from nvmath.distributed.process_group import ProcessGroup
+
+
+def nccl_empty_dlpack(
+    size: int,
+    device_id: int,
+    process_group: ProcessGroup,
+    make_symmetric: bool = False,
+    skip_symmetric_check: bool = False,
+    logger: logging.Logger | None = None,
+) -> Buffer:
+    """
+    Allocates a buffer with nccl4py. This function can be used to prepare a buffer
+    for symmetric memory registration, but **doesn't register the buffer as symmetric
+    memory** (because buffer registrations in NCCL are not global but on a
+    per-communicator basis, so for registration the buffer typically has to be passed
+    to the library that owns the communicator).
+
+    This is a collective call when ``make_symmetric=True`` or
+    ``skip_symmetric_check=False``.
+
+    Args:
+        size: Number of bytes to allocate.
+
+        device_id: Device on which to allocate.
+
+        process_group: nvmath.distributed process group.
+
+        make_symmetric: If the buffer is going to be registered with NCCL symmetric memory,
+            use ``make_symmetric=True`` to allocate the same size on every process (padding
+            the buffer to the maximum size given by all processes).
+
+        skip_symmetric_check: True if buffer is not needed for NCCL symmetric memory or
+            already known to be of the same size on every process.
+    """
+
+    logger = logger if logger is not None else logging.getLogger()
+
+    size = calculate_symmetric_size(size, process_group, make_symmetric, skip_symmetric_check, logger)
+
+    import nccl.core as nccl  # type: ignore
+
+    mem_buffer = nccl.buffer.mem_alloc(size, device_id)
+    return mem_buffer
+
+
+class NcclNDBufferAllocator:
+    __slots__ = ("ctx", "make_symmetric", "skip_symmetric_check")
+
+    def __init__(self, device_id: int, ctx: DistributedContext, make_symmetric: bool, skip_symmetric_check: bool):
+        assert ctx.device_id == device_id, (
+            "Internal error: attempting to allocate symmetric memory on a device not used by NCCL on this process"
+        )
+        self.ctx = ctx
+        self.make_symmetric = make_symmetric
+        self.skip_symmetric_check = skip_symmetric_check
+
+    def allocate(self, size: int, stream: Stream, logger: logging.Logger | None = None) -> Buffer:
+        return nccl_empty_dlpack(
+            size,
+            self.ctx.device_id,
+            self.ctx.process_group,
+            make_symmetric=self.make_symmetric,
+            skip_symmetric_check=self.skip_symmetric_check,
+            logger=logger,
+        )