nvidia-nccl-cu13 2.28.3__py3-none-manylinux_2_18_aarch64.whl → 2.28.7__py3-none-manylinux_2_18_aarch64.whl

nvidia/nccl/include/nccl.h

@@ -18,10 +18,10 @@
 
 #define NCCL_MAJOR 2
 #define NCCL_MINOR 28
-#define NCCL_PATCH 3
+#define NCCL_PATCH 7
 #define NCCL_SUFFIX ""
 
-#define NCCL_VERSION_CODE 22803
+#define NCCL_VERSION_CODE 22807
 #define NCCL_VERSION(X,Y,Z) (((X) <= 2 && (Y) <= 8) ? (X) * 1000 + (Y) * 100 + (Z) : (X) * 10000 + (Y) * 100 + (Z))
 
 #ifdef __cplusplus
@@ -69,6 +69,9 @@ typedef enum { ncclSuccess                 =  0,
 #define NCCL_SHRINK_DEFAULT 0x00 /* shrink the parent communicator */
 #define NCCL_SHRINK_ABORT 0x01   /* First, terminate ongoing parent operations, and then shrink the parent communicator */
 
+/* ncclCommRevoke flags */
+#define NCCL_REVOKE_DEFAULT 0x00 /* reserved for future use; must be 0 */
+
 /* Communicator configuration. Users can assign value to attributes to specify the
  * behavior of a communicator. */
 typedef struct ncclConfig_v22800 {
@@ -194,6 +197,15 @@ ncclResult_t pncclCommDestroy(ncclComm_t comm);
 ncclResult_t  ncclCommAbort(ncclComm_t comm);
 ncclResult_t pncclCommAbort(ncclComm_t comm);
 
+/* Revoke a communicator. ncclCommRevoke stops all in-flight operations
+ * and marks communicator state as ncclInProgress. The state will change to ncclSuccess
+ * when the communicator is quiescent; then, management operations (destroy, split,
+ * shrink) can proceed safely. Calling ncclCommFinalize after revoke is invalid.
+ * Additionally, resource sharing via splitShare/shrinkShare is disabled while revoked.
+ * revokeFlags must be NCCL_REVOKE_DEFAULT (0). */
+ncclResult_t  ncclCommRevoke(ncclComm_t comm, int revokeFlags);
+ncclResult_t pncclCommRevoke(ncclComm_t comm, int revokeFlags);
+
 /* Creates one or more communicators from an existing one.
  * Ranks with the same color will end up in the same communicator.
  * Within the new communicator, key will be used to order ranks.

nvidia/nccl/include/nccl_device/barrier.h

@@ -0,0 +1,47 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_DEVICE_BARRIER_H_
+#define _NCCL_DEVICE_BARRIER_H_
+#include "impl/core__types.h"
+#include "impl/lsa_barrier__types.h"
+#include "impl/gin_barrier__types.h"
+
+#if __CUDACC__
+template<typename Coop>
+struct ncclBarrierSession_internal;
+
+template<typename Coop>
+struct ncclBarrierSession: ncclBarrierSession_internal<Coop> {
+  // Full featured constructor:
+  NCCL_DEVICE_INLINE ncclBarrierSession(
+    Coop, ncclTeam innerTeam, ncclTeam outerTeam, ncclGin,
+    ncclLsaBarrierHandle innerBarHandle,
+    ncclGinBarrierHandle outerBarHandle,
+    uint32_t index,
+    bool multimem=false, ncclMultimemHandle innerMmHandle={}
+  );
+  // Convenience constructors for baked in teams:
+  NCCL_DEVICE_INLINE ncclBarrierSession(
+    Coop, ncclTeamTagWorld, ncclGin, uint32_t index, bool multimem=false
+  );
+  NCCL_DEVICE_INLINE ncclBarrierSession(
+    Coop, ncclTeamTagLsa, ncclDevComm const&, uint32_t index, bool multimem=false
+  );
+  NCCL_DEVICE_INLINE ncclBarrierSession(
+    Coop, ncclTeamTagRail, ncclGin, uint32_t index
+  );
+
+  ncclBarrierSession(ncclBarrierSession const&) = delete; // Sessions are not copyable
+
+  NCCL_DEVICE_INLINE ncclLsaBarrierSession<Coop>& lsaBarrier();
+  NCCL_DEVICE_INLINE ncclGinBarrierSession<Coop>& ginBarrier();
+
+  NCCL_DEVICE_INLINE void sync(Coop, cuda::memory_order, ncclGinFenceLevel);
+};
+#endif
+
+#endif // _NCCL_DEVICE_BARRIER_H_

nvidia/nccl/include/nccl_device/coop.h

@@ -30,7 +30,7 @@ struct ncclCoopTile { // An aligned pow2 set of threads within the warp.
     return (-1u>>(32-nThreadsPow2))<<(nccl::utility::lane() & -nThreadsPow2);
   }
   NCCL_DEVICE_INLINE void sync() {
-    __syncwarp(laneMask());
+    if (nThreadsPow2 > 1) __syncwarp(laneMask());
   }
 };
 #endif
@@ -43,7 +43,7 @@ typedef ncclCoopTile<32> ncclCoopWarp;
 #if __CUDACC__
 struct ncclCoopLanes { // Some lanes of this warp.
   uint32_t lmask;
-  
+
   NCCL_DEVICE_INLINE constexpr ncclCoopLanes(uint32_t lmask=-1u): lmask(lmask) {}
 
   NCCL_DEVICE_INLINE int thread_rank() const {
@@ -71,7 +71,7 @@ struct ncclCoopWarpSpan {
   NCCL_DEVICE_INLINE constexpr ncclCoopWarpSpan(int warp0, int nWarps, int id):
     warp0(warp0), nWarps(nWarps), id(id) {
   }
-  
+
   NCCL_DEVICE_INLINE int thread_rank() const {
     return threadIdx.x - 32*warp0;
   }
@@ -100,16 +100,16 @@ struct ncclCoopCta {
 
 #if __CUDACC__
 template<int nThreadsPow2>
-NCCL_DEVICE_INLINE uint32_t ncclCoopLaneMask(ncclCoopTile<nThreadsPow2> coop) {
+NCCL_DEVICE_INLINE uint32_t ncclCoopGetLaneMask(ncclCoopTile<nThreadsPow2> coop) {
   return coop.laneMask();
 }
-NCCL_DEVICE_INLINE uint32_t ncclCoopLaneMask(ncclCoopLanes coop) {
+NCCL_DEVICE_INLINE uint32_t ncclCoopGetLaneMask(ncclCoopLanes coop) {
   return coop.lmask;
 }
-NCCL_DEVICE_INLINE uint32_t ncclCoopLaneMask(ncclCoopWarpSpan coop) {
+NCCL_DEVICE_INLINE uint32_t ncclCoopGetLaneMask(ncclCoopWarpSpan coop) {
   return -1u;
 }
-NCCL_DEVICE_INLINE uint32_t ncclCoopLaneMask(ncclCoopCta coop) {
+NCCL_DEVICE_INLINE uint32_t ncclCoopGetLaneMask(ncclCoopCta coop) {
   return -1u;
 }
 #endif
@@ -126,6 +126,14 @@ NCCL_DEVICE_INLINE constexpr bool ncclCoopIsThread(ncclCoopWarpSpan) { return fa
 NCCL_DEVICE_INLINE constexpr bool ncclCoopIsThread(ncclCoopCta) { return false; }
 #endif
 
+#if __CUDACC__
+template<int nThreads>
+NCCL_DEVICE_INLINE constexpr bool ncclCoopWithinWarp(ncclCoopTile<nThreads>) { return true; }
+NCCL_DEVICE_INLINE constexpr bool ncclCoopWithinWarp(ncclCoopLanes) { return true; }
+NCCL_DEVICE_INLINE constexpr bool ncclCoopWithinWarp(ncclCoopWarpSpan) { return false; }
+NCCL_DEVICE_INLINE constexpr bool ncclCoopWithinWarp(ncclCoopCta) { return false; }
+#endif
+
 #if __CUDACC__
 // Pick threads of our warp that are safe to use collectively.
 NCCL_DEVICE_INLINE ncclCoopLanes ncclCoopCoalesced() {
@@ -149,4 +157,55 @@ NCCL_DEVICE_INLINE ncclCoopTile<nThreads> ncclCoopCoalesced(ncclCoopTile<nThread
 }
 #endif
 
+#if __CUDACC__
+template<int nThreads, typename T>
+NCCL_DEVICE_INLINE T ncclCoopBcast(ncclCoopTile<nThreads>, T value, int root, bool entrySync=true) {
+  constexpr int n = (sizeof(T)+4-1)/4;
+  union { uint32_t u[n]; T v; };
+  v = value;
+  #pragma unroll
+  for (int i=0; i < n; i++) u[i] = __shfl_sync(-1u, u[i], root, nThreads);
+  return v;
+}
+template<typename T>
+NCCL_DEVICE_INLINE T ncclCoopBcast(ncclCoopLanes coop, T value, int root, bool entrySync=true) {
+  uint32_t m = coop.lmask;
+  uint32_t r = root == 0 ? __ffs(m)-1 : __fns(m, 0, 1+root);
+  constexpr int n = (sizeof(T)+4-1)/4;
+  union { uint32_t u[n]; T v; };
+  v = value;
+  #pragma unroll
+  for (int i=0; i < n; i++) u[i] = __shfl_sync(m, u[i], r);
+  return v;
+}
+
+NCCL_DEVICE_INLINE ulong2* ncclCoopBcast_WarpSpan_stash() {
+  __shared__ ulong2 stash[15];
+  return stash;
+}
+
+template<typename T>
+NCCL_DEVICE_INLINE T ncclCoopBcast(ncclCoopWarpSpan coop, T value, int root, bool entrySync=true) {
+  static_assert(sizeof(T) <= sizeof(ncclCoopBcast_WarpSpan_stash()[0]), "Required");
+  if (entrySync) coop.sync();
+  if (coop.thread_rank() == root) *(T*)&ncclCoopBcast_WarpSpan_stash()[coop.id] = value;
+  coop.sync();
+  return *(T*)&ncclCoopBcast_WarpSpan_stash()[coop.id];
+}
+
+NCCL_DEVICE_INLINE ulong2* ncclCoopBcast_Cta_stash() {
+  __shared__ ulong2 stash;
+  return &stash;
+}
+
+template<typename T>
+NCCL_DEVICE_INLINE T ncclCoopBcast(ncclCoopCta coop, T value, int root, bool entrySync=true) {
+  static_assert(sizeof(T) <= sizeof(*ncclCoopBcast_Cta_stash()), "Required");
+  if (entrySync) coop.sync();
+  if (coop.thread_rank() == root) *(T*)ncclCoopBcast_Cta_stash() = value;
+  coop.sync();
+  return *(T*)ncclCoopBcast_Cta_stash();
+}
+#endif
+
 #endif

nvidia/nccl/include/nccl_device/core.h

@@ -24,9 +24,15 @@ typedef struct ncclMultimemHandle ncclMultimemHandle_t;
 typedef uint32_t ncclDevResourceHandle;
 typedef ncclDevResourceHandle ncclDevResourceHandle_t;
 
+typedef uint32_t ncclGinSignal_t;
+typedef uint32_t ncclGinCounter_t;
+
 struct ncclLsaBarrierHandle;
 typedef struct ncclLsaBarrierHandle ncclLsaBarrierHandle_t;
 
+struct ncclGinBarrierHandle;
+typedef struct ncclGinBarrierHandle ncclGinBarrierHandle_t;
+
 struct ncclLLA2AHandle;
 typedef struct ncclLLA2AHandle ncclLLA2AHandle_t;
 
@@ -59,13 +65,26 @@ struct ncclDevCommRequirements {
 
   bool lsaMultimem; // Enable multimem on lsa team
 
+  int barrierCount;
   int lsaBarrierCount;
+  int railGinBarrierCount;
+
+  int lsaLLA2ABlockCount, lsaLLA2ASlotCount;
+
+  bool ginForceEnable;
+  int ginContextCount; // This is a hint, the actual context count in the devcomm may not match.
+  int ginSignalCount; // Guaranteed to start at id=0
+  int ginCounterCount; // Guaranteed to start at id=0
 };
 
 struct ncclDevResourceRequirements {
   ncclDevResourceRequirements_t* next;
   size_t bufferSize, bufferAlign;
   ncclDevResourceHandle_t* outBufferHandle; // If non-null, target assigned during ncclDevCommCreate.
+  int ginSignalCount;
+  int ginCounterCount;
+  ncclGinSignal_t* outGinSignalStart;
+  ncclGinCounter_t* outGinCounterStart;
 };
 
 struct ncclTeamRequirements {

nvidia/nccl/include/nccl_device/gin/gdaki/gin_gdaki.h

@@ -0,0 +1,214 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_DEVICE_GIN_GDAKI_H_
+#define _NCCL_DEVICE_GIN_GDAKI_H_
+
+#ifndef DOCA_VERBS_USE_CUDA_WRAPPER
+#define DOCA_VERBS_USE_CUDA_WRAPPER
+#endif
+
+#ifndef DOCA_VERBS_USE_NET_WRAPPER
+#define DOCA_VERBS_USE_NET_WRAPPER
+#endif
+
+#ifdef NCCL_DEVICE_GIN_GDAKI_ENABLE_DEBUG
+#define DOCA_GPUNETIO_VERBS_ENABLE_DEBUG 1
+#endif
+
+#include "../gin_device_common.h"
+#include "gin_gdaki_device_host_common.h"
+#include "doca_gpunetio/doca_gpunetio_device.h"
+
+#ifdef NCCL_DEVICE_GIN_GDAKI_ENABLE_DEBUG
+#include <stdio.h>
+#endif
+
+template <>
+struct ncclGinApi_Put<NCCL_NET_DEVICE_GIN_GDAKI> {
+  template <typename Coop>
+  NCCL_DEVICE_INLINE static void call(ncclGinCtx ctx, Coop coop, int peer, bool hasWins,
+                                      ncclGinWindow_t dstWin, size_t dstOff, ncclGinWindow_t srcWin,
+                                      size_t srcOff, size_t bytes, bool hasSignal,
+                                      ncclGinSignal_t signalId, ncclGinSignalOp_t signalOp,
+                                      uint64_t signalOpArg, bool hasCounter,
+                                      ncclGinCounter_t counterId, bool hasDescriptor,
+                                      ncclGinDescriptorSmem* descriptor,
+                                      cuda::thread_scope required, cuda::thread_scope given) {
+    using nccl::utility::loadConst;
+
+    coop.sync();
+    if (coop.thread_rank() == 0) {
+      ncclGinGdakiGPUContext* gdaki = (struct ncclGinGdakiGPUContext*)ctx.handle;
+      doca_gpu_dev_verbs_qp* qp = loadConst(&gdaki->gdqp) + peer;
+      doca_gpu_dev_verbs_qp* companion_qp;
+      ncclGinGdakiMemHandle* dstMh = (ncclGinGdakiMemHandle*)dstWin;
+      ncclGinGdakiMemHandle* srcMh = (ncclGinGdakiMemHandle*)srcWin;
+
+      doca_gpu_dev_verbs_addr raddr, laddr;
+      if (hasWins) {
+        raddr.addr = dstOff;
+        raddr.key = loadConst(loadConst(&dstMh->rkeys) + peer);
+        laddr.addr = srcOff, laddr.key = loadConst(&srcMh->lkey);
+      }
+
+      doca_gpu_dev_verbs_addr sig_raddr, sig_laddr;
+      if (hasSignal) {
+        if (signalOp == ncclGinSignalInc) signalOpArg = 1;
+        sig_raddr.addr = sizeof(uint64_t) * signalId;
+        sig_raddr.key = loadConst(loadConst(&gdaki->signals_table.rkeys) + peer);
+        sig_laddr.addr = 0;
+        sig_laddr.key = loadConst(&gdaki->sink_buffer_lkey);
+      }
+
+      doca_gpu_dev_verbs_addr counter_raddr, counter_laddr;
+      if (hasCounter) {
+        companion_qp = loadConst(&gdaki->companion_gdqp) + peer;
+        counter_raddr.addr = sizeof(uint64_t) * counterId;
+        counter_raddr.key = loadConst(loadConst(&gdaki->counters_table.rkeys) + ctx.rank);
+        counter_laddr.addr = 0;
+        counter_laddr.key = loadConst(&gdaki->sink_buffer_lkey);
+      }
+
+      // cuda::thread_scope_system has the lowest value
+      if ((required == cuda::thread_scope_system) && (given > required)) {
+        doca_gpu_dev_verbs_fence_release<DOCA_GPUNETIO_VERBS_SYNC_SCOPE_SYS>();
+      }
+
+      if (hasWins) {
+        if (hasSignal && hasCounter) {
+          doca_gpu_dev_verbs_put_signal_counter<DOCA_GPUNETIO_VERBS_SIGNAL_OP_ADD>(
+            qp, raddr, laddr, bytes, sig_raddr, sig_laddr, signalOpArg, companion_qp, counter_raddr,
+            counter_laddr, 1);
+        } else if (hasSignal) {
+          doca_gpu_dev_verbs_put_signal<DOCA_GPUNETIO_VERBS_SIGNAL_OP_ADD>(
+            qp, raddr, laddr, bytes, sig_raddr, sig_laddr, signalOpArg);
+        } else if (hasCounter) {
+          doca_gpu_dev_verbs_put_counter(qp, raddr, laddr, bytes, companion_qp, counter_raddr,
+                                              counter_laddr, 1);
+        } else {
+          doca_gpu_dev_verbs_put(qp, raddr, laddr, bytes);
+        }
+      } else {
+        if (hasCounter) {
+          doca_gpu_dev_verbs_signal_counter<DOCA_GPUNETIO_VERBS_SIGNAL_OP_ADD>(
+            qp, sig_raddr, sig_laddr, signalOpArg, companion_qp, counter_raddr, counter_laddr, 1);
+        } else {
+          doca_gpu_dev_verbs_signal<DOCA_GPUNETIO_VERBS_SIGNAL_OP_ADD>(
+            qp, sig_raddr, sig_laddr, signalOpArg);
+        }
+      }
+
+#ifdef NCCL_DEVICE_GIN_GDAKI_ENABLE_DEBUG
+      doca_gpu_dev_verbs_wait(qp);
+      if (hasCounter) doca_gpu_dev_verbs_wait(companion_qp);
+#endif
+    }
+    coop.sync();
+  }
+};
+
+template <>
+struct ncclGinApi_PutValue<NCCL_NET_DEVICE_GIN_GDAKI> {
+  template <typename Coop, typename T>
+  NCCL_DEVICE_INLINE static void call(ncclGinCtx ctx, Coop coop, int peer, ncclGinWindow_t dstWin,
+                                      size_t dstOff, T srcVal, bool hasSignal,
+                                      ncclGinSignal_t signalId, ncclGinSignalOp_t signalOp,
+                                      uint64_t signalOpArg, bool hasDescriptor,
+                                      ncclGinDescriptorSmem* descriptor,
+                                      cuda::thread_scope required, cuda::thread_scope given) {
+    using nccl::utility::loadConst;
+
+    coop.sync();
+    if (coop.thread_rank() == 0) {
+      ncclGinGdakiGPUContext* gdaki = (struct ncclGinGdakiGPUContext*)ctx.handle;
+      doca_gpu_dev_verbs_qp* qp = loadConst(&gdaki->gdqp) + peer;
+      ncclGinGdakiMemHandle* dstMh = (ncclGinGdakiMemHandle*)dstWin;
+
+      doca_gpu_dev_verbs_addr raddr;
+      raddr.addr = dstOff;
+      raddr.key = loadConst(loadConst(&dstMh->rkeys) + peer);
+
+      doca_gpu_dev_verbs_addr sig_raddr, sig_laddr;
+      if (hasSignal) {
+        if (signalOp == ncclGinSignalInc) signalOpArg = 1;
+        sig_raddr.addr = sizeof(uint64_t) * signalId;
+        sig_raddr.key = loadConst(loadConst(&gdaki->signals_table.rkeys) + peer);
+        sig_laddr.addr = 0;
+        sig_laddr.key = loadConst(&gdaki->sink_buffer_lkey);
+      }
+
+      // cuda::thread_scope_system has the lowest value
+      if ((required == cuda::thread_scope_system) && (given > required)) {
+        doca_gpu_dev_verbs_fence_release<DOCA_GPUNETIO_VERBS_SYNC_SCOPE_SYS>();
+      }
+
+      if (hasSignal) {
+        doca_gpu_dev_verbs_p_signal<T, DOCA_GPUNETIO_VERBS_SIGNAL_OP_ADD>(
+          qp, raddr, srcVal, sig_raddr, sig_laddr, signalOpArg);
+      } else {
+        doca_gpu_dev_verbs_p(qp, raddr, srcVal);
+      }
+
+#ifdef NCCL_DEVICE_GIN_GDAKI_ENABLE_DEBUG
+      doca_gpu_dev_verbs_wait(qp);
+#endif
+    }
+    coop.sync();
+  }
+};
+
+template <>
+struct ncclGinApi_ResetCounter<NCCL_NET_DEVICE_GIN_GDAKI> {
+  NCCL_DEVICE_INLINE static void call(ncclGinCtx ctx, ncclGinCounter_t counterId) {
+    using nccl::utility::loadConst;
+    ncclGinGdakiGPUContext* gdaki = (ncclGinGdakiGPUContext*)ctx.handle;
+    loadConst(&gdaki->counters_table.buffer)[counterId] = 0;
+  }
+};
+
+template <>
+struct ncclGinApi_ResetSignal<NCCL_NET_DEVICE_GIN_GDAKI> {
+  NCCL_DEVICE_INLINE static void call(ncclGinCtx ctx, ncclGinSignal_t signalId) {
+    using nccl::utility::loadConst;
+    ncclGinGdakiGPUContext* gdaki = (ncclGinGdakiGPUContext*)ctx.handle;
+    loadConst(&gdaki->signals_table.buffer)[signalId] = 0;
+  }
+};
+
+template <>
+struct ncclGinApi_GetCounterPtr<NCCL_NET_DEVICE_GIN_GDAKI> {
+  NCCL_DEVICE_INLINE static uint64_t* call(ncclGinCtx ctx, ncclGinCounter_t counterId) {
+    using nccl::utility::loadConst;
+    ncclGinGdakiGPUContext* gdaki = (ncclGinGdakiGPUContext*)ctx.handle;
+    return loadConst(&gdaki->counters_table.buffer) + counterId;
+  }
+};
+
+template <>
+struct ncclGinApi_GetSignalPtr<NCCL_NET_DEVICE_GIN_GDAKI> {
+  NCCL_DEVICE_INLINE static uint64_t* call(ncclGinCtx ctx, ncclGinSignal_t signalId) {
+    using nccl::utility::loadConst;
+    ncclGinGdakiGPUContext* gdaki = (ncclGinGdakiGPUContext*)ctx.handle;
+    return loadConst(&gdaki->signals_table.buffer) + signalId;
+  }
+};
+
+template <>
+struct ncclGinApi_Flush<NCCL_NET_DEVICE_GIN_GDAKI> {
+  template <typename Coop>
+  NCCL_DEVICE_INLINE static void call(ncclGinCtx ctx, Coop coop, cuda::memory_order ord) {
+    using nccl::utility::loadConst;
+    ncclGinGdakiGPUContext* gdaki = (ncclGinGdakiGPUContext*)ctx.handle;
+    doca_gpu_dev_verbs_qp* qps = loadConst(&gdaki->gdqp);
+#pragma unroll 1
+    for (int peer = coop.thread_rank(); peer < ctx.nRanks; peer += coop.size()) {
+      doca_gpu_dev_verbs_wait(qps + peer);
+    }
+  }
+};
+
+#endif /* _NCCL_DEVICE_GIN_GDAKI_H_ */

nvidia/nccl/include/nccl_device/gin/gdaki/gin_gdaki_device_host_common.h

@@ -0,0 +1,36 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_DEVICE_GIN_GDAKI_DEVICE_HOST_COMMON_H_
+#define _NCCL_DEVICE_GIN_GDAKI_DEVICE_HOST_COMMON_H_
+
+#include <linux/types.h>
+
+#define NCCL_GIN_GDAKI_VERSION 100
+
+template <typename T>
+struct ncclGinGdakiGlobalGPUBufferTable {
+  T *buffer;
+  __be32 *rkeys;
+  __be32 lkey;
+};
+
+struct ncclGinGdakiGPUContext {
+  struct doca_gpu_dev_verbs_qp *gdqp;
+  struct doca_gpu_dev_verbs_qp *companion_gdqp;
+  struct ncclGinGdakiGlobalGPUBufferTable<uint64_t> counters_table;
+  struct ncclGinGdakiGlobalGPUBufferTable<uint64_t> signals_table;
+
+  // Local buffer we don't consume but is required for some operations.
+  __be32 sink_buffer_lkey;
+};
+
+struct ncclGinGdakiMemHandle {
+  __be32 *rkeys;
+  __be32 lkey;
+};
+
+#endif /* _NCCL_DEVICE_GIN_GDAKI_DEVICE_HOST_COMMON_H_ */

nvidia/nccl/include/nccl_device/gin/gin_device_api.h

@@ -0,0 +1,18 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+#ifndef _NCCL_GIN_DEVICE_API_H_
+#define _NCCL_GIN_DEVICE_API_H_
+
+#include "gin_device_common.h"
+
+#if NCCL_GIN_GDAKI_ENABLE
+#include "gdaki/gin_gdaki.h"
+#endif
+#if NCCL_GIN_PROXY_ENABLE
+#include "proxy/gin_proxy.h"
+#endif
+
+#endif

nvidia/nccl/include/nccl_device/gin/gin_device_common.h

@@ -0,0 +1,120 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_GIN_DEVICE_COMMON_H_
+#define _NCCL_GIN_DEVICE_COMMON_H_
+
+#include "../net_device.h"
+#include "../utility.h"
+#include "gin_device_host_common.h"
+
+#if CUDA_VERSION >= 12080 && __CUDA_ARCH__ >= 900
+#define NCCL_GIN_HAS_FENCE_ACQUIRE_RELEASE_PTX 1
+#endif
+
+#ifndef NCCL_GIN_PROXY_ENABLE
+#define NCCL_GIN_PROXY_ENABLE 1
+#endif
+
+#ifndef NCCL_GIN_GDAKI_ENABLE
+#if CUDA_VERSION >= 12020 && __CUDA_ARCH__ >= 700
+#define NCCL_GIN_GDAKI_ENABLE 1
+#else
+#define NCCL_GIN_GDAKI_ENABLE 0
+#endif
+#endif
+
+#define NCCL_GIN_BACKEND_MASK_ALL                                               \
+  (((NCCL_GIN_PROXY_ENABLE) ? 1u : 0u) << (unsigned)NCCL_NET_DEVICE_GIN_PROXY | \
+   ((NCCL_GIN_GDAKI_ENABLE) ? 1u : 0u) << (unsigned)NCCL_NET_DEVICE_GIN_GDAKI)
+
+struct ncclGinCtx {
+  ncclNetDeviceType backend;
+  int rank;
+  int nRanks;
+  void* handle;
+};
+
+template <unsigned backendMask>
+struct ncclGinCtx_M : ncclGinCtx {};
+
+struct ncclGinDescriptorSmem {
+  alignas(16) char space[64];
+};
+
+#if __CUDACC__
+template <ncclNetDeviceType backend>
+struct ncclGinApi_Put {
+  template <typename Coop>
+  NCCL_DEVICE_INLINE static void call(ncclGinCtx, Coop coop, int peer, bool hasWins,
+                                      ncclGinWindow_t dstWin, size_t dstOff, ncclGinWindow_t srcWin,
+                                      size_t srcOff, size_t bytes, bool hasSignal,
+                                      ncclGinSignal_t signalId, ncclGinSignalOp_t signalOp,
+                                      uint64_t signalOpArg, bool hasCounter,
+                                      ncclGinCounter_t counterId, bool hasDescriptor,
+                                      ncclGinDescriptorSmem* descriptor,
+                                      cuda::thread_scope required, cuda::thread_scope given);
+};
+
+template <ncclNetDeviceType backend>
+struct ncclGinApi_PutValue {
+  template <typename Coop, typename T>
+  NCCL_DEVICE_INLINE static void call(ncclGinCtx, Coop coop, int peer, ncclGinWindow_t dstWin,
+                                      size_t dstOff, T srcData, bool hasSignal,
+                                      ncclGinSignal_t signalId, ncclGinSignalOp_t signalOp,
+                                      uint64_t signalOpArg, bool hasDescriptor,
+                                      ncclGinDescriptorSmem* descriptor,
+                                      cuda::thread_scope required, cuda::thread_scope given);
+};
+
+template <ncclNetDeviceType backend>
+struct ncclGinApi_GetSignalPtr {
+  NCCL_DEVICE_INLINE static uint64_t* call(ncclGinCtx, int peer, ncclGinSignal_t signalId);
+};
+template <ncclNetDeviceType backend>
+struct ncclGinApi_GetCounterPtr {
+  NCCL_DEVICE_INLINE static uint64_t* call(ncclGinCtx, int peer, ncclGinCounter_t counterId);
+};
+
+template <ncclNetDeviceType backend>
+struct ncclGinApi_ResetSignal {
+  NCCL_DEVICE_INLINE static void call(ncclGinCtx, ncclGinSignal_t signalId);
+};
+
+template <ncclNetDeviceType backend>
+struct ncclGinApi_ResetCounter {
+  NCCL_DEVICE_INLINE static void call(ncclGinCtx, ncclGinCounter_t counterId);
+};
+
+template <ncclNetDeviceType backend>
+struct ncclGinApi_Flush {
+  template <typename Coop>
+  NCCL_DEVICE_INLINE static void call(ncclGinCtx, Coop, cuda::memory_order ord);
+};
+#endif
+
+#if __CUDACC__
+template <template <ncclNetDeviceType> typename ApiFn, unsigned beMask, typename... Arg>
+NCCL_DEVICE_INLINE static decltype(auto) ncclGinCall(ncclGinCtx_M<beMask> ctx, Arg&&... arg) {
+  bool singleton = (beMask & (beMask - 1)) == 0;  // Only one bit set
+  switch (singleton ? __popc(beMask - 1) : (int)ctx.backend) {
+#if NCCL_GIN_PROXY_ENABLE
+    case (int)NCCL_NET_DEVICE_GIN_PROXY:
+      if (!(1 & (beMask >> (int)NCCL_NET_DEVICE_GIN_PROXY))) __builtin_unreachable();
+      return ApiFn<NCCL_NET_DEVICE_GIN_PROXY>::call(ctx, static_cast<Arg&&>(arg)...);
+#endif
+#if NCCL_GIN_GDAKI_ENABLE
+    case (int)NCCL_NET_DEVICE_GIN_GDAKI:
+      if (!(1 & (beMask >> (int)NCCL_NET_DEVICE_GIN_GDAKI))) __builtin_unreachable();
+      return ApiFn<NCCL_NET_DEVICE_GIN_GDAKI>::call(ctx, static_cast<Arg&&>(arg)...);
+#endif
+    default:
+      __builtin_unreachable();
+  }
+}
+#endif
+
+#endif

nvidia/nccl/include/nccl_device/gin/gin_device_host_common.h

@@ -0,0 +1,24 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_GIN_DEVICE_HOST_COMMON_H_
+#define _NCCL_GIN_DEVICE_HOST_COMMON_H_
+
+#include <cuda.h>
+#include "../net_device.h"
+#include "../core.h"  // for ncclGin{Signal|Counter}_t
+
+#define NCCL_GIN_MAX_CONTEXTS 4
+
+typedef struct ncclGinGpuCtx *ncclGinGpuCtx_t;
+typedef void *ncclGinWindow_t;
+
+typedef enum ncclGinSignalOp_t {
+  ncclGinSignalInc = 0,
+  ncclGinSignalAdd,
+} ncclGinSignalOp_t;
+
+#endif