nvidia-nccl-cu13 2.28.3__py3-none-manylinux_2_18_aarch64.whl

nvidia/nccl/include/nccl_device/coop.h

@@ -0,0 +1,152 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_DEVICE_COOP_H_
+#define _NCCL_DEVICE_COOP_H_
+#include "utility.h"
+
+// ncclCoop[Foo]: NCCL's versions of CUDA's Cooperative Groups. They conform
+// to just this subset of the CUDA API:
+//   int Coop::thread_rank();
+//   int Coop::size();
+//   int Coop::num_threads(); // same as size()
+//   void Coop::sync();
+
+#if __CUDACC__
+template<int nThreadsPow2>
+struct ncclCoopTile { // An aligned pow2 set of threads within the warp.
+  static_assert(nccl::utility::isPow2(nThreadsPow2) && nThreadsPow2 <= 32, "Condition required");
+
+  NCCL_DEVICE_INLINE int thread_rank() const {
+    return nccl::utility::lane() % nThreadsPow2;
+  }
+  NCCL_DEVICE_INLINE constexpr int size() const { return nThreadsPow2; }
+  NCCL_DEVICE_INLINE constexpr int num_threads() const { return nThreadsPow2; }
+
+  NCCL_DEVICE_INLINE uint32_t laneMask() const {
+    return (-1u>>(32-nThreadsPow2))<<(nccl::utility::lane() & -nThreadsPow2);
+  }
+  NCCL_DEVICE_INLINE void sync() {
+    __syncwarp(laneMask());
+  }
+};
+#endif
+
+#if __CUDACC__
+typedef ncclCoopTile<1> ncclCoopThread;
+typedef ncclCoopTile<32> ncclCoopWarp;
+#endif
+
+#if __CUDACC__
+struct ncclCoopLanes { // Some lanes of this warp.
+  uint32_t lmask;
+  
+  NCCL_DEVICE_INLINE constexpr ncclCoopLanes(uint32_t lmask=-1u): lmask(lmask) {}
+
+  NCCL_DEVICE_INLINE int thread_rank() const {
+    return __popc(lmask & nccl::utility::lanemask_lt());
+  }
+  NCCL_DEVICE_INLINE int size() const {
+    return __popc(lmask);
+  }
+  NCCL_DEVICE_INLINE int num_threads() const {
+    return __popc(lmask);
+  }
+  NCCL_DEVICE_INLINE void sync() {
+    __syncwarp(lmask);
+  }
+};
+#endif
+
+#if __CUDACC__
+// A set of consecutive warps that the user has also supplied with a unique
+// id from [0..15]. It is an error for two different warp spans with the same
+// id to be in a collective concurrently.
+struct ncclCoopWarpSpan {
+  uint32_t warp0:8, nWarps:8, id:8;
+
+  NCCL_DEVICE_INLINE constexpr ncclCoopWarpSpan(int warp0, int nWarps, int id):
+    warp0(warp0), nWarps(nWarps), id(id) {
+  }
+  
+  NCCL_DEVICE_INLINE int thread_rank() const {
+    return threadIdx.x - 32*warp0;
+  }
+  NCCL_DEVICE_INLINE int size() const {
+    return 32*nWarps;
+  }
+  NCCL_DEVICE_INLINE int num_threads() const {
+    return 32*nWarps;
+  }
+
+  NCCL_DEVICE_INLINE void sync() {
+    //asm volatile("barrier.sync %0, %1;" :: "r"(1+id), "r"(32*nWarps) : "memory");
+    __barrier_sync_count(1+id, 32*nWarps);
+  }
+};
+#endif
+
+#if __CUDACC__
+struct ncclCoopCta {
+  NCCL_DEVICE_INLINE int thread_rank() const { return threadIdx.x; }
+  NCCL_DEVICE_INLINE int size() const { return blockDim.x; }
+  NCCL_DEVICE_INLINE int num_threads() const { return blockDim.x; }
+  NCCL_DEVICE_INLINE void sync() { __syncthreads(); }
+};
+#endif
+
+#if __CUDACC__
+template<int nThreadsPow2>
+NCCL_DEVICE_INLINE uint32_t ncclCoopLaneMask(ncclCoopTile<nThreadsPow2> coop) {
+  return coop.laneMask();
+}
+NCCL_DEVICE_INLINE uint32_t ncclCoopLaneMask(ncclCoopLanes coop) {
+  return coop.lmask;
+}
+NCCL_DEVICE_INLINE uint32_t ncclCoopLaneMask(ncclCoopWarpSpan coop) {
+  return -1u;
+}
+NCCL_DEVICE_INLINE uint32_t ncclCoopLaneMask(ncclCoopCta coop) {
+  return -1u;
+}
+#endif
+
+#if __CUDACC__
+// ncclCoopIsThread:
+// At compile time do we know the given coop is a single thread only.
+template<int nThreads>
+NCCL_DEVICE_INLINE constexpr bool ncclCoopIsThread(ncclCoopTile<nThreads>) {
+  return nThreads == 1;
+}
+NCCL_DEVICE_INLINE constexpr bool ncclCoopIsThread(ncclCoopLanes) { return false; }
+NCCL_DEVICE_INLINE constexpr bool ncclCoopIsThread(ncclCoopWarpSpan) { return false; }
+NCCL_DEVICE_INLINE constexpr bool ncclCoopIsThread(ncclCoopCta) { return false; }
+#endif
+
+#if __CUDACC__
+// Pick threads of our warp that are safe to use collectively.
+NCCL_DEVICE_INLINE ncclCoopLanes ncclCoopCoalesced() {
+  return ncclCoopLanes{__activemask()};
+}
+#endif
+
+#if __CUDACC__
+// Pick threads of our warp that are safe to use collectively given that this
+// is a collective on the provided cooperative group.
+template<typename Coop>
+NCCL_DEVICE_INLINE ncclCoopTile<32> ncclCoopCoalesced(Coop) {
+  return ncclCoopTile<32>();
+}
+NCCL_DEVICE_INLINE ncclCoopLanes ncclCoopCoalesced(ncclCoopLanes coop) {
+  return coop;
+}
+template<int nThreads>
+NCCL_DEVICE_INLINE ncclCoopTile<nThreads> ncclCoopCoalesced(ncclCoopTile<nThreads> coop) {
+  return coop;
+}
+#endif
+
+#endif

nvidia/nccl/include/nccl_device/core.h

@@ -0,0 +1,150 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_DEVICE_CORE_H_
+#define _NCCL_DEVICE_CORE_H_
+#include <nccl.h>
+#include "coop.h"
+#include "utility.h"
+
+struct ncclDevComm;
+typedef struct ncclDevComm ncclDevComm_t;
+
+struct ncclTeam;
+typedef struct ncclTeam ncclTeam_t;
+
+// typedef struct ncclWindow_vidmem* ncclWindow_t; // in nccl.h
+
+struct ncclMultimemHandle;
+typedef struct ncclMultimemHandle ncclMultimemHandle_t;
+
+typedef uint32_t ncclDevResourceHandle;
+typedef ncclDevResourceHandle ncclDevResourceHandle_t;
+
+struct ncclLsaBarrierHandle;
+typedef struct ncclLsaBarrierHandle ncclLsaBarrierHandle_t;
+
+struct ncclLLA2AHandle;
+typedef struct ncclLLA2AHandle ncclLLA2AHandle_t;
+
+struct ncclTeam {
+  int nRanks, rank, stride;
+};
+
+#if __cplusplus
+template<typename T> struct ncclSymPtr;
+#endif
+
+#if __cplusplus
+struct ncclTeamTagWorld {};
+struct ncclTeamTagLsa {};
+struct ncclTeamTagRail {};
+#endif
+
+struct ncclDevCommRequirements;
+typedef struct ncclDevCommRequirements ncclDevCommRequirements_t;
+
+struct ncclDevResourceRequirements;
+typedef struct ncclDevResourceRequirements ncclDevResourceRequirements_t;
+
+struct ncclTeamRequirements;
+typedef struct ncclTeamRequirements ncclTeamRequirements_t;
+
+struct ncclDevCommRequirements {
+  ncclDevResourceRequirements_t* resourceRequirementsList;
+  ncclTeamRequirements_t* teamRequirementsList;
+
+  bool lsaMultimem; // Enable multimem on lsa team
+
+  int lsaBarrierCount;
+};
+
+struct ncclDevResourceRequirements {
+  ncclDevResourceRequirements_t* next;
+  size_t bufferSize, bufferAlign;
+  ncclDevResourceHandle_t* outBufferHandle; // If non-null, target assigned during ncclDevCommCreate.
+};
+
+struct ncclTeamRequirements {
+  ncclTeamRequirements_t* next;
+  ncclTeam_t team;
+  bool multimem;
+  ncclMultimemHandle_t* outMultimemHandle; // If non-null, target assigned during ncclDevCommCreate.
+};
+
+NCCL_EXTERN_C __host__ ncclResult_t ncclDevCommCreate(ncclComm_t, ncclDevCommRequirements_t const*, ncclDevComm_t* outDevComm);
+NCCL_EXTERN_C __host__ ncclResult_t ncclDevCommDestroy(ncclComm_t, ncclDevComm_t const* devComm);
+
+////////////////////////////////////////////////////////////////////////////////
+// Team API:
+
+#if __cplusplus
+NCCL_HOST_DEVICE_INLINE ncclTeam ncclTeamWorld(ncclDevComm const&);
+#endif
+NCCL_EXTERN_C __host__ ncclTeam_t ncclTeamWorld(ncclComm_t);
+
+#if __cplusplus
+NCCL_HOST_DEVICE_INLINE ncclTeam ncclTeamLsa(ncclDevComm const&);
+#endif
+NCCL_EXTERN_C __host__ ncclTeam_t ncclTeamLsa(ncclComm_t);
+
+NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE bool ncclTeamRankIsMember(ncclTeam_t a, ncclTeam_t b, int bPeer);
+NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE int ncclTeamRankToTeam(ncclTeam_t a, ncclTeam_t b, int bPeer);
+
+#if __cplusplus
+NCCL_HOST_DEVICE_INLINE int ncclTeamRankToWorld(ncclDevComm const&, ncclTeam, int rank);
+#endif
+NCCL_EXTERN_C __host__ int ncclTeamRankToWorld(ncclComm_t, ncclTeam_t, int rank);
+
+#if __cplusplus
+NCCL_HOST_DEVICE_INLINE int ncclTeamRankToLsa(ncclDevComm const&, ncclTeam, int rank);
+#endif
+NCCL_EXTERN_C __host__ int ncclTeamRankToLsa(ncclComm_t, ncclTeam_t, int rank);
+
+NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE ncclTeam_t ncclTeamInnerFactor(ncclTeam_t parent, int innerSize);
+NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE ncclTeam_t ncclTeamOuterFactor(ncclTeam_t parent, int innerSize);
+
+// Interpret each team as a set of ranks. This function assumes that `subset`
+// is a subset of `parent`. Thus the number of ranks in the set difference of
+// `parent` minus `subset` is `super.nRanks - subset.nRanks`. Given `index` this
+// function returns the index'th element of `parent` minus `subset`.
+NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE int ncclTeamRankInDifference(ncclTeam_t parent, ncclTeam_t subset, int index);
+
+// Equivalent to ncclTeamOuterFactor of lsa team.
+#if __cplusplus
+NCCL_HOST_DEVICE_INLINE ncclTeam ncclTeamRail(ncclDevComm const&);
+#endif
+NCCL_EXTERN_C __host__ ncclTeam_t ncclTeamRail(ncclComm_t);
+
+// Get offset of resource buffer within `comm.resourceWindow`.
+NCCL_EXTERN_C NCCL_HOST_DEVICE_INLINE size_t ncclGetResourceBufferOffset(ncclDevResourceHandle_t);
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE ncclSymPtr<char> ncclGetResourceBuffer(ncclDevComm const&, ncclDevResourceHandle);
+#endif
+
+////////////////////////////////////////////////////////////////////////////////
+// Window API:
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetLocalPointer(ncclWindow_t w, size_t offset);
+NCCL_DEVICE_INLINE void* ncclGetLsaPointer(ncclWindow_t w, size_t offset, int peer);
+NCCL_DEVICE_INLINE void* ncclGetPeerPointer(ncclWindow_t w, size_t offset, int peer);
+NCCL_DEVICE_INLINE void* ncclGetPeerPointer(ncclWindow_t w, size_t offset, ncclTeam tm, int peer);
+NCCL_DEVICE_INLINE void* ncclGetMultimemPointer(ncclWindow_t w, size_t offset, ncclMultimemHandle mmHandle);
+NCCL_DEVICE_INLINE void* ncclGetLsaMultimemPointer(ncclWindow_t w, size_t offset, ncclDevComm const&);
+#endif
+
+#if __CUDACC__
+// Convenience for combining ncclGet***Pointer() with resource handle.
+NCCL_DEVICE_INLINE void* ncclGetResourceBufferLocalPointer(ncclDevComm const&, ncclDevResourceHandle);
+NCCL_DEVICE_INLINE void* ncclGetResourceBufferLsaPointer(ncclDevComm const&, ncclDevResourceHandle, int peer);
+NCCL_DEVICE_INLINE void* ncclGetResourceBufferPeerPointer(ncclDevComm const&, ncclDevResourceHandle, ncclTeam, int peer);
+NCCL_DEVICE_INLINE void* ncclGetResourceBufferMultimemPointer(ncclDevComm const&, ncclDevResourceHandle, ncclMultimemHandle);
+NCCL_DEVICE_INLINE void* ncclGetResourceBufferLsaMultimemPointer(ncclDevComm const&, ncclDevResourceHandle);
+#endif
+
+#endif // _NCCL_DEVICE_CORE_H_

nvidia/nccl/include/nccl_device/impl/comm__funcs.h

@@ -0,0 +1,10 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_DEVICE_COMM__FUNCS_H_
+#define _NCCL_DEVICE_COMM__FUNCS_H_
+#include "comm__types.h"
+#endif // _NCCL_DEVICE_COMM__FUNCS_H_

nvidia/nccl/include/nccl_device/impl/comm__types.h

@@ -0,0 +1,40 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_DEVICE_COMM__TYPES_H_
+#define _NCCL_DEVICE_COMM__TYPES_H_
+#include "../comm.h"
+#include "core__types.h"
+#include "mem_barrier__types.h"
+#include "ll_a2a__types.h"
+
+struct ncclDevCommWindowTable;
+#if __cplusplus
+struct ncclDevCommWindowTable {
+  struct Entry {
+    uintptr_t base, size;
+    ncclWindow_t window;
+  } entries[32];
+  struct ncclDevCommWindowTable* next;
+};
+#endif
+
+struct ncclDevComm {
+  int rank, nRanks;
+  uint32_t nRanks_rcp32;
+  int lsaRank, lsaSize;
+  uint32_t lsaSize_rcp32;
+
+  struct ncclDevCommWindowTable* windowTable;
+
+  ncclWindow_t resourceWindow;
+  struct ncclWindow_vidmem resourceWindow_inlined;
+
+  ncclMultimemHandle_t lsaMultimem;
+  ncclLsaBarrierHandle_t lsaBarrier;
+};
+
+#endif // _NCCL_DEVICE_COMM__TYPES_H_

nvidia/nccl/include/nccl_device/impl/core__funcs.h

@@ -0,0 +1,210 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_DEVICE_CORE__FUNCS_H_
+#define _NCCL_DEVICE_CORE__FUNCS_H_
+#include "core__types.h"
+#include "comm__types.h"
+#include "ptr__types.h"
+
+#if __cplusplus
+NCCL_HOST_DEVICE_INLINE ncclTeam ncclTeamWorld(ncclDevComm const &comm) {
+  ncclTeam ans;
+  ans.nRanks = comm.nRanks;
+  ans.rank = comm.rank;
+  ans.stride = 1;
+  return ans;
+}
+#endif
+
+#if __cplusplus
+NCCL_HOST_DEVICE_INLINE ncclTeam ncclTeamLsa(ncclDevComm const &comm) {
+  ncclTeam ans;
+  ans.nRanks = comm.lsaSize;
+  ans.rank = comm.lsaRank;
+  ans.stride = 1;
+  return ans;
+}
+#endif
+
+#if __cplusplus
+NCCL_HOST_DEVICE_INLINE ncclTeam ncclTeamRail(ncclDevComm const& comm) {
+  ncclTeam ans;
+  ans.nRanks = nccl::utility::idivFast32(comm.nRanks, comm.lsaSize, comm.lsaSize_rcp32);
+  ans.rank = nccl::utility::idivFast32(comm.rank, comm.lsaSize, comm.lsaSize_rcp32);
+  ans.stride = comm.lsaSize;
+  return ans;
+}
+#endif
+
+NCCL_HOST_DEVICE_INLINE bool ncclTeamRankIsMember(ncclTeam_t a, ncclTeam_t b, int brank) {
+  int wrank = (brank - b.rank)*b.stride;
+  uint32_t adelta = wrank/a.stride;
+  uint32_t amod = wrank%a.stride;
+  int arank = a.rank + adelta;
+  return 0 <= arank && arank < a.nRanks && amod == 0;
+}
+
+NCCL_HOST_DEVICE_INLINE int ncclTeamRankToTeam(ncclTeam_t a, ncclTeam_t b, int brank) {
+  int wrank = (brank - b.rank)*b.stride;
+  uint32_t adelta = wrank/a.stride;
+  //uint32_t amod = wrank%a.stride;
+  int arank = a.rank + adelta;
+  return arank;
+}
+
+#if __cplusplus
+NCCL_HOST_DEVICE_INLINE int ncclTeamRankToWorld(ncclDevComm const& comm, ncclTeam tm, int rank) {
+  return comm.rank + (rank - tm.rank)*tm.stride;
+}
+#endif
+
+#if __cplusplus
+NCCL_HOST_DEVICE_INLINE int ncclTeamRankToLsa(ncclDevComm const& comm, ncclTeam tm, int rank) {
+  return comm.lsaRank + (rank - tm.rank)*tm.stride;
+}
+#endif
+
+NCCL_HOST_DEVICE_INLINE ncclTeam_t ncclTeamInnerFactor(ncclTeam_t parent, int innerSize) {
+  ncclTeam_t ans;
+  ans.nRanks = innerSize;
+  ans.rank = parent.rank%innerSize;
+  ans.stride = parent.stride;
+  return ans;
+}
+
+NCCL_HOST_DEVICE_INLINE ncclTeam_t ncclTeamOuterFactor(ncclTeam_t parent, int innerSize) {
+  ncclTeam_t ans;
+  ans.nRanks = parent.nRanks/innerSize;
+  ans.rank = parent.rank/innerSize;
+  ans.stride = parent.stride*innerSize;
+  return ans;
+}
+
+NCCL_HOST_DEVICE_INLINE int ncclTeamRankInDifference(ncclTeam_t parent, ncclTeam_t subset, int index) {
+  int stride = subset.stride/parent.stride;
+  int below = parent.rank - subset.rank*stride;
+  if (stride < 0) {
+    stride = -stride;
+    below -= (subset.nRanks-1)*stride;
+  }
+  if (index < below) {
+    return index;
+  } else if (index-below < (subset.nRanks-1)*(stride-1)) {
+    return below + 1 + ((index-below)/(stride-1))*stride + (index-below)%(stride-1);
+  } else {
+    return below + 1 + (subset.nRanks-1)*stride + (index - below - (subset.nRanks-1)*(stride-1));
+  }
+}
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetLocalPointer(ncclWindow_t w, size_t offset) {
+  char* base = nccl::utility::loadConst(&w->lsaFlatBase);
+  uint32_t stride4G = nccl::utility::loadConst(&w->stride4G);
+  int i = nccl::utility::loadConst(&w->lsaRank);
+  return (void*)(nccl::utility::add4G(base, i*stride4G) + offset);
+}
+#endif
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetLsaPointer(ncclWindow_t w, size_t offset, int peer) {
+  char* base = nccl::utility::loadConst(&w->lsaFlatBase);
+  uint32_t stride4G = nccl::utility::loadConst(&w->stride4G);
+  int i = peer;
+  return (void*)(nccl::utility::add4G(base, i*stride4G) + offset);
+}
+#endif
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetPeerPointer(ncclWindow_t w, size_t offset, int peer) {
+  char* base = nccl::utility::loadConst(&w->lsaFlatBase);
+  uint32_t stride4G = nccl::utility::loadConst(&w->stride4G);
+  int worldRank = nccl::utility::loadConst(&w->worldRank);
+  int lsaRank = nccl::utility::loadConst(&w->lsaRank);
+  int i = lsaRank + (peer - worldRank);
+  return (void*)(nccl::utility::add4G(base, i*stride4G) + offset);
+}
+#endif
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetPeerPointer(ncclWindow_t w, size_t offset, ncclTeam tm, int peer) {
+  char* base = nccl::utility::loadConst(&w->lsaFlatBase);
+  uint32_t stride4G = nccl::utility::loadConst(&w->stride4G);
+  int lsaRank = nccl::utility::loadConst(&w->lsaRank);
+  int i = lsaRank + (peer - tm.rank)*tm.stride;
+  return (void*)(nccl::utility::add4G(base, i*stride4G) + offset);
+}
+#endif
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetMultimemPointer(ncclWindow_t w, size_t offset, ncclMultimemHandle mm) {
+  void* ptr = mm.mcBasePtr;
+  ptr = reinterpret_cast<char(*)[4096]>(ptr) + nccl::utility::loadConst(&w->mcOffset4K);
+  return (void*)((char*)ptr + offset);
+}
+#endif
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetLsaMultimemPointer(ncclWindow_t w, size_t offset, ncclDevComm const& comm) {
+  return ncclGetMultimemPointer(w, offset, comm.lsaMultimem);
+}
+#endif
+
+NCCL_HOST_DEVICE_INLINE size_t ncclGetResourceBufferOffset(ncclDevResourceHandle_t h) {
+  return ((size_t)h)*128;
+}
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetResourceBufferLocalPointer(ncclDevComm const& comm, ncclDevResourceHandle h) {
+  void* lsaFlatBase = comm.resourceWindow_inlined.lsaFlatBase;
+  uint32_t stride4G = comm.resourceWindow_inlined.stride4G;
+  void* local = nccl::utility::add4G(lsaFlatBase, comm.lsaRank*stride4G);
+  return (void*)(reinterpret_cast<char(*)[128]>(local) + h);
+}
+#endif
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetResourceBufferLsaPointer(ncclDevComm const& comm, ncclDevResourceHandle h, int peer) {
+  int r = peer;
+  void* lsaFlatBase = comm.resourceWindow_inlined.lsaFlatBase;
+  uint32_t stride4G = comm.resourceWindow_inlined.stride4G;
+  void* local = nccl::utility::add4G(lsaFlatBase, r*stride4G);
+  return (void*)(reinterpret_cast<char(*)[128]>(local) + h);
+}
+#endif
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetResourceBufferPeerPointer(ncclDevComm const& comm, ncclDevResourceHandle h, ncclTeam team, int peer) {
+  int r = comm.lsaRank + (peer - team.rank)*team.stride;
+  void* lsaFlatBase = comm.resourceWindow_inlined.lsaFlatBase;
+  uint32_t stride4G = comm.resourceWindow_inlined.stride4G;
+  void* local = nccl::utility::add4G(lsaFlatBase, r*stride4G);
+  return (void*)(reinterpret_cast<char(*)[128]>(local) + h);
+}
+#endif
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetResourceBufferMultimemPointer(ncclDevComm const& comm, ncclDevResourceHandle h, ncclMultimemHandle mm) {
+  void* ptr = mm.mcBasePtr;
+  ptr = reinterpret_cast<char(*)[4096]>(ptr) + comm.resourceWindow_inlined.mcOffset4K;
+  ptr = reinterpret_cast<char(*)[128]>(ptr) + h;
+  return ptr;
+}
+#endif
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE void* ncclGetResourceBufferLsaMultimemPointer(ncclDevComm const& comm, ncclDevResourceHandle h) {
+  return ncclGetResourceBufferMultimemPointer(comm, h, comm.lsaMultimem);
+}
+#endif
+
+#if __CUDACC__
+NCCL_DEVICE_INLINE ncclSymPtr<char> ncclGetResourceBuffer(ncclDevComm const& comm, ncclDevResourceHandle h) {
+  return ncclSymPtr<char>(comm.resourceWindow, size_t(h)*128);
+}
+#endif
+
+#endif

nvidia/nccl/include/nccl_device/impl/core__types.h

@@ -0,0 +1,26 @@
+/*************************************************************************
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ *
+ * See LICENSE.txt for license information
+ ************************************************************************/
+
+#ifndef _NCCL_DEVICE_CORE__TYPES_H_
+#define _NCCL_DEVICE_CORE__TYPES_H_
+#include "../core.h"
+
+// nccl.h has: typedef ncclWindow_vidmem* ncclWindow_t;
+struct ncclWindow_vidmem {
+  void* winHost;
+  //ncclGinWindow_t ginWin;
+  char* lsaFlatBase; // pointer to first byte for rank 0 of lsa team
+  int lsaRank;
+  int worldRank;
+  uint32_t stride4G;
+  uint32_t mcOffset4K;
+};
+
+struct ncclMultimemHandle {
+  void* mcBasePtr;
+};
+
+#endif