ibverbs 2026.7.17__tar.gz

ibverbs-2026.7.17/CLAUDE.md

@@ -0,0 +1,131 @@
+# CLAUDE.md — ibverbs
+
+Guidance for working on the `ibverbs` package (low-level libibverbs bindings).
+
+## What this is
+
+A Cython wrapper over `libibverbs`. The goal is a **faithful, low-level** verbs
+API in Python that a higher-level RDMA library can be built on — not a
+transport itself. Keep it thin: one Python object per verbs object, minimal
+policy, no hidden threads, no runtime dependencies, and **no torch/CUDA
+linkage** (GPUDirect works via raw addresses / dma-buf fds).
+
+## Layout
+
+```
+ibverbs/
+  pyproject.toml          # build (setuptools+Cython), metadata, pytest markers
+  setup.py                # Extension built via `pkg-config libibverbs`
+  src/ibverbs/
+    __init__.py           # public re-exports + __version__
+    _libverbs.pxd         # `cdef extern` declarations of <infiniband/verbs.h>
+    _ibverbs.pyx          # all cdef classes + posting logic (the real binding)
+    enums.py              # IntEnum/IntFlag mirrors of the rdma-core ABI
+    helpers.py            # QPInfo, RC connect helpers, reg_tensor (pure Python)
+    cuda.py               # optional GPUDirect: register_tensor/GpuMR (lazy libcuda)
+  tests/
+    conftest.py           # device/port fixtures, RoCE GID selection, markers
+    _rc.py                # HostBuffer + Endpoint + connected-pair helpers
+    test_*.py             # unit + integration (see README feature table)
+```
+
+## Build & test
+
+```bash
+# Dev venv lives at repo-root .venv (Python 3.14, has torch+cython+pytest+numpy).
+uv pip install --python ../.venv -e . --no-build-isolation   # rebuild after .pyx/.pxd edits
+../.venv/bin/python -m pytest -rs -q                          # run the suite
+```
+
+- Editing `.py` (enums/helpers/tests): no rebuild needed.
+- Editing `.pyx` / `.pxd`: **must rebuild** (`uv pip install -e . --no-build-isolation`).
+- The Cython-generated `_ibverbs.c` and the `.so` are git-ignored.
+
+## Linkage model: dlopen, not direct link (portability)
+
+The extension is compiled against the rdma-core headers but is **not** linked
+against `libibverbs`. `setup.py` uses `pkg-config --cflags-only-I` for the
+header path and links only `libdl`. At import, `_load_libibverbs()` in
+`_ibverbs.pyx` `dlopen`s `libibverbs.so.1` (RTLD_NOW|RTLD_GLOBAL) and `dlsym`s
+each exported verb into a typed function pointer (`_v_*`). Consequences:
+
+- The `.so`'s only `NEEDED` is `libc` — no external dep for `auditwheel`, so a
+  clean manylinux wheel; missing libibverbs → clean `ImportError`.
+- `ibv_reg_dmabuf_mr` is loaded **optionally** (may be NULL on rdma-core < 34);
+  `PD.reg_dmabuf_mr` raises a clear error if it's unavailable.
+- Built as **abi3** (Limited API, floor 3.9) → one wheel for CPython 3.9+.
+- Only the five `static inline` data-path verbs
+  (`ibv_post_send/recv`, `ibv_poll_cq`, `ibv_req_notify_cq`,
+  `ibv_post_srq_recv`) are still declared extern-from-header: they dispatch
+  through the provider op table and reference no exported symbol, so they
+  compile inline with zero symbol resolution (the fast path stays fast). This
+  is why a pure-ctypes binding is a poor fit and we use a compiled extension.
+
+When adding a verb: declare its function-pointer type (`fp_*`), a `_v_*`
+global, and a `_req_sym`/`dlsym` line in `_load_libibverbs`, then call `_v_*`.
+
+## How the C API is declared (`_libverbs.pxd`)
+
+- Structs are declared **partially** — only the fields we touch. Cython trusts
+  the real header for layout, so this is safe even for stack-allocated structs.
+- Enum-typed fields/params are declared as plain `int` (C converts implicitly);
+  this keeps the file small and robust across rdma-core versions.
+- `ibv_reg_mr` and `ibv_query_port` are **macros** in the header, so we never
+  reference them by name: we `dlsym` `ibv_reg_mr_iova2` (called with
+  `iova == addr`) and the raw `ibv_query_port` symbol (called with a zeroed
+  attr struct, which is what the header's `___ibv_query_port` compat shim does).
+- `ibv_send_wr.wr` is a C **union** (`rdma` / `atomic` / `ud`); declared with
+  named nested `cdef struct`s that are never emitted (extern), only used for
+  field-access type-checking.
+
+## Conventions in `_ibverbs.pyx`
+
+- **RAII:** each `cdef class` frees its handle in `__dealloc__` and in an
+  idempotent `close()`; all are context managers.
+- **Lifetime:** children hold a Python reference to their parent (MR→PD,
+  QP→PD+CQs+SRQ, CQ→Context/CompChannel) so the GC can't free a parent first.
+- **GIL:** build C structs while holding the GIL, then wrap only the actual
+  `ibv_*` call in `with nogil:`. Coercing a Python object inside `nogil` is a
+  compile error — convert to a C local first.
+- **Errors:** verbs failures raise `VerbsError(OSError)` carrying `errno`; a bad
+  *work completion* status does **not** raise (it's returned in the `WC`);
+  `WC.raise_for_status()` is opt-in.
+- **Byte order:** `imm_data` crosses the wire in network order — `htonl` on
+  send, `ntohl` on completion, so Python always sees host order.
+- **CQ ↔ event mapping:** `create_cq` passes `<void*>cq` as the CQ context, so
+  `CompChannel.get_cq_event()` can hand back the exact Python `CQ`.
+
+## Adding a wrapped verb
+
+1. Declare the C signature/struct fields in `_libverbs.pxd` (partial is fine).
+2. Add the method/class in `_ibverbs.pyx`; follow the RAII + GIL + error rules
+   above; add any new enum values to `enums.py`.
+3. Re-export new public names in `__init__.py` (and its `__all__`).
+4. Write a test in `tests/` — a unit test if it needs no wire, otherwise add to
+   the loopback/integration set. Rebuild, then `pytest`.
+
+## RDMA gotchas learned here (don't regress these)
+
+- **RoCE GID selection matters.** Link-local (`fe80::`) GIDs do **not** loop
+  back or route to themselves on this fabric; use a global **RoCE v2** GID.
+  `tests/conftest.find_roce_gid` scores candidates accordingly.
+- **Union aliasing.** In `post_send`, only populate the `wr` union member for
+  the actual opcode — writing `atomic.*` over `rdma.*` clobbers the rkey.
+- **`char*` comparison.** Comparing device names must use `strcmp`, not `==`
+  (which compares pointers in Cython).
+- **GPUDirect here.** `nvidia_peermem` may fail to load (kernel/driver skew);
+  the dma-buf path (`reg_dmabuf_mr` + `cuMemGetHandleForAddressRange`) is the
+  reliable route and needs no kernel module. `ibverbs.cuda.register_tensor`
+  wraps it — it stays torch-free (duck-typed on `data_ptr()`) and imports no
+  CUDA at build. Requirements learned the hard way: torch memory must be
+  VMM-backed (`PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`); the dma-buf
+  export needs the **base page-aligned down and length page-aligned up**
+  (`register_tensor` handles this via `offset`/`iova`); and `ibv_mr.addr` is
+  **not** meaningful for dma-buf MRs, so `GpuMR` carries the real device VA.
+- Loopback (two QPs on one port) is a real NIC round-trip and is the primary
+  integration signal; two-NIC tests skip when the NICs aren't mutually routable.
+
+## Hardware this was validated on
+
+12× Mellanox mlx5 NICs (RoCEv2, Ethernet link layer), 8× NVIDIA H100, CUDA
+driver 580.x, rdma-core 61, kernel 6.16.

ibverbs-2026.7.17/LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2026, Tristan Rice
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ibverbs-2026.7.17/MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md CLAUDE.md LICENSE MANIFEST.in pyproject.toml setup.py
+include src/ibverbs/*.pyx src/ibverbs/*.pxd
+recursive-include tests *.py

ibverbs-2026.7.17/PKG-INFO

@@ -0,0 +1,226 @@
+Metadata-Version: 2.4
+Name: ibverbs
+Version: 2026.7.17
+Summary: Low-level Pythonic bindings for libibverbs (RDMA), with GPUDirect support
+Author: Tristan Rice
+License-Expression: BSD-3-Clause
+Project-URL: Homepage, https://github.com/tristanr/rdma4py
+Project-URL: Repository, https://github.com/tristanr/rdma4py
+Keywords: rdma,ibverbs,infiniband,roce,gpudirect,networking
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Cython
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: System :: Networking
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Provides-Extra: test
+Requires-Dist: pytest>=7; extra == "test"
+Requires-Dist: numpy; extra == "test"
+Provides-Extra: gpu
+Requires-Dist: torch; extra == "gpu"
+Dynamic: license-file
+
+# ibverbs
+
+Low-level, Pythonic bindings for **libibverbs** (RDMA), designed as a
+foundation for building high-performance RDMA libraries in Python — including
+**GPUDirect** transfers to/from GPU memory.
+
+The bindings are a thin, faithful wrapper over the verbs API (device, PD, MR,
+CQ, QP, SRQ, AH, work requests, completions, async events) plus a small,
+optional set of RC connection helpers. They are written in Cython so the data
+path (`post_send` / `post_recv` / `poll`) compiles to direct C calls and
+releases the GIL, and so the `static inline` verbs fast-path functions are
+called correctly (they can't be reached through `dlsym`).
+
+- **No runtime dependencies.** Only libibverbs, which is `dlopen`ed at import.
+- **No torch / CUDA linkage.** GPUDirect works by registering an integer
+  device address or an exported dma-buf fd; CUDA stays entirely in your code.
+- **One `abi3` wheel for all of CPython 3.9+** on Linux.
+
+## Portability
+
+The extension does **not** link `libibverbs`. It is compiled against the
+rdma-core headers (for struct layouts and the `static inline` data-path verbs)
+but resolves the exported verbs at import time with `dlopen`/`dlsym`. As a
+result:
+
+- The compiled module's only `NEEDED` library is `libc` — no external
+  dependency for `auditwheel`, so a single **manylinux** wheel is portable
+  across distros.
+- It is built against the **CPython Limited API (abi3)**, so one wheel works on
+  CPython **3.9 through 3.14+** — no per-version builds.
+- A missing `libibverbs` yields a clean `ImportError`, not a loader crash.
+- Newer verbs are optional: `ibv_reg_dmabuf_mr` (rdma-core ≥ 34) is loaded if
+  present and only errors if you actually call `reg_dmabuf_mr`, so the wheel
+  still imports on older systems.
+
+At runtime you only need `libibverbs.so.1` (any `rdma-core` from the last
+several years). The data path (`post_send`/`poll`/…) stays compiled inline and
+dispatches through the provider op table, so `dlopen` costs nothing on the hot
+path.
+
+## Requirements
+
+- Linux with an RDMA-capable NIC (tested on Mellanox/NVIDIA **mlx5**, RoCEv2).
+- **Runtime:** `libibverbs.so.1` (`rdma-core` — `libibverbs1` on Debian/Ubuntu,
+  `libibverbs` on RHEL/Fedora). No compiler or headers needed to *use* a wheel.
+- **Build from source only:** a C compiler, Cython, and the `rdma-core`
+  development headers (`libibverbs-dev` / `rdma-core-devel`).
+
+## Install
+
+```bash
+pip install ibverbs        # prebuilt abi3 manylinux wheel (once published)
+```
+
+Building from source (needs the rdma-core dev headers + a compiler):
+
+```bash
+pip install "Cython>=3.0" "setuptools>=77" wheel
+pip install ./ibverbs       # or: pip install -e ./ibverbs
+```
+
+## Quickstart
+
+```python
+import ibverbs as ib
+
+# 1. Open a device and set up resources.
+dev = ib.get_device_list()[0]
+ctx = dev.open()
+pd = ctx.alloc_pd()
+cq = ctx.create_cq(64)
+
+# 2. Register memory (host or GPU address — any integer VA works).
+import numpy as np
+buf = np.zeros(4096, dtype=np.uint8)
+access = ib.AccessFlags.LOCAL_WRITE | ib.AccessFlags.REMOTE_WRITE | ib.AccessFlags.REMOTE_READ
+mr = pd.reg_mr(buf.ctypes.data, buf.nbytes, access)
+
+# 3. Create a reliable-connected QP.
+qp = pd.create_qp(ib.QPInitAttr(send_cq=cq, recv_cq=cq, qp_type=ib.QPType.RC))
+
+# 4. Exchange connection info with the peer out-of-band, then connect.
+port = 1
+port_attr = ctx.query_port(port)
+gid = ctx.query_gid(port, gid_index)              # pick a routable RoCEv2 GID
+local = ib.local_qp_info(qp, port_attr, gid, port=port, psn=0)
+# ... send local.to_bytes() to peer, receive remote_bytes ...
+remote = ib.QPInfo.from_bytes(remote_bytes)
+ib.connect_rc(qp, remote, port=port, sgid_index=gid_index, access=access)
+
+# 5. Post an RDMA write and reap the completion.
+qp.post_send(ib.SendWR(
+    wr_id=1, sg_list=[ib.SGE(mr, 4096)], opcode=ib.WROpcode.RDMA_WRITE,
+    send_flags=ib.SendFlags.SIGNALED, remote_addr=peer_addr, rkey=peer_rkey))
+for wc in cq.poll(16):
+    wc.raise_for_status()
+```
+
+Every resource is a context manager and frees its handle on `close()` /
+garbage collection; children hold references to their parents, so destruction
+order is always safe.
+
+## GPUDirect with torch tensors
+
+The library never imports torch or links CUDA. The optional `ibverbs.cuda`
+helper (which only lazily `dlopen`s `libcuda`) registers a CUDA tensor for RDMA
+in one call — handling the dma-buf export and page alignment for you:
+
+```python
+import os
+# torch's CUDA memory must be VMM-backed to be dma-buf exportable. Set this
+# BEFORE torch initializes CUDA:
+os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+
+import torch
+import ibverbs as ib
+import ibverbs.cuda
+
+src = torch.arange(4096, dtype=torch.float32, device="cuda:0")
+dst = torch.zeros(4096, dtype=torch.float32, device="cuda:0")
+
+access = ib.AccessFlags.LOCAL_WRITE | ib.AccessFlags.REMOTE_WRITE
+src_mr = ib.cuda.register_tensor(pd, src, access)   # retains src until close()
+dst_mr = ib.cuda.register_tensor(pd, dst, access)
+
+# RDMA-write one GPU buffer into another, with no host staging on the data path.
+torch.cuda.synchronize(src.device)  # source-producing CUDA work must be done
+qp.post_send(ib.SendWR(
+    wr_id=1, sg_list=[src_mr.sge()], opcode=ib.WROpcode.RDMA_WRITE,
+    send_flags=ib.SendFlags.SIGNALED,
+    remote_addr=dst_mr.addr, rkey=dst_mr.rkey))
+for wc in qp.send_cq.poll(16):
+    wc.raise_for_status()
+
+# On the receiver, after the peer has signaled that its write is complete,
+# order the inbound NIC writes before launching CUDA work that consumes dst.
+with torch.cuda.device(dst.device):
+    ib.cuda.flush_gpudirect_writes()
+```
+
+`GpuMR` wraps the `MR` with the correct device address (`ibv_mr.addr` is not
+meaningful for dma-buf MRs), retains the tensor allocation until `close()`, and
+exposes `.sge()`, `.addr`, `.lkey`, `.rkey`.
+
+CUDA work and NIC work are separate ordering domains. Synchronize the stream
+that produced an outbound tensor before posting it. For inbound `SEND`, RDMA
+read, or RDMA write, wait for the corresponding completion or protocol-level
+notification, then call `flush_gpudirect_writes()` in the destination CUDA
+context before consuming the tensor. A one-sided RDMA write does not create a
+remote CQ entry by itself; use write-with-immediate or an out-of-band message
+to notify the receiver.
+
+Under the hood there are two registration paths, chosen automatically:
+
+```python
+# dma-buf fd (default; no kernel module needed):
+mr = pd.reg_dmabuf_mr(offset, length, iova=device_va, fd=dmabuf_fd, access=access)
+# raw device pointer (requires the nvidia_peermem kernel module):
+mr = pd.reg_mr(tensor.data_ptr(), nbytes, access)
+```
+
+For a **host** (CPU) torch tensor or numpy array, `ib.reg_tensor(pd, tensor,
+access)` registers it directly and retains the allocation. Both tensor helpers
+require contiguous, non-empty tensors; split any single SGE larger than
+`2**32 - 1` bytes into multiple entries. `tests/test_gpudirect.py` performs real
+GPU-to-GPU RDMA writes, reads, and sends verified with `torch.equal`.
+
+## Feature coverage
+
+| Area | Supported |
+|------|-----------|
+| Device / port / GID query | ✅ `get_device_list`, `Context.query_device/query_port/query_gid` |
+| Protection domains | ✅ `alloc_pd` |
+| Memory regions | ✅ `reg_mr`, `reg_dmabuf_mr` (GPUDirect) |
+| Completion queues | ✅ `create_cq`, `poll`, comp channels + `req_notify`/`ack_events` |
+| Queue pairs | ✅ RC / UC / UD; `modify`, `query`, `to_init`/`to_rtr`/`to_rts` |
+| Work requests | ✅ SEND(/_IMM), RDMA_WRITE(/_IMM), RDMA_READ, ATOMIC_CMP_AND_SWP, ATOMIC_FETCH_AND_ADD, scatter/gather, inline/signaled/fenced/solicited flags |
+| Shared receive queues | ✅ `create_srq`, `post_recv`, `modify`, `query` |
+| Address handles | ✅ `create_ah` (UD) |
+| Async events | ✅ `get_async_event` / `ack_async_event`, `async_fd` |
+| Connection helpers | ✅ `QPInfo`, `local_qp_info`, `connect_rc` |
+
+Out of scope for v1 (candidates for later): the extended `ibv_wr_*` / `qp_ex`
+post API, device memory (`ibv_alloc_dm`), memory windows, and flow steering.
+
+## Testing
+
+The suite exercises real hardware and skips features the host lacks:
+
+```bash
+pip install -e "./ibverbs[test,gpu]"     # pytest, numpy, torch
+cd ibverbs && pytest -rs                  # -rs shows skip reasons
+pytest -m "not gpu"                       # skip GPUDirect tests
+pytest -m integration                     # only real-hardware tests
+```
+
+Markers: `integration` (needs an RDMA NIC), `gpu` (needs CUDA + torch).
+
+## License
+
+BSD-3-Clause. See the repository `LICENSE`.

ibverbs-2026.7.17/README.md

@@ -0,0 +1,201 @@
+# ibverbs
+
+Low-level, Pythonic bindings for **libibverbs** (RDMA), designed as a
+foundation for building high-performance RDMA libraries in Python — including
+**GPUDirect** transfers to/from GPU memory.
+
+The bindings are a thin, faithful wrapper over the verbs API (device, PD, MR,
+CQ, QP, SRQ, AH, work requests, completions, async events) plus a small,
+optional set of RC connection helpers. They are written in Cython so the data
+path (`post_send` / `post_recv` / `poll`) compiles to direct C calls and
+releases the GIL, and so the `static inline` verbs fast-path functions are
+called correctly (they can't be reached through `dlsym`).
+
+- **No runtime dependencies.** Only libibverbs, which is `dlopen`ed at import.
+- **No torch / CUDA linkage.** GPUDirect works by registering an integer
+  device address or an exported dma-buf fd; CUDA stays entirely in your code.
+- **One `abi3` wheel for all of CPython 3.9+** on Linux.
+
+## Portability
+
+The extension does **not** link `libibverbs`. It is compiled against the
+rdma-core headers (for struct layouts and the `static inline` data-path verbs)
+but resolves the exported verbs at import time with `dlopen`/`dlsym`. As a
+result:
+
+- The compiled module's only `NEEDED` library is `libc` — no external
+  dependency for `auditwheel`, so a single **manylinux** wheel is portable
+  across distros.
+- It is built against the **CPython Limited API (abi3)**, so one wheel works on
+  CPython **3.9 through 3.14+** — no per-version builds.
+- A missing `libibverbs` yields a clean `ImportError`, not a loader crash.
+- Newer verbs are optional: `ibv_reg_dmabuf_mr` (rdma-core ≥ 34) is loaded if
+  present and only errors if you actually call `reg_dmabuf_mr`, so the wheel
+  still imports on older systems.
+
+At runtime you only need `libibverbs.so.1` (any `rdma-core` from the last
+several years). The data path (`post_send`/`poll`/…) stays compiled inline and
+dispatches through the provider op table, so `dlopen` costs nothing on the hot
+path.
+
+## Requirements
+
+- Linux with an RDMA-capable NIC (tested on Mellanox/NVIDIA **mlx5**, RoCEv2).
+- **Runtime:** `libibverbs.so.1` (`rdma-core` — `libibverbs1` on Debian/Ubuntu,
+  `libibverbs` on RHEL/Fedora). No compiler or headers needed to *use* a wheel.
+- **Build from source only:** a C compiler, Cython, and the `rdma-core`
+  development headers (`libibverbs-dev` / `rdma-core-devel`).
+
+## Install
+
+```bash
+pip install ibverbs        # prebuilt abi3 manylinux wheel (once published)
+```
+
+Building from source (needs the rdma-core dev headers + a compiler):
+
+```bash
+pip install "Cython>=3.0" "setuptools>=77" wheel
+pip install ./ibverbs       # or: pip install -e ./ibverbs
+```
+
+## Quickstart
+
+```python
+import ibverbs as ib
+
+# 1. Open a device and set up resources.
+dev = ib.get_device_list()[0]
+ctx = dev.open()
+pd = ctx.alloc_pd()
+cq = ctx.create_cq(64)
+
+# 2. Register memory (host or GPU address — any integer VA works).
+import numpy as np
+buf = np.zeros(4096, dtype=np.uint8)
+access = ib.AccessFlags.LOCAL_WRITE | ib.AccessFlags.REMOTE_WRITE | ib.AccessFlags.REMOTE_READ
+mr = pd.reg_mr(buf.ctypes.data, buf.nbytes, access)
+
+# 3. Create a reliable-connected QP.
+qp = pd.create_qp(ib.QPInitAttr(send_cq=cq, recv_cq=cq, qp_type=ib.QPType.RC))
+
+# 4. Exchange connection info with the peer out-of-band, then connect.
+port = 1
+port_attr = ctx.query_port(port)
+gid = ctx.query_gid(port, gid_index)              # pick a routable RoCEv2 GID
+local = ib.local_qp_info(qp, port_attr, gid, port=port, psn=0)
+# ... send local.to_bytes() to peer, receive remote_bytes ...
+remote = ib.QPInfo.from_bytes(remote_bytes)
+ib.connect_rc(qp, remote, port=port, sgid_index=gid_index, access=access)
+
+# 5. Post an RDMA write and reap the completion.
+qp.post_send(ib.SendWR(
+    wr_id=1, sg_list=[ib.SGE(mr, 4096)], opcode=ib.WROpcode.RDMA_WRITE,
+    send_flags=ib.SendFlags.SIGNALED, remote_addr=peer_addr, rkey=peer_rkey))
+for wc in cq.poll(16):
+    wc.raise_for_status()
+```
+
+Every resource is a context manager and frees its handle on `close()` /
+garbage collection; children hold references to their parents, so destruction
+order is always safe.
+
+## GPUDirect with torch tensors
+
+The library never imports torch or links CUDA. The optional `ibverbs.cuda`
+helper (which only lazily `dlopen`s `libcuda`) registers a CUDA tensor for RDMA
+in one call — handling the dma-buf export and page alignment for you:
+
+```python
+import os
+# torch's CUDA memory must be VMM-backed to be dma-buf exportable. Set this
+# BEFORE torch initializes CUDA:
+os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
+
+import torch
+import ibverbs as ib
+import ibverbs.cuda
+
+src = torch.arange(4096, dtype=torch.float32, device="cuda:0")
+dst = torch.zeros(4096, dtype=torch.float32, device="cuda:0")
+
+access = ib.AccessFlags.LOCAL_WRITE | ib.AccessFlags.REMOTE_WRITE
+src_mr = ib.cuda.register_tensor(pd, src, access)   # retains src until close()
+dst_mr = ib.cuda.register_tensor(pd, dst, access)
+
+# RDMA-write one GPU buffer into another, with no host staging on the data path.
+torch.cuda.synchronize(src.device)  # source-producing CUDA work must be done
+qp.post_send(ib.SendWR(
+    wr_id=1, sg_list=[src_mr.sge()], opcode=ib.WROpcode.RDMA_WRITE,
+    send_flags=ib.SendFlags.SIGNALED,
+    remote_addr=dst_mr.addr, rkey=dst_mr.rkey))
+for wc in qp.send_cq.poll(16):
+    wc.raise_for_status()
+
+# On the receiver, after the peer has signaled that its write is complete,
+# order the inbound NIC writes before launching CUDA work that consumes dst.
+with torch.cuda.device(dst.device):
+    ib.cuda.flush_gpudirect_writes()
+```
+
+`GpuMR` wraps the `MR` with the correct device address (`ibv_mr.addr` is not
+meaningful for dma-buf MRs), retains the tensor allocation until `close()`, and
+exposes `.sge()`, `.addr`, `.lkey`, `.rkey`.
+
+CUDA work and NIC work are separate ordering domains. Synchronize the stream
+that produced an outbound tensor before posting it. For inbound `SEND`, RDMA
+read, or RDMA write, wait for the corresponding completion or protocol-level
+notification, then call `flush_gpudirect_writes()` in the destination CUDA
+context before consuming the tensor. A one-sided RDMA write does not create a
+remote CQ entry by itself; use write-with-immediate or an out-of-band message
+to notify the receiver.
+
+Under the hood there are two registration paths, chosen automatically:
+
+```python
+# dma-buf fd (default; no kernel module needed):
+mr = pd.reg_dmabuf_mr(offset, length, iova=device_va, fd=dmabuf_fd, access=access)
+# raw device pointer (requires the nvidia_peermem kernel module):
+mr = pd.reg_mr(tensor.data_ptr(), nbytes, access)
+```
+
+For a **host** (CPU) torch tensor or numpy array, `ib.reg_tensor(pd, tensor,
+access)` registers it directly and retains the allocation. Both tensor helpers
+require contiguous, non-empty tensors; split any single SGE larger than
+`2**32 - 1` bytes into multiple entries. `tests/test_gpudirect.py` performs real
+GPU-to-GPU RDMA writes, reads, and sends verified with `torch.equal`.
+
+## Feature coverage
+
+| Area | Supported |
+|------|-----------|
+| Device / port / GID query | ✅ `get_device_list`, `Context.query_device/query_port/query_gid` |
+| Protection domains | ✅ `alloc_pd` |
+| Memory regions | ✅ `reg_mr`, `reg_dmabuf_mr` (GPUDirect) |
+| Completion queues | ✅ `create_cq`, `poll`, comp channels + `req_notify`/`ack_events` |
+| Queue pairs | ✅ RC / UC / UD; `modify`, `query`, `to_init`/`to_rtr`/`to_rts` |
+| Work requests | ✅ SEND(/_IMM), RDMA_WRITE(/_IMM), RDMA_READ, ATOMIC_CMP_AND_SWP, ATOMIC_FETCH_AND_ADD, scatter/gather, inline/signaled/fenced/solicited flags |
+| Shared receive queues | ✅ `create_srq`, `post_recv`, `modify`, `query` |
+| Address handles | ✅ `create_ah` (UD) |
+| Async events | ✅ `get_async_event` / `ack_async_event`, `async_fd` |
+| Connection helpers | ✅ `QPInfo`, `local_qp_info`, `connect_rc` |
+
+Out of scope for v1 (candidates for later): the extended `ibv_wr_*` / `qp_ex`
+post API, device memory (`ibv_alloc_dm`), memory windows, and flow steering.
+
+## Testing
+
+The suite exercises real hardware and skips features the host lacks:
+
+```bash
+pip install -e "./ibverbs[test,gpu]"     # pytest, numpy, torch
+cd ibverbs && pytest -rs                  # -rs shows skip reasons
+pytest -m "not gpu"                       # skip GPUDirect tests
+pytest -m integration                     # only real-hardware tests
+```
+
+Markers: `integration` (needs an RDMA NIC), `gpu` (needs CUDA + torch).
+
+## License
+
+BSD-3-Clause. See the repository `LICENSE`.

ibverbs-2026.7.17/pyproject.toml

@@ -0,0 +1,67 @@
+[build-system]
+requires = ["setuptools>=77", "Cython>=3.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "ibverbs"
+dynamic = ["version"]
+description = "Low-level Pythonic bindings for libibverbs (RDMA), with GPUDirect support"
+readme = "README.md"
+requires-python = ">=3.9"
+license = "BSD-3-Clause"
+license-files = ["LICENSE"]
+authors = [{ name = "Tristan Rice" }]
+keywords = ["rdma", "ibverbs", "infiniband", "roce", "gpudirect", "networking"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: POSIX :: Linux",
+    "Programming Language :: Cython",
+    "Programming Language :: Python :: 3",
+    "Topic :: System :: Networking",
+]
+# No runtime dependencies: the library only needs libibverbs at load time.
+dependencies = []
+
+[tool.setuptools.dynamic]
+version = { attr = "ibverbs._version.__version__" }
+
+[project.optional-dependencies]
+test = ["pytest>=7", "numpy"]
+gpu = ["torch"]
+
+[project.urls]
+Homepage = "https://github.com/tristanr/rdma4py"
+Repository = "https://github.com/tristanr/rdma4py"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+include-package-data = false
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+markers = [
+    "integration: exercises real RDMA hardware (loopback / two-NIC)",
+    "gpu: requires a CUDA GPU and torch (GPUDirect)",
+]
+testpaths = ["tests"]
+
+# Single abi3 wheel per platform (works on CPython 3.9+). The extension is
+# built against rdma-core headers but dlopens libibverbs at runtime, so the
+# build image only needs the -devel headers; the wheel has no external NEEDED
+# library and is trivially manylinux-compliant.
+[tool.cibuildwheel]
+# cibuildwheel selects a CPython build environment; setup.py retags the
+# resulting extension as cp39-abi3 via Py_LIMITED_API.
+build = "cp39-*"
+build-frontend = "build"
+skip = ["*-musllinux*"]
+test-requires = ["pytest", "numpy"]
+# Hardware/GPU tests self-skip when no NIC/CUDA is present in CI.
+test-command = "pytest -rs -m 'not gpu' {package}/tests"
+
+[tool.cibuildwheel.linux]
+# manylinux images are AlmaLinux-based (dnf/yum).
+before-all = "dnf install -y rdma-core-devel || yum install -y rdma-core-devel"

ibverbs-2026.7.17/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+