efa 2026.7.18__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- efa-2026.7.18/CLAUDE.md +61 -0
- efa-2026.7.18/LICENSE +28 -0
- efa-2026.7.18/MANIFEST.in +3 -0
- efa-2026.7.18/PKG-INFO +216 -0
- efa-2026.7.18/README.md +190 -0
- efa-2026.7.18/pyproject.toml +68 -0
- efa-2026.7.18/setup.cfg +4 -0
- efa-2026.7.18/setup.py +71 -0
- efa-2026.7.18/src/efa/__init__.py +143 -0
- efa-2026.7.18/src/efa/_efa.c +69222 -0
- efa-2026.7.18/src/efa/_efa.pyx +2164 -0
- efa-2026.7.18/src/efa/_libefa.pxd +461 -0
- efa-2026.7.18/src/efa/_version.py +1 -0
- efa-2026.7.18/src/efa/cuda.py +251 -0
- efa-2026.7.18/src/efa/enums.py +208 -0
- efa-2026.7.18/src/efa/helpers.py +237 -0
- efa-2026.7.18/src/efa.egg-info/PKG-INFO +216 -0
- efa-2026.7.18/src/efa.egg-info/SOURCES.txt +33 -0
- efa-2026.7.18/src/efa.egg-info/dependency_links.txt +1 -0
- efa-2026.7.18/src/efa.egg-info/requires.txt +7 -0
- efa-2026.7.18/src/efa.egg-info/top_level.txt +1 -0
- efa-2026.7.18/tests/_srd.py +123 -0
- efa-2026.7.18/tests/conftest.py +48 -0
- efa-2026.7.18/tests/test_ah.py +55 -0
- efa-2026.7.18/tests/test_cq.py +188 -0
- efa-2026.7.18/tests/test_cuda_helpers.py +80 -0
- efa-2026.7.18/tests/test_device.py +75 -0
- efa-2026.7.18/tests/test_enums.py +90 -0
- efa-2026.7.18/tests/test_gpudirect.py +281 -0
- efa-2026.7.18/tests/test_helpers.py +124 -0
- efa-2026.7.18/tests/test_import.py +13 -0
- efa-2026.7.18/tests/test_loopback.py +448 -0
- efa-2026.7.18/tests/test_pd_mr.py +93 -0
- efa-2026.7.18/tests/test_qp.py +171 -0
- efa-2026.7.18/tests/test_two_device.py +108 -0
efa-2026.7.18/CLAUDE.md
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# EFA Package Guide
|
|
2
|
+
|
|
3
|
+
`efa` is a thin Cython wrapper over `libibverbs` and EFA's direct-verbs API.
|
|
4
|
+
Keep the package low-level: one Python object per verbs resource, explicit
|
|
5
|
+
state transitions, no hidden threads, no torch import, and no CUDA linkage.
|
|
6
|
+
|
|
7
|
+
## Layout
|
|
8
|
+
|
|
9
|
+
```text
|
|
10
|
+
efa/
|
|
11
|
+
setup.py
|
|
12
|
+
pyproject.toml
|
|
13
|
+
src/efa/
|
|
14
|
+
_libefa.pxd
|
|
15
|
+
_efa.pyx
|
|
16
|
+
enums.py
|
|
17
|
+
helpers.py
|
|
18
|
+
cuda.py
|
|
19
|
+
tests/
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
## Build And Test
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
uv pip install --python .venv/bin/python -e . --no-build-isolation
|
|
26
|
+
.venv/bin/python -m pytest -rs
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Rebuild after changing `_efa.pyx` or `_libefa.pxd`. Generated C, extension,
|
|
30
|
+
cache, wheel, and sdist files are ignored.
|
|
31
|
+
|
|
32
|
+
## Binding Conventions
|
|
33
|
+
|
|
34
|
+
- Resolve exported `ibv_*` and `efadv_*` symbols with `dlopen`/`dlsym`.
|
|
35
|
+
- Call header-only provider dispatch functions directly from Cython.
|
|
36
|
+
- Hold Python parent references for every live C resource.
|
|
37
|
+
- Make `close()` idempotent and release resources in `__dealloc__`.
|
|
38
|
+
- Convert Python values before entering `nogil`.
|
|
39
|
+
- Preserve positive return-code errors; do not assume every failure uses
|
|
40
|
+
`errno`.
|
|
41
|
+
- Convert immediate data to network byte order when posting and back to host
|
|
42
|
+
order on completion.
|
|
43
|
+
- Treat optional EFA 1.3/1.4 entry points as optional symbols with clear
|
|
44
|
+
runtime errors.
|
|
45
|
+
|
|
46
|
+
## EFA Details
|
|
47
|
+
|
|
48
|
+
SRD uses `IBV_QPT_DRIVER` plus `EFADV_QP_DRIVER_TYPE_SRD`. It follows the UD
|
|
49
|
+
RESET to INIT to RTR to RTS transitions, and each send supplies an AH, remote
|
|
50
|
+
QPN, and qkey. One-sided operations also require the responder to have an AH
|
|
51
|
+
for the requester; otherwise EFA reports vendor status `0x0e`.
|
|
52
|
+
|
|
53
|
+
All EFA sends are signaled unless `sq_sig_all` was enabled. RDMA operations
|
|
54
|
+
use the extended `ibv_wr_*` API. SEND size is bounded by
|
|
55
|
+
`PortAttr.max_msg_sz`; RDMA size is bounded by
|
|
56
|
+
`EfaDeviceAttr.max_rdma_size`.
|
|
57
|
+
|
|
58
|
+
GPU registration prefers dma-buf and falls back to `nvidia_peermem`.
|
|
59
|
+
Outbound CUDA work must complete before posting. Inbound data needs
|
|
60
|
+
`cuFlushGPUDirectRDMAWrites` after the network completion or notification and
|
|
61
|
+
before CUDA consumption.
|
efa-2026.7.18/LICENSE
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026, Tristan Rice
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
efa-2026.7.18/PKG-INFO
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: efa
|
|
3
|
+
Version: 2026.7.18
|
|
4
|
+
Summary: Low-level Pythonic bindings for AWS EFA (SRD RDMA via libibverbs/libefa), with GPUDirect support
|
|
5
|
+
Author: Tristan Rice
|
|
6
|
+
License-Expression: BSD-3-Clause
|
|
7
|
+
Project-URL: Documentation, https://d4l3k.github.io/rdma4py/efa/
|
|
8
|
+
Project-URL: Homepage, https://github.com/d4l3k/rdma4py
|
|
9
|
+
Project-URL: Repository, https://github.com/d4l3k/rdma4py
|
|
10
|
+
Keywords: efa,srd,rdma,ibverbs,aws,gpudirect,networking
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Programming Language :: Cython
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: System :: Networking
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Provides-Extra: test
|
|
21
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
22
|
+
Requires-Dist: numpy; extra == "test"
|
|
23
|
+
Provides-Extra: gpu
|
|
24
|
+
Requires-Dist: torch; extra == "gpu"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# efa
|
|
28
|
+
|
|
29
|
+
Low-level Python bindings for AWS Elastic Fabric Adapter (EFA), including
|
|
30
|
+
Scalable Reliable Datagram (SRD), one-sided RDMA, and GPUDirect transfers
|
|
31
|
+
to and from torch CUDA tensors.
|
|
32
|
+
|
|
33
|
+
The package wraps `libibverbs` and EFA's `libefa` direct-verbs API in Cython.
|
|
34
|
+
Its data path calls the provider's inline verbs directly, releases the GIL
|
|
35
|
+
around blocking and posting operations, and does not import torch or link
|
|
36
|
+
against CUDA.
|
|
37
|
+
|
|
38
|
+
- No Python runtime dependencies.
|
|
39
|
+
- SRD and UD queue pairs with SEND, RDMA read, and RDMA write operations.
|
|
40
|
+
- Host buffers, CUDA device pointers, and dma-buf memory registration.
|
|
41
|
+
- Classic and extended CQs, sender GID and unsolicited-write metadata, and
|
|
42
|
+
direct EFA CQ, SQ, RQ, MR, and AH queries.
|
|
43
|
+
- One Linux `abi3` wheel for CPython 3.9 and newer.
|
|
44
|
+
|
|
45
|
+
## Requirements
|
|
46
|
+
|
|
47
|
+
- Linux on an AWS instance with one or more EFA devices attached.
|
|
48
|
+
- `libibverbs.so.1` and `libefa.so.1` at runtime. The AWS EFA installer and
|
|
49
|
+
current `rdma-core` distributions provide both.
|
|
50
|
+
- The EFA device nodes available inside the process or container, normally
|
|
51
|
+
`/dev/infiniband/uverbs*`.
|
|
52
|
+
- A C compiler, Cython, and the `rdma-core` development headers only when
|
|
53
|
+
building from source.
|
|
54
|
+
|
|
55
|
+
For GPUDirect, the EFA device and instance type must support RDMA read/write,
|
|
56
|
+
and the NVIDIA driver must support dma-buf export or `nvidia_peermem`.
|
|
57
|
+
|
|
58
|
+
## Install
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
pip install efa
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
To build from this checkout:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
pip install "Cython>=3.0" "setuptools>=77" wheel
|
|
68
|
+
pip install ./efa
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## SRD Quickstart
|
|
72
|
+
|
|
73
|
+
SRD is reliable and connectionless. Each process creates a ready-to-send QP,
|
|
74
|
+
exchanges a 24-byte `EndpointInfo` out of band, and resolves the remote GID to
|
|
75
|
+
an address handle:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import numpy as np
|
|
79
|
+
import efa
|
|
80
|
+
|
|
81
|
+
dev = efa.get_efa_device_list()[0]
|
|
82
|
+
ctx = dev.open()
|
|
83
|
+
pd = ctx.alloc_pd()
|
|
84
|
+
cq = ctx.create_cq(256)
|
|
85
|
+
qp = pd.create_qp(
|
|
86
|
+
efa.QPInitAttr(send_cq=cq, recv_cq=cq)
|
|
87
|
+
).prepare(qkey=0x1234)
|
|
88
|
+
|
|
89
|
+
local_info = efa.local_endpoint_info(qp, qkey=0x1234)
|
|
90
|
+
# Exchange local_info.to_bytes() with the other process.
|
|
91
|
+
remote_info = efa.EndpointInfo.from_bytes(remote_bytes)
|
|
92
|
+
peer = remote_info.peer(pd)
|
|
93
|
+
|
|
94
|
+
buf = np.zeros(4096, dtype=np.uint8)
|
|
95
|
+
access = (
|
|
96
|
+
efa.AccessFlags.LOCAL_WRITE
|
|
97
|
+
| efa.AccessFlags.REMOTE_WRITE
|
|
98
|
+
| efa.AccessFlags.REMOTE_READ
|
|
99
|
+
)
|
|
100
|
+
mr = efa.reg_tensor(pd, buf, access)
|
|
101
|
+
|
|
102
|
+
qp.post_send(efa.SendWR(
|
|
103
|
+
wr_id=1,
|
|
104
|
+
sg_list=[mr.sge()],
|
|
105
|
+
opcode=efa.WROpcode.SEND,
|
|
106
|
+
send_flags=efa.SendFlags.SIGNALED,
|
|
107
|
+
dest=peer,
|
|
108
|
+
))
|
|
109
|
+
for wc in cq.poll(16):
|
|
110
|
+
wc.raise_for_status()
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Every handle is an idempotent context manager. A QP retains its PD and CQs,
|
|
114
|
+
an MR retains its PD and backing tensor, and an SGE retains the MR it addresses.
|
|
115
|
+
|
|
116
|
+
### One-sided RDMA
|
|
117
|
+
|
|
118
|
+
EFA requires the responder to have an address handle for the requester before
|
|
119
|
+
it accepts RDMA reads or writes. Both processes should therefore resolve the
|
|
120
|
+
other process's `EndpointInfo`, even if traffic is currently one-way. A
|
|
121
|
+
missing reverse AH completes with `REM_OP_ERR` and EFA vendor status `0x0e`
|
|
122
|
+
(`REMOTE_ERROR_UNKNOWN_PEER`).
|
|
123
|
+
|
|
124
|
+
Large buffers can be split at the device's `max_rdma_size`:
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
wrs = efa.write_wrs(local_mr, peer, remote_addr, remote_rkey)
|
|
128
|
+
qp.post_send(wrs)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
`read_wrs` provides the corresponding RDMA-read operation. Every generated WR
|
|
132
|
+
is signaled, as required by EFA.
|
|
133
|
+
|
|
134
|
+
## GPUDirect With Torch
|
|
135
|
+
|
|
136
|
+
The optional `efa.cuda` module is torch-free and duck-types objects exposing
|
|
137
|
+
`data_ptr()`, `numel()`, and `element_size()`. For torch's dma-buf path, enable
|
|
138
|
+
VMM-backed allocations before CUDA initializes:
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
import os
|
|
142
|
+
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
|
|
143
|
+
|
|
144
|
+
import torch
|
|
145
|
+
import efa
|
|
146
|
+
import efa.cuda
|
|
147
|
+
|
|
148
|
+
src = torch.arange(1 << 20, dtype=torch.float32, device="cuda:0")
|
|
149
|
+
dst = torch.zeros_like(src)
|
|
150
|
+
|
|
151
|
+
src_mr = efa.cuda.register_tensor(pd, src, access)
|
|
152
|
+
dst_mr = efa.cuda.register_tensor(pd, dst, access)
|
|
153
|
+
|
|
154
|
+
torch.cuda.synchronize(src.device)
|
|
155
|
+
qp.post_send(efa.SendWR(
|
|
156
|
+
wr_id=2,
|
|
157
|
+
sg_list=[src_mr.sge()],
|
|
158
|
+
opcode=efa.WROpcode.RDMA_WRITE,
|
|
159
|
+
send_flags=efa.SendFlags.SIGNALED,
|
|
160
|
+
remote_addr=remote_dst_addr,
|
|
161
|
+
rkey=remote_dst_rkey,
|
|
162
|
+
dest=peer,
|
|
163
|
+
))
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
After the receiver observes a completion or protocol-level write
|
|
167
|
+
notification, it must order inbound NIC writes before CUDA consumes the
|
|
168
|
+
destination:
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
with torch.cuda.device(dst.device):
|
|
172
|
+
efa.cuda.flush_gpudirect_writes()
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
`register_tensor` first exports a dma-buf fd and calls `ibv_reg_dmabuf_mr`.
|
|
176
|
+
If that path is unavailable, it falls back to `ibv_reg_mr`, which requires
|
|
177
|
+
`nvidia_peermem`. The returned `GpuMR` retains the tensor allocation and keeps
|
|
178
|
+
the actual CUDA virtual address because `ibv_mr.addr` is not meaningful for a
|
|
179
|
+
dma-buf MR.
|
|
180
|
+
|
|
181
|
+
## Direct EFA API
|
|
182
|
+
|
|
183
|
+
| Area | API |
|
|
184
|
+
| --- | --- |
|
|
185
|
+
| Device capabilities | `Context.query_efa_device` |
|
|
186
|
+
| EFA CQ creation | `Context.create_cq_ex` |
|
|
187
|
+
| Sender and unsolicited metadata | `CQEx.poll`, `WC.sgid`, `WC.unsolicited` |
|
|
188
|
+
| CQ layout | `CQ.query_efa`, `CQEx.query_efa` |
|
|
189
|
+
| SQ/RQ layout | `QP.query_wqs` |
|
|
190
|
+
| MR interconnect IDs | `MR.query_efa` |
|
|
191
|
+
| Address-handle number | `AH.ahn` |
|
|
192
|
+
| SRD QP creation | `PD.create_qp` with `QPType.SRD` |
|
|
193
|
+
|
|
194
|
+
When using unsolicited RDMA write-with-immediate completions, create every
|
|
195
|
+
communicating QP with `QPInitAttr(..., unsolicited_write_recv=True)` and use an
|
|
196
|
+
extended CQ created with `unsolicited=True`. EFA requires peers to negotiate
|
|
197
|
+
the same QP feature set.
|
|
198
|
+
|
|
199
|
+
The direct layout queries expose process-local addresses for advanced
|
|
200
|
+
consumers. They do not transfer ownership of provider memory.
|
|
201
|
+
|
|
202
|
+
## Testing
|
|
203
|
+
|
|
204
|
+
```bash
|
|
205
|
+
pip install -e "./efa[test,gpu]"
|
|
206
|
+
cd efa
|
|
207
|
+
pytest -rs
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Tests marked `integration` exercise real EFA hardware. Tests marked `gpu`
|
|
211
|
+
perform torch-verified GPU-to-GPU, GPU-to-host, and host-to-GPU transfers.
|
|
212
|
+
Unavailable hardware capabilities are skipped explicitly.
|
|
213
|
+
|
|
214
|
+
## License
|
|
215
|
+
|
|
216
|
+
BSD-3-Clause. See `LICENSE`.
|
efa-2026.7.18/README.md
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
# efa
|
|
2
|
+
|
|
3
|
+
Low-level Python bindings for AWS Elastic Fabric Adapter (EFA), including
|
|
4
|
+
Scalable Reliable Datagram (SRD), one-sided RDMA, and GPUDirect transfers
|
|
5
|
+
to and from torch CUDA tensors.
|
|
6
|
+
|
|
7
|
+
The package wraps `libibverbs` and EFA's `libefa` direct-verbs API in Cython.
|
|
8
|
+
Its data path calls the provider's inline verbs directly, releases the GIL
|
|
9
|
+
around blocking and posting operations, and does not import torch or link
|
|
10
|
+
against CUDA.
|
|
11
|
+
|
|
12
|
+
- No Python runtime dependencies.
|
|
13
|
+
- SRD and UD queue pairs with SEND, RDMA read, and RDMA write operations.
|
|
14
|
+
- Host buffers, CUDA device pointers, and dma-buf memory registration.
|
|
15
|
+
- Classic and extended CQs, sender GID and unsolicited-write metadata, and
|
|
16
|
+
direct EFA CQ, SQ, RQ, MR, and AH queries.
|
|
17
|
+
- One Linux `abi3` wheel for CPython 3.9 and newer.
|
|
18
|
+
|
|
19
|
+
## Requirements
|
|
20
|
+
|
|
21
|
+
- Linux on an AWS instance with one or more EFA devices attached.
|
|
22
|
+
- `libibverbs.so.1` and `libefa.so.1` at runtime. The AWS EFA installer and
|
|
23
|
+
current `rdma-core` distributions provide both.
|
|
24
|
+
- The EFA device nodes available inside the process or container, normally
|
|
25
|
+
`/dev/infiniband/uverbs*`.
|
|
26
|
+
- A C compiler, Cython, and the `rdma-core` development headers only when
|
|
27
|
+
building from source.
|
|
28
|
+
|
|
29
|
+
For GPUDirect, the EFA device and instance type must support RDMA read/write,
|
|
30
|
+
and the NVIDIA driver must support dma-buf export or `nvidia_peermem`.
|
|
31
|
+
|
|
32
|
+
## Install
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install efa
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
To build from this checkout:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install "Cython>=3.0" "setuptools>=77" wheel
|
|
42
|
+
pip install ./efa
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## SRD Quickstart
|
|
46
|
+
|
|
47
|
+
SRD is reliable and connectionless. Each process creates a ready-to-send QP,
|
|
48
|
+
exchanges a 24-byte `EndpointInfo` out of band, and resolves the remote GID to
|
|
49
|
+
an address handle:
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
import numpy as np
|
|
53
|
+
import efa
|
|
54
|
+
|
|
55
|
+
dev = efa.get_efa_device_list()[0]
|
|
56
|
+
ctx = dev.open()
|
|
57
|
+
pd = ctx.alloc_pd()
|
|
58
|
+
cq = ctx.create_cq(256)
|
|
59
|
+
qp = pd.create_qp(
|
|
60
|
+
efa.QPInitAttr(send_cq=cq, recv_cq=cq)
|
|
61
|
+
).prepare(qkey=0x1234)
|
|
62
|
+
|
|
63
|
+
local_info = efa.local_endpoint_info(qp, qkey=0x1234)
|
|
64
|
+
# Exchange local_info.to_bytes() with the other process.
|
|
65
|
+
remote_info = efa.EndpointInfo.from_bytes(remote_bytes)
|
|
66
|
+
peer = remote_info.peer(pd)
|
|
67
|
+
|
|
68
|
+
buf = np.zeros(4096, dtype=np.uint8)
|
|
69
|
+
access = (
|
|
70
|
+
efa.AccessFlags.LOCAL_WRITE
|
|
71
|
+
| efa.AccessFlags.REMOTE_WRITE
|
|
72
|
+
| efa.AccessFlags.REMOTE_READ
|
|
73
|
+
)
|
|
74
|
+
mr = efa.reg_tensor(pd, buf, access)
|
|
75
|
+
|
|
76
|
+
qp.post_send(efa.SendWR(
|
|
77
|
+
wr_id=1,
|
|
78
|
+
sg_list=[mr.sge()],
|
|
79
|
+
opcode=efa.WROpcode.SEND,
|
|
80
|
+
send_flags=efa.SendFlags.SIGNALED,
|
|
81
|
+
dest=peer,
|
|
82
|
+
))
|
|
83
|
+
for wc in cq.poll(16):
|
|
84
|
+
wc.raise_for_status()
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Every handle is an idempotent context manager. A QP retains its PD and CQs,
|
|
88
|
+
an MR retains its PD and backing tensor, and an SGE retains the MR it addresses.
|
|
89
|
+
|
|
90
|
+
### One-sided RDMA
|
|
91
|
+
|
|
92
|
+
EFA requires the responder to have an address handle for the requester before
|
|
93
|
+
it accepts RDMA reads or writes. Both processes should therefore resolve the
|
|
94
|
+
other process's `EndpointInfo`, even if traffic is currently one-way. A
|
|
95
|
+
missing reverse AH completes with `REM_OP_ERR` and EFA vendor status `0x0e`
|
|
96
|
+
(`REMOTE_ERROR_UNKNOWN_PEER`).
|
|
97
|
+
|
|
98
|
+
Large buffers can be split at the device's `max_rdma_size`:
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
wrs = efa.write_wrs(local_mr, peer, remote_addr, remote_rkey)
|
|
102
|
+
qp.post_send(wrs)
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
`read_wrs` provides the corresponding RDMA-read operation. Every generated WR
|
|
106
|
+
is signaled, as required by EFA.
|
|
107
|
+
|
|
108
|
+
## GPUDirect With Torch
|
|
109
|
+
|
|
110
|
+
The optional `efa.cuda` module is torch-free and duck-types objects exposing
|
|
111
|
+
`data_ptr()`, `numel()`, and `element_size()`. For torch's dma-buf path, enable
|
|
112
|
+
VMM-backed allocations before CUDA initializes:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
import os
|
|
116
|
+
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"
|
|
117
|
+
|
|
118
|
+
import torch
|
|
119
|
+
import efa
|
|
120
|
+
import efa.cuda
|
|
121
|
+
|
|
122
|
+
src = torch.arange(1 << 20, dtype=torch.float32, device="cuda:0")
|
|
123
|
+
dst = torch.zeros_like(src)
|
|
124
|
+
|
|
125
|
+
src_mr = efa.cuda.register_tensor(pd, src, access)
|
|
126
|
+
dst_mr = efa.cuda.register_tensor(pd, dst, access)
|
|
127
|
+
|
|
128
|
+
torch.cuda.synchronize(src.device)
|
|
129
|
+
qp.post_send(efa.SendWR(
|
|
130
|
+
wr_id=2,
|
|
131
|
+
sg_list=[src_mr.sge()],
|
|
132
|
+
opcode=efa.WROpcode.RDMA_WRITE,
|
|
133
|
+
send_flags=efa.SendFlags.SIGNALED,
|
|
134
|
+
remote_addr=remote_dst_addr,
|
|
135
|
+
rkey=remote_dst_rkey,
|
|
136
|
+
dest=peer,
|
|
137
|
+
))
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
After the receiver observes a completion or protocol-level write
|
|
141
|
+
notification, it must order inbound NIC writes before CUDA consumes the
|
|
142
|
+
destination:
|
|
143
|
+
|
|
144
|
+
```python
|
|
145
|
+
with torch.cuda.device(dst.device):
|
|
146
|
+
efa.cuda.flush_gpudirect_writes()
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
`register_tensor` first exports a dma-buf fd and calls `ibv_reg_dmabuf_mr`.
|
|
150
|
+
If that path is unavailable, it falls back to `ibv_reg_mr`, which requires
|
|
151
|
+
`nvidia_peermem`. The returned `GpuMR` retains the tensor allocation and keeps
|
|
152
|
+
the actual CUDA virtual address because `ibv_mr.addr` is not meaningful for a
|
|
153
|
+
dma-buf MR.
|
|
154
|
+
|
|
155
|
+
## Direct EFA API
|
|
156
|
+
|
|
157
|
+
| Area | API |
|
|
158
|
+
| --- | --- |
|
|
159
|
+
| Device capabilities | `Context.query_efa_device` |
|
|
160
|
+
| EFA CQ creation | `Context.create_cq_ex` |
|
|
161
|
+
| Sender and unsolicited metadata | `CQEx.poll`, `WC.sgid`, `WC.unsolicited` |
|
|
162
|
+
| CQ layout | `CQ.query_efa`, `CQEx.query_efa` |
|
|
163
|
+
| SQ/RQ layout | `QP.query_wqs` |
|
|
164
|
+
| MR interconnect IDs | `MR.query_efa` |
|
|
165
|
+
| Address-handle number | `AH.ahn` |
|
|
166
|
+
| SRD QP creation | `PD.create_qp` with `QPType.SRD` |
|
|
167
|
+
|
|
168
|
+
When using unsolicited RDMA write-with-immediate completions, create every
|
|
169
|
+
communicating QP with `QPInitAttr(..., unsolicited_write_recv=True)` and use an
|
|
170
|
+
extended CQ created with `unsolicited=True`. EFA requires peers to negotiate
|
|
171
|
+
the same QP feature set.
|
|
172
|
+
|
|
173
|
+
The direct layout queries expose process-local addresses for advanced
|
|
174
|
+
consumers. They do not transfer ownership of provider memory.
|
|
175
|
+
|
|
176
|
+
## Testing
|
|
177
|
+
|
|
178
|
+
```bash
|
|
179
|
+
pip install -e "./efa[test,gpu]"
|
|
180
|
+
cd efa
|
|
181
|
+
pytest -rs
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Tests marked `integration` exercise real EFA hardware. Tests marked `gpu`
|
|
185
|
+
perform torch-verified GPU-to-GPU, GPU-to-host, and host-to-GPU transfers.
|
|
186
|
+
Unavailable hardware capabilities are skipped explicitly.
|
|
187
|
+
|
|
188
|
+
## License
|
|
189
|
+
|
|
190
|
+
BSD-3-Clause. See `LICENSE`.
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "Cython>=3.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "efa"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Low-level Pythonic bindings for AWS EFA (SRD RDMA via libibverbs/libefa), with GPUDirect support"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = "BSD-3-Clause"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Tristan Rice" }]
|
|
14
|
+
keywords = ["efa", "srd", "rdma", "ibverbs", "aws", "gpudirect", "networking"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: POSIX :: Linux",
|
|
19
|
+
"Programming Language :: Cython",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Topic :: System :: Networking",
|
|
22
|
+
]
|
|
23
|
+
# No runtime dependencies: the library only needs libibverbs + libefa at load time.
|
|
24
|
+
dependencies = []
|
|
25
|
+
|
|
26
|
+
[tool.setuptools.dynamic]
|
|
27
|
+
version = { attr = "efa._version.__version__" }
|
|
28
|
+
|
|
29
|
+
[project.optional-dependencies]
|
|
30
|
+
test = ["pytest>=7", "numpy"]
|
|
31
|
+
gpu = ["torch"]
|
|
32
|
+
|
|
33
|
+
[project.urls]
|
|
34
|
+
Documentation = "https://d4l3k.github.io/rdma4py/efa/"
|
|
35
|
+
Homepage = "https://github.com/d4l3k/rdma4py"
|
|
36
|
+
Repository = "https://github.com/d4l3k/rdma4py"
|
|
37
|
+
|
|
38
|
+
[tool.setuptools]
|
|
39
|
+
package-dir = { "" = "src" }
|
|
40
|
+
include-package-data = false
|
|
41
|
+
|
|
42
|
+
[tool.setuptools.packages.find]
|
|
43
|
+
where = ["src"]
|
|
44
|
+
|
|
45
|
+
[tool.pytest.ini_options]
|
|
46
|
+
markers = [
|
|
47
|
+
"integration: exercises real EFA hardware (loopback / two-NIC)",
|
|
48
|
+
"gpu: requires a CUDA GPU and torch (GPUDirect)",
|
|
49
|
+
]
|
|
50
|
+
testpaths = ["tests"]
|
|
51
|
+
|
|
52
|
+
# Single abi3 wheel per platform (works on CPython 3.9+). The extension is
|
|
53
|
+
# built against rdma-core headers but dlopens libibverbs/libefa at runtime, so
|
|
54
|
+
# the build image only needs the -devel headers; the wheel has no external
|
|
55
|
+
# NEEDED library and is trivially manylinux-compliant.
|
|
56
|
+
[tool.cibuildwheel]
|
|
57
|
+
# cibuildwheel selects a CPython build environment; setup.py retags the
|
|
58
|
+
# resulting extension as cp39-abi3 via Py_LIMITED_API.
|
|
59
|
+
build = "cp39-*"
|
|
60
|
+
build-frontend = "build"
|
|
61
|
+
skip = ["*-musllinux*"]
|
|
62
|
+
test-requires = ["pytest", "numpy"]
|
|
63
|
+
# Hardware/GPU tests self-skip when no EFA NIC/CUDA is present in CI.
|
|
64
|
+
test-command = "pytest -rs -m 'not gpu' {package}/tests"
|
|
65
|
+
|
|
66
|
+
[tool.cibuildwheel.linux]
|
|
67
|
+
# manylinux images are AlmaLinux-based (dnf/yum).
|
|
68
|
+
before-all = "dnf install -y rdma-core-devel || yum install -y rdma-core-devel"
|
efa-2026.7.18/setup.cfg
ADDED
efa-2026.7.18/setup.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Build the efa Cython extension.
|
|
2
|
+
|
|
3
|
+
The extension is compiled against the ``rdma-core`` headers (for struct layouts
|
|
4
|
+
and the ``static inline`` data-path verbs / extended work-request API) but is
|
|
5
|
+
**not** linked against ``libibverbs`` or ``libefa``: the exported symbols are
|
|
6
|
+
resolved at import time with ``dlopen``/``dlsym`` (see ``_efa.pyx``). The
|
|
7
|
+
header include path is discovered with ``pkg-config`` when available.
|
|
8
|
+
|
|
9
|
+
It is built against the CPython **Limited API** (abi3), so a single wheel works
|
|
10
|
+
across CPython 3.9+.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import shlex
|
|
16
|
+
import subprocess
|
|
17
|
+
|
|
18
|
+
from setuptools import Extension, setup
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
from Cython.Build import cythonize
|
|
22
|
+
except ImportError as exc: # pragma: no cover - build-time only
|
|
23
|
+
raise SystemExit(
|
|
24
|
+
"Cython is required to build efa. Install it with `pip install Cython`."
|
|
25
|
+
) from exc
|
|
26
|
+
|
|
27
|
+
# CPython Limited API floor == requires-python floor (3.9).
|
|
28
|
+
LIMITED_API_VERSION = "0x03090000"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _pkg_config(*args: str) -> list[str]:
|
|
32
|
+
try:
|
|
33
|
+
out = subprocess.check_output(["pkg-config", *args], text=True)
|
|
34
|
+
except (OSError, subprocess.CalledProcessError):
|
|
35
|
+
return []
|
|
36
|
+
return shlex.split(out)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _include_dirs() -> list[str]:
|
|
40
|
+
dirs = []
|
|
41
|
+
for flag in _pkg_config("--cflags-only-I", "libefa", "libibverbs"):
|
|
42
|
+
if flag.startswith("-I") and flag[2:] not in dirs:
|
|
43
|
+
dirs.append(flag[2:])
|
|
44
|
+
return dirs
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
extensions = [
|
|
48
|
+
Extension(
|
|
49
|
+
"efa._efa",
|
|
50
|
+
["src/efa/_efa.pyx"],
|
|
51
|
+
include_dirs=_include_dirs(),
|
|
52
|
+
# No libibverbs/libefa link: they are dlopen'd at runtime. libdl
|
|
53
|
+
# provides dlopen/dlsym (a no-op stub on glibc >= 2.34).
|
|
54
|
+
libraries=["dl"],
|
|
55
|
+
define_macros=[("Py_LIMITED_API", LIMITED_API_VERSION)],
|
|
56
|
+
py_limited_api=True,
|
|
57
|
+
),
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
setup(
|
|
61
|
+
ext_modules=cythonize(
|
|
62
|
+
extensions,
|
|
63
|
+
language_level="3",
|
|
64
|
+
compiler_directives={
|
|
65
|
+
"annotation_typing": False,
|
|
66
|
+
"embedsignature": True,
|
|
67
|
+
"binding": True,
|
|
68
|
+
},
|
|
69
|
+
),
|
|
70
|
+
options={"bdist_wheel": {"py_limited_api": "cp39"}},
|
|
71
|
+
)
|