lmcache-cli 0.4.8.dev31__py3-none-any.whl → 0.4.8.dev42__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. lmcache/_version.py +3 -3
  2. lmcache/cli/commands/tool/__init__.py +3 -2
  3. lmcache/cli/commands/tool/transfer_channel_benchmark.py +53 -0
  4. lmcache/tools/transfer_channel_benchmark/README.md +128 -0
  5. lmcache/tools/transfer_channel_benchmark/__init__.py +1 -0
  6. lmcache/tools/transfer_channel_benchmark/__main__.py +39 -0
  7. lmcache/tools/transfer_channel_benchmark/benchmark.py +385 -0
  8. lmcache/tools/transfer_channel_benchmark/config.py +252 -0
  9. lmcache/v1/distributed/api.py +64 -0
  10. lmcache/v1/distributed/config.py +200 -9
  11. lmcache/v1/distributed/l2_adapters/base.py +25 -1
  12. lmcache/v1/distributed/l2_adapters/s3_l2_adapter.py +235 -4
  13. lmcache/v1/distributed/l2_adapters/serde_wrapper.py +13 -1
  14. lmcache/v1/distributed/memory_manager/l1_memory_manager.py +30 -8
  15. lmcache/v1/distributed/storage_manager.py +22 -0
  16. lmcache/v1/distributed/transfer_channel/__init__.py +105 -0
  17. lmcache/v1/distributed/transfer_channel/abstract.py +149 -0
  18. lmcache/v1/distributed/transfer_channel/api.py +47 -0
  19. lmcache/v1/distributed/transfer_channel/factory.py +100 -0
  20. lmcache/v1/distributed/transfer_channel/impl/__init__.py +9 -0
  21. lmcache/v1/distributed/transfer_channel/impl/nixl_impl.py +591 -0
  22. lmcache/v1/memory_management.py +455 -0
  23. lmcache/v1/mp_coordinator/app.py +33 -5
  24. lmcache/v1/mp_coordinator/blend_client.py +37 -16
  25. lmcache/v1/mp_coordinator/blend_directory.py +196 -40
  26. lmcache/v1/mp_coordinator/config.py +33 -0
  27. lmcache/v1/mp_coordinator/http_apis/blend_directory_api.py +5 -1
  28. lmcache/v1/mp_coordinator/http_apis/l2_api.py +19 -75
  29. lmcache/v1/mp_coordinator/l2/event_listener.py +22 -45
  30. lmcache/v1/mp_coordinator/l2/eviction_manager.py +117 -79
  31. lmcache/v1/mp_coordinator/l2/resync_manager.py +149 -0
  32. lmcache/v1/mp_coordinator/l2/usage_manager.py +57 -70
  33. lmcache/v1/mp_coordinator/registry.py +10 -0
  34. lmcache/v1/mp_coordinator/schemas.py +80 -32
  35. lmcache/v1/mp_observability/event.py +4 -0
  36. lmcache/v1/mp_observability/subscribers/tracing/cb_server.py +5 -0
  37. lmcache/v1/multiprocess/custom_types.py +16 -1
  38. lmcache/v1/multiprocess/engine_context.py +1 -1
  39. lmcache/v1/multiprocess/http_apis/l2_api.py +241 -0
  40. lmcache/v1/multiprocess/modules/blend_v3.py +29 -10
  41. lmcache/v1/multiprocess/modules/p2p_controller.py +245 -0
  42. lmcache/v1/multiprocess/mq.py +5 -0
  43. lmcache/v1/multiprocess/protocols/__init__.py +2 -0
  44. lmcache/v1/multiprocess/protocols/base.py +5 -0
  45. lmcache/v1/multiprocess/protocols/p2p.py +61 -0
  46. {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/METADATA +1 -1
  47. {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/RECORD +51 -35
  48. {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/WHEEL +0 -0
  49. {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/entry_points.txt +0 -0
  50. {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/licenses/LICENSE +0 -0
  51. {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/top_level.txt +0 -0
lmcache/_version.py CHANGED
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.4.8.dev31'
22
- __version_tuple__ = version_tuple = (0, 4, 8, 'dev31')
21
+ __version__ = version = '0.4.8.dev42'
22
+ __version_tuple__ = version_tuple = (0, 4, 8, 'dev42')
23
23
 
24
- __commit_id__ = commit_id = 'g71d2c23f4'
24
+ __commit_id__ = commit_id = 'g9ffb104f3'
@@ -15,7 +15,7 @@ import sys
15
15
 
16
16
  # First Party
17
17
  from lmcache.cli.commands.base import BaseCommand
18
- from lmcache.cli.commands.tool import cache_simulator
18
+ from lmcache.cli.commands.tool import cache_simulator, transfer_channel_benchmark
19
19
 
20
20
 
21
21
  class ToolCommand(BaseCommand):
@@ -46,9 +46,10 @@ class ToolCommand(BaseCommand):
46
46
  inner = parser.add_subparsers(
47
47
  dest="tool_name",
48
48
  required=True,
49
- metavar="{cache-simulator}",
49
+ metavar="{cache-simulator,transfer-channel-benchmark}",
50
50
  )
51
51
  cache_simulator.register(inner)
52
+ transfer_channel_benchmark.register(inner)
52
53
 
53
54
  def execute(self, args: argparse.Namespace) -> None:
54
55
  """Dispatch is handled per-tool via parser.set_defaults(func=...).
@@ -0,0 +1,53 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """``lmcache tool transfer-channel-benchmark`` sub-subcommand wiring.
3
+
4
+ Argument definitions and execution logic live in the benchmark module:
5
+
6
+ * :func:`~lmcache.tools.transfer_channel_benchmark.benchmark.add_benchmark_arguments`
7
+ * :func:`~lmcache.tools.transfer_channel_benchmark.benchmark.run_benchmark`
8
+ """
9
+
10
+ # Standard
11
+ import argparse
12
+ import sys
13
+
14
+
15
+ def register(subparsers: argparse._SubParsersAction) -> None:
16
+ """Register the ``transfer-channel-benchmark`` sub-subcommand.
17
+
18
+ Only the torch-free ``config`` module is imported here, so registering this
19
+ tool (which happens on every ``lmcache`` invocation) does not require torch
20
+ or the distributed runtime. Those are imported in ``execute``.
21
+
22
+ Args:
23
+ subparsers: The subparsers action from the ``lmcache tool`` parser.
24
+ """
25
+ # First Party
26
+ from lmcache.tools.transfer_channel_benchmark.config import (
27
+ add_benchmark_arguments,
28
+ )
29
+
30
+ parser = subparsers.add_parser(
31
+ "transfer-channel-benchmark",
32
+ help="Benchmark transfer channel read throughput (server/client).",
33
+ description=(
34
+ "Throughput benchmark for the LMCache transfer channel. Run one "
35
+ "process with --role server and another with --role client."
36
+ ),
37
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
38
+ )
39
+ add_benchmark_arguments(parser)
40
+ parser.set_defaults(func=execute)
41
+
42
+
43
+ def execute(args: argparse.Namespace) -> None:
44
+ """Run the benchmark and exit non-zero on failure.
45
+
46
+ Args:
47
+ args: Parsed CLI arguments.
48
+ """
49
+ # First Party
50
+ from lmcache.tools.transfer_channel_benchmark.benchmark import run_benchmark
51
+
52
+ if not run_benchmark(args):
53
+ sys.exit(1)
@@ -0,0 +1,128 @@
1
+ # Transfer Channel Throughput Benchmark
2
+
3
+ Measures read throughput (GB/s) of the LMCache **transfer channel**
4
+ (`lmcache/v1/distributed/transfer_channel/`) for batched peer-to-peer reads.
5
+
6
+ Unlike a raw-tensor microbenchmark, this tool uses LMCache's `L1MemoryManager`
7
+ to initialize the registered memory region and to **allocate the transferred
8
+ objects** on both sides, so it exercises the same memory path production uses.
9
+
10
+ ## How it works
11
+
12
+ The benchmark runs as **two separate processes**:
13
+
14
+ - **server** — uses an `L1MemoryManager` to allocate a registered L1 buffer and
15
+ a pool of source memory objects, registers the buffer with the transfer
16
+ channel, and publishes the source object catalog (`offset`/`size` per object)
17
+ over a small ZMQ side-channel.
18
+ - **client** — fetches the catalog, allocates its own destination objects via an
19
+ `L1MemoryManager`, connects the transfer channel, and repeatedly reads a random
20
+ `--num-objects` subset of the source objects, then reports throughput.
21
+
22
+ A side-channel is needed because the transfer channel handshake only exchanges
23
+ the whole-buffer registration, not per-object offsets.
24
+
25
+ > Only the `nixl` transfer channel type is registered today. The tool is generic
26
+ > over `--transfer-channel-type`; an unknown type raises a clear error.
27
+
28
+ ## Usage
29
+
30
+ Run via the module or the `lmcache` CLI. Start the server first.
31
+
32
+ ### `python -m`
33
+
34
+ ```bash
35
+ # terminal 1 — server
36
+ python -m lmcache.tools.transfer_channel_benchmark \
37
+ --role server --transfer-channel-type nixl \
38
+ --url 0.0.0.0:7600 --control-url 0.0.0.0:7610 \
39
+ --buffer-size 2GB --page-size 512KB --object-size 10MB
40
+
41
+ # terminal 2 — client
42
+ python -m lmcache.tools.transfer_channel_benchmark \
43
+ --role client --transfer-channel-type nixl \
44
+ --url 127.0.0.1:7600 --control-url 127.0.0.1:7610 \
45
+ --listen-url 0.0.0.0:7601 \
46
+ --page-size 512KB --object-size 10MB \
47
+ --num-objects 10 --iters 3 --warmup 1
48
+ ```
49
+
50
+ ### `lmcache tool`
51
+
52
+ ```bash
53
+ lmcache tool transfer-channel-benchmark --role server --url 0.0.0.0:7600 \
54
+ --control-url 0.0.0.0:7610 --buffer-size 2GB --object-size 10MB
55
+ lmcache tool transfer-channel-benchmark --role client --url 127.0.0.1:7600 \
56
+ --control-url 127.0.0.1:7610 --object-size 10MB --num-objects 10
57
+ ```
58
+
59
+ The server always writes a deterministic per-object byte pattern (object index
60
+ mod 256) into its source objects, so adding `--verify` on the **client** checks
61
+ the transferred bytes against that pattern (no `--verify` needed on the server).
62
+
63
+ ## Key options
64
+
65
+ | Option | Role | Meaning |
66
+ | --- | --- | --- |
67
+ | `--role {server,client}` | both | Which side to run (required). |
68
+ | `--transfer-channel-type` | both | Implementation to benchmark (default `nixl`). |
69
+ | `--nixl-backend` | both | nixl backend, e.g. `UCX` (nixl-specific). |
70
+ | `--url` | both | Server binds its transfer-channel server here; client dials it. |
71
+ | `--listen-url` | client | Client's own (mandatory) transfer-channel server bind. |
72
+ | `--control-url` | both | Catalog side-channel: server binds, client connects. |
73
+ | `--buffer-size` | server | Registered L1 source buffer size (e.g. `8GB`). |
74
+ | `--page-size` | both | Page / alignment size; **must match** on both sides. |
75
+ | `--object-size` | both | Per-object size; multiple of `--page-size`. |
76
+ | `--num-objects` | client | Objects transferred per read. |
77
+ | `--num-source-objects` | server | Source pool size (default `5 * --num-objects`). |
78
+ | `--iters` / `--warmup` | client | Measured / warmup read iterations. |
79
+ | `--seed` | client | RNG seed for the read subset. |
80
+ | `--verify` | client | Verify transferred bytes against the server's known pattern. |
81
+ | `--use-lazy` | both | Use the lazy L1 allocator (experimental for registration). |
82
+ | `--server-timeout` | server | Seconds to serve catalog requests before exiting. |
83
+
84
+ ## Notes
85
+
86
+ - `--page-size` must be identical on server and client (enforced via the catalog
87
+ handshake) so remote page-index math lines up.
88
+ - With the default (non-lazy) allocator the whole `--buffer-size` is allocated up
89
+ front and may be CUDA-pinned; keep it within available host memory.
90
+ - Requires a working transfer channel runtime (for `nixl`, a UCX backend).
91
+
92
+ ## Performance: NUMA placement
93
+
94
+ On a multi-NUMA host with NICs spread across nodes (e.g. an 8-NIC, 2-socket
95
+ box), run **both** the server and client under `numactl --interleave=all`:
96
+
97
+ ```bash
98
+ numactl --interleave=all \
99
+ python -m lmcache.tools.transfer_channel_benchmark --role server ...
100
+ ```
101
+
102
+ Without it the registered buffer is allocated on a single NUMA node, so only the
103
+ rails local to that node reach full bandwidth and the rest are throttled by the
104
+ cross-socket link — roughly halving throughput. In testing on a 2-NUMA / 8-rail
105
+ host this was the difference between **~110 GB/s** (no interleave) and
106
+ **~210 GB/s** (`--interleave=all`), with everything else identical. Page size
107
+ only matters in the small regime (descriptor-bound below ~64KB); from ~128KB up
108
+ the transfer is bandwidth-bound and flat.
109
+
110
+ ## Troubleshooting
111
+
112
+ - **Client hangs after "connected" or during connect, then raises
113
+ `TimeoutError` after ~60s.** The transfer-channel handshake could not reach the
114
+ server. Check that `--url`/`--control-url` on the client point at the *server's*
115
+ reachable address (a common mistake is a typo such as a trailing dot:
116
+ `10.0.0.5.:7600`), that the server is running, and that the ports are open
117
+ between hosts:
118
+ ```bash
119
+ nc -vz <server-host> 7600 # transfer channel
120
+ nc -vz <server-host> 7610 # catalog side-channel
121
+ ```
122
+ The handshake has a fixed 60s timeout, so a misconfiguration fails with a clear
123
+ error instead of hanging forever.
124
+ - **`page_size`/`object_size` mismatch error.** The client validates these against
125
+ the server's catalog; pass the same `--page-size` and `--object-size` on both.
126
+ - **Allocation `RuntimeError` on the server.** The source pool
127
+ (`--num-source-objects` × `--object-size`) must fit in `--buffer-size`; increase
128
+ the buffer or reduce the pool/object size.
@@ -0,0 +1 @@
1
+ # SPDX-License-Identifier: Apache-2.0
@@ -0,0 +1,39 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """``python -m lmcache.tools.transfer_channel_benchmark`` entry point."""
3
+
4
+ # Standard
5
+ import argparse
6
+ import sys
7
+
8
+ # First Party
9
+ # Only the torch-free config module is imported eagerly so that `-h` works
10
+ # without torch installed. The runtime (which needs torch) is imported after
11
+ # arguments are parsed.
12
+ from lmcache.tools.transfer_channel_benchmark.config import add_benchmark_arguments
13
+
14
+
15
+ def main() -> int:
16
+ """Parse arguments and run the transfer channel benchmark.
17
+
18
+ Returns:
19
+ Process exit code (0 on success, 1 on failure).
20
+ """
21
+ parser = argparse.ArgumentParser(
22
+ prog="python -m lmcache.tools.transfer_channel_benchmark",
23
+ description="Throughput benchmark for the LMCache transfer channel.",
24
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
25
+ )
26
+ add_benchmark_arguments(parser)
27
+ args = parser.parse_args()
28
+
29
+ # Imported here (not at module load) so this entry point can show --help
30
+ # without requiring torch; importing it raises a clear error if torch is
31
+ # missing.
32
+ # First Party
33
+ from lmcache.tools.transfer_channel_benchmark.benchmark import run_benchmark
34
+
35
+ return 0 if run_benchmark(args) else 1
36
+
37
+
38
+ if __name__ == "__main__":
39
+ sys.exit(main())
@@ -0,0 +1,385 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """Core logic for the transfer channel throughput benchmark.
3
+
4
+ A *server* process uses an :class:`L1MemoryManager` to initialize a registered
5
+ L1 region, allocates a pool of source memory objects in it, registers the
6
+ region with a transfer channel, and publishes the source object catalog
7
+ (offset/size per object) over a small ZMQ side-channel.
8
+
9
+ A *client* process fetches the catalog, allocates its own destination objects
10
+ via an :class:`L1MemoryManager`, connects the transfer channel, and issues
11
+ batched reads of a random subset of the source objects, reporting aggregate
12
+ read throughput in GB/s.
13
+ """
14
+
15
+ # Standard
16
+ import argparse
17
+ import json
18
+ import random
19
+ import time
20
+
21
+ # Third Party
22
+ import zmq
23
+
24
+ # torch is an optional dependency for this benchmark tool. It (and the
25
+ # distributed runtime imported below, which also needs torch) is imported here
26
+ # rather than in config.py so the `lmcache` CLI can register this tool without
27
+ # torch installed. Raise a clear, actionable error if it is missing.
28
+ try:
29
+ # Third Party
30
+ import torch
31
+ except ImportError as err:
32
+ raise ImportError(
33
+ "PyTorch is required to run the transfer channel benchmark but is not "
34
+ "installed. Install it (e.g. `pip install torch`) and retry."
35
+ ) from err
36
+
37
+ # First Party
38
+ from lmcache.logging import init_logger
39
+ from lmcache.tools.transfer_channel_benchmark.config import (
40
+ BenchmarkConfig,
41
+ build_config,
42
+ )
43
+ from lmcache.v1.distributed.api import MemoryLayoutDesc
44
+ from lmcache.v1.distributed.config import L1MemoryManagerConfig
45
+ from lmcache.v1.distributed.error import L1Error
46
+ from lmcache.v1.distributed.memory_manager import L1MemoryManager
47
+ from lmcache.v1.distributed.transfer_channel import (
48
+ TransferChannelAddress,
49
+ delete_transfer_channel_context,
50
+ initialize_transfer_channel_context,
51
+ )
52
+
53
+ logger = init_logger(__name__)
54
+
55
+ _CATALOG_REQUEST = b"catalog"
56
+ _LARGE_LAZY_INIT = 20 * 1024**3
57
+ _CLIENT_BUFFER_SLACK = 64 * 1024**2
58
+
59
+
60
+ ############################################################
61
+ # Helpers
62
+ ############################################################
63
+ def _zmq_endpoint(url: str) -> str:
64
+ """Return a ZMQ tcp endpoint for a ``host:port`` (or pass-through url)."""
65
+ return url if "://" in url else f"tcp://{url}"
66
+
67
+
68
+ def _gbps(num_bytes: int, seconds: float) -> float:
69
+ """Return throughput in GB/s (decimal GB) for ``num_bytes`` over ``seconds``."""
70
+ return (num_bytes / 1e9) / seconds if seconds > 0 else float("inf")
71
+
72
+
73
+ def _create_l1_manager(cfg: BenchmarkConfig, size_bytes: int) -> L1MemoryManager:
74
+ """Create an L1 memory manager sized to ``size_bytes`` for this benchmark."""
75
+ manager_config = L1MemoryManagerConfig(
76
+ size_in_bytes=size_bytes,
77
+ use_lazy=cfg.use_lazy,
78
+ init_size_in_bytes=min(_LARGE_LAZY_INIT, size_bytes),
79
+ align_bytes=cfg.page_size,
80
+ )
81
+ return L1MemoryManager(manager_config)
82
+
83
+
84
+ def _object_layout(cfg: BenchmarkConfig) -> MemoryLayoutDesc:
85
+ """Return the layout for a single ``object_size``-byte uint8 object."""
86
+ return MemoryLayoutDesc(
87
+ shapes=[torch.Size([cfg.object_size])], dtypes=[torch.uint8]
88
+ )
89
+
90
+
91
+ ############################################################
92
+ # Server
93
+ ############################################################
94
+ def server_main(cfg: BenchmarkConfig) -> None:
95
+ """Run the benchmark server: register a buffer and serve its catalog.
96
+
97
+ Args:
98
+ cfg: The benchmark configuration (``role == "server"``).
99
+
100
+ Raises:
101
+ RuntimeError: If the source pool cannot be allocated in the buffer.
102
+ """
103
+ pool_bytes = cfg.num_source_objects * cfg.object_size
104
+ if pool_bytes > cfg.buffer_size:
105
+ raise RuntimeError(
106
+ f"source pool ({pool_bytes / 1e9:.2f} GB = {cfg.num_source_objects} "
107
+ f"x {cfg.object_size / 1024**2:.1f} MiB) does not fit in the "
108
+ f"{cfg.buffer_size / 1e9:.2f} GB buffer; increase --buffer-size or "
109
+ f"reduce --num-source-objects / --object-size."
110
+ )
111
+
112
+ logger.debug(
113
+ "Allocating L1 buffer (%.2f GB) and %d source objects",
114
+ cfg.buffer_size / 1e9,
115
+ cfg.num_source_objects,
116
+ )
117
+ manager = _create_l1_manager(cfg, cfg.buffer_size)
118
+ source_objs: list = []
119
+ try:
120
+ error, source_objs = manager.allocate(
121
+ _object_layout(cfg), cfg.num_source_objects
122
+ )
123
+ if error != L1Error.SUCCESS:
124
+ raise RuntimeError(
125
+ f"Failed to allocate {cfg.num_source_objects} source objects: "
126
+ f"{error}. Increase --buffer-size or reduce "
127
+ f"--num-source-objects / --object-size."
128
+ )
129
+
130
+ # Always write a deterministic per-object byte pattern (object index
131
+ # mod 256) so that a client run with --verify can validate the
132
+ # transferred bytes without any extra coordination. This is a one-time
133
+ # cost at startup and does not affect the measured read throughput.
134
+ for index, obj in enumerate(source_objs):
135
+ assert obj.tensor is not None, "Failed to allocate tensor!"
136
+ obj.tensor.fill_(index % 256)
137
+
138
+ catalog = [
139
+ (obj.shm_offset, obj.shm_byte_length, index)
140
+ for index, obj in enumerate(source_objs)
141
+ ]
142
+
143
+ initialize_transfer_channel_context(
144
+ cfg.transfer_channel_type,
145
+ l1_memory_desc=manager.get_l1_memory_desc(),
146
+ listen_url=cfg.url,
147
+ advertise_url=cfg.url,
148
+ backends=[cfg.nixl_backend],
149
+ )
150
+ try:
151
+ _serve_catalog(cfg, catalog)
152
+ finally:
153
+ delete_transfer_channel_context()
154
+ finally:
155
+ if source_objs:
156
+ manager.free(source_objs)
157
+ manager.close()
158
+
159
+
160
+ def _serve_catalog(cfg: BenchmarkConfig, catalog: list[tuple[int, int, int]]) -> None:
161
+ """Serve the source object catalog over a ZMQ REP socket until timeout."""
162
+ payload = json.dumps(
163
+ {
164
+ "page_size": cfg.page_size,
165
+ "object_size": cfg.object_size,
166
+ "objects": catalog,
167
+ }
168
+ ).encode()
169
+
170
+ socket = zmq.Context.instance().socket(zmq.REP)
171
+ socket.setsockopt(zmq.LINGER, 0)
172
+ socket.bind(_zmq_endpoint(cfg.control_url))
173
+ print(
174
+ f"[server] ready: transfer channel on {cfg.url}, "
175
+ f"catalog on {cfg.control_url} ({len(catalog)} source objects)",
176
+ flush=True,
177
+ )
178
+
179
+ poller = zmq.Poller()
180
+ poller.register(socket, zmq.POLLIN)
181
+ deadline = time.monotonic() + cfg.server_timeout
182
+ try:
183
+ while time.monotonic() < deadline:
184
+ events = dict(poller.poll(timeout=1000))
185
+ if socket in events:
186
+ socket.recv()
187
+ socket.send(payload)
188
+ except KeyboardInterrupt:
189
+ pass
190
+ finally:
191
+ socket.close(linger=0)
192
+ print("[server] shut down", flush=True)
193
+
194
+
195
+ ############################################################
196
+ # Client
197
+ ############################################################
198
+ def client_main(cfg: BenchmarkConfig) -> bool:
199
+ """Run the benchmark client: read a subset of source objects and report.
200
+
201
+ Args:
202
+ cfg: The benchmark configuration (``role == "client"``).
203
+
204
+ Returns:
205
+ ``True`` on success.
206
+
207
+ Raises:
208
+ ValueError: If the server's page/object size or pool size is
209
+ incompatible with this client's configuration.
210
+ RuntimeError: If allocation or a read transfer fails.
211
+ """
212
+ catalog = _fetch_catalog(cfg)
213
+ if catalog["page_size"] != cfg.page_size:
214
+ raise ValueError(
215
+ f"page_size mismatch: client {cfg.page_size} vs server "
216
+ f"{catalog['page_size']}; they must match."
217
+ )
218
+ if catalog["object_size"] != cfg.object_size:
219
+ raise ValueError(
220
+ f"object_size mismatch: client {cfg.object_size} vs server "
221
+ f"{catalog['object_size']}; they must match."
222
+ )
223
+ source_objs = catalog["objects"]
224
+ if len(source_objs) < cfg.num_objects:
225
+ raise ValueError(
226
+ f"server has only {len(source_objs)} source objects, fewer than "
227
+ f"--num-objects ({cfg.num_objects})."
228
+ )
229
+
230
+ client_buffer = cfg.num_objects * cfg.object_size + _CLIENT_BUFFER_SLACK
231
+ manager = _create_l1_manager(cfg, client_buffer)
232
+ dst_objs: list = []
233
+ try:
234
+ error, dst_objs = manager.allocate(_object_layout(cfg), cfg.num_objects)
235
+ if error != L1Error.SUCCESS:
236
+ raise RuntimeError(
237
+ f"Failed to allocate {cfg.num_objects} destination objects: {error}."
238
+ )
239
+
240
+ ctx = initialize_transfer_channel_context(
241
+ cfg.transfer_channel_type,
242
+ l1_memory_desc=manager.get_l1_memory_desc(),
243
+ listen_url=cfg.listen_url,
244
+ advertise_url=cfg.listen_url,
245
+ backends=[cfg.nixl_backend],
246
+ )
247
+ try:
248
+ local_addrs = ctx.get_transfer_channel_address(
249
+ [(obj.shm_offset, obj.shm_byte_length) for obj in dst_objs]
250
+ )
251
+ rng = random.Random(cfg.seed)
252
+ chosen = rng.sample(source_objs, cfg.num_objects)
253
+ remote_addrs = [
254
+ TransferChannelAddress(offset=offset, size=size)
255
+ for offset, size, _ in chosen
256
+ ]
257
+ chosen_indices = [index for _, _, index in chosen]
258
+
259
+ connect_start = time.perf_counter()
260
+ client = ctx.get_transfer_channel_client(cfg.url)
261
+ print(
262
+ f"[client] connected in {time.perf_counter() - connect_start:.1f}s; "
263
+ f"reading {cfg.num_objects} objects x "
264
+ f"{cfg.object_size / 1024**2:.1f} MiB",
265
+ flush=True,
266
+ )
267
+
268
+ def one_read() -> float:
269
+ start = time.perf_counter()
270
+ task_id = client.submit_read(local_addrs, remote_addrs)
271
+ result = client.query_read_status(task_id)
272
+ while not result.is_finished():
273
+ result = client.query_read_status(task_id)
274
+ elapsed = time.perf_counter() - start
275
+ succeeded = sum(result.succeeded_mask)
276
+ if succeeded != cfg.num_objects:
277
+ raise RuntimeError(
278
+ f"read failed: {succeeded}/{cfg.num_objects} objects succeeded."
279
+ )
280
+ return elapsed
281
+
282
+ for _ in range(cfg.warmup):
283
+ one_read()
284
+ times = [one_read() for _ in range(cfg.iters)]
285
+
286
+ if cfg.verify:
287
+ _verify(dst_objs, chosen_indices)
288
+
289
+ _report(cfg, times)
290
+ return True
291
+ finally:
292
+ delete_transfer_channel_context()
293
+ del ctx
294
+ finally:
295
+ if dst_objs:
296
+ manager.free(dst_objs)
297
+ manager.close()
298
+
299
+
300
+ def _fetch_catalog(cfg: BenchmarkConfig) -> dict:
301
+ """Fetch the source object catalog from the server's control socket."""
302
+ socket = zmq.Context.instance().socket(zmq.REQ)
303
+ socket.setsockopt(zmq.LINGER, 0)
304
+ socket.setsockopt(zmq.RCVTIMEO, int(cfg.server_timeout * 1000))
305
+ socket.connect(_zmq_endpoint(cfg.control_url))
306
+ try:
307
+ socket.send(_CATALOG_REQUEST)
308
+ try:
309
+ reply = socket.recv()
310
+ except zmq.Again as exc:
311
+ raise RuntimeError(
312
+ f"timed out fetching catalog from {cfg.control_url}; is the "
313
+ f"server running?"
314
+ ) from exc
315
+ finally:
316
+ socket.close(linger=0)
317
+ return json.loads(reply)
318
+
319
+
320
+ def _verify(dst_objs: list, chosen_indices: list[int]) -> None:
321
+ """Check each destination object holds the expected per-object byte pattern.
322
+
323
+ Raises:
324
+ RuntimeError: If any destination object does not match.
325
+ """
326
+ for obj, index in zip(dst_objs, chosen_indices, strict=False):
327
+ expected = index % 256
328
+ tensor = obj.tensor
329
+ if tensor is None or not bool((tensor == expected).all()):
330
+ raise RuntimeError(
331
+ f"verify FAILED for source object {index}: expected all bytes "
332
+ f"== {expected}."
333
+ )
334
+ print(
335
+ f"[client] verify OK: {len(dst_objs)} objects match expected pattern",
336
+ flush=True,
337
+ )
338
+
339
+
340
+ def _report(cfg: BenchmarkConfig, times: list[float]) -> None:
341
+ """Print best/median/mean latency and throughput for the measured reads."""
342
+ total_bytes = cfg.num_objects * cfg.object_size
343
+ ordered = sorted(times)
344
+ best = ordered[0]
345
+ median = ordered[len(ordered) // 2]
346
+ mean = sum(ordered) / len(ordered)
347
+ print("\n==== Transfer channel read throughput ====", flush=True)
348
+ print(f" channel type : {cfg.transfer_channel_type}", flush=True)
349
+ print(
350
+ f" payload per read : {total_bytes / 1e9:.3f} GB "
351
+ f"({cfg.num_objects} objs x {cfg.object_size / 1024**2:.1f} MiB)",
352
+ flush=True,
353
+ )
354
+ print(f" iterations : {cfg.iters} (warmup {cfg.warmup})", flush=True)
355
+ print(
356
+ f" best : {best * 1e3:8.2f} ms {_gbps(total_bytes, best):7.2f} GB/s",
357
+ flush=True,
358
+ )
359
+ print(
360
+ f" median : {median * 1e3:8.2f} ms {_gbps(total_bytes, median):7.2f} GB/s",
361
+ flush=True,
362
+ )
363
+ print(
364
+ f" mean : {mean * 1e3:8.2f} ms {_gbps(total_bytes, mean):7.2f} GB/s",
365
+ flush=True,
366
+ )
367
+
368
+
369
+ ############################################################
370
+ # Entry
371
+ ############################################################
372
+ def run_benchmark(args: argparse.Namespace) -> bool:
373
+ """Dispatch to the server or client role.
374
+
375
+ Args:
376
+ args: Parsed CLI arguments produced by ``add_benchmark_arguments``.
377
+
378
+ Returns:
379
+ ``True`` on success, ``False`` otherwise.
380
+ """
381
+ cfg = build_config(args)
382
+ if cfg.role == "server":
383
+ server_main(cfg)
384
+ return True
385
+ return client_main(cfg)