lmcache-cli 0.4.8.dev31__py3-none-any.whl → 0.4.8.dev42__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lmcache/_version.py +3 -3
- lmcache/cli/commands/tool/__init__.py +3 -2
- lmcache/cli/commands/tool/transfer_channel_benchmark.py +53 -0
- lmcache/tools/transfer_channel_benchmark/README.md +128 -0
- lmcache/tools/transfer_channel_benchmark/__init__.py +1 -0
- lmcache/tools/transfer_channel_benchmark/__main__.py +39 -0
- lmcache/tools/transfer_channel_benchmark/benchmark.py +385 -0
- lmcache/tools/transfer_channel_benchmark/config.py +252 -0
- lmcache/v1/distributed/api.py +64 -0
- lmcache/v1/distributed/config.py +200 -9
- lmcache/v1/distributed/l2_adapters/base.py +25 -1
- lmcache/v1/distributed/l2_adapters/s3_l2_adapter.py +235 -4
- lmcache/v1/distributed/l2_adapters/serde_wrapper.py +13 -1
- lmcache/v1/distributed/memory_manager/l1_memory_manager.py +30 -8
- lmcache/v1/distributed/storage_manager.py +22 -0
- lmcache/v1/distributed/transfer_channel/__init__.py +105 -0
- lmcache/v1/distributed/transfer_channel/abstract.py +149 -0
- lmcache/v1/distributed/transfer_channel/api.py +47 -0
- lmcache/v1/distributed/transfer_channel/factory.py +100 -0
- lmcache/v1/distributed/transfer_channel/impl/__init__.py +9 -0
- lmcache/v1/distributed/transfer_channel/impl/nixl_impl.py +591 -0
- lmcache/v1/memory_management.py +455 -0
- lmcache/v1/mp_coordinator/app.py +33 -5
- lmcache/v1/mp_coordinator/blend_client.py +37 -16
- lmcache/v1/mp_coordinator/blend_directory.py +196 -40
- lmcache/v1/mp_coordinator/config.py +33 -0
- lmcache/v1/mp_coordinator/http_apis/blend_directory_api.py +5 -1
- lmcache/v1/mp_coordinator/http_apis/l2_api.py +19 -75
- lmcache/v1/mp_coordinator/l2/event_listener.py +22 -45
- lmcache/v1/mp_coordinator/l2/eviction_manager.py +117 -79
- lmcache/v1/mp_coordinator/l2/resync_manager.py +149 -0
- lmcache/v1/mp_coordinator/l2/usage_manager.py +57 -70
- lmcache/v1/mp_coordinator/registry.py +10 -0
- lmcache/v1/mp_coordinator/schemas.py +80 -32
- lmcache/v1/mp_observability/event.py +4 -0
- lmcache/v1/mp_observability/subscribers/tracing/cb_server.py +5 -0
- lmcache/v1/multiprocess/custom_types.py +16 -1
- lmcache/v1/multiprocess/engine_context.py +1 -1
- lmcache/v1/multiprocess/http_apis/l2_api.py +241 -0
- lmcache/v1/multiprocess/modules/blend_v3.py +29 -10
- lmcache/v1/multiprocess/modules/p2p_controller.py +245 -0
- lmcache/v1/multiprocess/mq.py +5 -0
- lmcache/v1/multiprocess/protocols/__init__.py +2 -0
- lmcache/v1/multiprocess/protocols/base.py +5 -0
- lmcache/v1/multiprocess/protocols/p2p.py +61 -0
- {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/METADATA +1 -1
- {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/RECORD +51 -35
- {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/WHEEL +0 -0
- {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/entry_points.txt +0 -0
- {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/licenses/LICENSE +0 -0
- {lmcache_cli-0.4.8.dev31.dist-info → lmcache_cli-0.4.8.dev42.dist-info}/top_level.txt +0 -0
lmcache/_version.py
CHANGED
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.4.8.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 4, 8, '
|
|
21
|
+
__version__ = version = '0.4.8.dev42'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 4, 8, 'dev42')
|
|
23
23
|
|
|
24
|
-
__commit_id__ = commit_id = '
|
|
24
|
+
__commit_id__ = commit_id = 'g9ffb104f3'
|
|
@@ -15,7 +15,7 @@ import sys
|
|
|
15
15
|
|
|
16
16
|
# First Party
|
|
17
17
|
from lmcache.cli.commands.base import BaseCommand
|
|
18
|
-
from lmcache.cli.commands.tool import cache_simulator
|
|
18
|
+
from lmcache.cli.commands.tool import cache_simulator, transfer_channel_benchmark
|
|
19
19
|
|
|
20
20
|
|
|
21
21
|
class ToolCommand(BaseCommand):
|
|
@@ -46,9 +46,10 @@ class ToolCommand(BaseCommand):
|
|
|
46
46
|
inner = parser.add_subparsers(
|
|
47
47
|
dest="tool_name",
|
|
48
48
|
required=True,
|
|
49
|
-
metavar="{cache-simulator}",
|
|
49
|
+
metavar="{cache-simulator,transfer-channel-benchmark}",
|
|
50
50
|
)
|
|
51
51
|
cache_simulator.register(inner)
|
|
52
|
+
transfer_channel_benchmark.register(inner)
|
|
52
53
|
|
|
53
54
|
def execute(self, args: argparse.Namespace) -> None:
|
|
54
55
|
"""Dispatch is handled per-tool via parser.set_defaults(func=...).
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""``lmcache tool transfer-channel-benchmark`` sub-subcommand wiring.
|
|
3
|
+
|
|
4
|
+
Argument definitions and execution logic live in the benchmark module:
|
|
5
|
+
|
|
6
|
+
* :func:`~lmcache.tools.transfer_channel_benchmark.benchmark.add_benchmark_arguments`
|
|
7
|
+
* :func:`~lmcache.tools.transfer_channel_benchmark.benchmark.run_benchmark`
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
# Standard
|
|
11
|
+
import argparse
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def register(subparsers: argparse._SubParsersAction) -> None:
|
|
16
|
+
"""Register the ``transfer-channel-benchmark`` sub-subcommand.
|
|
17
|
+
|
|
18
|
+
Only the torch-free ``config`` module is imported here, so registering this
|
|
19
|
+
tool (which happens on every ``lmcache`` invocation) does not require torch
|
|
20
|
+
or the distributed runtime. Those are imported in ``execute``.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
subparsers: The subparsers action from the ``lmcache tool`` parser.
|
|
24
|
+
"""
|
|
25
|
+
# First Party
|
|
26
|
+
from lmcache.tools.transfer_channel_benchmark.config import (
|
|
27
|
+
add_benchmark_arguments,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
parser = subparsers.add_parser(
|
|
31
|
+
"transfer-channel-benchmark",
|
|
32
|
+
help="Benchmark transfer channel read throughput (server/client).",
|
|
33
|
+
description=(
|
|
34
|
+
"Throughput benchmark for the LMCache transfer channel. Run one "
|
|
35
|
+
"process with --role server and another with --role client."
|
|
36
|
+
),
|
|
37
|
+
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
38
|
+
)
|
|
39
|
+
add_benchmark_arguments(parser)
|
|
40
|
+
parser.set_defaults(func=execute)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def execute(args: argparse.Namespace) -> None:
|
|
44
|
+
"""Run the benchmark and exit non-zero on failure.
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
args: Parsed CLI arguments.
|
|
48
|
+
"""
|
|
49
|
+
# First Party
|
|
50
|
+
from lmcache.tools.transfer_channel_benchmark.benchmark import run_benchmark
|
|
51
|
+
|
|
52
|
+
if not run_benchmark(args):
|
|
53
|
+
sys.exit(1)
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# Transfer Channel Throughput Benchmark
|
|
2
|
+
|
|
3
|
+
Measures read throughput (GB/s) of the LMCache **transfer channel**
|
|
4
|
+
(`lmcache/v1/distributed/transfer_channel/`) for batched peer-to-peer reads.
|
|
5
|
+
|
|
6
|
+
Unlike a raw-tensor microbenchmark, this tool uses LMCache's `L1MemoryManager`
|
|
7
|
+
to initialize the registered memory region and to **allocate the transferred
|
|
8
|
+
objects** on both sides, so it exercises the same memory path production uses.
|
|
9
|
+
|
|
10
|
+
## How it works
|
|
11
|
+
|
|
12
|
+
The benchmark runs as **two separate processes**:
|
|
13
|
+
|
|
14
|
+
- **server** — uses an `L1MemoryManager` to allocate a registered L1 buffer and
|
|
15
|
+
a pool of source memory objects, registers the buffer with the transfer
|
|
16
|
+
channel, and publishes the source object catalog (`offset`/`size` per object)
|
|
17
|
+
over a small ZMQ side-channel.
|
|
18
|
+
- **client** — fetches the catalog, allocates its own destination objects via an
|
|
19
|
+
`L1MemoryManager`, connects the transfer channel, and repeatedly reads a random
|
|
20
|
+
`--num-objects` subset of the source objects, then reports throughput.
|
|
21
|
+
|
|
22
|
+
A side-channel is needed because the transfer channel handshake only exchanges
|
|
23
|
+
the whole-buffer registration, not per-object offsets.
|
|
24
|
+
|
|
25
|
+
> Only the `nixl` transfer channel type is registered today. The tool is generic
|
|
26
|
+
> over `--transfer-channel-type`; an unknown type raises a clear error.
|
|
27
|
+
|
|
28
|
+
## Usage
|
|
29
|
+
|
|
30
|
+
Run via the module or the `lmcache` CLI. Start the server first.
|
|
31
|
+
|
|
32
|
+
### `python -m`
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
# terminal 1 — server
|
|
36
|
+
python -m lmcache.tools.transfer_channel_benchmark \
|
|
37
|
+
--role server --transfer-channel-type nixl \
|
|
38
|
+
--url 0.0.0.0:7600 --control-url 0.0.0.0:7610 \
|
|
39
|
+
--buffer-size 2GB --page-size 512KB --object-size 10MB
|
|
40
|
+
|
|
41
|
+
# terminal 2 — client
|
|
42
|
+
python -m lmcache.tools.transfer_channel_benchmark \
|
|
43
|
+
--role client --transfer-channel-type nixl \
|
|
44
|
+
--url 127.0.0.1:7600 --control-url 127.0.0.1:7610 \
|
|
45
|
+
--listen-url 0.0.0.0:7601 \
|
|
46
|
+
--page-size 512KB --object-size 10MB \
|
|
47
|
+
--num-objects 10 --iters 3 --warmup 1
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
### `lmcache tool`
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
lmcache tool transfer-channel-benchmark --role server --url 0.0.0.0:7600 \
|
|
54
|
+
--control-url 0.0.0.0:7610 --buffer-size 2GB --object-size 10MB
|
|
55
|
+
lmcache tool transfer-channel-benchmark --role client --url 127.0.0.1:7600 \
|
|
56
|
+
--control-url 127.0.0.1:7610 --object-size 10MB --num-objects 10
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The server always writes a deterministic per-object byte pattern (object index
|
|
60
|
+
mod 256) into its source objects, so adding `--verify` on the **client** checks
|
|
61
|
+
the transferred bytes against that pattern (no `--verify` needed on the server).
|
|
62
|
+
|
|
63
|
+
## Key options
|
|
64
|
+
|
|
65
|
+
| Option | Role | Meaning |
|
|
66
|
+
| --- | --- | --- |
|
|
67
|
+
| `--role {server,client}` | both | Which side to run (required). |
|
|
68
|
+
| `--transfer-channel-type` | both | Implementation to benchmark (default `nixl`). |
|
|
69
|
+
| `--nixl-backend` | both | nixl backend, e.g. `UCX` (nixl-specific). |
|
|
70
|
+
| `--url` | both | Server binds its transfer-channel server here; client dials it. |
|
|
71
|
+
| `--listen-url` | client | Client's own (mandatory) transfer-channel server bind. |
|
|
72
|
+
| `--control-url` | both | Catalog side-channel: server binds, client connects. |
|
|
73
|
+
| `--buffer-size` | server | Registered L1 source buffer size (e.g. `8GB`). |
|
|
74
|
+
| `--page-size` | both | Page / alignment size; **must match** on both sides. |
|
|
75
|
+
| `--object-size` | both | Per-object size; multiple of `--page-size`. |
|
|
76
|
+
| `--num-objects` | client | Objects transferred per read. |
|
|
77
|
+
| `--num-source-objects` | server | Source pool size (default `5 * --num-objects`). |
|
|
78
|
+
| `--iters` / `--warmup` | client | Measured / warmup read iterations. |
|
|
79
|
+
| `--seed` | client | RNG seed for the read subset. |
|
|
80
|
+
| `--verify` | client | Verify transferred bytes against the server's known pattern. |
|
|
81
|
+
| `--use-lazy` | both | Use the lazy L1 allocator (experimental for registration). |
|
|
82
|
+
| `--server-timeout` | server | Seconds to serve catalog requests before exiting. |
|
|
83
|
+
|
|
84
|
+
## Notes
|
|
85
|
+
|
|
86
|
+
- `--page-size` must be identical on server and client (enforced via the catalog
|
|
87
|
+
handshake) so remote page-index math lines up.
|
|
88
|
+
- With the default (non-lazy) allocator the whole `--buffer-size` is allocated up
|
|
89
|
+
front and may be CUDA-pinned; keep it within available host memory.
|
|
90
|
+
- Requires a working transfer channel runtime (for `nixl`, a UCX backend).
|
|
91
|
+
|
|
92
|
+
## Performance: NUMA placement
|
|
93
|
+
|
|
94
|
+
On a multi-NUMA host with NICs spread across nodes (e.g. an 8-NIC, 2-socket
|
|
95
|
+
box), run **both** the server and client under `numactl --interleave=all`:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
numactl --interleave=all \
|
|
99
|
+
python -m lmcache.tools.transfer_channel_benchmark --role server ...
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Without it the registered buffer is allocated on a single NUMA node, so only the
|
|
103
|
+
rails local to that node reach full bandwidth and the rest are throttled by the
|
|
104
|
+
cross-socket link — roughly halving throughput. In testing on a 2-NUMA / 8-rail
|
|
105
|
+
host this was the difference between **~110 GB/s** (no interleave) and
|
|
106
|
+
**~210 GB/s** (`--interleave=all`), with everything else identical. Page size
|
|
107
|
+
only matters in the small regime (descriptor-bound below ~64KB); from ~128KB up
|
|
108
|
+
the transfer is bandwidth-bound and flat.
|
|
109
|
+
|
|
110
|
+
## Troubleshooting
|
|
111
|
+
|
|
112
|
+
- **Client hangs after "connected" or during connect, then raises
|
|
113
|
+
`TimeoutError` after ~60s.** The transfer-channel handshake could not reach the
|
|
114
|
+
server. Check that `--url`/`--control-url` on the client point at the *server's*
|
|
115
|
+
reachable address (a common mistake is a typo such as a trailing dot:
|
|
116
|
+
`10.0.0.5.:7600`), that the server is running, and that the ports are open
|
|
117
|
+
between hosts:
|
|
118
|
+
```bash
|
|
119
|
+
nc -vz <server-host> 7600 # transfer channel
|
|
120
|
+
nc -vz <server-host> 7610 # catalog side-channel
|
|
121
|
+
```
|
|
122
|
+
The handshake has a fixed 60s timeout, so a misconfiguration fails with a clear
|
|
123
|
+
error instead of hanging forever.
|
|
124
|
+
- **`page_size`/`object_size` mismatch error.** The client validates these against
|
|
125
|
+
the server's catalog; pass the same `--page-size` and `--object-size` on both.
|
|
126
|
+
- **Allocation `RuntimeError` on the server.** The source pool
|
|
127
|
+
(`--num-source-objects` × `--object-size`) must fit in `--buffer-size`; increase
|
|
128
|
+
the buffer or reduce the pool/object size.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""``python -m lmcache.tools.transfer_channel_benchmark`` entry point."""
|
|
3
|
+
|
|
4
|
+
# Standard
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
# First Party
|
|
9
|
+
# Only the torch-free config module is imported eagerly so that `-h` works
|
|
10
|
+
# without torch installed. The runtime (which needs torch) is imported after
|
|
11
|
+
# arguments are parsed.
|
|
12
|
+
from lmcache.tools.transfer_channel_benchmark.config import add_benchmark_arguments
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main() -> int:
|
|
16
|
+
"""Parse arguments and run the transfer channel benchmark.
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
Process exit code (0 on success, 1 on failure).
|
|
20
|
+
"""
|
|
21
|
+
parser = argparse.ArgumentParser(
|
|
22
|
+
prog="python -m lmcache.tools.transfer_channel_benchmark",
|
|
23
|
+
description="Throughput benchmark for the LMCache transfer channel.",
|
|
24
|
+
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
25
|
+
)
|
|
26
|
+
add_benchmark_arguments(parser)
|
|
27
|
+
args = parser.parse_args()
|
|
28
|
+
|
|
29
|
+
# Imported here (not at module load) so this entry point can show --help
|
|
30
|
+
# without requiring torch; importing it raises a clear error if torch is
|
|
31
|
+
# missing.
|
|
32
|
+
# First Party
|
|
33
|
+
from lmcache.tools.transfer_channel_benchmark.benchmark import run_benchmark
|
|
34
|
+
|
|
35
|
+
return 0 if run_benchmark(args) else 1
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
if __name__ == "__main__":
|
|
39
|
+
sys.exit(main())
|
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Core logic for the transfer channel throughput benchmark.
|
|
3
|
+
|
|
4
|
+
A *server* process uses an :class:`L1MemoryManager` to initialize a registered
|
|
5
|
+
L1 region, allocates a pool of source memory objects in it, registers the
|
|
6
|
+
region with a transfer channel, and publishes the source object catalog
|
|
7
|
+
(offset/size per object) over a small ZMQ side-channel.
|
|
8
|
+
|
|
9
|
+
A *client* process fetches the catalog, allocates its own destination objects
|
|
10
|
+
via an :class:`L1MemoryManager`, connects the transfer channel, and issues
|
|
11
|
+
batched reads of a random subset of the source objects, reporting aggregate
|
|
12
|
+
read throughput in GB/s.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
# Standard
|
|
16
|
+
import argparse
|
|
17
|
+
import json
|
|
18
|
+
import random
|
|
19
|
+
import time
|
|
20
|
+
|
|
21
|
+
# Third Party
|
|
22
|
+
import zmq
|
|
23
|
+
|
|
24
|
+
# torch is an optional dependency for this benchmark tool. It (and the
|
|
25
|
+
# distributed runtime imported below, which also needs torch) is imported here
|
|
26
|
+
# rather than in config.py so the `lmcache` CLI can register this tool without
|
|
27
|
+
# torch installed. Raise a clear, actionable error if it is missing.
|
|
28
|
+
try:
|
|
29
|
+
# Third Party
|
|
30
|
+
import torch
|
|
31
|
+
except ImportError as err:
|
|
32
|
+
raise ImportError(
|
|
33
|
+
"PyTorch is required to run the transfer channel benchmark but is not "
|
|
34
|
+
"installed. Install it (e.g. `pip install torch`) and retry."
|
|
35
|
+
) from err
|
|
36
|
+
|
|
37
|
+
# First Party
|
|
38
|
+
from lmcache.logging import init_logger
|
|
39
|
+
from lmcache.tools.transfer_channel_benchmark.config import (
|
|
40
|
+
BenchmarkConfig,
|
|
41
|
+
build_config,
|
|
42
|
+
)
|
|
43
|
+
from lmcache.v1.distributed.api import MemoryLayoutDesc
|
|
44
|
+
from lmcache.v1.distributed.config import L1MemoryManagerConfig
|
|
45
|
+
from lmcache.v1.distributed.error import L1Error
|
|
46
|
+
from lmcache.v1.distributed.memory_manager import L1MemoryManager
|
|
47
|
+
from lmcache.v1.distributed.transfer_channel import (
|
|
48
|
+
TransferChannelAddress,
|
|
49
|
+
delete_transfer_channel_context,
|
|
50
|
+
initialize_transfer_channel_context,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
logger = init_logger(__name__)
|
|
54
|
+
|
|
55
|
+
_CATALOG_REQUEST = b"catalog"
|
|
56
|
+
_LARGE_LAZY_INIT = 20 * 1024**3
|
|
57
|
+
_CLIENT_BUFFER_SLACK = 64 * 1024**2
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
############################################################
|
|
61
|
+
# Helpers
|
|
62
|
+
############################################################
|
|
63
|
+
def _zmq_endpoint(url: str) -> str:
|
|
64
|
+
"""Return a ZMQ tcp endpoint for a ``host:port`` (or pass-through url)."""
|
|
65
|
+
return url if "://" in url else f"tcp://{url}"
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _gbps(num_bytes: int, seconds: float) -> float:
|
|
69
|
+
"""Return throughput in GB/s (decimal GB) for ``num_bytes`` over ``seconds``."""
|
|
70
|
+
return (num_bytes / 1e9) / seconds if seconds > 0 else float("inf")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _create_l1_manager(cfg: BenchmarkConfig, size_bytes: int) -> L1MemoryManager:
|
|
74
|
+
"""Create an L1 memory manager sized to ``size_bytes`` for this benchmark."""
|
|
75
|
+
manager_config = L1MemoryManagerConfig(
|
|
76
|
+
size_in_bytes=size_bytes,
|
|
77
|
+
use_lazy=cfg.use_lazy,
|
|
78
|
+
init_size_in_bytes=min(_LARGE_LAZY_INIT, size_bytes),
|
|
79
|
+
align_bytes=cfg.page_size,
|
|
80
|
+
)
|
|
81
|
+
return L1MemoryManager(manager_config)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _object_layout(cfg: BenchmarkConfig) -> MemoryLayoutDesc:
|
|
85
|
+
"""Return the layout for a single ``object_size``-byte uint8 object."""
|
|
86
|
+
return MemoryLayoutDesc(
|
|
87
|
+
shapes=[torch.Size([cfg.object_size])], dtypes=[torch.uint8]
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
############################################################
|
|
92
|
+
# Server
|
|
93
|
+
############################################################
|
|
94
|
+
def server_main(cfg: BenchmarkConfig) -> None:
|
|
95
|
+
"""Run the benchmark server: register a buffer and serve its catalog.
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
cfg: The benchmark configuration (``role == "server"``).
|
|
99
|
+
|
|
100
|
+
Raises:
|
|
101
|
+
RuntimeError: If the source pool cannot be allocated in the buffer.
|
|
102
|
+
"""
|
|
103
|
+
pool_bytes = cfg.num_source_objects * cfg.object_size
|
|
104
|
+
if pool_bytes > cfg.buffer_size:
|
|
105
|
+
raise RuntimeError(
|
|
106
|
+
f"source pool ({pool_bytes / 1e9:.2f} GB = {cfg.num_source_objects} "
|
|
107
|
+
f"x {cfg.object_size / 1024**2:.1f} MiB) does not fit in the "
|
|
108
|
+
f"{cfg.buffer_size / 1e9:.2f} GB buffer; increase --buffer-size or "
|
|
109
|
+
f"reduce --num-source-objects / --object-size."
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
logger.debug(
|
|
113
|
+
"Allocating L1 buffer (%.2f GB) and %d source objects",
|
|
114
|
+
cfg.buffer_size / 1e9,
|
|
115
|
+
cfg.num_source_objects,
|
|
116
|
+
)
|
|
117
|
+
manager = _create_l1_manager(cfg, cfg.buffer_size)
|
|
118
|
+
source_objs: list = []
|
|
119
|
+
try:
|
|
120
|
+
error, source_objs = manager.allocate(
|
|
121
|
+
_object_layout(cfg), cfg.num_source_objects
|
|
122
|
+
)
|
|
123
|
+
if error != L1Error.SUCCESS:
|
|
124
|
+
raise RuntimeError(
|
|
125
|
+
f"Failed to allocate {cfg.num_source_objects} source objects: "
|
|
126
|
+
f"{error}. Increase --buffer-size or reduce "
|
|
127
|
+
f"--num-source-objects / --object-size."
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
# Always write a deterministic per-object byte pattern (object index
|
|
131
|
+
# mod 256) so that a client run with --verify can validate the
|
|
132
|
+
# transferred bytes without any extra coordination. This is a one-time
|
|
133
|
+
# cost at startup and does not affect the measured read throughput.
|
|
134
|
+
for index, obj in enumerate(source_objs):
|
|
135
|
+
assert obj.tensor is not None, "Failed to allocate tensor!"
|
|
136
|
+
obj.tensor.fill_(index % 256)
|
|
137
|
+
|
|
138
|
+
catalog = [
|
|
139
|
+
(obj.shm_offset, obj.shm_byte_length, index)
|
|
140
|
+
for index, obj in enumerate(source_objs)
|
|
141
|
+
]
|
|
142
|
+
|
|
143
|
+
initialize_transfer_channel_context(
|
|
144
|
+
cfg.transfer_channel_type,
|
|
145
|
+
l1_memory_desc=manager.get_l1_memory_desc(),
|
|
146
|
+
listen_url=cfg.url,
|
|
147
|
+
advertise_url=cfg.url,
|
|
148
|
+
backends=[cfg.nixl_backend],
|
|
149
|
+
)
|
|
150
|
+
try:
|
|
151
|
+
_serve_catalog(cfg, catalog)
|
|
152
|
+
finally:
|
|
153
|
+
delete_transfer_channel_context()
|
|
154
|
+
finally:
|
|
155
|
+
if source_objs:
|
|
156
|
+
manager.free(source_objs)
|
|
157
|
+
manager.close()
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _serve_catalog(cfg: BenchmarkConfig, catalog: list[tuple[int, int, int]]) -> None:
|
|
161
|
+
"""Serve the source object catalog over a ZMQ REP socket until timeout."""
|
|
162
|
+
payload = json.dumps(
|
|
163
|
+
{
|
|
164
|
+
"page_size": cfg.page_size,
|
|
165
|
+
"object_size": cfg.object_size,
|
|
166
|
+
"objects": catalog,
|
|
167
|
+
}
|
|
168
|
+
).encode()
|
|
169
|
+
|
|
170
|
+
socket = zmq.Context.instance().socket(zmq.REP)
|
|
171
|
+
socket.setsockopt(zmq.LINGER, 0)
|
|
172
|
+
socket.bind(_zmq_endpoint(cfg.control_url))
|
|
173
|
+
print(
|
|
174
|
+
f"[server] ready: transfer channel on {cfg.url}, "
|
|
175
|
+
f"catalog on {cfg.control_url} ({len(catalog)} source objects)",
|
|
176
|
+
flush=True,
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
poller = zmq.Poller()
|
|
180
|
+
poller.register(socket, zmq.POLLIN)
|
|
181
|
+
deadline = time.monotonic() + cfg.server_timeout
|
|
182
|
+
try:
|
|
183
|
+
while time.monotonic() < deadline:
|
|
184
|
+
events = dict(poller.poll(timeout=1000))
|
|
185
|
+
if socket in events:
|
|
186
|
+
socket.recv()
|
|
187
|
+
socket.send(payload)
|
|
188
|
+
except KeyboardInterrupt:
|
|
189
|
+
pass
|
|
190
|
+
finally:
|
|
191
|
+
socket.close(linger=0)
|
|
192
|
+
print("[server] shut down", flush=True)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
############################################################
|
|
196
|
+
# Client
|
|
197
|
+
############################################################
|
|
198
|
+
def client_main(cfg: BenchmarkConfig) -> bool:
|
|
199
|
+
"""Run the benchmark client: read a subset of source objects and report.
|
|
200
|
+
|
|
201
|
+
Args:
|
|
202
|
+
cfg: The benchmark configuration (``role == "client"``).
|
|
203
|
+
|
|
204
|
+
Returns:
|
|
205
|
+
``True`` on success.
|
|
206
|
+
|
|
207
|
+
Raises:
|
|
208
|
+
ValueError: If the server's page/object size or pool size is
|
|
209
|
+
incompatible with this client's configuration.
|
|
210
|
+
RuntimeError: If allocation or a read transfer fails.
|
|
211
|
+
"""
|
|
212
|
+
catalog = _fetch_catalog(cfg)
|
|
213
|
+
if catalog["page_size"] != cfg.page_size:
|
|
214
|
+
raise ValueError(
|
|
215
|
+
f"page_size mismatch: client {cfg.page_size} vs server "
|
|
216
|
+
f"{catalog['page_size']}; they must match."
|
|
217
|
+
)
|
|
218
|
+
if catalog["object_size"] != cfg.object_size:
|
|
219
|
+
raise ValueError(
|
|
220
|
+
f"object_size mismatch: client {cfg.object_size} vs server "
|
|
221
|
+
f"{catalog['object_size']}; they must match."
|
|
222
|
+
)
|
|
223
|
+
source_objs = catalog["objects"]
|
|
224
|
+
if len(source_objs) < cfg.num_objects:
|
|
225
|
+
raise ValueError(
|
|
226
|
+
f"server has only {len(source_objs)} source objects, fewer than "
|
|
227
|
+
f"--num-objects ({cfg.num_objects})."
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
client_buffer = cfg.num_objects * cfg.object_size + _CLIENT_BUFFER_SLACK
|
|
231
|
+
manager = _create_l1_manager(cfg, client_buffer)
|
|
232
|
+
dst_objs: list = []
|
|
233
|
+
try:
|
|
234
|
+
error, dst_objs = manager.allocate(_object_layout(cfg), cfg.num_objects)
|
|
235
|
+
if error != L1Error.SUCCESS:
|
|
236
|
+
raise RuntimeError(
|
|
237
|
+
f"Failed to allocate {cfg.num_objects} destination objects: {error}."
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
ctx = initialize_transfer_channel_context(
|
|
241
|
+
cfg.transfer_channel_type,
|
|
242
|
+
l1_memory_desc=manager.get_l1_memory_desc(),
|
|
243
|
+
listen_url=cfg.listen_url,
|
|
244
|
+
advertise_url=cfg.listen_url,
|
|
245
|
+
backends=[cfg.nixl_backend],
|
|
246
|
+
)
|
|
247
|
+
try:
|
|
248
|
+
local_addrs = ctx.get_transfer_channel_address(
|
|
249
|
+
[(obj.shm_offset, obj.shm_byte_length) for obj in dst_objs]
|
|
250
|
+
)
|
|
251
|
+
rng = random.Random(cfg.seed)
|
|
252
|
+
chosen = rng.sample(source_objs, cfg.num_objects)
|
|
253
|
+
remote_addrs = [
|
|
254
|
+
TransferChannelAddress(offset=offset, size=size)
|
|
255
|
+
for offset, size, _ in chosen
|
|
256
|
+
]
|
|
257
|
+
chosen_indices = [index for _, _, index in chosen]
|
|
258
|
+
|
|
259
|
+
connect_start = time.perf_counter()
|
|
260
|
+
client = ctx.get_transfer_channel_client(cfg.url)
|
|
261
|
+
print(
|
|
262
|
+
f"[client] connected in {time.perf_counter() - connect_start:.1f}s; "
|
|
263
|
+
f"reading {cfg.num_objects} objects x "
|
|
264
|
+
f"{cfg.object_size / 1024**2:.1f} MiB",
|
|
265
|
+
flush=True,
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
def one_read() -> float:
|
|
269
|
+
start = time.perf_counter()
|
|
270
|
+
task_id = client.submit_read(local_addrs, remote_addrs)
|
|
271
|
+
result = client.query_read_status(task_id)
|
|
272
|
+
while not result.is_finished():
|
|
273
|
+
result = client.query_read_status(task_id)
|
|
274
|
+
elapsed = time.perf_counter() - start
|
|
275
|
+
succeeded = sum(result.succeeded_mask)
|
|
276
|
+
if succeeded != cfg.num_objects:
|
|
277
|
+
raise RuntimeError(
|
|
278
|
+
f"read failed: {succeeded}/{cfg.num_objects} objects succeeded."
|
|
279
|
+
)
|
|
280
|
+
return elapsed
|
|
281
|
+
|
|
282
|
+
for _ in range(cfg.warmup):
|
|
283
|
+
one_read()
|
|
284
|
+
times = [one_read() for _ in range(cfg.iters)]
|
|
285
|
+
|
|
286
|
+
if cfg.verify:
|
|
287
|
+
_verify(dst_objs, chosen_indices)
|
|
288
|
+
|
|
289
|
+
_report(cfg, times)
|
|
290
|
+
return True
|
|
291
|
+
finally:
|
|
292
|
+
delete_transfer_channel_context()
|
|
293
|
+
del ctx
|
|
294
|
+
finally:
|
|
295
|
+
if dst_objs:
|
|
296
|
+
manager.free(dst_objs)
|
|
297
|
+
manager.close()
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _fetch_catalog(cfg: BenchmarkConfig) -> dict:
|
|
301
|
+
"""Fetch the source object catalog from the server's control socket."""
|
|
302
|
+
socket = zmq.Context.instance().socket(zmq.REQ)
|
|
303
|
+
socket.setsockopt(zmq.LINGER, 0)
|
|
304
|
+
socket.setsockopt(zmq.RCVTIMEO, int(cfg.server_timeout * 1000))
|
|
305
|
+
socket.connect(_zmq_endpoint(cfg.control_url))
|
|
306
|
+
try:
|
|
307
|
+
socket.send(_CATALOG_REQUEST)
|
|
308
|
+
try:
|
|
309
|
+
reply = socket.recv()
|
|
310
|
+
except zmq.Again as exc:
|
|
311
|
+
raise RuntimeError(
|
|
312
|
+
f"timed out fetching catalog from {cfg.control_url}; is the "
|
|
313
|
+
f"server running?"
|
|
314
|
+
) from exc
|
|
315
|
+
finally:
|
|
316
|
+
socket.close(linger=0)
|
|
317
|
+
return json.loads(reply)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _verify(dst_objs: list, chosen_indices: list[int]) -> None:
|
|
321
|
+
"""Check each destination object holds the expected per-object byte pattern.
|
|
322
|
+
|
|
323
|
+
Raises:
|
|
324
|
+
RuntimeError: If any destination object does not match.
|
|
325
|
+
"""
|
|
326
|
+
for obj, index in zip(dst_objs, chosen_indices, strict=False):
|
|
327
|
+
expected = index % 256
|
|
328
|
+
tensor = obj.tensor
|
|
329
|
+
if tensor is None or not bool((tensor == expected).all()):
|
|
330
|
+
raise RuntimeError(
|
|
331
|
+
f"verify FAILED for source object {index}: expected all bytes "
|
|
332
|
+
f"== {expected}."
|
|
333
|
+
)
|
|
334
|
+
print(
|
|
335
|
+
f"[client] verify OK: {len(dst_objs)} objects match expected pattern",
|
|
336
|
+
flush=True,
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _report(cfg: BenchmarkConfig, times: list[float]) -> None:
|
|
341
|
+
"""Print best/median/mean latency and throughput for the measured reads."""
|
|
342
|
+
total_bytes = cfg.num_objects * cfg.object_size
|
|
343
|
+
ordered = sorted(times)
|
|
344
|
+
best = ordered[0]
|
|
345
|
+
median = ordered[len(ordered) // 2]
|
|
346
|
+
mean = sum(ordered) / len(ordered)
|
|
347
|
+
print("\n==== Transfer channel read throughput ====", flush=True)
|
|
348
|
+
print(f" channel type : {cfg.transfer_channel_type}", flush=True)
|
|
349
|
+
print(
|
|
350
|
+
f" payload per read : {total_bytes / 1e9:.3f} GB "
|
|
351
|
+
f"({cfg.num_objects} objs x {cfg.object_size / 1024**2:.1f} MiB)",
|
|
352
|
+
flush=True,
|
|
353
|
+
)
|
|
354
|
+
print(f" iterations : {cfg.iters} (warmup {cfg.warmup})", flush=True)
|
|
355
|
+
print(
|
|
356
|
+
f" best : {best * 1e3:8.2f} ms {_gbps(total_bytes, best):7.2f} GB/s",
|
|
357
|
+
flush=True,
|
|
358
|
+
)
|
|
359
|
+
print(
|
|
360
|
+
f" median : {median * 1e3:8.2f} ms {_gbps(total_bytes, median):7.2f} GB/s",
|
|
361
|
+
flush=True,
|
|
362
|
+
)
|
|
363
|
+
print(
|
|
364
|
+
f" mean : {mean * 1e3:8.2f} ms {_gbps(total_bytes, mean):7.2f} GB/s",
|
|
365
|
+
flush=True,
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
############################################################
|
|
370
|
+
# Entry
|
|
371
|
+
############################################################
|
|
372
|
+
def run_benchmark(args: argparse.Namespace) -> bool:
|
|
373
|
+
"""Dispatch to the server or client role.
|
|
374
|
+
|
|
375
|
+
Args:
|
|
376
|
+
args: Parsed CLI arguments produced by ``add_benchmark_arguments``.
|
|
377
|
+
|
|
378
|
+
Returns:
|
|
379
|
+
``True`` on success, ``False`` otherwise.
|
|
380
|
+
"""
|
|
381
|
+
cfg = build_config(args)
|
|
382
|
+
if cfg.role == "server":
|
|
383
|
+
server_main(cfg)
|
|
384
|
+
return True
|
|
385
|
+
return client_main(cfg)
|