checkpoint-engine 0.3.0rc0__py3-none-any.whl

checkpoint_engine/worker.py

@@ -0,0 +1,168 @@
+import gc
+import traceback
+from collections.abc import Callable
+from typing import TypedDict
+
+import torch
+import zmq
+
+from checkpoint_engine.device_utils import DeviceManager, npu_generate_uuid
+
+
+def _rebuild_ipc(handle: tuple[Callable, tuple], device_id: int | None = None) -> torch.Tensor:
+    func, args = handle
+    list_args = list(args)
+    if device_id is not None:
+        # the key is to change device id to the current device id
+        # in case two processes have different CUDA_VISIBLE_DEVICES
+        list_args[6] = device_id
+    buffer = func(*list_args)
+    return buffer
+
+
+class FlattenedTensorMetadata(TypedDict):
+    name: str
+    shape: torch.Size
+    dtype: torch.dtype
+    # specify the start offset of this tensor in shared ipc_buffer tensor
+    offset: int
+
+
+def _extract_weights(
+    payload: list[FlattenedTensorMetadata], buffer: torch.Tensor
+) -> list[tuple[str, torch.Tensor]]:
+    assert buffer is not None
+    weights: list[tuple[str, torch.Tensor]] = []
+    for item in payload:
+        shape = item["shape"]
+        if isinstance(shape, list | tuple):
+            shape = torch.Size(shape)
+        assert isinstance(shape, torch.Size)
+        dtype, offset = item["dtype"], item["offset"]
+        size = dtype.itemsize * shape.numel()
+        tensor = buffer[offset : offset + size].view(dtype=dtype).view(shape)
+        weights.append((item["name"], tensor))
+    return weights
+
+
+def update_weights_from_ipc(
+    zmq_ctx: zmq.Context,
+    zmq_handle: str,
+    device_id: int,
+    *,
+    run: Callable[[list[tuple[str, torch.Tensor]]], None],
+    post_hook: Callable[[], None] | None = None,
+):
+    socket = zmq_ctx.socket(zmq.REP)
+    socket.connect(zmq_handle)
+    buffer: torch.Tensor | None = None
+    device_manager = DeviceManager()
+    try:
+        ipc_handle: tuple[Callable, tuple] = socket.recv_pyobj()
+        assert isinstance(ipc_handle, tuple)
+        buffer = _rebuild_ipc(ipc_handle, device_id)
+        assert buffer.dtype == torch.uint8
+        socket.send(b"")
+    except Exception as e:
+        msg = "".join(traceback.format_exception(type(e), e, e.__traceback__))
+        socket.send_string(msg)
+        socket.recv()  # wait for ack
+        raise
+    try:
+        while True:
+            payload: list[FlattenedTensorMetadata] | Exception | None = socket.recv_pyobj()
+            if payload is None:  # done signal
+                if post_hook is not None:
+                    post_hook()
+                device_manager.device_module.synchronize()
+                socket.send(b"")
+                break
+            if isinstance(payload, list):  # still updating weights
+                try:
+                    run(_extract_weights(payload, buffer))
+                    device_manager.device_module.synchronize()
+                    socket.send(b"")
+                except Exception as e:  # noqa: BLE001
+                    # Send exception back to Parameter Server.
+                    # Don't raise here. Because all workers should quit in the same way by receiving the exception from PS
+                    msg = "".join(traceback.format_exception(type(e), e, e.__traceback__))
+                    socket.send_string(msg)
+            elif isinstance(
+                payload, Exception
+            ):  # error occurred, got force quit signal from Parameter Server
+                raise payload
+            else:
+                raise TypeError(f"Unexpected payload type: {type(payload)}")
+
+    finally:
+        socket.close()
+        del buffer
+        gc.collect()
+        device_manager.device_module.empty_cache()
+
+
+class VllmColocateWorkerExtension:
+    """
+    Worker extension for vLLM to update weights from checkpoint-engine.
+
+    This class provides a worker extension mechanism that allows vLLM workers to receive
+    and apply weight updates from the checkpoint-engine via IPC (Inter-Process Communication).
+    The methods in this worker extension will be injected into the vLLM worker class and
+    are callable from the `collective_rpc` API, enabling seamless weight updates for both
+    vLLM V0 and V1 versions.
+
+    Note:
+        This class is defined in a separate module. The fully qualified name
+        `checkpoint_engine.worker.VllmColocateWorkerExtension` should be passed as the
+        `worker_extension_cls` argument when initializing the vLLM worker.
+    """
+
+    def update_weights_from_ipc(self, zmq_handles: dict[str, str]):
+        """
+        Update model weights from checkpoint-engine via IPC communication.
+
+        This method establishes a ZMQ connection to the checkpoint-engine and receives
+        weight updates through a shared memory buffer. The update process includes:
+        1. Receiving IPC handles to reconstruct shared memory tensors
+        2. Extracting flattened metadata describing tensor weights in the shared memory tensor
+        3. Loading weights into the model
+        4. Post-processing weights after loading
+
+        Args:
+            zmq_handles: A dictionary mapping device UUIDs to ZMQ socket handles.
+                        The device UUID is platform-specific:
+                        - For CUDA: UUID from `current_platform.get_device_uuid()`
+                        - For NPU: Format "NPU-{generated_uuid}"
+
+        Raises:
+            ValueError: If the device type is not supported (not CUDA or NPU).
+            AssertionError: If the device is not properly initialized.
+
+        Note:
+            This method is called by vLLM's collective RPC mechanism. The ZMQ context
+            is lazily initialized on first call and reused for subsequent updates.
+        """
+        from vllm.model_executor.model_loader.utils import process_weights_after_loading
+        from vllm.platforms import current_platform
+
+        # vllm-ascend not init device
+        if current_platform.device_type == "npu" and self.device is None:
+            self.device = torch.device(f"npu:{self.local_rank}")
+        assert self.device is not None
+        if not hasattr(self, "_zmq_ctx") or self._zmq_ctx is None:
+            self._zmq_ctx = zmq.Context()
+        if current_platform.device_type == "cuda":
+            device_uuid = current_platform.get_device_uuid(self.device.index)
+        elif current_platform.device_type == "npu":
+            device_uuid = f"NPU-{npu_generate_uuid()}"
+        else:
+            raise ValueError(f"Unsupported device type: {current_platform.device_type}")
+        update_weights_from_ipc(
+            self._zmq_ctx,
+            zmq_handles[device_uuid],
+            device_id=self.device.index,
+            run=self.model_runner.model.load_weights,
+            post_hook=lambda: process_weights_after_loading(
+                self.model_runner.model, self.model_config, self.device
+            ),
+        )

checkpoint_engine-0.3.0rc0.dist-info/METADATA

@@ -0,0 +1,236 @@
+Metadata-Version: 2.4
+Name: checkpoint-engine
+Version: 0.3.0rc0
+Summary: checkpoint-engine is a lightweight, decoupling and efficient weight update middleware
+Project-URL: Homepage, https://github.com/MoonshotAI/checkpoint-engine
+Project-URL: Repository, https://github.com/MoonshotAI/checkpoint-engine
+Project-URL: Documentation, https://github.com/MoonshotAI/checkpoint-engine
+Project-URL: Issue Tracker, https://github.com/MoonshotAI/checkpoint-engine/issues
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENCE
+Requires-Dist: torch>=2.5.0
+Requires-Dist: fastapi
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: safetensors
+Requires-Dist: pyzmq
+Requires-Dist: uvicorn
+Requires-Dist: loguru
+Requires-Dist: numpy
+Requires-Dist: httpx
+Provides-Extra: p2p
+Requires-Dist: mooncake-transfer-engine>=0.3.5; extra == "p2p"
+Dynamic: license-file
+
+# Checkpoint Engine
+Checkpoint-engine is a simple middleware to update model weights in LLM inference engines -- a critical step in reinforcement learning.
+We provide an efficient and lightweight implementation for inplace weight update:
+updating our [Kimi-K2](https://github.com/MoonshotAI/Kimi-K2) model (1 Trillion parameters) across thousands of GPUs takes about 20s.
+
+
+<div align="center">
+  <picture>
+      <img src="figures/checkpoint-engine.png" width="80%" alt="ckpt-engine">
+  </picture>
+</div>
+
+## Architecture
+
+The core weight update logic is in `ParameterServer` class, a service colocated with inference engines. It provides two implementations of weight update: Broadcast and P2P.
+
+- **Broadcast**: Used when a large number of inference instances need to update weights in synchronous. This is the fastest implementation and should be used as the default update method. See `_update_per_bucket` with `ranks == None or []`.
+- **P2P**: Used when new inference instances are dynamically added (due to restarts or dynamic availability) while the existing instances are already serving requests. Under this scenario, to avoid affecting the workloads on existing instances, we use the [`mooncake-transfer-engine`](https://github.com/kvcache-ai/Mooncake?tab=readme-ov-file#use-python-package) to P2P send weights from CPUs in existing instances to GPUs in new instances. See `_update_per_bucket` with `ranks` specified.
+
+### Optimized Weight Broadcast
+In the *Broadcast* implementation, the checkpoint-engine holds references to sharded weights in CPU memory, and need to efficiently broadcast them to a cluster of inference instances, often under a different sharding pattern.
+We arrange the data transfer into 3 stages:
+1. H2D: moving weights to GPU memory. These weights may come from disk or the training engine.
+2. broadcast: broadcast among checkpoint engine workers; the data results in a CUDA IPC buffer shared with inference engine.
+3. reload: inference engine decides what subset of weights to copy from the broadcasted data.
+
+Checkpoint-engine orchestrates the entire transfer process. It first gathers necessary metadata to create a plan, including deciding the proper bucket size for data transfer.
+It then executes the transfer, where it controls the inference engine through a ZeroMQ socket. To maximize performance, it organizes the data transfers into a pipeline with overlapped communication and copy, illustrated below. The details can be found in [Kimi-K2 Technical Report](https://arxiv.org/abs/2507.20534).
+
+
+<div align="center">
+  <picture>
+      <img src="figures/pipeline.png" width="80%" alt="pipeline">
+  </picture>
+</div>
+
+Pipelining naturally requires more GPU memory. When memory is not enough, checkpoint-engine will fallback to serial execution.
+
+### Optimized P2P Bucket Assignment
+In the *P2P* implementation, checkpoint-engine needs to send weights from existing instances to new instances.
+To minimize the overall transfer time, checkpoint-engine optimizes the bucket assignment for each sender-receiver pair.
+The optimization goal is to make full use of the available network bandwidth for each sender and receiver.
+See [issue #25](https://github.com/MoonshotAI/checkpoint-engine/issues/25)
+
+## Benchmark
+
+| Model                                | Device Info  | GatherMetas | Update (Broadcast) | Update (P2P)            |
+| :----------------------------------- | :----------- | :---------- |:-------------------| :---------------------- |
+| GLM-4.5-Air (BF16)                   | 8xH800 TP8   | 0.12s       | 3.47s (3.02GiB)    | 4.12s (3.02GiB)         |
+| Qwen3-235B-A22B-Instruct-2507 (BF16) | 8xH800 TP8   | 0.33s       | 6.22s (2.67GiB)    | 7.10s (2.68GiB)         |
+| DeepSeek-V3.1 (FP8)                  | 16xH20 TP16  | 1.17s       | 10.19s (5.39GiB)   | 11.80s (5.41GiB)        |
+| Kimi-K2-Instruct (FP8)               | 16xH20 TP16  | 1.33s       | 14.36s (5.89GiB)   | 17.49s (5.91GiB)        |
+| DeepSeek-V3.1 (FP8)                  | 256xH20 TP16 | 0.80s       | 11.33s (8.00GiB)   | 11.81s (8.00GiB)        |
+| Kimi-K2-Instruct (FP8)               | 256xH20 TP16 | 1.22s       | 16.04s (8.00GiB)   | 16.75s (8.00GiB)        |
+
+All results above are tested by [`examples/update.py`](./examples/update.py) and use [vLLM v0.10.2rc1](https://github.com/vllm-project/vllm/tree/v0.10.2rc1) as inference engine. Some notes:
+
+* FP8 test needs additional vLLM patches, see [FP8 quantization](#fp8-quantization).
+* Device Info: we tested various combination of devices and parallelism setups. For example, a 256-GPU TP16 setup means that we deploy 16 vLLM instances, each with 16-way tensor parallelism.
+* Since update duration is related to IPC bucket size, we provide the bucket size in the table.
+* The P2P time were tested for updating no more than two nodes (16 GPUs) (`ParameterServer.update(ranks=range(0, 16))`) out of the entire cluster.
+* We bind each GPU to its corresponding NUMA node to ensure stable H2D transfer speeds.
+
+## Installation
+
+Use the fastest broadcast implementation
+
+```Bash
+pip install checkpoint-engine
+```
+
+Use the flexible P2P implementation, notice this will install `mooncake-transfer-engine` to support RDMA transfer between different ranks.
+
+```Bash
+pip install 'checkpoint-engine[p2p]'
+```
+
+## Getting Started
+
+Prepare an H800 or H20 machine with 8 GPUs with vLLM. Be sure to include [/collective_rpc API endpoint](https://github.com/vllm-project/vllm/commit/f7cf5b512ee41f36613deb2471a44de5f304f70d) commit (available in main branch) since checkpoint-engine will use this endpoint to update weights. vLLM version `v0.10.2` is fully tested and recommended.
+
+```Bash
+mkdir -p /opt/vLLM && cd /opt/vLLM
+uv venv --python 3.12 --seed
+source .venv/bin/activate
+uv pip install vllm==0.10.2
+```
+
+Install checkpoint-engine
+
+```Bash
+uv pip install 'checkpoint-engine[p2p]'
+```
+
+We use `Qwen/Qwen3-235B-A22B-Instruct-2507` (BF16) as the test model
+
+```Bash
+hf download Qwen/Qwen3-235B-A22B-Instruct-2507 --local-dir /opt/models/Qwen/Qwen3-235B-A22B-Instruct-2507/
+```
+
+Start vLLM in dev mode and set `--load-format dummy`. Notice that we also set `--worker-extension-cls=checkpoint_engine.worker.VllmColocateWorkerExtension`
+
+```Bash
+VLLM_SERVER_DEV_MODE=1 python3 -m vllm.entrypoints.openai.api_server --host 0.0.0.0 --port 19730 --trust-remote-code \
+    --tensor-parallel-size=8 --max-model-len 4096 --load-format dummy \
+    --served-model-name checkpoint-engine-demo --model /opt/models/Qwen/Qwen3-235B-A22B-Instruct-2507/ \
+    --worker-extension-cls checkpoint_engine.worker.VllmColocateWorkerExtension
+```
+
+Meanwhile, use this command to update weights by checkpoint-engine. No need to wait for vLLM to get ready.
+
+```Bash
+torchrun --nproc-per-node 8 examples/update.py --update-method all --checkpoint-path /opt/models/Qwen/Qwen3-235B-A22B-Instruct-2507/
+```
+
+### Reuse weights from existing instances
+
+New checkpoint-engine instances can join existing instances and reuse their weights. This is simple to achieve.
+
+First, start the existing instances with `--save-metas-file global_metas.pkl` to save global metas to a file and use `--sleep-time 300` to make sure they stay alive.
+
+```Bash
+torchrun --nproc-per-node 8 examples/update.py --checkpoint-path $MODEL_PATH \
+    --sleep-time 300 --save-metas-file global_metas.pkl
+```
+
+After a checkpoint is registered, new instances can obtain a copy of the checkpoint by setting `--load-metas-file global_metas.pkl`.
+
+```Bash
+torchrun --nproc-per-node 8 examples/update.py --load-metas-file global_metas.pkl
+```
+
+### FP8 quantization
+
+FP8 quantization currently do not natively work in vLLM when updating weights.
+We provide a simple patch in [`patches/vllm_fp8.patch`](./patches/vllm_fp8.patch) to handle the correct weight update.
+Notice this patch is only tested in DeepSeek-V3.1 and Kimi-K2. Other models may meet some compatible issues.
+
+A [PR](https://github.com/vllm-project/vllm/pull/24488) is opened to the vLLM project and waiting to discuss and review.
+
+### Test
+
+Run a simple correctness test for checkpoint_engine
+
+```bash
+pytest tests/test_update.py
+```
+
+`test_update.py` are only designed to run with `pytest`. Please don't run it directly with `torchrun`.
+
+Other unit tests can also be done with pytest. Only test_update.py requires GPUs, other tests can be run on CPUs. Only to run CPU tests, use:
+
+```bash
+pytest tests/ -m "not gpu"
+```
+
+### Environment Variables
+- `PS_MAX_BUCKET_SIZE_GB`: An integer is used to set the maximum bucket size for checkpoint-engine. If not set, 8GB is used as default.
+- `PS_P2P_STORE_RDMA_DEVICES`: Comma-separated RDMA devices' names for P2P transfer. If not set, checkpoint-engine will fall back to use `NCCL_IB_HCA` to detect RDMA devices.
+- `NCCL_IB_HCA`: Available patterns can be found from [NCCL documentation](https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html#id8). If also not set, all RDMA devices will be used and divided evenly among the ranks.
+
+## SGLang Integration
+
+Checkpoint Engine provides efficient distributed checkpoint loading for SGLang inference servers, significantly reducing model loading time for large models and multi-node setups.
+
+### Quick Start
+
+**1. Install checkpoint-engine:**
+```bash
+pip install 'checkpoint-engine[p2p]'
+```
+
+**2. Launch SGLang server:**
+```bash
+python -m sglang.launch_server \
+    --model-path $MODEL_PATH \
+    --tp 8 \
+    --load-format dummy \
+    --wait-for-initial-weights
+```
+
+**3. Run checkpoint engine:**
+```bash
+python -m sglang.srt.checkpoint_engine.update \
+    --update-method broadcast \
+    --checkpoint-path $MODEL_PATH \
+    --inference-parallel-size 8
+```
+
+### Multi-Node Setup
+
+For 2-node setup, run the same commands on both nodes with appropriate `--host` and distributed training parameters.
+
+### Key Options
+
+**SGLang Server:**
+- `--wait-for-initial-weights`: Wait for checkpoint engine before becoming ready
+- `--load-format dummy`: Enable overlapping initialization tasks
+
+**Checkpoint Engine:**
+- `--update-method`: Choose `broadcast`, `p2p`, or `all`
+- `--inference-parallel-size`: Number of parallel processes
+- `--checkpoint-path`: Model checkpoint directory
+
+## Limitations and Future Work
+
+- This project is currently tested with vLLM and SGLang. Integration with other frameworks is planned for future releases.
+- The perfect three-stage pipeline mentioned in our paper is currently not implemented. This could be useful for architectures where H2D and broadcast do not conflict in PCIE.
+
+## Acknowledgments
+
+This open source project uses the same vLLM interface in https://github.com/vllm-project/vllm/pull/24295 . Thanks for the comments and insights from [youkaichao](https://github.com/youkaichao).

checkpoint_engine-0.3.0rc0.dist-info/RECORD

@@ -0,0 +1,10 @@
+checkpoint_engine/__init__.py,sha256=Zj4I008kn9R6fYr0lVBzcQSnvckLpX2s1ljCOOqV1c8,87
+checkpoint_engine/_version.py,sha256=v0iyeXv9HxMc4JmYu_bJTIGKXRQVfpijACyjq2P_sk0,714
+checkpoint_engine/device_utils.py,sha256=iKrof60j3CY3fStRTq3DRTt_kE1vYoEWHhAeyh0lByA,3020
+checkpoint_engine/ps.py,sha256=eIvg_eI7HMedacoQQer62NRnGDjANtxsHVxgM93ccXQ,66977
+checkpoint_engine/worker.py,sha256=f6kS1ushIXxkRCEHXM5wVofUer9OxRiVY03vmKYLzgo,6757
+checkpoint_engine-0.3.0rc0.dist-info/licenses/LICENCE,sha256=D3gPmHKpGtF1yxYNhqjtBtZY_brZjDotJTzpnmClzlY,1067
+checkpoint_engine-0.3.0rc0.dist-info/METADATA,sha256=iVd2qPdNyTPPX3XIEiuM0ASk8As72zSGfFIYicpZG3E,11562
+checkpoint_engine-0.3.0rc0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+checkpoint_engine-0.3.0rc0.dist-info/top_level.txt,sha256=66sik_1eLakLYmcllOEJzFaNbSfjsueuP0tHYEzhMSs,18
+checkpoint_engine-0.3.0rc0.dist-info/RECORD,,

checkpoint_engine-0.3.0rc0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

checkpoint_engine-0.3.0rc0.dist-info/licenses/LICENCE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Moonshot AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE

checkpoint_engine-0.3.0rc0.dist-info/top_level.txt

@@ -0,0 +1 @@
+checkpoint_engine