vllm-ascend 0.11.0rc2__tar.gz → 0.11.0rc3__tar.gz

{vllm_ascend-0.11.0rc2 → vllm_ascend-0.11.0rc3}/Dockerfile

@@ -19,20 +19,29 @@ FROM quay.io/ascend/cann:8.3.rc2-910b-ubuntu22.04-py3.11
 
 ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
 ARG COMPILE_CUSTOM_KERNELS=1
+ARG MOONCAKE_TAG="v0.3.7.post2"
 
 # Define environments
 ENV DEBIAN_FRONTEND=noninteractive
 ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS}
 
-RUN apt-get update -y && \
-    apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake libnuma-dev && \
-    rm -rf /var/cache/apt/* && \
-    rm -rf /var/lib/apt/lists/*
-
 WORKDIR /workspace
 
 COPY . /vllm-workspace/vllm-ascend/
 
+# Install Mooncake dependencies
+RUN apt-get update -y && \
+    apt-get install -y git vim wget net-tools gcc g++ cmake libnuma-dev && \
+    git clone --depth 1 --branch ${MOONCAKE_TAG} https://github.com/kvcache-ai/Mooncake /vllm-workspace/Mooncake && \
+    cp /vllm-workspace/vllm-ascend/tools/mooncake_installer.sh /vllm-workspace/Mooncake/ && \
+    cd /vllm-workspace/Mooncake && bash mooncake_installer.sh -y && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/lib64 && \
+    mkdir -p build && cd build && cmake .. -DUSE_ASCEND_DIRECT=ON && \
+    make -j$(nproc) && make install && \
+    rm -fr /vllm-workspace/Mooncake/build && \
+    rm -rf /var/cache/apt/* && \
+    rm -rf /var/lib/apt/lists/*
+
 RUN pip config set global.index-url ${PIP_INDEX_URL}
 
 # Install vLLM
@@ -54,7 +63,7 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
     python3 -m pip cache purge
 
 # Install modelscope (for fast download) and ray (for multinode)
-RUN python3 -m pip install modelscope 'ray>=2.47.1' 'protobuf>3.20.0' && \
+RUN python3 -m pip install modelscope 'ray>=2.47.1,<=2.48.0' 'protobuf>3.20.0' && \
     python3 -m pip cache purge
 
 CMD ["/bin/bash"]

{vllm_ascend-0.11.0rc2 → vllm_ascend-0.11.0rc3}/Dockerfile.a3

@@ -19,21 +19,29 @@ FROM quay.io/ascend/cann:8.3.rc2-a3-ubuntu22.04-py3.11
 
 ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
 ARG COMPILE_CUSTOM_KERNELS=1
+ARG MOONCAKE_TAG=v0.3.7.post2
 
+COPY . /vllm-workspace/vllm-ascend/
 # Define environments
 ENV DEBIAN_FRONTEND=noninteractive
 ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS}
 
-RUN apt-get update -y && \
-    apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake libnuma-dev && \
-    rm -rf /var/cache/apt/* && \
-    rm -rf /var/lib/apt/lists/*
+RUN pip config set global.index-url ${PIP_INDEX_URL}
 
 WORKDIR /workspace
 
-COPY . /vllm-workspace/vllm-ascend/
-
-RUN pip config set global.index-url ${PIP_INDEX_URL}
+# Install Mooncake dependencies
+RUN apt-get update -y && \
+    apt-get install -y git vim wget net-tools gcc g++ cmake libnuma-dev && \
+    git clone --depth 1 --branch ${MOONCAKE_TAG} https://github.com/kvcache-ai/Mooncake /vllm-workspace/Mooncake && \
+    cp /vllm-workspace/vllm-ascend/tools/mooncake_installer.sh /vllm-workspace/Mooncake/ && \
+    cd /vllm-workspace/Mooncake && bash mooncake_installer.sh -y && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/lib64 && \
+    mkdir -p build && cd build && cmake .. -DUSE_ASCEND_DIRECT=ON && \
+    make -j$(nproc) && make install && \
+    rm -fr /vllm-workspace/Mooncake/build && \
+    rm -rf /var/cache/apt/* && \
+    rm -rf /var/lib/apt/lists/*
 
 # Install vLLM
 ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
@@ -54,7 +62,7 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
     python3 -m pip cache purge
 
 # Install modelscope (for fast download) and ray (for multinode)
-RUN python3 -m pip install modelscope 'ray>=2.47.1' 'protobuf>3.20.0' && \
+RUN python3 -m pip install modelscope 'ray>=2.47.1,<=2.48.0' 'protobuf>3.20.0' && \
     python3 -m pip cache purge
 
 CMD ["/bin/bash"]

{vllm_ascend-0.11.0rc2 → vllm_ascend-0.11.0rc3}/Dockerfile.a3.openEuler

@@ -19,23 +19,36 @@ FROM quay.io/ascend/cann:8.3.rc2-a3-openeuler24.03-py3.11
 
 ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
 ARG COMPILE_CUSTOM_KERNELS=1
+ARG MOONCAKE_TAG="v0.3.7.post2"
 
 ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS}
 
-RUN yum update -y && \
-    yum install -y python3-pip git vim wget net-tools gcc gcc-c++ make cmake numactl-devel && \
-    rm -rf /var/cache/yum
-
 RUN pip config set global.index-url ${PIP_INDEX_URL}
 
 WORKDIR /workspace
 
 COPY . /vllm-workspace/vllm-ascend/
 
+SHELL ["/bin/bash", "-c"]
+
+RUN yum update -y && \
+    yum install -y git vim wget net-tools gcc gcc-c++ make cmake numactl-devel && \
+    git clone --depth 1 --branch ${MOONCAKE_TAG} https://github.com/kvcache-ai/Mooncake /vllm-workspace/Mooncake && \
+    cp /vllm-workspace/vllm-ascend/tools/mooncake_installer.sh /vllm-workspace/Mooncake/ && \
+    ARCH=$(uname -m) && \
+    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/${ARCH}-linux/devlib:/usr/local/Ascend/ascend-toolkit/latest/${ARCH}-linux/lib64:$LD_LIBRARY_PATH && \
+    export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/c++/12:/usr/include/c++/12/${ARCH}-openEuler-linux && \
+    cd /vllm-workspace/Mooncake && \
+    bash mooncake_installer.sh -y && \
+    mkdir -p build && cd build && cmake .. -DUSE_ASCEND_DIRECT=ON && \
+    make -j$(nproc) && make install && \
+    rm -fr /vllm-workspace/Mooncake/build && \
+    rm -rf /var/cache/yum/*
+
 # Install vLLM
 ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
 ARG VLLM_TAG=v0.11.0
-
 RUN git clone --depth 1 $VLLM_REPO --branch $VLLM_TAG /vllm-workspace/vllm
 # In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it.
 RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index https://download.pytorch.org/whl/cpu/ && \
@@ -52,7 +65,7 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
     python3 -m pip cache purge
 
 # Install modelscope (for fast download) and ray (for multinode)
-RUN python3 -m pip install modelscope 'ray>=2.47.1' 'protobuf>3.20.0' && \
+RUN python3 -m pip install modelscope 'ray>=2.47.1,<=2.48.0' 'protobuf>3.20.0' && \
     python3 -m pip cache purge
 
 CMD ["/bin/bash"]

{vllm_ascend-0.11.0rc2 → vllm_ascend-0.11.0rc3}/Dockerfile.openEuler

@@ -19,12 +19,10 @@ FROM quay.io/ascend/cann:8.3.rc2-910b-openeuler24.03-py3.11
 
 ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple"
 ARG COMPILE_CUSTOM_KERNELS=1
+ARG MOONCAKE_TAG="v0.3.7.post2"
 
 ENV COMPILE_CUSTOM_KERNELS=${COMPILE_CUSTOM_KERNELS}
 
-RUN yum update -y && \
-    yum install -y python3-pip git vim wget net-tools gcc gcc-c++ make cmake numactl-devel && \
-    rm -rf /var/cache/yum
 
 RUN pip config set global.index-url ${PIP_INDEX_URL}
 
@@ -32,10 +30,26 @@ WORKDIR /workspace
 
 COPY . /vllm-workspace/vllm-ascend/
 
+SHELL ["/bin/bash", "-c"]
+
+RUN yum update -y && \
+    yum install -y git vim wget net-tools gcc gcc-c++ make cmake numactl-devel && \
+    git clone --depth 1 --branch ${MOONCAKE_TAG} https://github.com/kvcache-ai/Mooncake /vllm-workspace/Mooncake && \
+    cp /vllm-workspace/vllm-ascend/tools/mooncake_installer.sh /vllm-workspace/Mooncake/ && \
+    ARCH=$(uname -m) && \
+    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    export LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/${ARCH}-linux/devlib:/usr/local/Ascend/ascend-toolkit/latest/${ARCH}-linux/lib64:$LD_LIBRARY_PATH && \
+    export CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH:/usr/include/c++/12:/usr/include/c++/12/${ARCH}-openEuler-linux && \
+    cd /vllm-workspace/Mooncake && \
+    bash mooncake_installer.sh -y && \
+    mkdir -p build && cd build && cmake .. -DUSE_ASCEND_DIRECT=ON && \
+    make -j$(nproc) && make install && \
+    rm -fr /vllm-workspace/Mooncake/build && \
+    rm -rf /var/cache/yum/*
+
 # Install vLLM
 ARG VLLM_REPO=https://github.com/vllm-project/vllm.git
-ARG VLLM_TAG=v0.11.0
-
+ARG VLLM_TAG=v0.11.2
 RUN git clone --depth 1 $VLLM_REPO --branch $VLLM_TAG /vllm-workspace/vllm
 # In x86, triton will be installed by vllm. But in Ascend, triton doesn't work correctly. we need to uninstall it.
 RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/[audio] --extra-index https://download.pytorch.org/whl/cpu/ && \
@@ -52,7 +66,7 @@ RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi
     python3 -m pip cache purge
 
 # Install modelscope (for fast download) and ray (for multinode)
-RUN python3 -m pip install modelscope 'ray>=2.47.1' 'protobuf>3.20.0' && \
+RUN python3 -m pip install modelscope 'ray>=2.47.1,<=2.48.0' 'protobuf>3.20.0' && \
     python3 -m pip cache purge
 
 CMD ["/bin/bash"]

{vllm_ascend-0.11.0rc2/vllm_ascend.egg-info → vllm_ascend-0.11.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vllm_ascend
-Version: 0.11.0rc2
+Version: 0.11.0rc3
 Summary: vLLM Ascend backend plugin
 Home-page: https://github.com/vllm-project/vllm-ascend
 Author: vLLM-Ascend team
@@ -37,6 +37,7 @@ Requires-Dist: msgpack
 Requires-Dist: quart
 Requires-Dist: numba
 Requires-Dist: torch-npu==2.7.1
+Requires-Dist: transformers<=4.57.1
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{vllm_ascend-0.11.0rc2 → vllm_ascend-0.11.0rc3}/docs/source/tutorials/index.md

@@ -8,6 +8,7 @@ single_npu_multimodal
 single_npu_audio
 single_npu_qwen3_embedding
 single_npu_qwen3_quantization
+single_node_pd_disaggregation_llmdatadist
 multi_npu_qwen3_next
 multi_npu
 multi_npu_moge

vllm_ascend-0.11.0rc3/docs/source/tutorials/single_node_pd_disaggregation_llmdatadist.md

@@ -0,0 +1,181 @@
+# Prefill-Decode Disaggregation Llmdatadist Verification (Qwen2.5-VL)
+
+## Getting Start
+
+vLLM-Ascend now supports prefill-decode (PD) disaggregation. This guide takes one-by-one steps to verify these features with constrained resources.
+
+Using the Qwen2.5-VL-7B-Instruct model as an example, use vllm-ascend v0.11.0rc1 (with vLLM v0.11.0) on 1 Atlas 800T A2 server to deploy the "1P1D" architecture. Assume the IP address is 192.0.0.1.
+
+## Verify Communication Environment
+
+### Verification Process
+
+1. Single Node Verification:
+
+Execute the following commands in sequence. The results must all be `success` and the status must be `UP`:
+
+```bash
+# Check the remote switch ports
+for i in {0..7}; do hccn_tool -i $i -lldp -g | grep Ifname; done
+# Get the link status of the Ethernet ports (UP or DOWN)
+for i in {0..7}; do hccn_tool -i $i -link -g ; done
+# Check the network health status
+for i in {0..7}; do hccn_tool -i $i -net_health -g ; done
+# View the network detected IP configuration
+for i in {0..7}; do hccn_tool -i $i -netdetect -g ; done
+# View gateway configuration
+for i in {0..7}; do hccn_tool -i $i -gateway -g ; done
+# View NPU network configuration
+cat /etc/hccn.conf
+```
+
+2. Get NPU IP Addresses
+
+```bash
+for i in {0..7}; do hccn_tool -i $i -ip -g;done
+```
+
+## Generate Ranktable
+
+The rank table is a JSON file that specifies the mapping of Ascend NPU ranks to nodes. For more details, please refer to the [vllm-ascend examples](https://github.com/vllm-project/vllm-ascend/blob/main/examples/disaggregated_prefill_v1/README.md). Execute the following commands for reference.
+
+```shell
+cd vllm-ascend/examples/disaggregate_prefill_v1/
+bash gen_ranktable.sh --ips 192.0.0.1 \
+  --npus-per-node  2 --network-card-name eth0 --prefill-device-cnt 1 --decode-device-cnt 1
+```
+
+If you want to run "2P1D", please set npus-per-node to 3 and prefill-device-cnt to 2. The rank table will be generated at /vllm-workspace/vllm-ascend/examples/disaggregate_prefill_v1/ranktable.json
+
+|Parameter  | Meaning |
+| --- | --- |
+| --ips | Each node's local IP address (prefiller nodes should be in front of decoder nodes) |
+| --npus-per-node | Each node's NPU clips |
+| --network-card-name | The physical machines' NIC |
+|--prefill-device-cnt  | NPU clips used for prefill |
+|--decode-device-cnt |NPU clips used for decode |
+
+## Prefiller/Decoder Deployment
+
+We can run the following scripts to launch a server on the prefiller/decoder NPU, respectively.
+
+:::::{tab-set}
+
+::::{tab-item} Prefiller
+
+```shell
+export ASCEND_RT_VISIBLE_DEVICES=0
+export HCCL_IF_IP=192.0.0.1 # node ip
+export GLOO_SOCKET_IFNAME="eth0"  # network card name
+export TP_SOCKET_IFNAME="eth0"
+export HCCL_SOCKET_IFNAME="eth0"
+export DISAGGREGATED_PREFILL_RANK_TABLE_PATH="/path/to/your/generated/ranktable.json"
+export OMP_PROC_BIND=false
+export OMP_NUM_THREADS=10
+export VLLM_ASCEND_LLMDD_RPC_PORT=5959
+
+vllm serve /model/Qwen2.5-VL-7B-Instruct  \
+  --host 0.0.0.0 \
+  --port 13700 \
+  --tensor-parallel-size 1 \
+  --no-enable-prefix-caching \
+  --seed 1024 \
+  --served-model-name qwen25vl \
+  --max-model-len 40000  \
+  --max-num-batched-tokens 40000  \
+  --trust-remote-code \
+  --gpu-memory-utilization 0.9  \
+  --kv-transfer-config  \
+  '{"kv_connector": "LLMDataDistCMgrConnector",
+    "kv_buffer_device": "npu",
+    "kv_role": "kv_producer",
+    "kv_parallel_size": 1,
+    "kv_port": "20001",
+    "engine_id": "0",
+    "kv_connector_module_path": "vllm_ascend.distributed.llmdatadist_c_mgr_connector"
+  }'
+```
+
+::::
+
+::::{tab-item} Decoder
+
+```shell
+export ASCEND_RT_VISIBLE_DEVICES=1
+export HCCL_IF_IP=192.0.0.1  # node ip
+export GLOO_SOCKET_IFNAME="eth0"  # network card name
+export TP_SOCKET_IFNAME="eth0"
+export HCCL_SOCKET_IFNAME="eth0"
+export DISAGGREGATED_PREFILL_RANK_TABLE_PATH="/path/to/your/generated/ranktable.json"
+export OMP_PROC_BIND=false
+export OMP_NUM_THREADS=10
+export VLLM_ASCEND_LLMDD_RPC_PORT=5979
+
+vllm serve /model/Qwen2.5-VL-7B-Instruct  \
+  --host 0.0.0.0 \
+  --port 13701 \
+  --no-enable-prefix-caching \
+  --tensor-parallel-size 1 \
+  --seed 1024 \
+  --served-model-name qwen25vl \
+  --max-model-len 40000  \
+  --max-num-batched-tokens 40000  \
+  --trust-remote-code \
+  --gpu-memory-utilization 0.9  \
+  --kv-transfer-config  \
+  '{"kv_connector": "LLMDataDistCMgrConnector",
+  "kv_buffer_device": "npu",
+  "kv_role": "kv_consumer",
+  "kv_parallel_size": 1,
+  "kv_port": "20001",
+  "engine_id": "0",
+  "kv_connector_module_path": "vllm_ascend.distributed.llmdatadist_c_mgr_connector"
+  }'
+```
+
+::::
+
+:::::
+
+If you want to run "2P1D", please set ASCEND_RT_VISIBLE_DEVICES, VLLM_ASCEND_LLMDD_RPC_PORT and port to different values for each P process.
+
+## Example Proxy for Deployment
+
+Run a proxy server on the same node with the prefiller service instance. You can get the proxy program in the repository's examples: [load\_balance\_proxy\_server\_example.py](https://github.com/vllm-project/vllm-ascend/blob/main/examples/disaggregated_prefill_v1/load_balance_proxy_server_example.py)
+
+```shell
+python load_balance_proxy_server_example.py \
+    --host 192.0.0.1 \
+    --port 8080 \
+    --prefiller-hosts 192.0.0.1 \
+    --prefiller-port 13700 \
+    --decoder-hosts 192.0.0.1 \
+    --decoder-ports 13701
+```
+
+|Parameter  | Meaning |
+| --- | --- |
+| --port | Port of proxy |
+| --prefiller-port | All ports of prefill |
+| --decoder-ports | All ports of decoder |
+
+## Verification
+
+Check service health using the proxy server endpoint.
+
+```shell
+curl http://192.0.0.1:8080/v1/chat/completions \
+    -H "Content-Type: application/json" \
+    -d '{
+        "model": "qwen25vl",
+        "messages": [
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": [
+                {"type": "image_url", "image_url": {"url": "https://modelscope.oss-cn-beijing.aliyuncs.com/resource/qwen.png"}},
+                {"type": "text", "text": "What is the text in the illustrate?"}
+            ]}
+            ],
+        "max_tokens": 100,
+        "temperature": 0
+    }'
+```

{vllm_ascend-0.11.0rc2 → vllm_ascend-0.11.0rc3}/pyproject.toml

@@ -12,6 +12,7 @@ requires = [
     "scipy",
     "setuptools>=64",
     "setuptools-scm>=8",
+    "transformers<=4.57.1",
     "torch-npu==2.7.1",
     "torch==2.7.1",
     "torchvision",

{vllm_ascend-0.11.0rc2 → vllm_ascend-0.11.0rc3}/requirements.txt

@@ -26,3 +26,5 @@ numba
 #--pre
 #--extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi
 torch-npu==2.7.1
+
+transformers<=4.57.1

{vllm_ascend-0.11.0rc2 → vllm_ascend-0.11.0rc3}/tests/ut/quantization/test_w8a8.py

@@ -326,7 +326,6 @@ class TestAscendC8KVCacheMethod(TestBase):
         self.attention_type.ENCODER = "encoder"
 
     def test_create_weights(self):
-        """测试 create_weights 是否正确注册参数"""
         AscendC8KVCacheMethod.create_weights(self.layer)
 
         self.layer.register_parameter.assert_any_call("key_antiquant_scale",