vllm-ascend 0.13.0rc2__tar.gz → 0.14.0rc1__tar.gz

vllm_ascend-0.14.0rc1/.github/CODEOWNERS

@@ -0,0 +1,65 @@
+# See https://help.github.com/articles/about-codeowners/
+# for more info about CODEOWNERS file
+
+# Infra, CI
+/.gemini @wangxiyuan @Yikun
+/.github @wangxiyuan @Yikun
+/tools @wangxiyuan @Yikun
+/.gitignore @wangxiyuan
+/.gitmodules @wangxiyuan @zzzzwwjj
+/.pre-commit-config.yaml @wangxiyuan
+/codecov.yml @wangxiyuan
+/Dockerfile* @wangxiyuan
+/format.sh @wangxiyuan
+/mypy.ini @wangxiyuan
+/requirements* @wangxiyuan
+/setup.py @wangxiyuan
+/typos.toml @wangxiyuan
+
+# benchmark
+/benchmarks @wangxiyuan
+
+# c++ source code
+/cmake @zzzzwwjj
+/csrc @zzzzwwjj
+/CMakeLists.txt @zzzzwwjj
+
+# docs
+/docs  @wangxiyuan @Yikun @LCAIZJ
+/.readthedocs.yaml  @wangxiyuan @Yikun
+/README*  @wangxiyuan @Yikun
+
+# example
+/examples @wangxiyuan
+
+# tests
+/tests @wangxiyuan
+
+# python source code
+/vllm_ascend/attention @weijinqian0
+/vllm_ascend/compilation @yiz-liu
+/vllm_ascend/core @wangxiyuan @MengqingCao
+/vllm_ascend/device @weijinqian0 @zzzzwwjj
+/vllm_ascend/device_allocator @wangxiyuan @weijinqian0
+/vllm_ascend/distributed @MengqingCao @LCAIZJ
+/vllm_ascend/eplb @wangxiyuan
+/vllm_ascend/kv_offload @nalinaly
+/vllm_ascend/lora @paulyu12
+/vllm_ascend/model_loader @wangxiyuan
+/vllm_ascend/ops @zzzzwwjj @realliujiaxu
+/vllm_ascend/patch @wangxiyuan
+/vllm_ascend/quantization @wangxiyuan
+/vllm_ascend/sample @realliujiaxu
+/vllm_ascend/spec_decode @wangxiyuan
+/vllm_ascend/worker @MengqingCao
+/vllm_ascend/xlite @wangxiyuan
+/vllm_ascend/ascend_config.py @wangxiyuan
+/vllm_ascend/ascend_forward_context.py @wangxiyuan
+/vllm_ascend/batch_invariant.py @wangxiyuan
+/vllm_ascend/cpu_binding.py @wangxiyuan
+/vllm_ascend/envs.py @wangxiyuan
+/vllm_ascend/flash_common3_context.py @wangxiyuan
+/vllm_ascend/meta_registration.py @wangxiyuan
+/vllm_ascend/platform.py @wangxiyuan
+/vllm_ascend/profiling_config.py @wangxiyuan
+/vllm_ascend/utils.py @wangxiyuan

vllm_ascend-0.14.0rc1/.github/Dockerfile.buildwheel

@@ -0,0 +1,45 @@
+#
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This file is a part of the vllm-ascend project.
+#
+ARG PY_VERSION=3.11
+FROM quay.io/ascend/manylinux:8.5.0-910b-manylinux_2_28-py${PY_VERSION}
+
+ARG SOC_VERSION="ascend910b1"
+
+# Define environments
+ENV DEBIAN_FRONTEND=noninteractive
+ENV SOC_VERSION=$SOC_VERSION
+RUN yum update -y && \
+    yum install -y python3-pip git vim wget net-tools gcc gcc-c++ make cmake numactl-devel && \
+    rm -rf /var/cache/yum
+
+WORKDIR /workspace
+
+COPY . /workspace/vllm-ascend/
+
+# Install req
+RUN python3 -m pip install -r vllm-ascend/requirements.txt --extra-index https://download.pytorch.org/whl/cpu/ && \
+    python3 -m pip install twine attrs psutil
+
+# Install vllm-ascend
+RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    source /usr/local/Ascend/nnal/atb/set_env.sh && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
+    cd vllm-ascend && \
+    python3 setup.py bdist_wheel && \
+    ls -l dist 
+
+CMD ["/bin/bash"]

vllm_ascend-0.14.0rc1/.github/ISSUE_TEMPLATE/110-user-story.yml

@@ -0,0 +1,37 @@
+name: 📚 User Story
+description: Apply for an user story to be displayed on https://docs.vllm.ai/projects/ascend/en/latest/community/user_stories/index.html
+title: "[User Story]: "
+labels: ["user-story"]
+
+body:
+- type: textarea
+  attributes:
+    label: 📚 Title
+    description: >
+      A clear title about what your user story is about.
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: About / Introduction
+    description: >
+      A brief introduction about the background of your use case, like your scenario, hardware size etc.
+- type: textarea
+  attributes:
+    label: Business Challenges
+    description: >
+      Tell us how what kind of challenge you faced in this user story.
+- type: textarea
+  attributes:
+    label: Solving challenges with vLLM Ascend and benefits
+    description: >
+      Tell us how vLLM Ascend helped you overcome the challenges, including details like how you use it, what version you used, hardware info, etc. And what kind of benefit do you get from using vLLM Ascend
+- type: textarea
+  attributes:
+    label: Extra Info
+    description: >
+      Any extra information you want to include in this story
+- type: markdown
+  attributes:
+    value: >
+      Thanks for contributing 🎉!

vllm_ascend-0.14.0rc1/.github/ISSUE_TEMPLATE/600-new-model.yml

@@ -0,0 +1,33 @@
+name: 🤗 Support request for new model supported from huggingface/modelscope/modelers on Ascend
+description: Submit a proposal/request for a new model from huggingface/modelscope/modelers on Ascend
+title: "[New Model]: "
+labels: ["new model"]
+
+body:
+- type: markdown
+  attributes:
+    value: >
+      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-ascend/issues?q=is%3Aissue+sort%3Acreated-desc+).
+
+      #### We also highly recommend you read https://docs.vllm.ai/projects/ascend/en/latest/user_guide/supported_models.html first to know which model already supported.
+- type: textarea
+  attributes:
+    label: The model to consider.
+    description: >
+      A huggingface/modelscope/modelers url, pointing to the model, e.g. https://huggingface.co/openai-community/gpt2 .
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: The closest model vllm already supports.
+    description: >
+      Here is the list of models already supported by vllm: https://docs.vllm.ai/projects/ascend/en/latest/user_guide/supported_models.html . Which model is the most similar to the model you want to add support for?
+- type: textarea
+  attributes:
+    label: What's your difficulty of supporting the model you want?
+    description: >
+      For example, any new operators or new architecture?
+- type: markdown
+  attributes:
+    value: >
+      Thanks for contributing 🎉!

vllm_ascend-0.14.0rc1/.github/actionlint.yaml

@@ -0,0 +1,27 @@
+self-hosted-runner:
+  # Labels of self-hosted runner in array of strings.
+  labels:
+    - linux-aarch64-a2-0
+    - linux-aarch64-a2-1
+    - linux-aarch64-a2-2
+    - linux-aarch64-a2-4
+    - linux-aarch64-a2-8
+    - linux-arm64-npu-static-8
+    - linux-aarch64-310p-1
+    - linux-aarch64-310p-2
+    - linux-aarch64-310p-4
+    - ubuntu-24.04-arm
+    - linux-aarch64-a3-1
+    - linux-aarch64-a3-2
+    - linux-aarch64-a3-4
+    - linux-aarch64-a3-8
+    - linux-amd64-cpu-0
+    - linux-amd64-cpu-8
+    - linux-amd64-cpu-16
+    - linux-aarch64-a3-0
+    - linux-amd64-cpu-8-hk
+    - linux-amd64-cpu-16-hk
+    - linux-aarch64-a2b3-0
+    - linux-aarch64-a2b3-1
+    - linux-aarch64-a2b3-2
+    - linux-aarch64-a2b3-4

vllm_ascend-0.14.0rc1/.github/workflows/_e2e_nightly_multi_node.yaml

@@ -0,0 +1,268 @@
+name: 'e2e nightly test multi_node'
+
+on:
+  workflow_call:
+    inputs:
+      soc_version:
+        required: true
+        type: string
+        description: use a2 or a3
+      runner:
+        required: false
+        type: string
+        default: linux-aarch64-a3-0
+      image:
+        required: false
+        type: string
+        description: base image for pods
+        default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-910b-ubuntu22.04-py3.11"
+      config_file_path:
+        required: true
+        type: string
+        description: the model config for multi_node test
+      replicas:
+        required: false
+        default: "1"
+        type: string
+        description: replicas of the k8s cluster
+      size:
+        required: false
+        default: "2"
+        type: string
+        description: how many pods will be pulled up via lws.yaml, indicates number of nodes we need
+      vllm_version:
+        required: false
+        default: "v0.14.1"
+        type: string
+        description: vllm version to use
+      vllm_ascend_remote_url:
+        required: false
+        default: https://github.com/vllm-project/vllm-ascend.git
+        type: string
+        description: used for pr level tests
+      vllm_ascend_ref:
+        required: false
+        default: main
+        type: string
+        description: used for pr level tests
+    secrets:
+      KUBECONFIG_B64:
+        required: true
+
+
+# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
+# declared as "shell: bash -el {0}" on steps that need to be properly activated.
+# It's used to activate ascend-toolkit environment variables.
+defaults:
+  run:
+    shell: bash -el {0}
+
+# only cancel in-progress runs of the same workflow
+# and ignore the lint / 8 cards test type
+concurrency:
+  group: ascend-nightly-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.soc_version }}
+  cancel-in-progress: true
+
+jobs:
+  e2e:
+    name: ${{ inputs.config_file_path }}
+    # This is the runner with no NPU for k8s controller
+    runs-on: ${{ inputs.runner }}
+    container:
+      image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:nightly-cpu
+      env:
+        KUBECONFIG: /tmp/kubeconfig
+        NAMESPACE: vllm-project
+        LEADER_POD: vllm-0
+    steps:
+        - name: Decode kubeconfig from secrets
+          run: |
+            # Decode and save kubeconfig
+            echo "${{ secrets.KUBECONFIG_B64 }}" | base64 -d > $KUBECONFIG
+
+        - name: Checkout code
+          uses: actions/checkout@v6
+
+        - name: Prepare scripts
+          run: |
+            # prepare for lws entrypoint scripts
+            install -D tests/e2e/nightly/multi_node/scripts/run.sh /root/.cache/tests/run.sh
+
+        - name: Clear resources
+          run: |
+            set -euo pipefail
+
+            CRD_NAME="${CRD_NAME:-vllm}"
+            TIMEOUT=${TIMEOUT:-120}
+            SLEEP_INTERVAL=2
+
+            echo "Deleting leaderworkerset [$CRD_NAME] in namespace [$NAMESPACE]..."
+            kubectl delete leaderworkerset "$CRD_NAME" -n "$NAMESPACE" --ignore-not-found
+
+            echo "Waiting for all pods starting with 'vllm' to be deleted..."
+            START_TIME=$(date +%s)
+
+            while true; do
+              NOW=$(date +%s)
+              ELAPSED=$((NOW - START_TIME))
+
+              if [[ $ELAPSED -ge $TIMEOUT ]]; then
+                echo "Timeout reached ($TIMEOUT seconds), some pods still exist:"
+                kubectl get pods -n "$NAMESPACE" | grep '^vllm' || true
+                exit 1
+              fi
+
+              PODS_EXIST=$(kubectl get pods -n "$NAMESPACE" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n' | grep '^vllm' || true)
+
+              if [[ -z "$PODS_EXIST" ]]; then
+                echo "All vllm pods deleted."
+                break
+              else
+                echo "Waiting for pods to be deleted: $PODS_EXIST"
+                sleep $SLEEP_INTERVAL
+              fi
+            done
+
+        - name: Launch cluster
+          id: launcher
+          run: |
+            set -e
+
+            size="${{ inputs.size }}"
+            replicas="${{ inputs.replicas }}"
+            image="${{ inputs.image }}"
+            config_file_path="${{ inputs.config_file_path }}"
+            fail_tag=FAIL_TAG_"${{ inputs.config_file_path }}"
+            echo "FAIL_TAG=${fail_tag}" >> $GITHUB_ENV
+
+            required_params=("size" "replicas" "image" "config_file_path")
+            for param in "${required_params[@]}"; do
+              if [ -z "${!param}" ]; then
+                echo "Error: Parameter '$param' is required but empty"
+                exit 1
+              fi
+            done
+
+            if [ "${{ inputs.soc_version }}" = "a3" ]; then
+              npu_per_node=16
+              TEMPLATE_FILE="tests/e2e/nightly/multi_node/scripts/lws.yaml.jinja2"
+            else
+              npu_per_node=8
+              TEMPLATE_FILE="tests/e2e/nightly/multi_node/scripts/lws-a2.yaml.jinja2"
+            fi
+
+            jinja2 $TEMPLATE_FILE \
+              -D size="$size" \
+              -D replicas="$replicas" \
+              -D image="$image" \
+              -D config_file_path="$config_file_path" \
+              -D npu_per_node="$npu_per_node" \
+              -D fail_tag="$fail_tag" \
+              --outfile lws.yaml
+
+            kubectl apply -f ./lws.yaml
+
+        - name: Waiting for pod ready
+          run: |
+            POD_PREFIX="${POD_PREFIX:-vllm-0}"
+            SIZE="${{ inputs.size }}"
+            TIMEOUT=1200  # default timeout 20 minutes
+
+            echo "Waiting for Pods in namespace [$NAMESPACE] to become Running and Ready (timeout ${TIMEOUT}s)..."
+
+            START_TIME=$(date +%s)
+
+            while true; do
+              NOW=$(date +%s)
+              ELAPSED=$((NOW - START_TIME))
+              if [[ $ELAPSED -ge $TIMEOUT ]]; then
+                echo "Timeout reached after ${ELAPSED}s"
+                echo "Dumping pod status for debugging:"
+                kubectl get pods -n "$NAMESPACE"
+                kubectl describe pod "$LEADER_POD" -n "$NAMESPACE"
+                exit 1
+              fi
+
+              # 1) check follower pods
+              ALL_FOLLOWERS_READY=true
+              for ((i=1; i<SIZE; i++)); do
+                POD="${POD_PREFIX}-${i}"
+                PHASE=$(kubectl get pod "$POD" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound")
+                READY=$(kubectl get pod "$POD" -n "$NAMESPACE" -o jsonpath='{.status.containerStatuses[*].ready}' 2>/dev/null)
+
+                echo "Follower [$POD] phase=$PHASE ready=$READY"
+
+                if [[ "$PHASE" != "Running" || "$READY" != "true" ]]; then
+                  echo "Follower [$POD] not Ready yet..."
+                  ALL_FOLLOWERS_READY=false
+                  break
+                fi
+              done
+
+              # 2) check leader pod
+              LEADER_PHASE=$(kubectl get pod "$LEADER_POD" -n "$NAMESPACE" -o jsonpath='{.status.phase}' 2>/dev/null || echo "NotFound")
+              LEADER_READY=$(kubectl get pod "$LEADER_POD" -n "$NAMESPACE" -o jsonpath='{.status.containerStatuses[*].ready}' 2>/dev/null)
+
+              echo "Leader [$LEADER_POD] phase=$LEADER_PHASE ready=$LEADER_READY"
+
+              if [[ "$LEADER_PHASE" != "Running" || "$LEADER_READY" != "true" ]]; then
+                echo "Leader not Ready yet..."
+                ALL_FOLLOWERS_READY=false
+              fi
+
+              if [[ "$ALL_FOLLOWERS_READY" == "true" ]]; then
+                echo "All follower pods and leader pod are Running and Ready — continuing."
+                break
+              fi
+
+              sleep 2
+            done
+
+        - name: Stream logs
+          run: |
+            set -euo pipefail
+
+            size="${{ inputs.size }}"
+            pids=()
+
+            cleanup() {
+              echo "Cleaning up background log streams..."
+              for pid in "${pids[@]}"; do
+                kill "$pid" 2>/dev/null || true
+              done
+            }
+            trap cleanup EXIT
+
+            for i in $(seq 1 $((size - 1))); do
+              POD="vllm-0-${i}"
+
+              echo "==== Collecting logs from worker pod: $POD ===="
+              kubectl logs -f "$POD" -n "$NAMESPACE" \
+                > "/tmp/${POD}_logs.txt" 2>&1 &
+
+              pids+=($!)
+            done
+
+            echo "==== Streaming logs from leader pod: $LEADER_POD ===="
+            echo "Looking for logs containing: $FAIL_TAG"
+
+            kubectl logs -f "$LEADER_POD" -n "$NAMESPACE" | while IFS= read -r line; do
+              echo "$line"
+              if echo "$line" | grep -q "$FAIL_TAG"; then
+                exit 1
+              fi
+            done
+
+        - name: Upload logs
+          if: always()
+          uses: actions/upload-artifact@v6
+          with:
+            name: ${{ inputs.config_file_path }}-pod-logs
+            path: /tmp/vllm*_logs.txt
+            retention-days: 7
+
+        - name: Post process
+          if: always()
+          run: |
+            kubectl get pods -n $NAMESPACE --ignore-not-found=true
+            kubectl delete -f ./lws.yaml --ignore-not-found=true || true

vllm_ascend-0.14.0rc1/.github/workflows/_e2e_nightly_single_node.yaml

@@ -0,0 +1,130 @@
+#
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This file is a part of the vllm-ascend project.
+#
+
+name: 'e2e nightly test'
+
+on:
+  workflow_call:
+    inputs:
+      vllm:
+        required: true
+        type: string
+      runner:
+        required: true
+        type: string
+      image:
+        required: false
+        type: string
+        default: "swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.5.0-910b-ubuntu22.04-py3.11"
+      tests:
+        required: true
+        type: string
+      name:
+        required: false
+        type: string
+
+# Bash shells do not use ~/.profile or ~/.bashrc so these shells need to be explicitly
+# declared as "shell: bash -el {0}" on steps that need to be properly activated.
+# It's used to activate ascend-toolkit environment variables.
+defaults:
+  run:
+    shell: bash -el {0}
+
+# only cancel in-progress runs of the same workflow
+# and ignore the lint / 1 card / 4 cards test type
+concurrency:
+  group: ascend-nightly-${{ github.workflow_ref }}-${{ github.ref }}-${{ inputs.tests }}
+  cancel-in-progress: true
+
+jobs:
+  e2e-nightly:
+    name: ${{ inputs.tests }}
+    runs-on: ${{ inputs.runner }}
+    timeout-minutes: 600
+    container:
+      image: ${{ inputs.image }}
+    env:
+      TRANSFORMERS_OFFLINE: 1
+      VLLM_USE_MODELSCOPE: True
+    steps:
+      - name: Check npu and CANN info
+        run: |
+          npu-smi info
+          cat /usr/local/Ascend/ascend-toolkit/latest/"$(uname -i)"-linux/ascend_toolkit_install.info
+
+      - name: Show vLLM and vLLM-Ascend version
+        working-directory: /vllm-workspace
+        run: |
+          echo "Installed vLLM-related Python packages:"
+          pip list | grep vllm || echo "No vllm packages found."
+
+          echo ""
+          echo "============================"
+          echo "vLLM Git information"
+          echo "============================"
+          cd vllm
+          if [ -d .git ]; then
+            echo "Branch:      $(git rev-parse --abbrev-ref HEAD)"
+            echo "Commit hash: $(git rev-parse HEAD)"
+            echo "Author:      $(git log -1 --pretty=format:'%an <%ae>')"
+            echo "Date:        $(git log -1 --pretty=format:'%ad' --date=iso)"
+            echo "Message:     $(git log -1 --pretty=format:'%s')"
+            echo "Tags:        $(git tag --points-at HEAD || echo 'None')"
+            echo "Remote:      $(git remote -v | head -n1)"
+            echo ""
+          else
+            echo "No .git directory found in vllm"
+          fi
+          cd ..
+
+          echo ""
+          echo "============================"
+          echo "vLLM-Ascend Git information"
+          echo "============================"
+          cd vllm-ascend
+          if [ -d .git ]; then
+            echo "Branch:      $(git rev-parse --abbrev-ref HEAD)"
+            echo "Commit hash: $(git rev-parse HEAD)"
+            echo "Author:      $(git log -1 --pretty=format:'%an <%ae>')"
+            echo "Date:        $(git log -1 --pretty=format:'%ad' --date=iso)"
+            echo "Message:     $(git log -1 --pretty=format:'%s')"
+            echo "Tags:        $(git tag --points-at HEAD || echo 'None')"
+            echo "Remote:      $(git remote -v | head -n1)"
+            echo ""
+          else
+            echo "No .git directory found in vllm-ascend"
+          fi
+          cd ..
+
+      - name: Install clang
+        shell: bash -l {0}
+        run: |
+          apt-get update && apt-get -y install clang-15
+          update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 20
+          update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 20
+
+      - name: Run vllm-project/vllm-ascend test
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_USE_MODELSCOPE: True
+          VLLM_CI_RUNNER: ${{ inputs.runner }}
+          BENCHMARK_HOME: /vllm-workspace/vllm-ascend/benchmark
+        working-directory: /vllm-workspace/vllm-ascend
+        run: |
+          # ignore test_dispatch_ffn_combine until the test is fixed
+          pytest -sv ${{ inputs.tests }} \
+          --ignore=tests/e2e/nightly/single_node/ops/singlecard_ops/test_fused_moe.py