vllm-cpu 0.9.2.post2__cp311-cp311-manylinux_2_17_aarch64.whl

vllm/worker/xpu_worker.py

@@ -0,0 +1,186 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""A XPU worker class."""
+import gc
+import os
+from typing import List, Optional, Tuple
+
+import intel_extension_for_pytorch  # noqa: F401
+import oneccl_bindings_for_pytorch  # noqa: F401
+import torch
+import torch.distributed
+
+from vllm.config import VllmConfig
+from vllm.distributed import (ensure_model_parallel_initialized,
+                              init_distributed_environment)
+from vllm.distributed.parallel_state import get_pp_group
+from vllm.logger import init_logger
+from vllm.model_executor import set_random_seed
+from vllm.platforms import current_platform
+from vllm.worker.cache_engine import CacheEngine
+from vllm.worker.worker import Worker
+from vllm.worker.worker_base import LoRANotSupportedWorkerBase, WorkerBase
+from vllm.worker.xpu_model_runner import XPUModelRunner
+
+logger = init_logger(__name__)
+
+
+class XPUWorker(LoRANotSupportedWorkerBase, Worker):
+    """A worker class that executes (a partition of) the model on a GPU.
+
+    Each worker is associated with a single XPU device. The worker is 
+    responsible for maintaining the KV cache and executing the model on the 
+    XPU. In case of distributed inference, each worker is assigned a partition
+    of the model.
+    """
+
+    def __init__(
+        self,
+        vllm_config: VllmConfig,
+        local_rank: int,
+        rank: int,
+        distributed_init_method: str,
+        is_driver_worker: bool = False,
+    ) -> None:
+        WorkerBase.__init__(self, vllm_config=vllm_config)
+        device_config = self.device_config
+        parallel_config = self.parallel_config
+        assert device_config.device_type == "xpu"
+        assert current_platform.is_xpu()
+
+        self.parallel_config.rank = rank
+
+        self.local_rank = local_rank
+        self.rank = rank
+        self.distributed_init_method = distributed_init_method
+        self.is_driver_worker = is_driver_worker
+        if parallel_config and is_driver_worker:
+            assert rank % parallel_config.tensor_parallel_size == 0, \
+                   "Driver worker should be rank 0 of tensor parallel group."
+
+        self.model_runner = XPUModelRunner(  # type: ignore
+            vllm_config=vllm_config,
+            kv_cache_dtype=self.cache_config.cache_dtype,
+            is_driver_worker=is_driver_worker,
+        )
+        # Uninitialized cache engine. Will be initialized by
+        # initialize_cache.
+        self.cache_engine: List[CacheEngine]
+        self.gpu_cache: Optional[List[List[torch.Tensor]]]
+
+    def init_device(self) -> None:
+        if self.device_config.device.type == "xpu" and current_platform.is_xpu(
+        ):
+            self.device = torch.device(f"xpu:{self.local_rank}")
+            torch.xpu.set_device(self.device)
+            torch.xpu.empty_cache()
+            self.init_gpu_memory = torch.xpu.get_device_properties(
+                self.local_rank).total_memory
+        else:
+            raise RuntimeError(
+                f"Not support device type: {self.device_config.device}")
+        # Initialize the distributed environment.
+        self.init_worker_distributed_environment()
+        # Initialize the model.
+        set_random_seed(self.model_config.seed)
+
+    # keep this method for `empty_cache` and `synchronize` api
+    @torch.inference_mode()
+    def determine_num_available_blocks(self) -> Tuple[int, int]:
+        """Profiles the peak memory usage of the model to determine how many
+        KV blocks may be allocated without OOMs.
+
+        The engine will first conduct a profiling of the existing memory usage.
+        Then, it calculate the maximum possible number of GPU and CPU blocks
+        that can be allocated with the remaining free memory.
+
+        Tip:
+            You may limit the usage of GPU memory
+            by adjusting the `gpu_memory_utilization` parameter.
+        """
+        # Profile the memory usage of the model and get the maximum number of
+        # cache blocks that can be allocated with the remaining free memory.
+        torch.xpu.empty_cache()
+
+        # Execute a forward pass with dummy inputs to profile the memory usage
+        # of the model.
+        self.model_runner.profile_run()
+
+        # Calculate the number of blocks that can be allocated with the
+        # profiled peak memory.
+        torch.xpu.synchronize()
+        used_memory = torch.xpu.memory_allocated()
+        total_gpu_memory = torch.xpu.get_device_properties(
+            self.local_rank).total_memory
+        free_gpu_memory = total_gpu_memory - used_memory
+
+        # NOTE(woosuk): Here we assume that the other processes using the same
+        # GPU did not change their memory usage during the profiling.
+        peak_memory = self.init_gpu_memory - free_gpu_memory
+        assert peak_memory > 0, (
+            "Error in memory profiling. "
+            f"Initial free memory {self.init_gpu_memory}, current free memory"
+            f" {free_gpu_memory}. This happens when the GPU memory was "
+            "not properly cleaned up before initializing the vLLM instance.")
+
+        cache_block_size = self.get_cache_block_size_bytes()
+        num_gpu_blocks = int(
+            (total_gpu_memory * self.cache_config.gpu_memory_utilization -
+             peak_memory) // cache_block_size)
+        num_cpu_blocks = int(self.cache_config.swap_space_bytes //
+                             cache_block_size)
+        num_gpu_blocks = max(num_gpu_blocks, 0)
+        num_cpu_blocks = max(num_cpu_blocks, 0)
+        gc.collect()
+        torch.xpu.empty_cache()
+        return num_gpu_blocks, num_cpu_blocks
+
+    def _warm_up_model(self) -> None:
+        # IPEX don't support capture graph yet
+        pass
+
+    def init_worker_distributed_environment(self) -> None:
+        """Initialize the distributed environment."""
+
+        parallel_config = self.parallel_config
+        rank = self.rank
+        distributed_init_method = self.distributed_init_method
+
+        if torch.distributed.is_initialized():
+            torch_world_size = torch.distributed.get_world_size()
+            if torch_world_size != parallel_config.world_size:
+                raise RuntimeError(
+                    "torch.distributed is already initialized but the torch "
+                    "world size does not match parallel_config.world_size "
+                    f"({torch_world_size} vs. {parallel_config.world_size}).")
+        elif not distributed_init_method:
+            raise ValueError(
+                "distributed_init_method must be set if torch.distributed "
+                "is not already initialized")
+        else:
+            # use sockets as default Level zero IPC exchange backend. By
+            # default oneccl will use `drmfd` as mechanism which need extra
+            # dependency (libdrm and drm headers) on your system.
+            ENV_CCL_ATL_TRANSPORT = os.getenv("CCL_ATL_TRANSPORT", "ofi")
+            ENV_LOCAL_WORLD_SIZE = os.getenv("LOCAL_WORLD_SIZE",
+                                             str(parallel_config.world_size))
+            os.environ["CCL_ATL_TRANSPORT"] = ENV_CCL_ATL_TRANSPORT
+            os.environ["LOCAL_WORLD_SIZE"] = ENV_LOCAL_WORLD_SIZE
+            os.environ["LOCAL_RANK"] = str(self.local_rank)
+            init_distributed_environment(
+                world_size=parallel_config.world_size,
+                rank=rank,
+                distributed_init_method=distributed_init_method,
+                local_rank=self.local_rank,
+                backend="ccl")
+
+        ensure_model_parallel_initialized(
+            parallel_config.tensor_parallel_size,
+            parallel_config.pipeline_parallel_size)
+        # global all_reduce needed for overall oneccl warm up
+        torch.distributed.all_reduce(torch.zeros(1).xpu())
+
+        if parallel_config.pipeline_parallel_size > 1:
+            # Add pp group init to avoid
+            # p2p communication as the first call
+            get_pp_group().all_reduce(torch.zeros(1).xpu())

vllm_cpu-0.9.2.post2.dist-info/METADATA

@@ -0,0 +1,339 @@
+Metadata-Version: 2.4
+Name: vllm-cpu
+Version: 0.9.2.post2
+Summary: vLLM CPU inference engine (without AVX512 support, Runs on both x86_64 and Arm64 Processors)
+Author-email: Mekayel Anik <mekayel.anik@gmail.com>
+Maintainer-email: Mekayel Anik <mekayel.anik@gmail.com>
+License: GPL-3.0-only
+Project-URL: Bug Tracker, https://github.com/MekayelAnik/vllm-cpu/issues
+Project-URL: Homepage, https://github.com/vllm-project/vllm
+Project-URL: Documentation, https://docs.vllm.ai/en/latest/
+Project-URL: Slack, https://slack.vllm.ai/
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Information Technology
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: <3.13,>=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: regex
+Requires-Dist: cachetools
+Requires-Dist: psutil
+Requires-Dist: sentencepiece
+Requires-Dist: numpy
+Requires-Dist: requests>=2.26.0
+Requires-Dist: tqdm
+Requires-Dist: blake3
+Requires-Dist: py-cpuinfo
+Requires-Dist: transformers>=4.51.1
+Requires-Dist: huggingface-hub[hf_xet]>=0.33.0
+Requires-Dist: tokenizers>=0.21.1
+Requires-Dist: protobuf
+Requires-Dist: fastapi[standard]>=0.115.0
+Requires-Dist: aiohttp
+Requires-Dist: openai<=1.90.0,>=1.52.0
+Requires-Dist: pydantic>=2.10
+Requires-Dist: prometheus_client>=0.18.0
+Requires-Dist: pillow
+Requires-Dist: prometheus-fastapi-instrumentator>=7.0.0
+Requires-Dist: tiktoken>=0.6.0
+Requires-Dist: lm-format-enforcer<0.11,>=0.10.11
+Requires-Dist: llguidance<0.8.0,>=0.7.11; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64"
+Requires-Dist: outlines==0.1.11
+Requires-Dist: lark==1.2.2
+Requires-Dist: xgrammar==0.1.19; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64"
+Requires-Dist: typing_extensions>=4.10
+Requires-Dist: filelock>=3.16.1
+Requires-Dist: partial-json-parser
+Requires-Dist: pyzmq>=25.0.0
+Requires-Dist: msgspec
+Requires-Dist: gguf>=0.13.0
+Requires-Dist: importlib_metadata; python_version < "3.10"
+Requires-Dist: mistral_common[opencv]>=1.6.2
+Requires-Dist: opencv-python-headless>=4.11.0
+Requires-Dist: pyyaml
+Requires-Dist: six>=1.16.0; python_version > "3.11"
+Requires-Dist: setuptools<80,>=77.0.3; python_version > "3.11"
+Requires-Dist: einops
+Requires-Dist: compressed-tensors==0.10.2
+Requires-Dist: depyf==0.18.0
+Requires-Dist: cloudpickle
+Requires-Dist: watchfiles
+Requires-Dist: python-json-logger
+Requires-Dist: scipy
+Requires-Dist: ninja
+Requires-Dist: pybase64
+Requires-Dist: numba==0.60.0; python_version == "3.9"
+Requires-Dist: numba==0.61.2; python_version > "3.9"
+Requires-Dist: packaging>=24.2
+Requires-Dist: setuptools<80.0.0,>=77.0.3
+Requires-Dist: torch==2.6.0+cpu; platform_machine == "x86_64"
+Requires-Dist: torch==2.7.0; platform_system == "Darwin"
+Requires-Dist: torch==2.7.0; platform_machine == "ppc64le" or platform_machine == "aarch64"
+Requires-Dist: torchaudio; platform_machine != "ppc64le" and platform_machine != "s390x"
+Requires-Dist: torchaudio==2.7.0; platform_machine == "ppc64le"
+Requires-Dist: torchvision; platform_machine != "ppc64le" and platform_machine != "s390x"
+Requires-Dist: torchvision==0.22.0; platform_machine == "ppc64le"
+Requires-Dist: datasets
+Requires-Dist: intel-openmp==2024.2.1; platform_machine == "x86_64"
+Requires-Dist: intel_extension_for_pytorch==2.6.0; platform_machine == "x86_64"
+Requires-Dist: py-libnuma; platform_system != "Darwin"
+Requires-Dist: psutil; platform_system != "Darwin"
+Requires-Dist: triton==3.2.0; platform_machine == "x86_64"
+Provides-Extra: bench
+Requires-Dist: pandas; extra == "bench"
+Requires-Dist: datasets; extra == "bench"
+Provides-Extra: tensorizer
+Requires-Dist: tensorizer>=2.9.0; extra == "tensorizer"
+Provides-Extra: fastsafetensors
+Requires-Dist: fastsafetensors>=0.1.10; extra == "fastsafetensors"
+Provides-Extra: runai
+Requires-Dist: runai-model-streamer; extra == "runai"
+Requires-Dist: runai-model-streamer-s3; extra == "runai"
+Requires-Dist: boto3; extra == "runai"
+Provides-Extra: audio
+Requires-Dist: librosa; extra == "audio"
+Requires-Dist: soundfile; extra == "audio"
+Provides-Extra: video
+Dynamic: provides-extra
+Dynamic: requires-dist
+
+<!-- markdownlint-disable MD001 MD041 -->
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/vllm-project/vllm/main/docs/assets/logos/vllm-logo-text-dark.png">
+    <img alt="vLLM" src="https://raw.githubusercontent.com/vllm-project/vllm/main/docs/assets/logos/vllm-logo-text-light.png" width=55%>
+  </picture>
+</p>
+
+<h3 align="center">
+Easy, fast, and cheap LLM serving for everyone
+</h3>
+
+<p align="center">
+  <a href="https://github.com/MekayelAnik/vllm-cpu/stargazers">
+    <img src="https://img.shields.io/github/stars/MekayelAnik/vllm-cpu?style=for-the-badge&logo=github&logoColor=white&labelColor=2b3137&color=f0c14b" alt="GitHub Stars">
+  </a>
+  <a href="https://github.com/MekayelAnik/vllm-cpu/network/members">
+    <img src="https://img.shields.io/github/forks/MekayelAnik/vllm-cpu?style=for-the-badge&logo=github&logoColor=white&labelColor=2b3137&color=6cc644" alt="GitHub Forks">
+  </a>
+  <a href="https://github.com/MekayelAnik/vllm-cpu/issues">
+    <img src="https://img.shields.io/github/issues/MekayelAnik/vllm-cpu?style=for-the-badge&logo=github&logoColor=white&labelColor=2b3137&color=d73a49" alt="GitHub Issues">
+  </a>
+  <a href="https://github.com/MekayelAnik/vllm-cpu/pulls">
+    <img src="https://img.shields.io/github/issues-pr/MekayelAnik/vllm-cpu?style=for-the-badge&logo=github&logoColor=white&labelColor=2b3137&color=2188ff" alt="GitHub PRs">
+  </a>
+</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/vllm-cpu/">
+    <img src="https://img.shields.io/pypi/v/vllm-cpu?style=for-the-badge&logo=pypi&logoColor=white&labelColor=2b3137&color=3775a9" alt="PyPI Version">
+  </a>
+  <a href="https://pypi.org/project/vllm-cpu/">
+    <img src="https://img.shields.io/pypi/dm/vllm-cpu?style=for-the-badge&logo=pypi&logoColor=white&labelColor=2b3137&color=9c27b0" alt="PyPI Downloads">
+  </a>
+  <a href="https://github.com/MekayelAnik/vllm-cpu/blob/main/LICENSE">
+    <img src="https://img.shields.io/github/license/MekayelAnik/vllm-cpu?style=for-the-badge&logo=gnu&logoColor=white&labelColor=2b3137&color=a32d2a" alt="License">
+  </a>
+</p>
+
+<p align="center">
+  <a href="https://hub.docker.com/r/mekayelanik/vllm-cpu">
+    <img src="https://img.shields.io/docker/pulls/mekayelanik/vllm-cpu?style=for-the-badge&logo=docker&logoColor=white&labelColor=2b3137&color=0db7ed" alt="Docker Pulls">
+  </a>
+  <a href="https://hub.docker.com/r/mekayelanik/vllm-cpu">
+    <img src="https://img.shields.io/docker/stars/mekayelanik/vllm-cpu?style=for-the-badge&logo=docker&logoColor=white&labelColor=2b3137&color=f0c14b" alt="Docker Stars">
+  </a>
+  <a href="https://hub.docker.com/r/mekayelanik/vllm-cpu">
+    <img src="https://img.shields.io/docker/v/mekayelanik/vllm-cpu?style=for-the-badge&logo=docker&logoColor=white&labelColor=2b3137&color=6cc644&label=version" alt="Docker Version">
+  </a>
+  <a href="https://hub.docker.com/r/mekayelanik/vllm-cpu">
+    <img src="https://img.shields.io/docker/image-size/mekayelanik/vllm-cpu?style=for-the-badge&logo=docker&logoColor=white&labelColor=2b3137&color=9c27b0" alt="Docker Image Size">
+  </a>
+</p>
+
+<p align="center">
+  <a href="https://github.com/MekayelAnik/vllm-cpu/commits/main">
+    <img src="https://img.shields.io/github/last-commit/MekayelAnik/vllm-cpu?style=for-the-badge&logo=git&logoColor=white&labelColor=2b3137&color=ff6f00" alt="Last Commit">
+  </a>
+  <a href="https://github.com/MekayelAnik/vllm-cpu/graphs/contributors">
+    <img src="https://img.shields.io/github/contributors/MekayelAnik/vllm-cpu?style=for-the-badge&logo=github&logoColor=white&labelColor=2b3137&color=00bcd4" alt="Contributors">
+  </a>
+  <a href="https://github.com/MekayelAnik/vllm-cpu">
+    <img src="https://img.shields.io/github/repo-size/MekayelAnik/vllm-cpu?style=for-the-badge&logo=github&logoColor=white&labelColor=2b3137&color=607d8b" alt="Repo Size">
+  </a>
+</p>
+
+---
+
+<div align="center">
+
+## Buy Me a Coffee
+
+**Your support encourages me to keep creating/supporting my open-source projects.** If you found value in this project, you can buy me a coffee to keep me up all the sleepless nights.
+
+<a href="https://07mekayel07.gumroad.com/coffee" target="_blank">
+<img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" width="217" height="60">
+</a>
+
+</div>
+
+---
+
+## About
+
+vLLM is a fast and easy-to-use library for LLM inference and serving. <b>This PyPl package has NO support for AVX512/VNNI/AVX512BF16/AMXBF16.</b> CPU inference will not have any instruction set acceleration. Only RAW CPU power will be used. <b>This package should be used for inference on ARM64 CPUs</b>
+
+Originally developed in the [Sky Computing Lab](https://sky.cs.berkeley.edu) at UC Berkeley, vLLM has evolved into a community-driven project with contributions from both academia and industry.
+
+vLLM is fast with:
+
+- State-of-the-art serving throughput
+- Efficient management of attention key and value memory with [**PagedAttention**](https://blog.vllm.ai/2023/06/20/vllm.html)
+- Continuous batching of incoming requests
+- Fast model execution with <b>VNNI on supported CPUs</b> Use this package <b>ONLY IF</b> your CPU doesn't have <b>avx512</b> instruction set
+- Quantizations: [GPTQ](https://arxiv.org/abs/2210.17323), [AWQ](https://arxiv.org/abs/2306.00978), [AutoRound](https://arxiv.org/abs/2309.05516), INT4, INT8, and FP8
+- Optimized CPU kernels, including integration with FlashAttention and FlashInfer
+- Speculative decoding
+- Chunked prefill
+
+vLLM is flexible and easy to use with:
+
+- Seamless integration with popular Hugging Face models
+- High-throughput serving with various decoding algorithms, including *parallel sampling*, *beam search*, and more
+- Tensor, pipeline, data and expert parallelism support for distributed inference
+- Streaming outputs
+- OpenAI-compatible API server
+- Support for x86_64, PowerPC CPUs, Arm CPUs and Applie Scilicon (CPU inference). <b>This package does not support any GPU inference.</b> For GPU inference support use the official [vLLM PypI](https://pypi.org/project/vllm/)
+- Prefix caching support
+- Multi-LoRA support
+
+vLLM seamlessly supports most popular open-source models on HuggingFace, including:
+
+- Transformer-like LLMs (e.g., Llama)
+- Mixture-of-Expert LLMs (e.g., Mixtral, Deepseek-V2 and V3)
+- Embedding Models (e.g., E5-Mistral)
+- Multi-modal LLMs (e.g., LLaVA)
+
+Find the full list of supported models [here](https://docs.vllm.ai/en/latest/models/supported_models.html).
+
+## Important Notes
+
+- Install this package on Linux envirenment only. For Windows you will have to use WSL2 or later
+- This package has a Container.io (Docker/Podman etc.) compatible image in [Docker Hub](https://hub.docker.com/r/mekayelanik/vllm-cpu)
+- [Apache Licence of main vLLM project](https://raw.githubusercontent.com/vllm-project/vllm/refs/heads/main/LICENSE)
+- [GPL License of this CPU specific vLLM package](https://raw.githubusercontent.com/MekayelAnik/vllm-cpu/refs/heads/main/LICENSE)
+- **For versions 0.8.5–0.12.0, use `.post2` releases** (e.g., `pip install vllm-cpu==0.12.0.post2`) — includes critical CPU platform detection fix
+
+## Platform Detection Fix (versions 0.8.5 - 0.12.0)
+
+If you encounter `RuntimeError: Failed to infer device type` or see `UnspecifiedPlatform` warnings with versions 0.8.5 to 0.12.0, run this one-time fix after installation:
+
+```python
+import os, sys, importlib.metadata as m
+v = next((d.metadata['Version'] for d in m.distributions() if d.metadata['Name'].startswith('vllm-cpu')), None)
+if v:
+    p = next((p for p in sys.path if 'site-packages' in p and os.path.isdir(p)), None)
+    if p:
+        d = os.path.join(p, 'vllm-0.0.0.dist-info'); os.makedirs(d, exist_ok=True)
+        open(os.path.join(d, 'METADATA'), 'w').write(f'Metadata-Version: 2.1\nName: vllm\nVersion: {v}+cpu\n')
+        print(f'Fixed: vllm version set to {v}+cpu')
+```
+
+This creates a package alias so vLLM detects the CPU platform correctly. Only needed once per environment. Versions 0.8.5.post2+ and 0.12.0+ include this fix automatically.
+
+## Getting Started
+
+Install vLLM with a single command:
+
+```bash
+pip install vllm-cpu --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple
+```
+
+This installs vllm-cpu with CPU-optimized PyTorch (no CUDA dependencies).
+
+### Alternative: Using uv (faster)
+
+```bash
+uv pip install vllm-cpu --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple
+```
+
+Install `uv` on Linux:
+```bash
+curl -LsSf https://astral.sh/uv/install.sh | sh
+```
+
+## Docker Images
+
+Pre-built Docker images are available on [Docker Hub](https://hub.docker.com/r/mekayelanik/vllm-cpu) and [GitHub Container Registry](https://ghcr.io/mekayelanik/vllm-cpu).
+
+```bash
+# Pull from Docker Hub
+docker pull mekayelanik/vllm-cpu:noavx512-latest
+
+# Or from GitHub Container Registry
+docker pull ghcr.io/mekayelanik/vllm-cpu:noavx512-latest
+
+# Run OpenAI-compatible API server
+docker run -p 8000:8000 \
+  -v $HOME/.cache/huggingface:/root/.cache/huggingface \
+  mekayelanik/vllm-cpu:noavx512-latest \
+  --model facebook/opt-125m
+```
+
+**Available tags:** `noavx512-latest`, `noavx512-<version>` (e.g., `noavx512-0.12.0`)
+
+**Platforms:** `linux/amd64`, `linux/arm64`
+
+## vllm-cpu
+This CPU specific vLLM has **5 optimized wheel packages** from the upstream vLLM source code:
+
+| Package | Optimizations | Target CPUs |
+|---------|--------------|-------------|
+| [`vllm-cpu`](https://pypi.org/project/vllm-cpu/) | Baseline (no AVX512) | All x86_64 and ARM64 CPUs |
+| [`vllm-cpu-avx512`](https://pypi.org/project/vllm-cpu-avx512/) | AVX512 | Intel Skylake-X and newer |
+| [`vllm-cpu-avx512vnni`](https://pypi.org/project/vllm-cpu-avx512vnni/) | AVX512 + VNNI | Intel Cascade Lake and newer |
+| [`vllm-cpu-avx512bf16`](https://pypi.org/project/vllm-cpu-avx512bf16/) | AVX512 + VNNI + BF16 | Intel Cooper Lake and newer |
+| [`vllm-cpu-amxbf16`](https://pypi.org/project/vllm-cpu-amxbf16/) | AVX512 + VNNI + BF16 + AMX | Intel Sapphire Rapids (4th gen Xeon+) |
+
+Each package is compiled with specific CPU instruction set flags for optimal inference performance.
+
+
+## Check available CPU instruction sets
+```
+lscpu | grep -i flags
+```
+## Example list of CPUs with their supported instruction sets
+ | CPU Architecture (Intel/AMD) | AVX2 | AVX-512 F (Base) | VNNI (INT8) | BF16 (BFloat16) (via AVX-512) | AMX-BF16 (via Tile Unit) |
+|---------|--------------|:-------------:|:-------------:|:-------------:|:-------------:|
+| Intel 4th Gen / AMD Ryzen Zen2 & Newer | Yes | No	| No | No | No |
+| Intel Skylake-SP / Skylake-X / AMD Zen 4 & Newer | Yes | Yes	| No	| No	| No |
+| Intel Cooper Lake (3rd Gen Xeon) / AMD Zen 4 (EPYC) / Ryzen Zen5 & Newer | Yes | Yes | Yes | Yes | No |
+|Intel Sapphire Rapids (4th Gen Xeon) & Newer	| Yes	| Yes	| Yes	| Yes | Yes |
+
+<b>***Currently no AMD CPU support AMXBF16. AMD expected to include AMXBF16 support from AMD Zen 7 CPUs</b>
+
+---
+
+<div align="center">
+
+## Buy Me a Coffee
+
+**Your support encourages me to keep creating/supporting my open-source projects.** If you found value in this project, you can buy me a coffee to keep me up all the sleepless nights.
+
+<a href="https://07mekayel07.gumroad.com/coffee" target="_blank">
+<img src="https://cdn.buymeacoffee.com/buttons/v2/default-yellow.png" alt="Buy Me A Coffee" width="217" height="60">
+</a>
+
+</div>
+
+---
+
+- [vLLM documentation](https://docs.vllm.ai/en/latest/) to learn more.
+- [GitHub Repo for this package](https://github.com/MekayelAnik/vllm-cpu)
+- [DockerHub](https://hub.docker.com/r/mekayelanik/vllm-cpu)
+- [List of Supported Models](https://docs.vllm.ai/en/latest/models/supported_models.html)