vec-inf 0.4.1__tar.gz → 0.6.0__tar.gz

{vec_inf-0.4.1 → vec_inf-0.6.0}/.github/workflows/code_checks.yml

@@ -30,13 +30,13 @@ jobs:
     steps:
       - uses: actions/checkout@v4.2.2
       - name: Install uv
-        uses: astral-sh/setup-uv@v5.2.2
+        uses: astral-sh/setup-uv@v6
         with:
           # Install a specific version of uv.
           version: "0.5.21"
           enable-cache: true
       - name: "Set up Python"
-        uses: actions/setup-python@v5.4.0
+        uses: actions/setup-python@v5.5.0
         with:
           python-version-file: ".python-version"
       - name: Install the project
@@ -46,6 +46,6 @@ jobs:
           source .venv/bin/activate
           pre-commit run --all-files
       - name: pip-audit (gh-action-pip-audit)
-        uses: pypa/gh-action-pip-audit@v1.0.8
+        uses: pypa/gh-action-pip-audit@v1.1.0
         with:
           virtual-environment: .venv/

vec_inf-0.6.0/.github/workflows/docker.yml

@@ -0,0 +1,56 @@
+name: docker
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+      - main
+    paths:
+      - Dockerfile
+      - .github/workflows/docker.yml
+      - uv.lock
+  pull_request:
+    branches:
+      - main
+    paths:
+      - Dockerfile
+      - .github/workflows/docker.yml
+      - uv.lock
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4.2.2
+
+      - name: Extract vLLM version
+        id: vllm-version
+        run: |
+          VERSION=$(grep -A 1 'name = "vllm"' uv.lock | grep version | cut -d '"' -f 2)
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804
+        with:
+          images: vectorinstitute/vector-inference
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ steps.meta.outputs.tags }}
+            vectorinstitute/vector-inference:${{ steps.vllm-version.outputs.version }}
+          labels: ${{ steps.meta.outputs.labels }}

vec_inf-0.6.0/.github/workflows/docs.yml

@@ -0,0 +1,176 @@
+name: docs
+permissions:
+  contents: write
+  pull-requests: write
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/docs.yml
+      - '**.py'
+      - '**.ipynb'
+      - '**.html'
+      - '**.js'
+      - '**.md'
+      - uv.lock
+      - pyproject.toml
+      - mkdocs.yml
+      - '**.png'
+      - '**.svg'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/docs.yml
+      - '**.py'
+      - '**.ipynb'
+      - '**.js'
+      - '**.html'
+      - uv.lock
+      - pyproject.toml
+      - '**.md'
+      - mkdocs.yml
+      - '**.png'
+      - '**.svg'
+  release:
+    types: [published]
+  # Allow manual trigger
+  workflow_dispatch:
+    inputs:
+      version:
+        description: 'Version to deploy (e.g., 0.5.0, latest)'
+        required: true
+        default: 'latest'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.2.2
+        with:
+          fetch-depth: 0  # Fetch all history for proper versioning
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.5.21"
+          enable-cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install the project
+        run: uv sync --all-extras --group docs
+
+      - name: Build docs
+        run: uv run mkdocs build
+
+      - name: Create .nojekyll file
+        run: touch site/.nojekyll
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs-site
+          path: site/
+          retention-days: 1
+
+  deploy:
+    needs: build
+    if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'workflow_dispatch' || github.event_name == 'release'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4.2.2
+        with:
+          fetch-depth: 0  # Fetch all history for proper versioning
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.5.21"
+          enable-cache: true
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install the project
+        run: uv sync --all-extras --group docs
+
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: docs-site
+          path: site
+
+      - name: Ensure .nojekyll exists
+        run: touch site/.nojekyll
+
+      - name: Determine version
+        id: version
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            # Use the version provided in the workflow dispatch
+            echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
+            echo "VERSION_ALIAS=latest" >> $GITHUB_OUTPUT
+          elif [[ "${{ github.event_name }}" == "release" ]]; then
+            # Use the tag from the release
+            VERSION="${{ github.ref_name }}"
+            # Remove 'v' prefix if present
+            VERSION="${VERSION#v}"
+            echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+            echo "VERSION_ALIAS=latest" >> $GITHUB_OUTPUT
+          elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then
+            # For pushes to main, tag as "main"
+            echo "VERSION=main" >> $GITHUB_OUTPUT
+            # No alias for main
+            echo "VERSION_ALIAS=" >> $GITHUB_OUTPUT
+          else
+            # Get version from pyproject.toml as fallback
+            VERSION=$(grep -m 1 '^version = ' pyproject.toml | sed 's/^version = "\(.*\)"$/\1/')
+            echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+            echo "VERSION_ALIAS=latest" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Deploy docs with mike
+        run: |
+          VERSION=${{ steps.version.outputs.VERSION }}
+          ALIAS=${{ steps.version.outputs.VERSION_ALIAS }}
+
+          # Add a temporary remote to fetch gh-pages if it exists
+          git remote add temp https://github.com/${{ github.repository }}.git || true
+          git fetch temp gh-pages || true
+
+          DEPLOY_ARGS="--push --update-aliases $VERSION"
+
+          if [[ ! -z "$ALIAS" ]]; then
+            DEPLOY_ARGS="$DEPLOY_ARGS $ALIAS"
+          fi
+
+          # Activate the virtual environment
+          source .venv/bin/activate
+
+          echo "Running: mike deploy $DEPLOY_ARGS"
+          mike deploy $DEPLOY_ARGS
+
+          # Set default version to latest only if we're deploying a version with the latest alias
+          if [[ ! -z "$ALIAS" && "$ALIAS" == "latest" ]]; then
+            mike set-default --push latest
+          fi
+
+          # Remove the temporary remote
+          git remote remove temp || true

{vec_inf-0.4.1 → vec_inf-0.6.0}/.github/workflows/publish.yml

@@ -12,16 +12,21 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install libcurl4-openssl-dev libssl-dev
-      - uses: actions/checkout@v4.1.1
-      - name: Install poetry
-        run: python3 -m pip install --upgrade pip && python3 -m pip install poetry
-      - uses: actions/setup-python@v5.0.0
+
+      - uses: actions/checkout@v4.2.2
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.6.6"
+          enable-cache: true
+
+      - uses: actions/setup-python@v5.5.0
         with:
           python-version: '3.10'
+
       - name: Build package
-        run: poetry build
+        run: uv build
+
       - name: Publish package
-        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
-        with:
-          user: __token__
-          password: ${{ secrets.PYPI_API_TOKEN }}
+        run: uv publish --token ${{ secrets.PYPI_API_TOKEN }}

{vec_inf-0.4.1 → vec_inf-0.6.0}/.github/workflows/unit_tests.yml

@@ -39,33 +39,43 @@ on:
 jobs:
   unit-tests:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
     steps:
       - uses: actions/checkout@v4.2.2
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5.2.2
+        uses: astral-sh/setup-uv@v6
         with:
           # Install a specific version of uv.
           version: "0.5.21"
           enable-cache: true
 
-      - name: "Set up Python"
-        uses: actions/setup-python@v5.4.0
+      - name: "Set up Python ${{ matrix.python-version }}"
+        uses: actions/setup-python@v5.5.0
         with:
-          python-version-file: ".python-version"
+          python-version: ${{ matrix.python-version }}
 
       - name: Install the project
-        run: uv sync --all-extras --dev
+        run: uv sync --dev
 
       - name: Install dependencies and check code
         run: |
           uv run pytest -m "not integration_test" --cov vec_inf --cov-report=xml tests
 
-      # Uncomment this once this repo is configured on Codecov
+      - name: Install the core package only
+        run: uv sync --no-dev
+
+      - name: Run package import tests
+        run: |
+          uv run pytest tests/test_imports.py
+
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v5.3.1
+        uses: codecov/codecov-action@v5.4.2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          slug: VectorInstitute/vec-inf
+          file: ./coverage.xml
+          name: codecov-umbrella
           fail_ci_if_error: true
           verbose: true

{vec_inf-0.4.1 → vec_inf-0.6.0}/.pre-commit-config.yaml

@@ -13,10 +13,11 @@ repos:
       args: [--fix=lf]
     - id: requirements-txt-fixer
     - id: check-yaml
+      args: [--unsafe]
     - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.9.6'
+    rev: 'v0.11.8'
     hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]

vec_inf-0.6.0/Dockerfile

@@ -0,0 +1,54 @@
+FROM nvidia/cuda:12.4.1-devel-ubuntu20.04
+
+# Non-interactive apt-get commands
+ARG DEBIAN_FRONTEND=noninteractive
+
+# No GPUs visible during build
+ARG CUDA_VISIBLE_DEVICES=none
+
+# Specify CUDA architectures -> 7.5: RTX 6000 & T4, 8.0: A100, 8.6+PTX
+ARG TORCH_CUDA_ARCH_LIST="7.5;8.0;8.6+PTX"
+
+# Set the Python version
+ARG PYTHON_VERSION=3.10.12
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    wget build-essential libssl-dev zlib1g-dev libbz2-dev \
+    libreadline-dev libsqlite3-dev libffi-dev libncursesw5-dev \
+    xz-utils tk-dev libxml2-dev libxmlsec1-dev liblzma-dev git vim \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python
+RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \
+    tar -xzf Python-$PYTHON_VERSION.tgz && \
+    cd Python-$PYTHON_VERSION && \
+    ./configure --enable-optimizations && \
+    make -j$(nproc) && \
+    make altinstall && \
+    cd .. && \
+    rm -rf Python-$PYTHON_VERSION.tgz Python-$PYTHON_VERSION
+
+# Install pip and core Python tools
+RUN wget https://bootstrap.pypa.io/get-pip.py && \
+    python3.10 get-pip.py && \
+    rm get-pip.py && \
+    python3.10 -m pip install --upgrade pip setuptools wheel uv
+
+# Set up project
+WORKDIR /vec-inf
+COPY . /vec-inf
+
+# Install project dependencies with build requirements
+RUN PIP_INDEX_URL="https://download.pytorch.org/whl/cu121" uv pip install --system -e .[dev]
+# Install FlashAttention
+RUN python3.10 -m pip install flash-attn --no-build-isolation
+# Install FlashInfer
+RUN python3.10 -m pip install flashinfer-python -i https://flashinfer.ai/whl/cu124/torch2.6/
+
+# Final configuration
+RUN mkdir -p /vec-inf/nccl && \
+    mv /root/.config/vllm/nccl/cu12/libnccl.so.2.18.1 /vec-inf/nccl/libnccl.so.2.18.1
+
+# Set the default command to start an interactive shell
+CMD ["bash"]

vec_inf-0.6.0/PKG-INFO

@@ -0,0 +1,193 @@
+Metadata-Version: 2.4
+Name: vec-inf
+Version: 0.6.0
+Summary: Efficient LLM inference on Slurm clusters using vLLM.
+Author-email: Marshall Wang <marshall.wang@vectorinstitute.ai>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.0
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: requests>=2.31.0
+Requires-Dist: rich>=13.7.0
+Provides-Extra: dev
+Requires-Dist: cupy-cuda12x==12.1.0; extra == 'dev'
+Requires-Dist: ray>=2.40.0; extra == 'dev'
+Requires-Dist: torch>=2.5.1; extra == 'dev'
+Requires-Dist: vllm-nccl-cu12<2.19,>=2.18; extra == 'dev'
+Requires-Dist: vllm>=0.7.3; extra == 'dev'
+Requires-Dist: xgrammar>=0.1.11; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Vector Inference: Easy inference on Slurm clusters
+
+----------------------------------------------------
+
+[![PyPI](https://img.shields.io/pypi/v/vec-inf)](https://pypi.org/project/vec-inf)
+[![downloads](https://img.shields.io/pypi/dm/vec-inf)](https://pypistats.org/packages/vec-inf)
+[![code checks](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml)
+[![docs](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml)
+[![codecov](https://codecov.io/github/VectorInstitute/vector-inference/branch/main/graph/badge.svg?token=NI88QSIGAC)](https://app.codecov.io/github/VectorInstitute/vector-inference/tree/main)
+![GitHub License](https://img.shields.io/github/license/VectorInstitute/vector-inference)
+
+This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using [vLLM](https://docs.vllm.ai/en/latest/). **All scripts in this repository runs natively on the Vector Institute cluster environment**. To adapt to other environments, update the environment variables in [`vec_inf/client/slurm_vars.py`](vec_inf/client/slurm_vars.py), and the model config for cached model weights in [`vec_inf/config/models.yaml`](vec_inf/config/models.yaml) accordingly.
+
+## Installation
+If you are using the Vector cluster environment, and you don't need any customization to the inference server environment, run the following to install package:
+
+```bash
+pip install vec-inf
+```
+Otherwise, we recommend using the provided [`Dockerfile`](Dockerfile) to set up your own environment with the package
+
+## Usage
+
+Vector Inference provides 2 user interfaces, a CLI and an API
+
+### CLI
+
+The `launch` command allows users to deploy a model as a slurm job. If the job successfully launches, a URL endpoint is exposed for the user to send requests for inference.
+
+We will use the Llama 3.1 model as example, to launch an OpenAI compatible inference server for Meta-Llama-3.1-8B-Instruct, run:
+
+```bash
+vec-inf launch Meta-Llama-3.1-8B-Instruct
+```
+You should see an output like the following:
+
+<img width="600" alt="launch_image" src="https://github.com/user-attachments/assets/a72a99fd-4bf2-408e-8850-359761d96c4f">
+
+
+#### Overrides
+
+Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be
+overriden. For example, if `qos` is to be overriden:
+
+```bash
+vec-inf launch Meta-Llama-3.1-8B-Instruct --qos <new_qos>
+```
+
+To overwrite default vLLM engine arguments, you can specify the engine arguments in a comma separated string:
+
+```bash
+vec-inf launch Meta-Llama-3.1-8B-Instruct --vllm-args '--max-model-len=65536,--compilation-config=3'
+```
+
+For the full list of vLLM engine arguments, you can find them [here](https://docs.vllm.ai/en/stable/serving/engine_args.html), make sure you select the correct vLLM version.
+
+#### Custom models
+
+You can also launch your own custom model as long as the model architecture is [supported by vLLM](https://docs.vllm.ai/en/stable/models/supported_models.html), and make sure to follow the instructions below:
+* Your model weights directory naming convention should follow `$MODEL_FAMILY-$MODEL_VARIANT` ($MODEL_VARIANT is OPTIONAL).
+* Your model weights directory should contain HuggingFace format weights.
+* You should specify your model configuration by:
+  * Creating a custom configuration file for your model and specify its path via setting the environment variable `VEC_INF_CONFIG`. Check the [default parameters](vec_inf/config/models.yaml) file for the format of the config file. All the parameters for the model should be specified in that config file.
+  * Using launch command options to specify your model setup.
+* For other model launch parameters you can reference the default values for similar models using the [`list` command ](#list-command).
+
+Here is an example to deploy a custom [Qwen2.5-7B-Instruct-1M](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-1M) model which is not
+supported in the default list of models using a user custom config. In this case, the model weights are assumed to be downloaded to
+a `model-weights` directory inside the user's home directory. The weights directory of the model follows the naming convention so it
+would be named `Qwen2.5-7B-Instruct-1M`. The following yaml file would need to be created, lets say it is named `/h/<username>/my-model-config.yaml`.
+
+```yaml
+models:
+  Qwen2.5-7B-Instruct-1M:
+    model_family: Qwen2.5
+    model_variant: 7B-Instruct-1M
+    model_type: LLM
+    gpus_per_node: 1
+    num_nodes: 1
+    vocab_size: 152064
+    qos: m2
+    time: 08:00:00
+    partition: a40
+    model_weights_parent_dir: /h/<username>/model-weights
+    vllm_args:
+      --max-model-len: 1010000
+      --max-num-seqs: 256
+      --compilation-confi: 3
+```
+
+You would then set the `VEC_INF_CONFIG` path using:
+
+```bash
+export VEC_INF_CONFIG=/h/<username>/my-model-config.yaml
+```
+
+Note that there are other parameters that can also be added to the config but not shown in this example, check the [`ModelConfig`](vec_inf/client/config.py) for details.
+
+#### Other commands
+
+* `status`: Check the model status by providing its Slurm job ID, `--json-mode` supported.
+* `metrics`: Streams performance metrics to the console.
+* `shutdown`: Shutdown a model by providing its Slurm job ID.
+* `list`: List all available model names, or view the default/cached configuration of a specific model, `--json-mode` supported.
+
+For more details on the usage of these commands, refer to the [User Guide](https://vectorinstitute.github.io/vector-inference/user_guide/)
+
+### API
+
+Example:
+
+```python
+>>> from vec_inf.api import VecInfClient
+>>> client = VecInfClient()
+>>> response = client.launch_model("Meta-Llama-3.1-8B-Instruct")
+>>> job_id = response.slurm_job_id
+>>> status = client.get_status(job_id)
+>>> if status.status == ModelStatus.READY:
+...     print(f"Model is ready at {status.base_url}")
+>>> client.shutdown_model(job_id)
+```
+
+For details on the usage of the API, refer to the [API Reference](https://vectorinstitute.github.io/vector-inference/api/)
+
+## Check Job Configuration
+
+With every model launch, a Slurm script will be generated dynamically based on the job and model configuration. Once the Slurm job is queued, the generated Slurm script will be moved to the log directory for reproducibility, located at `$log_dir/$model_family/$model_name.$slurm_job_id/$model_name.$slurm_job_id.slurm`. In the same directory you can also find a JSON file with the same name that captures the launch configuration, and will have an entry of server URL once the server is ready.
+
+## Send inference requests
+
+Once the inference server is ready, you can start sending in inference requests. We provide example scripts for sending inference requests in [`examples`](examples) folder. Make sure to update the model server URL and the model weights location in the scripts. For example, you can run `python examples/inference/llm/chat_completions.py`, and you should expect to see an output like the following:
+
+```json
+{
+    "id":"chatcmpl-387c2579231948ffaf66cdda5439d3dc",
+    "choices": [
+        {
+            "finish_reason":"stop",
+            "index":0,
+            "logprobs":null,
+            "message": {
+                "content":"Arrr, I be Captain Chatbeard, the scurviest chatbot on the seven seas! Ye be wantin' to know me identity, eh? Well, matey, I be a swashbucklin' AI, here to provide ye with answers and swappin' tales, savvy?",
+                "role":"assistant",
+                "function_call":null,
+                "tool_calls":[],
+                "reasoning_content":null
+            },
+            "stop_reason":null
+        }
+    ],
+    "created":1742496683,
+    "model":"Meta-Llama-3.1-8B-Instruct",
+    "object":"chat.completion",
+    "system_fingerprint":null,
+    "usage": {
+        "completion_tokens":66,
+        "prompt_tokens":32,
+        "total_tokens":98,
+        "prompt_tokens_details":null
+    },
+    "prompt_logprobs":null
+}
+```
+**NOTE**: For multimodal models, currently only `ChatCompletion` is available, and only one image can be provided for each prompt.
+
+## SSH tunnel from your local device
+If you want to run inference from your local device, you can open a SSH tunnel to your cluster environment like the following:
+```bash
+ssh -L 8081:172.17.8.29:8081 username@v.vectorinstitute.ai -N
+```
+Where the last number in the URL is the GPU number (gpu029 in this case). The example provided above is for the vector cluster, change the variables accordingly for your environment