vec-inf 0.7.2__tar.gz → 0.8.0__tar.gz

{vec_inf-0.7.2 → vec_inf-0.8.0}/.github/workflows/code_checks.yml

@@ -28,7 +28,7 @@ jobs:
   run-code-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v5.0.0
+      - uses: actions/checkout@v6.0.1
       - name: Install uv
         uses: astral-sh/setup-uv@v7
         with:

vec_inf-0.8.0/.github/workflows/docker.yml

@@ -0,0 +1,85 @@
+name: docker
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+      - main
+    paths:
+      - vllm.Dockerfile
+      - sglang.Dockerfile
+      - .github/workflows/docker.yml
+      - uv.lock
+  pull_request:
+    branches:
+      - main
+      - f/sglang-support
+    paths:
+      - vllm.Dockerfile
+      - sglang.Dockerfile
+      - .github/workflows/docker.yml
+      - uv.lock
+
+jobs:
+  push_to_registry:
+    name: Build and push Docker images
+    runs-on:
+      - ubuntu-latest
+    strategy:
+      matrix:
+        backend: [vllm, sglang]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6.0.1
+
+      - name: Extract backend version
+        id: backend-version
+        run: |
+          VERSION=$(grep -A 1 "name = \"${{ matrix.backend }}\"" uv.lock | grep version | cut -d '"' -f 2)
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Maximize build space
+        run: |
+          echo "Disk space before cleanup:"
+          df -h
+          # Remove unnecessary pre-installed software
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo rm -rf /usr/local/share/boost
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          # Clean apt cache
+          sudo apt-get clean
+          # Remove docker images
+          docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+          echo "Disk space after cleanup:"
+          df -h
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051
+        with:
+          images: vectorinstitute/vector-inference-${{ matrix.backend }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
+        with:
+          context: .
+          file: ./${{ matrix.backend }}.Dockerfile
+          push: true
+          tags: |
+            ${{ steps.meta.outputs.tags }}
+            vectorinstitute/vector-inference-${{ matrix.backend }}:${{ steps.backend-version.outputs.version }}
+            vectorinstitute/vector-inference-${{ matrix.backend }}:latest
+          labels: ${{ steps.meta.outputs.labels }}

{vec_inf-0.7.2 → vec_inf-0.8.0}/.github/workflows/docs.yml

@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v5.0.0
+        uses: actions/checkout@v6.0.1
         with:
           fetch-depth: 0  # Fetch all history for proper versioning
 
@@ -67,7 +67,7 @@ jobs:
           python-version-file: ".python-version"
 
       - name: Install the project
-        run: uv sync --all-extras --group docs --prerelease=allow
+        run: uv sync --group docs --prerelease=allow
 
       - name: Build docs
         run: uv run --frozen mkdocs build
@@ -76,7 +76,7 @@ jobs:
         run: touch site/.nojekyll
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: docs-site
           path: site/
@@ -88,7 +88,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v5.0.0
+        uses: actions/checkout@v6.0.1
         with:
           fetch-depth: 0  # Fetch all history for proper versioning
 
@@ -104,7 +104,7 @@ jobs:
           python-version-file: ".python-version"
 
       - name: Install the project
-        run: uv sync --all-extras --group docs --frozen
+        run: uv sync --group docs --frozen
 
       - name: Configure Git Credentials
         run: |
@@ -112,7 +112,7 @@ jobs:
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
 
       - name: Download artifact
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v7
         with:
           name: docs-site
           path: site

{vec_inf-0.7.2 → vec_inf-0.8.0}/.github/workflows/publish.yml

@@ -13,7 +13,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install libcurl4-openssl-dev libssl-dev
 
-      - uses: actions/checkout@v5.0.0
+      - uses: actions/checkout@v6.0.1
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7

{vec_inf-0.7.2 → vec_inf-0.8.0}/.github/workflows/unit_tests.yml

@@ -43,7 +43,7 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
     steps:
-      - uses: actions/checkout@v5.0.0
+      - uses: actions/checkout@v6.0.1
 
       - name: Install uv
         uses: astral-sh/setup-uv@v7
@@ -58,16 +58,26 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install the project
+        env:
+          # Ensure uv uses the matrix interpreter instead of `.python-version` (3.10),
+          # otherwise the "3.11"/"3.12" jobs silently run on 3.10.
+          UV_PYTHON: ${{ matrix.python-version }}
         run: uv sync --dev --prerelease=allow
 
       - name: Install dependencies and check code
+        env:
+          UV_PYTHON: ${{ matrix.python-version }}
         run: |
           uv run --frozen pytest -m "not integration_test" --cov vec_inf --cov-report=xml tests
 
       - name: Install the core package only
+        env:
+          UV_PYTHON: ${{ matrix.python-version }}
         run: uv sync --no-dev
 
       - name: Run package import tests
+        env:
+          UV_PYTHON: ${{ matrix.python-version }}
         run: |
           uv run --frozen pytest tests/test_imports.py
 
@@ -76,7 +86,7 @@ jobs:
           gpg --keyserver keyserver.ubuntu.com --recv-keys 806BB28AED779869
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v5.5.1
+        uses: codecov/codecov-action@v5.5.2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./coverage.xml

{vec_inf-0.7.2 → vec_inf-0.8.0}/.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
     - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.14.3'
+    rev: 'v0.14.10'
     hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]
@@ -26,7 +26,7 @@ repos:
       types_or: [python, jupyter]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.18.2
+    rev: v1.19.1
     hooks:
     - id: mypy
       entry: python3 -m mypy --config-file pyproject.toml

{vec_inf-0.7.2 → vec_inf-0.8.0}/MODEL_TRACKING.md

@@ -94,6 +94,7 @@ This document tracks all model weights available in the `/model-weights` directo
 | Model | Configuration |
 |:------|:-------------|
 | `Llama-4-Scout-17B-16E-Instruct` | ❌ |
+| `Llama-4-Maverick-17B-128E-Instruct` | ❌ |
 
 ### Mistral AI: Mistral
 | Model | Configuration |
@@ -128,6 +129,7 @@ This document tracks all model weights available in the `/model-weights` directo
 |:------|:-------------|
 | `Qwen2.5-0.5B-Instruct` | ✅ |
 | `Qwen2.5-1.5B-Instruct` | ✅ |
+| `Qwen2.5-3B` | ❌ |
 | `Qwen2.5-3B-Instruct` | ✅ |
 | `Qwen2.5-7B-Instruct` | ✅ |
 | `Qwen2.5-14B-Instruct` | ✅ |
@@ -138,12 +140,14 @@ This document tracks all model weights available in the `/model-weights` directo
 | Model | Configuration |
 |:------|:-------------|
 | `Qwen2.5-Math-1.5B-Instruct` | ✅ |
+| `Qwen2.5-Math-7B` | ❌ |
 | `Qwen2.5-Math-7B-Instruct` | ✅ |
 | `Qwen2.5-Math-72B-Instruct` | ✅ |
 
 ### Qwen: Qwen2.5-Coder
 | Model | Configuration |
 |:------|:-------------|
+| `Qwen2.5-Coder-3B-Instruct` | ✅ |
 | `Qwen2.5-Coder-7B-Instruct` | ✅ |
 
 ### Qwen: QwQ
@@ -162,6 +166,12 @@ This document tracks all model weights available in the `/model-weights` directo
 | `Qwen2-Math-72B-Instruct` | ❌ |
 | `Qwen2-VL-7B-Instruct` | ❌ |
 
+### Qwen: Qwen2.5-VL
+| Model | Configuration |
+|:------|:-------------|
+| `Qwen2.5-VL-3B-Instruct` | ❌ |
+| `Qwen2.5-VL-7B-Instruct` | ✅ |
+
 ### Qwen: Qwen3
 | Model | Configuration |
 |:------|:-------------|
@@ -191,27 +201,76 @@ This document tracks all model weights available in the `/model-weights` directo
 | Model | Configuration |
 |:------|:-------------|
 | `gpt-oss-120b` | ✅ |
+| `gpt-oss-20b` | ✅ |
 
-### Other LLM Models
+
+#### AI21: Jamba
 | Model | Configuration |
 |:------|:-------------|
 | `AI21-Jamba-1.5-Mini` | ❌ |
-| `aya-expanse-32b` | ✅ (as Aya-Expanse-32B) |
+
+#### Cohere for AI: Aya
+| Model | Configuration |
+|:------|:-------------|
+| `aya-expanse-32b` | ✅ |
+
+#### OpenAI: GPT-2
+| Model | Configuration |
+|:------|:-------------|
 | `gpt2-large` | ❌ |
 | `gpt2-xl` | ❌ |
-| `gpt-oss-120b` | ❌ |
-| `instructblip-vicuna-7b` | ❌ |
+
+#### InternLM: InternLM2
+| Model | Configuration |
+|:------|:-------------|
 | `internlm2-math-plus-7b` | ❌ |
+
+#### Janus
+| Model | Configuration |
+|:------|:-------------|
 | `Janus-Pro-7B` | ❌ |
+
+#### Moonshot AI: Kimi
+| Model | Configuration |
+|:------|:-------------|
 | `Kimi-K2-Instruct` | ❌ |
+
+#### Mistral AI: Ministral
+| Model | Configuration |
+|:------|:-------------|
 | `Ministral-8B-Instruct-2410` | ❌ |
-| `Molmo-7B-D-0924` | ✅ |
+
+#### AI2: OLMo
+| Model | Configuration |
+|:------|:-------------|
 | `OLMo-1B-hf` | ❌ |
 | `OLMo-7B-hf` | ❌ |
 | `OLMo-7B-SFT` | ❌ |
+
+#### EleutherAI: Pythia
+| Model | Configuration |
+|:------|:-------------|
 | `pythia` | ❌ |
+
+#### Qwen: Qwen1.5
+| Model | Configuration |
+|:------|:-------------|
 | `Qwen1.5-72B-Chat` | ❌ |
+
+#### ReasonFlux
+| Model | Configuration |
+|:------|:-------------|
 | `ReasonFlux-PRM-7B` | ❌ |
+
+#### LMSYS: Vicuna
+| Model | Configuration |
+|:------|:-------------|
+| `vicuna-13b-v1.5` | ❌ |
+
+#### Google: T5 (Encoder-Decoder Models)
+**Note**: These are encoder-decoder (T5) models, not decoder-only LLMs.
+| Model | Configuration |
+|:------|:-------------|
 | `t5-large-lm-adapt` | ❌ |
 | `t5-xl-lm-adapt` | ❌ |
 | `mt5-xl-lm-adapt` | ❌ |
@@ -238,10 +297,10 @@ This document tracks all model weights available in the `/model-weights` directo
 ### Meta: Llama 3.2 Vision
 | Model | Configuration |
 |:------|:-------------|
-| `Llama-3.2-11B-Vision` | ✅ |
-| `Llama-3.2-11B-Vision-Instruct` | ✅ |
-| `Llama-3.2-90B-Vision` | ✅ |
-| `Llama-3.2-90B-Vision-Instruct` | ✅ |
+| `Llama-3.2-11B-Vision` | ❌ |
+| `Llama-3.2-11B-Vision-Instruct` | ✅ | (SGLang only)
+| `Llama-3.2-90B-Vision` | ❌ |
+| `Llama-3.2-90B-Vision-Instruct` | ✅ | (SGLang only)
 
 ### Mistral: Pixtral
 | Model | Configuration |
@@ -266,10 +325,19 @@ This document tracks all model weights available in the `/model-weights` directo
 | `deepseek-vl2` | ✅ |
 | `deepseek-vl2-small` | ✅ |
 
+### Google: MedGemma
+| Model | Configuration |
+|:------|:-------------|
+| `medgemma-4b-it` | ✅ |
+| `medgemma-27b-it` | ✅ |
+| `medgemma-27b-text-it` | ❌ |
+
 ### Other VLM Models
 | Model | Configuration |
 |:------|:-------------|
+| `instructblip-vicuna-7b` | ❌ |
 | `MiniCPM-Llama3-V-2_5` | ❌ |
+| `Molmo-7B-D-0924` | ✅ |
 
 ---
 
@@ -298,6 +366,8 @@ This document tracks all model weights available in the `/model-weights` directo
 | `data2vec` | ❌ |
 | `gte-modernbert-base` | ❌ |
 | `gte-Qwen2-7B-instruct` | ❌ |
+| `KaLM-Embedding-Gemma3-12B-2511` | ❌ |
+| `llama-embed-nemotron-8b` | ❌ |
 | `m2-bert-80M-32k-retrieval` | ❌ |
 | `m2-bert-80M-8k-retrieval` | ❌ |
 
@@ -313,7 +383,7 @@ This document tracks all model weights available in the `/model-weights` directo
 
 ---
 
-## Multimodal Models
+## Vision Models
 
 ### CLIP
 | Model | Configuration |

{vec_inf-0.7.2 → vec_inf-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vec-inf
-Version: 0.7.2
+Version: 0.8.0
 Summary: Efficient LLM inference on Slurm clusters using vLLM.
 Author-email: Marshall Wang <marshall.wang@vectorinstitute.ai>
 License-Expression: MIT
@@ -11,14 +11,14 @@ Requires-Dist: pydantic>=2.10.6
 Requires-Dist: pyyaml>=6.0.2
 Requires-Dist: requests>=2.31.0
 Requires-Dist: rich>=13.7.0
-Provides-Extra: dev
-Requires-Dist: cupy-cuda12x==12.1.0; extra == 'dev'
-Requires-Dist: flashinfer-python>=0.4.0; extra == 'dev'
-Requires-Dist: ray[default]>=2.50.0; extra == 'dev'
-Requires-Dist: sglang>=0.5.0; extra == 'dev'
-Requires-Dist: torch>=2.7.0; extra == 'dev'
-Requires-Dist: vllm>=0.10.0; extra == 'dev'
-Requires-Dist: xgrammar>=0.1.11; extra == 'dev'
+Provides-Extra: sglang
+Requires-Dist: orjson>=3.11.0; extra == 'sglang'
+Requires-Dist: sgl-kernel>=0.3.0; extra == 'sglang'
+Requires-Dist: sglang>=0.5.5; extra == 'sglang'
+Requires-Dist: torchao>=0.9.0; extra == 'sglang'
+Provides-Extra: vllm
+Requires-Dist: ray[default]>=2.51.0; extra == 'vllm'
+Requires-Dist: vllm>=0.11.2; extra == 'vllm'
 Description-Content-Type: text/markdown
 
 # Vector Inference: Easy inference on Slurm clusters
@@ -30,10 +30,11 @@ Description-Content-Type: text/markdown
 [![code checks](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml)
 [![docs](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml)
 [![codecov](https://codecov.io/github/VectorInstitute/vector-inference/branch/main/graph/badge.svg?token=NI88QSIGAC)](https://app.codecov.io/github/VectorInstitute/vector-inference/tree/main)
-[![vLLM](https://img.shields.io/badge/vLLM-0.10.1.1-blue)](https://docs.vllm.ai/en/v0.10.1.1/)
+[![vLLM](https://img.shields.io/badge/vLLM-0.12.0-blue)](https://docs.vllm.ai/en/v0.12.0/)
+[![SGLang](https://img.shields.io/badge/SGLang-0.5.5.post3-blue)](https://docs.sglang.io/index.html)
 ![GitHub License](https://img.shields.io/github/license/VectorInstitute/vector-inference)
 
-This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using [vLLM](https://docs.vllm.ai/en/latest/). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation).
+This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using open-source inference engines ([vLLM](https://docs.vllm.ai/en/v0.12.0/), [SGLang](https://docs.sglang.io/index.html)). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation).
 
 **NOTE**: Supported models on Killarney are tracked [here](./MODEL_TRACKING.md)
 
@@ -43,12 +44,12 @@ If you are using the Vector cluster environment, and you don't need any customiz
 ```bash
 pip install vec-inf
 ```
-Otherwise, we recommend using the provided [`Dockerfile`](Dockerfile) to set up your own environment with the package. The latest image has `vLLM` version `0.10.1.1`.
+Otherwise, we recommend using the provided [`vllm.Dockerfile`](vllm.Dockerfile) and [`sglang.Dockerfile`](sglang.Dockerfile) to set up your own environment with the package. The built images are available through [Docker Hub](https://hub.docker.com/orgs/vectorinstitute/repositories)
 
 If you'd like to use `vec-inf` on your own Slurm cluster, you would need to update the configuration files, there are 3 ways to do it:
 * Clone the repository and update the `environment.yaml` and the `models.yaml` file in [`vec_inf/config`](vec_inf/config/), then install from source by running `pip install .`.
 * The package would try to look for cached configuration files in your environment before using the default configuration. The default cached configuration directory path points to `/model-weights/vec-inf-shared`, you would need to create an `environment.yaml` and a `models.yaml` following the format of these files in [`vec_inf/config`](vec_inf/config/).
-* The package would also look for an enviroment variable `VEC_INF_CONFIG_DIR`. You can put your `environment.yaml` and `models.yaml` in a directory of your choice and set the enviroment variable `VEC_INF_CONFIG_DIR` to point to that location.
+* [OPTIONAL] The package could also look for an enviroment variable `VEC_INF_CONFIG_DIR`. You can put your `environment.yaml` and `models.yaml` in a directory of your choice and set the enviroment variable `VEC_INF_CONFIG_DIR` to point to that location.
 
 ## Usage
 
@@ -65,18 +66,18 @@ vec-inf launch Meta-Llama-3.1-8B-Instruct
 ```
 You should see an output like the following:
 
-<img width="720" alt="launch_image" src="https://github.com/user-attachments/assets/c1e0c60c-cf7a-49ed-a426-fdb38ebf88ee" />
+<img width="720" alt="launch_image" src="./docs/assets/launch.png" />
 
 **NOTE**: You can set the required fields in the environment configuration (`environment.yaml`), it's a mapping between required arguments and their corresponding environment variables. On the Vector **Killarney** Cluster environment, the required fields are:
   * `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`.
   * `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`.
 
-Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be overriden. You can also launch your own custom model as long as the model architecture is [supported by vLLM](https://docs.vllm.ai/en/stable/models/supported_models.html). For detailed instructions on how to customize your model launch, check out the [`launch` command section in User Guide](https://vectorinstitute.github.io/vector-inference/latest/user_guide/#launch-command)
+Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be overriden. You can also launch your own custom model as long as the model architecture is supported by the underlying inference engine. For detailed instructions on how to customize your model launch, check out the [`launch` command section in User Guide](https://vectorinstitute.github.io/vector-inference/latest/user_guide/#launch-command)
 
 #### Other commands
 
 * `batch-launch`: Launch multiple model inference servers at once, currently ONLY single node models supported,
-* `status`: Check the model status by providing its Slurm job ID.
+* `status`: Check the status of all `vec-inf` jobs, or a specific job by providing its job ID.
 * `metrics`: Streams performance metrics to the console.
 * `shutdown`: Shutdown a model by providing its Slurm job ID.
 * `list`: List all available model names, or view the default/cached configuration of a specific model.

{vec_inf-0.7.2 → vec_inf-0.8.0}/README.md

@@ -7,10 +7,11 @@
 [![code checks](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml)
 [![docs](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml)
 [![codecov](https://codecov.io/github/VectorInstitute/vector-inference/branch/main/graph/badge.svg?token=NI88QSIGAC)](https://app.codecov.io/github/VectorInstitute/vector-inference/tree/main)
-[![vLLM](https://img.shields.io/badge/vLLM-0.10.1.1-blue)](https://docs.vllm.ai/en/v0.10.1.1/)
+[![vLLM](https://img.shields.io/badge/vLLM-0.12.0-blue)](https://docs.vllm.ai/en/v0.12.0/)
+[![SGLang](https://img.shields.io/badge/SGLang-0.5.5.post3-blue)](https://docs.sglang.io/index.html)
 ![GitHub License](https://img.shields.io/github/license/VectorInstitute/vector-inference)
 
-This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using [vLLM](https://docs.vllm.ai/en/latest/). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation).
+This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using open-source inference engines ([vLLM](https://docs.vllm.ai/en/v0.12.0/), [SGLang](https://docs.sglang.io/index.html)). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation).
 
 **NOTE**: Supported models on Killarney are tracked [here](./MODEL_TRACKING.md)
 
@@ -20,12 +21,12 @@ If you are using the Vector cluster environment, and you don't need any customiz
 ```bash
 pip install vec-inf
 ```
-Otherwise, we recommend using the provided [`Dockerfile`](Dockerfile) to set up your own environment with the package. The latest image has `vLLM` version `0.10.1.1`.
+Otherwise, we recommend using the provided [`vllm.Dockerfile`](vllm.Dockerfile) and [`sglang.Dockerfile`](sglang.Dockerfile) to set up your own environment with the package. The built images are available through [Docker Hub](https://hub.docker.com/orgs/vectorinstitute/repositories)
 
 If you'd like to use `vec-inf` on your own Slurm cluster, you would need to update the configuration files, there are 3 ways to do it:
 * Clone the repository and update the `environment.yaml` and the `models.yaml` file in [`vec_inf/config`](vec_inf/config/), then install from source by running `pip install .`.
 * The package would try to look for cached configuration files in your environment before using the default configuration. The default cached configuration directory path points to `/model-weights/vec-inf-shared`, you would need to create an `environment.yaml` and a `models.yaml` following the format of these files in [`vec_inf/config`](vec_inf/config/).
-* The package would also look for an enviroment variable `VEC_INF_CONFIG_DIR`. You can put your `environment.yaml` and `models.yaml` in a directory of your choice and set the enviroment variable `VEC_INF_CONFIG_DIR` to point to that location.
+* [OPTIONAL] The package could also look for an enviroment variable `VEC_INF_CONFIG_DIR`. You can put your `environment.yaml` and `models.yaml` in a directory of your choice and set the enviroment variable `VEC_INF_CONFIG_DIR` to point to that location.
 
 ## Usage
 
@@ -42,18 +43,18 @@ vec-inf launch Meta-Llama-3.1-8B-Instruct
 ```
 You should see an output like the following:
 
-<img width="720" alt="launch_image" src="https://github.com/user-attachments/assets/c1e0c60c-cf7a-49ed-a426-fdb38ebf88ee" />
+<img width="720" alt="launch_image" src="./docs/assets/launch.png" />
 
 **NOTE**: You can set the required fields in the environment configuration (`environment.yaml`), it's a mapping between required arguments and their corresponding environment variables. On the Vector **Killarney** Cluster environment, the required fields are:
   * `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`.
   * `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`.
 
-Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be overriden. You can also launch your own custom model as long as the model architecture is [supported by vLLM](https://docs.vllm.ai/en/stable/models/supported_models.html). For detailed instructions on how to customize your model launch, check out the [`launch` command section in User Guide](https://vectorinstitute.github.io/vector-inference/latest/user_guide/#launch-command)
+Models that are already supported by `vec-inf` would be launched using the cached configuration (set in [slurm_vars.py](vec_inf/client/slurm_vars.py)) or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be overriden. You can also launch your own custom model as long as the model architecture is supported by the underlying inference engine. For detailed instructions on how to customize your model launch, check out the [`launch` command section in User Guide](https://vectorinstitute.github.io/vector-inference/latest/user_guide/#launch-command)
 
 #### Other commands
 
 * `batch-launch`: Launch multiple model inference servers at once, currently ONLY single node models supported,
-* `status`: Check the model status by providing its Slurm job ID.
+* `status`: Check the status of all `vec-inf` jobs, or a specific job by providing its job ID.
 * `metrics`: Streams performance metrics to the console.
 * `shutdown`: Shutdown a model by providing its Slurm job ID.
 * `list`: List all available model names, or view the default/cached configuration of a specific model.

{vec_inf-0.7.2 → vec_inf-0.8.0}/docs/index.md

@@ -1,6 +1,7 @@
 # Vector Inference: Easy inference on Slurm clusters
 
-This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using [vLLM](https://docs.vllm.ai/en/stable/). **This package runs natively on the Vector Institute cluster environment**. To adapt to other environments, follow the instructions in [Installation](#installation).
+This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using open-source inference engines ([vLLM](https://docs.vllm.ai/en/v0.12.0/), [SGLang](https://docs.sglang.io/index.html)). **This package runs natively on the Vector Institute cluster environments**. To adapt to other environments, follow the instructions in [Installation](#installation).
+
 
 **NOTE**: Supported models on Killarney are tracked [here](https://github.com/VectorInstitute/vector-inference/blob/main/MODEL_TRACKING.md)
 
@@ -12,9 +13,10 @@ If you are using the Vector cluster environment, and you don't need any customiz
 pip install vec-inf
 ```
 
-Otherwise, we recommend using the provided [`Dockerfile`](https://github.com/VectorInstitute/vector-inference/blob/main/Dockerfile) to set up your own environment with the package. The latest image has `vLLM` version `0.10.1.1`.
+Otherwise, we recommend using the provided [`vllm.Dockerfile`](https://github.com/VectorInstitute/vector-inference/blob/main/vllm.Dockerfile) and [`sglang.Dockerfile`](https://github.com/VectorInstitute/vector-inference/blob/main/sglang.Dockerfile) to set up your own environment with the package. The built images are available through [Docker Hub](https://hub.docker.com/orgs/vectorinstitute/repositories)
 
 If you'd like to use `vec-inf` on your own Slurm cluster, you would need to update the configuration files, there are 3 ways to do it:
+
 * Clone the repository and update the `environment.yaml` and the `models.yaml` file in [`vec_inf/config`](https://github.com/VectorInstitute/vector-inference/blob/main/vec_inf/config), then install from source by running `pip install .`.
 * The package would try to look for cached configuration files in your environment before using the default configuration. The default cached configuration directory path points to `/model-weights/vec-inf-shared`, you would need to create an `environment.yaml` and a `models.yaml` following the format of these files in [`vec_inf/config`](https://github.com/VectorInstitute/vector-inference/blob/main/vec_inf/config).
-* The package would also look for an enviroment variable `VEC_INF_CONFIG_DIR`. You can put your `environment.yaml` and `models.yaml` in a directory of your choice and set the enviroment variable `VEC_INF_CONFIG_DIR` to point to that location.
+* [OPTIONAL] The package would also look for an enviroment variable `VEC_INF_CONFIG_DIR`. You can put your `environment.yaml` and `models.yaml` in a directory of your choice and set the enviroment variable `VEC_INF_CONFIG_DIR` to point to that location.