vec-inf 0.7.0__tar.gz → 0.7.2__tar.gz

{vec_inf-0.7.0 → vec_inf-0.7.2}/.github/workflows/code_checks.yml

@@ -30,17 +30,17 @@ jobs:
     steps:
       - uses: actions/checkout@v5.0.0
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           # Install a specific version of uv.
           version: "0.5.21"
           enable-cache: true
       - name: "Set up Python"
-        uses: actions/setup-python@v5.5.0
+        uses: actions/setup-python@v6
         with:
           python-version-file: ".python-version"
       - name: Install the project
-        run: uv sync --dev
+        run: uv sync --dev --prerelease=allow
       - name: Install dependencies and check code
         run: |
           source .venv/bin/activate
@@ -49,3 +49,5 @@ jobs:
         uses: pypa/gh-action-pip-audit@v1.1.0
         with:
           virtual-environment: .venv/
+          # Temporary: ignore pip advisory until fixed in pip>=25.3
+          ignore-vulns: GHSA-4xh5-x5gv-qwph

{vec_inf-0.7.0 → vec_inf-0.7.2}/.github/workflows/docker.yml

@@ -33,7 +33,7 @@ jobs:
           echo "version=$VERSION" >> $GITHUB_OUTPUT
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1
+        uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}

{vec_inf-0.7.0 → vec_inf-0.7.2}/.github/workflows/docs.yml

@@ -56,27 +56,27 @@ jobs:
           fetch-depth: 0  # Fetch all history for proper versioning
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           version: "0.5.21"
           enable-cache: true
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version-file: ".python-version"
 
       - name: Install the project
-        run: uv sync --all-extras --group docs
+        run: uv sync --all-extras --group docs --prerelease=allow
 
       - name: Build docs
-        run: uv run mkdocs build
+        run: uv run --frozen mkdocs build
 
       - name: Create .nojekyll file
         run: touch site/.nojekyll
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v5
         with:
           name: docs-site
           path: site/
@@ -93,18 +93,18 @@ jobs:
           fetch-depth: 0  # Fetch all history for proper versioning
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           version: "0.5.21"
           enable-cache: true
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version-file: ".python-version"
 
       - name: Install the project
-        run: uv sync --all-extras --group docs
+        run: uv sync --all-extras --group docs --frozen
 
       - name: Configure Git Credentials
         run: |
@@ -112,7 +112,7 @@ jobs:
           git config user.email 41898282+github-actions[bot]@users.noreply.github.com
 
       - name: Download artifact
-        uses: actions/download-artifact@v5
+        uses: actions/download-artifact@v6
         with:
           name: docs-site
           path: site

{vec_inf-0.7.0 → vec_inf-0.7.2}/.github/workflows/publish.yml

@@ -16,12 +16,12 @@ jobs:
       - uses: actions/checkout@v5.0.0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           version: "0.6.6"
           enable-cache: true
 
-      - uses: actions/setup-python@v5.5.0
+      - uses: actions/setup-python@v6
         with:
           python-version: '3.10'
 

{vec_inf-0.7.0 → vec_inf-0.7.2}/.github/workflows/unit_tests.yml

@@ -46,40 +46,40 @@ jobs:
       - uses: actions/checkout@v5.0.0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
           # Install a specific version of uv.
           version: "0.5.21"
           enable-cache: true
 
       - name: "Set up Python ${{ matrix.python-version }}"
-        uses: actions/setup-python@v5.5.0
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install the project
-        run: uv sync --dev
+        run: uv sync --dev --prerelease=allow
 
       - name: Install dependencies and check code
         run: |
-          uv run pytest -m "not integration_test" --cov vec_inf --cov-report=xml tests
+          uv run --frozen pytest -m "not integration_test" --cov vec_inf --cov-report=xml tests
 
       - name: Install the core package only
         run: uv sync --no-dev
 
       - name: Run package import tests
         run: |
-          uv run pytest tests/test_imports.py
+          uv run --frozen pytest tests/test_imports.py
 
       - name: Import Codecov GPG public key
         run: |
           gpg --keyserver keyserver.ubuntu.com --recv-keys 806BB28AED779869
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v5.5.0
+        uses: codecov/codecov-action@v5.5.1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          file: ./coverage.xml
+          files: ./coverage.xml
           name: codecov-umbrella
           fail_ci_if_error: true
           verbose: true

{vec_inf-0.7.0 → vec_inf-0.7.2}/.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
     - id: check-toml
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 'v0.12.10'
+    rev: 'v0.14.3'
     hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]
@@ -26,7 +26,7 @@ repos:
       types_or: [python, jupyter]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.17.1
+    rev: v1.18.2
     hooks:
     - id: mypy
       entry: python3 -m mypy --config-file pyproject.toml

{vec_inf-0.7.0 → vec_inf-0.7.2}/Dockerfile

@@ -35,29 +35,33 @@ RUN wget https://bootstrap.pypa.io/get-pip.py && \
     rm get-pip.py && \
     python3.10 -m pip install --upgrade pip setuptools wheel uv
 
-# Install Infiniband/RDMA support
+# Install RDMA support
 RUN apt-get update && apt-get install -y \
     libibverbs1 libibverbs-dev ibverbs-utils \
     librdmacm1 librdmacm-dev rdmacm-utils \
+    rdma-core ibverbs-providers infiniband-diags perftest \
     && rm -rf /var/lib/apt/lists/*
 
 # Set up RDMA environment (these will persist in the final container)
 ENV LD_LIBRARY_PATH="/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH"
-ENV UCX_NET_DEVICES=all
 ENV NCCL_IB_DISABLE=0
+ENV NCCL_SOCKET_IFNAME="^lo,docker0"
+ENV NCCL_NET_GDR_LEVEL=PHB
+ENV NCCL_IB_TIMEOUT=22
+ENV NCCL_IB_RETRY_CNT=7
+ENV NCCL_DEBUG=INFO
 
 # Set up project
 WORKDIR /vec-inf
 COPY . /vec-inf
 
 # Install project dependencies with build requirements
-RUN PIP_INDEX_URL="https://download.pytorch.org/whl/cu128" uv pip install --system -e .[dev]
+RUN uv pip install --system -e .[dev] --prerelease=allow
 
-# Final configuration
-RUN mkdir -p /vec-inf/nccl && \
-    mv /root/.config/vllm/nccl/cu12/libnccl.so.2.18.1 /vec-inf/nccl/libnccl.so.2.18.1
-ENV VLLM_NCCL_SO_PATH=/vec-inf/nccl/libnccl.so.2.18.1
-ENV NCCL_DEBUG=INFO
+# Install a single, system NCCL (from NVIDIA CUDA repo in base image)
+RUN apt-get update && apt-get install -y --allow-change-held-packages\
+    libnccl2 libnccl-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 # Set the default command to start an interactive shell
 CMD ["bash"]

{vec_inf-0.7.0 → vec_inf-0.7.2}/MODEL_TRACKING.md

@@ -40,6 +40,7 @@ This document tracks all model weights available in the `/model-weights` directo
 | `gemma-2b-it` | ❌ |
 | `gemma-7b` | ❌ |
 | `gemma-7b-it` | ❌ |
+| `gemma-2-2b-it` | ✅ |
 | `gemma-2-9b` | ✅ |
 | `gemma-2-9b-it` | ✅ |
 | `gemma-2-27b` | ✅ |
@@ -165,8 +166,8 @@ This document tracks all model weights available in the `/model-weights` directo
 | Model | Configuration |
 |:------|:-------------|
 | `Qwen3-14B` | ✅ |
-| `Qwen3-8B` | ❌ |
-| `Qwen3-32B` | ❌ |
+| `Qwen3-8B` | ✅ |
+| `Qwen3-32B` | ✅ |
 | `Qwen3-235B-A22B` | ❌ |
 | `Qwen3-Embedding-8B` | ❌ |
 
@@ -186,6 +187,11 @@ This document tracks all model weights available in the `/model-weights` directo
 | `DeepSeek-Coder-V2-Lite-Instruct` | ❌ |
 | `deepseek-math-7b-instruct` | ❌ |
 
+### OpenAI: GPT-OSS
+| Model | Configuration |
+|:------|:-------------|
+| `gpt-oss-120b` | ✅ |
+
 ### Other LLM Models
 | Model | Configuration |
 |:------|:-------------|

vec_inf-0.7.0/README.md → vec_inf-0.7.2/PKG-INFO

@@ -1,3 +1,26 @@
+Metadata-Version: 2.4
+Name: vec-inf
+Version: 0.7.2
+Summary: Efficient LLM inference on Slurm clusters using vLLM.
+Author-email: Marshall Wang <marshall.wang@vectorinstitute.ai>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.0
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: requests>=2.31.0
+Requires-Dist: rich>=13.7.0
+Provides-Extra: dev
+Requires-Dist: cupy-cuda12x==12.1.0; extra == 'dev'
+Requires-Dist: flashinfer-python>=0.4.0; extra == 'dev'
+Requires-Dist: ray[default]>=2.50.0; extra == 'dev'
+Requires-Dist: sglang>=0.5.0; extra == 'dev'
+Requires-Dist: torch>=2.7.0; extra == 'dev'
+Requires-Dist: vllm>=0.10.0; extra == 'dev'
+Requires-Dist: xgrammar>=0.1.11; extra == 'dev'
+Description-Content-Type: text/markdown
+
 # Vector Inference: Easy inference on Slurm clusters
 
 ----------------------------------------------------
@@ -44,7 +67,7 @@ You should see an output like the following:
 
 <img width="720" alt="launch_image" src="https://github.com/user-attachments/assets/c1e0c60c-cf7a-49ed-a426-fdb38ebf88ee" />
 
-**NOTE**: On Vector Killarney Cluster environment, the following fields are required:
+**NOTE**: You can set the required fields in the environment configuration (`environment.yaml`), it's a mapping between required arguments and their corresponding environment variables. On the Vector **Killarney** Cluster environment, the required fields are:
   * `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`.
   * `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`.
 
@@ -74,6 +97,11 @@ Example:
 >>> status = client.get_status(job_id)
 >>> if status.status == ModelStatus.READY:
 ...     print(f"Model is ready at {status.base_url}")
+>>> # Alternatively, use wait_until_ready which will either return a StatusResponse or throw a ServerError
+>>> try:
+>>>     status = wait_until_ready(job_id)
+>>> except ServerError as e:
+>>>     print(f"Model launch failed: {e}")
 >>> client.shutdown_model(job_id)
 ```
 
@@ -124,6 +152,19 @@ Once the inference server is ready, you can start sending in inference requests.
 ## SSH tunnel from your local device
 If you want to run inference from your local device, you can open a SSH tunnel to your cluster environment like the following:
 ```bash
-ssh -L 8081:172.17.8.29:8081 username@v.vectorinstitute.ai -N
+ssh -L 8081:10.1.1.29:8081 username@v.vectorinstitute.ai -N
+```
+The example provided above is for the Vector Killarney cluster, change the variables accordingly for your environment. The IP address for the compute nodes on Killarney follow `10.1.1.XX` pattern, where `XX` is the GPU number (`kn029` -> `29` in this example).
+
+## Reference
+If you found Vector Inference useful in your research or applications, please cite using the following BibTeX template:
+```
+@software{vector_inference,
+  title        = {Vector Inference: Efficient LLM inference on Slurm clusters using vLLM},
+  author       = {Wang, Marshall},
+  organization = {Vector Institute},
+  year         = {<YEAR_OF_RELEASE>},
+  version      = {<VERSION_TAG>},
+  url          = {https://github.com/VectorInstitute/vector-inference}
+}
 ```
-Where the last number in the URL is the GPU number (gpu029 in this case). The example provided above is for the vector cluster, change the variables accordingly for your environment

vec_inf-0.7.0/PKG-INFO → vec_inf-0.7.2/README.md

@@ -1,25 +1,3 @@
-Metadata-Version: 2.4
-Name: vec-inf
-Version: 0.7.0
-Summary: Efficient LLM inference on Slurm clusters using vLLM.
-Author-email: Marshall Wang <marshall.wang@vectorinstitute.ai>
-License-Expression: MIT
-License-File: LICENSE
-Requires-Python: >=3.10
-Requires-Dist: click>=8.1.0
-Requires-Dist: pydantic>=2.10.6
-Requires-Dist: pyyaml>=6.0.2
-Requires-Dist: requests>=2.31.0
-Requires-Dist: rich>=13.7.0
-Provides-Extra: dev
-Requires-Dist: cupy-cuda12x==12.1.0; extra == 'dev'
-Requires-Dist: ray>=2.40.0; extra == 'dev'
-Requires-Dist: torch>=2.7.0; extra == 'dev'
-Requires-Dist: vllm-nccl-cu12<2.19,>=2.18; extra == 'dev'
-Requires-Dist: vllm>=0.10.0; extra == 'dev'
-Requires-Dist: xgrammar>=0.1.11; extra == 'dev'
-Description-Content-Type: text/markdown
-
 # Vector Inference: Easy inference on Slurm clusters
 
 ----------------------------------------------------
@@ -66,7 +44,7 @@ You should see an output like the following:
 
 <img width="720" alt="launch_image" src="https://github.com/user-attachments/assets/c1e0c60c-cf7a-49ed-a426-fdb38ebf88ee" />
 
-**NOTE**: On Vector Killarney Cluster environment, the following fields are required:
+**NOTE**: You can set the required fields in the environment configuration (`environment.yaml`), it's a mapping between required arguments and their corresponding environment variables. On the Vector **Killarney** Cluster environment, the required fields are:
   * `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`.
   * `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`.
 
@@ -96,6 +74,11 @@ Example:
 >>> status = client.get_status(job_id)
 >>> if status.status == ModelStatus.READY:
 ...     print(f"Model is ready at {status.base_url}")
+>>> # Alternatively, use wait_until_ready which will either return a StatusResponse or throw a ServerError
+>>> try:
+>>>     status = wait_until_ready(job_id)
+>>> except ServerError as e:
+>>>     print(f"Model launch failed: {e}")
 >>> client.shutdown_model(job_id)
 ```
 
@@ -146,6 +129,19 @@ Once the inference server is ready, you can start sending in inference requests.
 ## SSH tunnel from your local device
 If you want to run inference from your local device, you can open a SSH tunnel to your cluster environment like the following:
 ```bash
-ssh -L 8081:172.17.8.29:8081 username@v.vectorinstitute.ai -N
+ssh -L 8081:10.1.1.29:8081 username@v.vectorinstitute.ai -N
+```
+The example provided above is for the Vector Killarney cluster, change the variables accordingly for your environment. The IP address for the compute nodes on Killarney follow `10.1.1.XX` pattern, where `XX` is the GPU number (`kn029` -> `29` in this example).
+
+## Reference
+If you found Vector Inference useful in your research or applications, please cite using the following BibTeX template:
+```
+@software{vector_inference,
+  title        = {Vector Inference: Efficient LLM inference on Slurm clusters using vLLM},
+  author       = {Wang, Marshall},
+  organization = {Vector Institute},
+  year         = {<YEAR_OF_RELEASE>},
+  version      = {<VERSION_TAG>},
+  url          = {https://github.com/VectorInstitute/vector-inference}
+}
 ```
-Where the last number in the URL is the GPU number (gpu029 in this case). The example provided above is for the vector cluster, change the variables accordingly for your environment

{vec_inf-0.7.0 → vec_inf-0.7.2}/docs/user_guide.md

@@ -37,7 +37,7 @@ You should see an output like the following:
 └─────────────────────────┴───────────────────────────────────────────┘
 ```
 
-**NOTE**: On Vector Killarney Cluster environment, the following fields are required:
+**NOTE**: You can set the required fields in the environment configuration (`environment.yaml`), it's a mapping between required arguments and their corresponding environment variables. On the Vector **Killarney** Cluster environment, the required fields are:
   * `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`.
   * `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`.
 
@@ -334,9 +334,9 @@ Once the inference server is ready, you can start sending in inference requests.
 
 If you want to run inference from your local device, you can open a SSH tunnel to your cluster environment like the following:
 ```bash
-ssh -L 8081:172.17.8.29:8081 username@v.vectorinstitute.ai -N
+ssh -L 8081:10.1.1.29:8081 username@v.vectorinstitute.ai -N
 ```
-Where the last number in the URL is the GPU number (gpu029 in this case). The example provided above is for the vector cluster, change the variables accordingly for your environment
+The example provided above is for the Vector Killarney cluster, change the variables accordingly for your environment. The IP address for the compute nodes on Killarney follow `10.1.1.XX` pattern, where `XX` is the GPU number (`kn029` -> `29` in this example). Similarly, for Bon Echo it's `172.17.8.XX`, where `XX` is from `gpuXX`.
 
 ## Python API Usage
 

{vec_inf-0.7.0 → vec_inf-0.7.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vec-inf"
-version = "0.7.0"
+version = "0.7.2"
 description = "Efficient LLM inference on Slurm clusters using vLLM."
 readme = "README.md"
 authors = [{name = "Marshall Wang", email = "marshall.wang@vectorinstitute.ai"}]
@@ -42,9 +42,10 @@ dev = [
     "xgrammar>=0.1.11",
     "torch>=2.7.0",
     "vllm>=0.10.0",
-    "vllm-nccl-cu12>=2.18,<2.19",
-    "ray>=2.40.0",
-    "cupy-cuda12x==12.1.0"
+    "ray[default]>=2.50.0",
+    "cupy-cuda12x==12.1.0",
+    "flashinfer-python>=0.4.0",
+    "sglang>=0.5.0",
 ]
 
 [project.scripts]

{vec_inf-0.7.0 → vec_inf-0.7.2}/tests/vec_inf/cli/test_cli.py

@@ -39,6 +39,7 @@ def test_launch_command_success(runner):
             "mem_per_node": "32G",
             "model_weights_parent_dir": "/model-weights",
             "vocab_size": "128000",
+            "venv": "/path/to/venv",
             "vllm_args": {"max_model_len": 8192},
             "env": {"CACHE": "/cache"},
         }

{vec_inf-0.7.0 → vec_inf-0.7.2}/tests/vec_inf/cli/test_helper.py

@@ -35,6 +35,7 @@ class TestLaunchResponseFormatter:
             "mem_per_node": "32G",
             "model_weights_parent_dir": "/model-weights",
             "log_dir": "/tmp/logs",
+            "venv": "/path/to/venv",
             "vllm_args": {"max_model_len": 8192, "enable_prefix_caching": True},
             "env": {"CACHE": "/cache"},
         }
@@ -63,6 +64,7 @@ class TestLaunchResponseFormatter:
             "mem_per_node": "16G",
             "model_weights_parent_dir": "/weights",
             "log_dir": "/logs",
+            "venv": "/path/to/venv",
             "vllm_args": {},
             "env": {},
         }

{vec_inf-0.7.0 → vec_inf-0.7.2}/tests/vec_inf/client/test_slurm_script_generator.py

@@ -53,7 +53,7 @@ class TestSlurmScriptGenerator:
         singularity = basic_params.copy()
         singularity.update(
             {
-                "venv": "singularity",
+                "venv": "apptainer",
                 "bind": "/scratch:/scratch,/data:/data",
                 "env": {
                     "CACHE_DIR": "/cache",
@@ -109,7 +109,7 @@ class TestSlurmScriptGenerator:
     def test_init_singularity_no_bind(self, basic_params):
         """Test Singularity initialization without additional binds."""
         params = basic_params.copy()
-        params["venv"] = "singularity"
+        params["venv"] = "apptainer"
         generator = SlurmScriptGenerator(params)
 
         assert generator.params == params
@@ -173,7 +173,6 @@ class TestSlurmScriptGenerator:
         generator = SlurmScriptGenerator(basic_params)
         launch_cmd = generator._generate_launch_cmd()
 
-        assert "source /path/to/venv/bin/activate" in launch_cmd
         assert "vllm serve /path/to/model_weights/test-model" in launch_cmd
         assert "--served-model-name test-model" in launch_cmd
         assert "--tensor-parallel-size 4" in launch_cmd
@@ -185,7 +184,7 @@ class TestSlurmScriptGenerator:
         generator = SlurmScriptGenerator(singularity_params)
         launch_cmd = generator._generate_launch_cmd()
 
-        assert "exec --nv" in launch_cmd
+        assert "apptainer exec --nv" in launch_cmd
         assert "--bind /path/to/model_weights/test-model" in launch_cmd
         assert "--bind /scratch:/scratch,/data:/data" in launch_cmd
         assert "source" not in launch_cmd
@@ -306,9 +305,9 @@ class TestBatchSlurmScriptGenerator:
     def batch_singularity_params(self, batch_params):
         """Generate batch SLURM configuration parameters with Singularity."""
         singularity_params = batch_params.copy()
-        singularity_params["venv"] = "singularity"  # Set top-level venv to singularity
+        singularity_params["venv"] = "apptainer"  # Set top-level venv to apptainer
         for model_name in singularity_params["models"]:
-            singularity_params["models"][model_name]["venv"] = "singularity"
+            singularity_params["models"][model_name]["venv"] = "apptainer"
             singularity_params["models"][model_name]["bind"] = (
                 "/scratch:/scratch,/data:/data"
             )
@@ -341,9 +340,9 @@ class TestBatchSlurmScriptGenerator:
     def test_init_singularity_no_bind(self, batch_params):
         """Test Singularity initialization without additional binds."""
         params = batch_params.copy()
-        params["venv"] = "singularity"  # Set top-level venv to singularity
+        params["venv"] = "apptainer"  # Set top-level venv to apptainer
         for model_name in params["models"]:
-            params["models"][model_name]["venv"] = "singularity"
+            params["models"][model_name]["venv"] = "apptainer"
 
         generator = BatchSlurmScriptGenerator(params)