vec-inf 0.4.0.post1__tar.gz → 0.5.0__tar.gz

vec_inf-0.5.0/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,26 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+### Describe the bug
+A clear and concise description of what the bug is.
+
+### To Reproduce
+Code snippet or clear steps to reproduce behaviour.
+
+### Expected behavior
+A clear and concise description of what you expected to happen.
+
+### Screenshots
+If applicable, add screenshots to help explain your problem.
+
+### Version
+ - Version info such as v0.1.5
+
+### Additional context
+Add any other context about the problem here.

vec_inf-0.5.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1 @@
+blank_issues_enabled: false

vec_inf-0.5.0/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+### Is your feature request related to a problem? Please describe.
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+### Describe the solution you'd like
+A clear and concise description of what you want to happen.
+
+### Describe alternatives you've considered
+A clear and concise description of any alternative solutions or features you've considered.
+
+### Additional context
+Add any other context or screenshots about the feature request here.

vec_inf-0.5.0/.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

vec_inf-0.5.0/.github/pull_request_template.md

@@ -0,0 +1,8 @@
+# PR Type
+[Feature | Fix | Documentation | Other() ]
+
+# Short Description
+...
+
+# Tests Added
+...

vec_inf-0.5.0/.github/workflows/code_checks.yml

@@ -0,0 +1,51 @@
+name: code checks
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/code_checks.yml
+      - '**.py'
+      - uv.lock
+      - pyproject.toml
+      - '**.ipynb'
+  pull_request:
+    branches:
+      - main
+      - develop
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/code_checks.yml
+      - '**.py'
+      - uv.lock
+      - pyproject.toml
+      - '**.ipynb'
+
+jobs:
+  run-code-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.2.2
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5.3.1
+        with:
+          # Install a specific version of uv.
+          version: "0.5.21"
+          enable-cache: true
+      - name: "Set up Python"
+        uses: actions/setup-python@v5.4.0
+        with:
+          python-version-file: ".python-version"
+      - name: Install the project
+        run: uv sync --dev
+      - name: Install dependencies and check code
+        run: |
+          source .venv/bin/activate
+          pre-commit run --all-files
+      - name: pip-audit (gh-action-pip-audit)
+        uses: pypa/gh-action-pip-audit@v1.1.0
+        with:
+          virtual-environment: .venv/

vec_inf-0.5.0/.github/workflows/docker.yml

@@ -0,0 +1,55 @@
+name: docker
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+      - main
+    paths:
+      - Dockerfile
+      - .github/workflows/docker.yml
+  pull_request:
+    branches:
+      - main
+      - develop
+    paths:
+      - Dockerfile
+      - .github/workflows/docker.yml
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4.2.2
+
+      - name: Extract vLLM version
+        id: vllm-version
+        run: |
+          VERSION=$(grep -A 1 'name = "vllm"' uv.lock | grep version | cut -d '"' -f 2)
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804
+        with:
+          images: vectorinstitute/vector-inference
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: |
+            ${{ steps.meta.outputs.tags }}
+            vectorinstitute/vector-inference:${{ steps.vllm-version.outputs.version }}
+          labels: ${{ steps.meta.outputs.labels }}

vec_inf-0.5.0/.github/workflows/docs_build.yml

@@ -0,0 +1,44 @@
+name: docs (build)
+permissions:
+  contents: read
+  pull-requests: write
+
+on:
+  pull_request:
+    branches:
+      - main
+      - develop
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/docs_build.yml
+      - '**.py'
+      - '**.ipynb'
+      - '**.js'
+      - '**.html'
+      - uv.lock
+      - pyproject.toml
+      - '**.rst'
+      - '**.md'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.2.2
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@f94ec6bedd8674c4426838e6b50417d36b6ab231
+        with:
+          version: "0.5.21"
+          enable-cache: true
+
+      - name: "Set up Python"
+        uses: actions/setup-python@8039c45ed9a312fba91f3399cd0605ba2ebfe93c
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install the project
+        run: uv sync --dev --group docs
+
+      - name: Build docs
+        run: cd docs && rm -rf source/reference/api/_autosummary && uv run make html

vec_inf-0.5.0/.github/workflows/docs_deploy.yml

@@ -0,0 +1,59 @@
+name: docs
+permissions:
+  contents: write
+  pull-requests: write
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/code_checks.yml
+      - .github/workflows/docs_build.yml
+      - .github/workflows/docs_deploy.yml
+      - .github/workflows/integration_tests.yml
+      - '**.py'
+      - '**.ipynb'
+      - '**.html'
+      - '**.js'
+      - uv.lock
+      - pyproject.toml
+      - '**.rst'
+      - '**.md'
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.2.2
+        with:
+          submodules: 'true'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@f94ec6bedd8674c4426838e6b50417d36b6ab231
+        with:
+          # Install a specific version of uv.
+          version: "0.5.21"
+          enable-cache: true
+
+      - name: "Set up Python"
+        uses: actions/setup-python@8039c45ed9a312fba91f3399cd0605ba2ebfe93c
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install the project
+        run: uv sync --dev --group docs
+
+      - name: Build docs
+        run: |
+          cd docs
+          rm -rf source/reference/api/_autosummary
+          uv run make html
+          touch build/html/.nojekyll
+
+      - name: Deploy to Github pages
+        uses: JamesIves/github-pages-deploy-action@6c2d9db40f9296374acc17b90404b6e8864128c8
+        with:
+          branch: github_pages
+          folder: docs/build/html

vec_inf-0.5.0/.github/workflows/publish.yml

@@ -0,0 +1,32 @@
+name: publish package
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install apt dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libcurl4-openssl-dev libssl-dev
+
+      - uses: actions/checkout@v4.2.2
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.6.6"
+          enable-cache: true
+
+      - uses: actions/setup-python@v5.4.0
+        with:
+          python-version: '3.10'
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish package
+        run: uv publish --token ${{ secrets.PYPI_API_TOKEN }}

vec_inf-0.5.0/.github/workflows/unit_tests.yml

@@ -0,0 +1,81 @@
+name: unit tests
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/code_checks.yml
+      - .github/workflows/docs_build.yml
+      - .github/workflows/docs_deploy.yml
+      - .github/workflows/unit_tests.yml
+      - .github/workflows/integration_tests.yml
+      - '**.py'
+      - '**.ipynb'
+      - uv.lock
+      - pyproject.toml
+      - '**.rst'
+      - '**.md'
+  pull_request:
+    branches:
+      - main
+      - develop
+    paths:
+      - .pre-commit-config.yaml
+      - .github/workflows/code_checks.yml
+      - .github/workflows/docs_build.yml
+      - .github/workflows/docs_deploy.yml
+      - .github/workflows/unit_tests.yml
+      - .github/workflows/integration_tests.yml
+      - '**.py'
+      - '**.ipynb'
+      - uv.lock
+      - pyproject.toml
+      - '**.rst'
+      - '**.md'
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4.2.2
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5.3.1
+        with:
+          # Install a specific version of uv.
+          version: "0.5.21"
+          enable-cache: true
+
+      - name: "Set up Python ${{ matrix.python-version }}"
+        uses: actions/setup-python@v5.4.0
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install the project
+        run: uv sync --dev
+
+      - name: Install dependencies and check code
+        run: |
+          uv run pytest -m "not integration_test" --cov vec_inf --cov-report=xml tests
+
+      - name: Install the core package only
+        run: uv sync --no-dev
+
+      - name: Run package import tests
+        run: |
+          uv run pytest tests/test_imports.py
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5.4.0
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.xml
+          name: codecov-umbrella
+          fail_ci_if_error: true
+          verbose: true

vec_inf-0.5.0/.gitignore

@@ -0,0 +1,154 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pycharm
+.idea/
+
+# VS Code
+.vscode/
+
+# MacOS
+.DS_Store
+
+# Slurm logs
+*.out
+*.err
+
+# Server url files
+*_url
+
+logs/
+
+local/
+slurm/
+scripts/
+
+# vLLM bug reporting files
+collect_env.py
+
+# build files
+dist/

vec_inf-0.5.0/.pre-commit-config.yaml

@@ -0,0 +1,61 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0  # Use the ref you want to point at
+    hooks:
+    - id: trailing-whitespace
+    - id: check-ast
+    - id: check-builtin-literals
+    - id: check-docstring-first
+    - id: check-executables-have-shebangs
+    - id: debug-statements
+    - id: end-of-file-fixer
+    - id: mixed-line-ending
+      args: [--fix=lf]
+    - id: requirements-txt-fixer
+    - id: check-yaml
+    - id: check-toml
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: 'v0.11.0'
+    hooks:
+    - id: ruff
+      args: [--fix, --exit-non-zero-on-fix]
+      types_or: [python, jupyter]
+    - id: ruff-format
+      types_or: [python, jupyter]
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.15.0
+    hooks:
+    - id: mypy
+      entry: python3 -m mypy --config-file pyproject.toml
+      language: system
+      types: [python]
+      exclude: "tests"
+
+  - repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.9.1
+    hooks:
+    - id: nbqa-ruff
+      args: [--fix, --exit-non-zero-on-fix]
+
+  - repo: local
+    hooks:
+    - id: pytest
+      name: pytest
+      entry: python3 -m pytest -m "not integration_test"
+      language: system
+      pass_filenames: false
+      always_run: true
+
+ci:
+  autofix_commit_msg: |
+    [pre-commit.ci] Add auto fixes from pre-commit.com hooks
+
+    for more information, see https://pre-commit.ci
+  autofix_prs: true
+  autoupdate_branch: ''
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
+  autoupdate_schedule: weekly
+  skip: [pytest,mypy]
+  submodules: false

vec_inf-0.5.0/.python-version

@@ -0,0 +1 @@
+3.10

vec_inf-0.5.0/Dockerfile

@@ -0,0 +1,52 @@
+FROM nvidia/cuda:12.3.1-devel-ubuntu20.04
+
+# Non-interactive apt-get commands
+ARG DEBIAN_FRONTEND=noninteractive
+
+# No GPUs visible during build
+ARG CUDA_VISIBLE_DEVICES=none
+
+# Specify CUDA architectures -> 7.5: RTX 6000 & T4, 8.0: A100, 8.6+PTX
+ARG TORCH_CUDA_ARCH_LIST="7.5;8.0;8.6+PTX"
+
+# Set the Python version
+ARG PYTHON_VERSION=3.10.12
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    wget build-essential libssl-dev zlib1g-dev libbz2-dev \
+    libreadline-dev libsqlite3-dev libffi-dev libncursesw5-dev \
+    xz-utils tk-dev libxml2-dev libxmlsec1-dev liblzma-dev git vim \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python
+RUN wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \
+    tar -xzf Python-$PYTHON_VERSION.tgz && \
+    cd Python-$PYTHON_VERSION && \
+    ./configure --enable-optimizations && \
+    make -j$(nproc) && \
+    make altinstall && \
+    cd .. && \
+    rm -rf Python-$PYTHON_VERSION.tgz Python-$PYTHON_VERSION
+
+# Install pip and core Python tools
+RUN wget https://bootstrap.pypa.io/get-pip.py && \
+    python3.10 get-pip.py && \
+    rm get-pip.py && \
+    python3.10 -m pip install --upgrade pip setuptools wheel uv
+
+# Set up project
+WORKDIR /vec-inf
+COPY . /vec-inf
+
+# Install project dependencies with build requirements
+RUN PIP_INDEX_URL="https://download.pytorch.org/whl/cu121" uv pip install --system -e .[dev]
+# Install Flash Attention
+RUN python3.10 -m pip install flash-attn --no-build-isolation
+
+# Final configuration
+RUN mkdir -p /vec-inf/nccl && \
+    mv /root/.config/vllm/nccl/cu12/libnccl.so.2.18.1 /vec-inf/nccl/libnccl.so.2.18.1
+
+# Set the default command to start an interactive shell
+CMD ["bash"]