sendnn-inference 2.0.0__tar.gz

sendnn_inference-2.0.0/.github/CODEOWNERS

@@ -0,0 +1,27 @@
+# See https://help.github.com/articles/about-codeowners/
+# for more info about CODEOWNERS file
+
+# These owners will be the default owners for everything in the repo
+# and will be requested for review when someone opens a pull request
+# unless a later match takes precedence
+* @joerunde @prashantgupta24
+
+# This lists cover the "core" components of SenDNN-Inference that require careful review
+/sendnn_inference @yannicks1 @tdoublep @nikolaospapandreou @sducouedic
+
+# TODO: separate code into different packages:
+#   /sendnn_inference/v1/worker/continuous_batching @nikolaospapandreou
+#   /sendnn_inference/v1/worker/static_batching ...
+
+# Tests
+/tests @rafvasq @prashantgupta24 @sducouedic
+
+# Examples
+/examples @yannicks1 @prashantgupta24 @sducouedic
+
+# Docs
+/docs @rafvasq
+mkdocs.yaml @rafvasq
+
+# CI
+/.github @joerunde @ckadner @rafvasq

sendnn_inference-2.0.0/.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,46 @@
+name: 🐛 Bug report
+description: Report a bug in SenDNN Inference
+title: "[Bug]: "
+labels: ["bug"]
+
+body:
+- type: markdown
+  attributes:
+    value: |
+      Before submitting, please search [existing issues](https://github.com/torch-spyre/sendnn-inference/issues) to avoid duplicates.
+
+- type: textarea
+  attributes:
+    label: Describe the bug
+    description: A clear and concise description of what the bug is.
+  validations:
+    required: true
+
+- type: textarea
+  attributes:
+    label: How to reproduce
+    description: |
+      Steps to reproduce the behavior. Include a minimal code example if possible.
+      
+    placeholder: |
+      ```
+      # Your code here
+      ```
+      1. Run command '...'
+      2. See error
+  validations:
+    required: true
+
+- type: textarea
+  attributes:
+    label: Additional context
+    description: Any other context, logs, etc. of the problem.
+
+- type: checkboxes
+  attributes:
+    label: Checklist
+    options:
+      - label: I have searched for similar issues.
+        required: true
+
+# Made with Bob

sendnn_inference-2.0.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,13 @@
+blank_issues_enabled: false
+contact_links:
+  - name: 📖 Documentation
+    url: https://docs.vllm.ai/projects/spyre/
+    about: Read the SenDNN-Inference documentation
+  - name: 💬 Discussions
+    url: https://inviter.co/vllm-slack
+    about: Ask questions and discuss in the `#sig-spyre` channel.
+  - name: 🤝 Contributing Guide
+    url: https://docs.vllm.ai/projects/spyre/en/latest/contributing
+    about: Learn how to contribute to SenDNN-Inference
+
+# Made with Bob

sendnn_inference-2.0.0/.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,46 @@
+name: 🚀 Feature request
+description: Suggest a new feature for SenDNN Inference
+title: "[Feature]: "
+labels: ["enhancement"]
+
+body:
+- type: markdown
+  attributes:
+    value: |
+      Before submitting, please search [existing issues](https://github.com/torch-spyre/sendnn-inference/issues) to avoid duplicates.
+
+- type: textarea
+  attributes:
+    label: Feature description
+    description: |
+      A clear and concise description of the feature you'd like to see.
+      
+  validations:
+    required: true
+
+- type: textarea
+  attributes:
+    label: Motivation and context
+    description: |
+      Why is this feature important? How would it benefit SenDNN Inference users?
+      
+      Is this related to a specific problem or limitation?
+  validations:
+    required: true
+
+- type: textarea
+  attributes:
+    label: Proposed solution
+    description: |
+      If you have ideas on how this could be implemented, share them here.
+      
+      Include any relevant examples, APIs, or resources.
+
+- type: checkboxes
+  attributes:
+    label: Checklist
+    options:
+      - label: I have searched for similar feature requests
+        required: true
+
+# Made with Bob

sendnn_inference-2.0.0/.github/ISSUE_TEMPLATE/rfc.yml

@@ -0,0 +1,27 @@
+name: 💭 Request for comments
+description: Start a discussion for larger changes related to SenDNN Inference
+title: "[RFC]: "
+labels: []
+
+body:
+- type: markdown
+  attributes:
+    value: |
+      Before submitting, please search [existing issues](https://github.com/torch-spyre/sendnn-inference/issues) to avoid duplicates.
+
+- type: textarea
+  attributes:
+    label: To discuss
+    description: |
+      Describe what you want to discuss or get feedback on. Consider covering
+      the motivation, your proposed approach, and any alternatives you considered.
+  validations:
+    required: true
+
+- type: checkboxes
+  attributes:
+    label: Checklist
+    options:
+      - label: I have searched for similar feature requests or RFCs.
+        required: true
+

sendnn_inference-2.0.0/.github/actions/free-up-disk-space/action.yml

@@ -0,0 +1,46 @@
+name: "Free up disk space"
+description: "Removes non-essential tools, libraries and cached files from GitHub action runner node"
+
+runs:
+  using: "composite"
+  steps:
+    - name: "Remove non-essential tools and libraries"
+      shell: bash
+      run: |
+        # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+        echo "Disk usage before cleanup:"
+        df -h
+        echo "Removing non-essential tools and libraries ..."
+        sudo rm -rf /opt/ghc
+        sudo rm -rf /usr/local/.ghcup
+        sudo rm -rf /usr/share/dotnet
+        # sudo rm -rf /usr/local/share/boost
+        echo "Deleting libraries for Android (12G), CodeQL (5.3G), PowerShell (1.3G), Swift (1.7G) ..."
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf "${AGENT_TOOLSDIRECTORY}/CodeQL"
+        sudo rm -rf /usr/local/share/powershell
+        sudo rm -rf /usr/share/swift
+        # ref: https://github.com/jlumbroso/free-disk-space/blob/main/action.yml
+        echo "Deleting some larger apt packages:"
+        sudo apt-get remove -y '^aspnetcore-.*' || echo "::warning::The command [sudo apt-get remove -y '^aspnetcore-.*'] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^dotnet-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^dotnet-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^llvm-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^llvm-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y 'php.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y 'php.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^mongodb-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mongodb-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^mysql-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mysql-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing || echo "::warning::The command [sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y google-cloud-sdk --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-sdk --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y google-cloud-cli --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-cli --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed to complete successfully. Proceeding..."
+        sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed to complete successfully. Proceeding..."
+        echo "Disk usage after cleanup:"
+        df -h
+
+    - name: "Prune docker images"
+      shell: bash
+      run: |
+        echo "Pruning docker images ..."
+        docker image prune -a -f
+        docker system df
+        echo "Disk usage after pruning docker images:"
+        df -h

sendnn_inference-2.0.0/.github/pull_request_template.md

@@ -0,0 +1,21 @@
+<!-- markdownlint-disable -->
+
+## Description
+
+<!-- Provide a clear description of your changes. -->
+
+## Related Issues
+
+<!-- Link related issues, e.g., `Fixes #` or `Relates to #456` -->
+
+## Test Plan
+
+<!-- Describe how you tested your changes. Include commands or steps to reproduce. -->
+
+## Checklist
+
+- [ ] I have read the [contributing guidelines](https://docs.vllm.ai/projects/spyre/en/latest/contributing)
+- [ ] My code follows the project's code style (run `bash format.sh`)
+- [ ] I have added tests for my changes (if applicable)
+- [ ] I have updated the documentation (if applicable)
+- [ ] My commits include a `Signed-off-by:` line (DCO compliance)

sendnn_inference-2.0.0/.github/workflows/add_label_automerge.yml

@@ -0,0 +1,26 @@
+name: Add label on auto-merge enabled
+
+on:
+  pull_request_target:
+    types:
+      - auto_merge_enabled
+
+jobs:
+  add-label-on-auto-merge:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - name: Add label
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            github.rest.issues.addLabels({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              labels: ['ready']
+            })
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

sendnn_inference-2.0.0/.github/workflows/build_and_publish.yaml

@@ -0,0 +1,30 @@
+name: Build and Publish sendnn-inference Library
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    # Only run for sendnn-inference releases (v* tags).
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+
+    environment:
+      name: pypi
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+            version: "latest"
+            python-version: "3.12"
+      - name: Build package
+        run: uv build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

sendnn_inference-2.0.0/.github/workflows/build_docker.yml

@@ -0,0 +1,98 @@
+name: "Build Docker"
+
+# Manual dispatch only for now
+on:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      CACHE_IMAGE: "ghcr.io/torch-spyre/sendnn-inference:build-cache"
+      QUAY_REPOSITORY: "quay.io/ibm-aiu/sendnn-inference"
+
+    permissions:
+      packages: write
+      contents: read
+
+    steps:
+    - name: "Checkout"
+      uses: actions/checkout@v4
+      # We need to fetch history and tags so that we can correctly version the
+      # sendnn-inference package, since it uses setuptools_scm to version based on
+      # git tags.
+      with:
+        fetch-depth: 0
+
+    - name: "Free up disk space"
+      uses: ./.github/actions/free-up-disk-space
+
+    - name: "Set up QEMU"
+      uses: docker/setup-qemu-action@v3
+
+    - name: "Set up Docker Buildx"
+      uses: docker/setup-buildx-action@v3
+
+    - name: "Log in to Github Container registry"
+      uses: docker/login-action@v3
+      if: github.event_name != 'pull_request'
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: "Log in to quay"
+      uses: docker/login-action@v3
+      if: github.event_name != 'pull_request'
+      with:
+        registry: quay.io
+        username: ibm-aiu+aiu_pusher_bot
+        password: ${{ secrets.IBM_AIU_QUAY_TOKEN }}
+
+    - name: "Set build cache target"
+      run: |
+        # For push to `main` (PR merged), push a new cache image with all layers (cache-mode=max).
+        # For PR builds, use GitHub action cache which isolates cached layers by PR/branch.
+        # to optimize builds for subsequent pushes to the same PR/branch.
+        # Do not set a cache-to image for PR builds to not overwrite the `main` cache image and
+        # to not ping-pong cache images for two or more different PRs.
+        # Do not push cache images for each PR or multiple branches to not exceed GitHub package
+        # usage and traffic limitations.
+        # UPDATE 2024/02/26: GHA cache appears to have issues, cannot use `cache-to: gha,mode=min`
+        # if `cache-from: reg...,mode=max` but `cache-to: gha,mode=max` takes longer than uncached
+        # build and exhausts GHA cache size limits, so use cache `type=inline` (no external cache).
+        if [ "${{ github.event_name }}" == "pull_request" ]
+        then
+            #CACHE_TO="type=gha,mode=min"
+            CACHE_TO="type=inline"
+        else
+            CACHE_TO="type=registry,ref=${{ env.CACHE_IMAGE }},mode=max"
+        fi
+        echo "CACHE_TO=$CACHE_TO" >> "$GITHUB_ENV"
+
+    - name: "Generate tags"
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: |
+          ${{ env.QUAY_REPOSITORY }}
+        tags: |
+          type=raw,value=latest.amd64
+          type=semver,pattern={{version}}.amd64
+
+    - name: "Build and push"
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: docker/Dockerfile.amd64
+        target: release
+        provenance: false
+        tags: ${{ steps.meta.outputs.tags }}
+        cache-from: type=registry,ref=${{ env.CACHE_IMAGE }}
+        cache-to: ${{ env.CACHE_TO }}
+        push: ${{ github.event_name != 'pull_request' }}
+        platforms: linux/amd64

sendnn_inference-2.0.0/.github/workflows/check_uv_lock.yml

@@ -0,0 +1,48 @@
+name: Check uv.lock
+
+on:
+  pull_request:
+    paths:
+      - 'pyproject.toml'
+      - 'uv.lock'
+  push:
+    branches: [main]
+    paths:
+      - 'pyproject.toml'
+      - 'uv.lock'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+  contents: read
+
+jobs:
+  check-lock:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: "Get changed files"
+        id: changed-files
+        uses: tj-actions/changed-files@v46
+        with:
+          files_yaml: |
+            root:
+              - pyproject.toml
+              - uv.lock
+
+      - name: "Set up Python 3.12"
+        if: steps.changed-files.outputs.root_any_changed == 'true' || steps.changed-files.outputs.next_any_changed == 'true'
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.12"
+          enable-cache: true
+          ignore-nothing-to-cache: true
+          cache-dependency-glob: |
+            pyproject.toml
+
+      - name: "Check root uv.lock is up to date"
+        if: steps.changed-files.outputs.root_any_changed == 'true'
+        run: uv lock --check

sendnn_inference-2.0.0/.github/workflows/lint_scripts.yml

@@ -0,0 +1,37 @@
+name: Lint Scripts
+
+on:
+  # Don't use pull_request.paths filter since this workflow is required for
+  # all pull requests on main irrespective of file type or location
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+    paths:
+      - '**/*.sh'
+      - '.github/workflows/lint_scripts.yml'
+
+env:
+  LC_ALL: en_US.UTF-8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  lint-scripts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
+
+      - name: "Check shell scripts"
+        run: |
+          tools/lint_scripts.sh

sendnn_inference-2.0.0/.github/workflows/matchers/ruff.json

@@ -0,0 +1,17 @@
+{
+    "problemMatcher": [
+      {
+        "owner": "ruff",
+        "pattern": [
+          {
+            "regexp": "^(.+?):(\\d+):(\\d+): (\\w+): (.+)$",
+            "file": 1,
+            "line": 2,
+            "column": 3,
+            "code": 4,
+            "message": 5
+          }
+        ]
+      }
+    ]
+  }

sendnn_inference-2.0.0/.github/workflows/pre-commit.yml

@@ -0,0 +1,35 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+  contents: read
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+    
+    # Need to pull real deps from cache so that `ty` can type check against vllm
+    - name: "Set up Python 3.12"
+      uses: astral-sh/setup-uv@v5
+      with:
+        python-version: "3.12"
+        enable-cache: true
+        ignore-nothing-to-cache: true
+        cache-dependency-glob: |
+          pyproject.toml
+    - name: "Install dependencies"
+      run: VLLM_TARGET_DEVICE=empty uv sync --frozen
+    - run: echo "::add-matcher::.github/workflows/matchers/ruff.json"
+    - uses: j178/prek-action@v1
+      with:
+        extra_args: --all-files --hook-stage manual

sendnn_inference-2.0.0/.github/workflows/publish_to_test_pypi.yaml

@@ -0,0 +1,35 @@
+# This tests the publication process for sendnn-inference by pushing a pre-release to test.pypi.org
+# on all main commits
+name: Test publication on test.pypi.org
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/sendnn-inference
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+            # Fetch all history for setuptools_scm to work correctly
+            fetch-depth: 0
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+            version: "latest"
+            python-version: "3.12"
+      - name: Build package
+        run: uv build
+      - name: Publish package distributions to test PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+            repository-url: https://test.pypi.org/legacy/

sendnn_inference-2.0.0/.github/workflows/reminder_comment.yml

@@ -0,0 +1,23 @@
+name: PR Reminder Comment Bot
+on:
+  pull_request_target:
+    types: [opened]
+
+jobs:
+  pr_reminder:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Remind to run full CI on PR
+        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          script: |
+            github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: "👋 Hi! Thank you for contributing.\n Just a reminder: Make sure that your code passes all the linting checks, otherwise your PR won't be able to be merged. To do so, run `./format.sh`. \nNow you are good to go 🚀.\n\nWe also recommend installing prek and configuring it to check your code before every local commit."
+            })
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

sendnn_inference-2.0.0/.github/workflows/stale.yml

@@ -0,0 +1,52 @@
+name: Close inactive issues and PRs
+
+on:
+  schedule:
+    # Daily at 1:30 AM UTC
+    - cron: '30 1 * * *'
+
+jobs:
+  close-issues-and-pull-requests:
+    permissions:
+      issues: write
+      pull-requests: write
+      actions: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9.0.0
+        with:
+          # Increasing this value ensures that changes to this workflow
+          # propagate to all issues and PRs in days rather than months
+          operations-per-run: 1000
+
+          exempt-draft-pr: true
+          exempt-issue-labels: 'keep-open'
+          exempt-pr-labels: 'keep-open'
+
+          labels-to-add-when-unstale: 'unstale'
+          labels-to-remove-when-stale: 'unstale'
+
+          days-before-issue-stale: 90
+          days-before-issue-close: 30
+          stale-issue-label: 'stale'
+          stale-issue-message: >
+            This issue has been automatically marked as stale because it has not
+            had any activity within 90 days. It will be automatically closed if no
+            further activity occurs within 30 days. Leave a comment if
+            you feel this issue should remain open. Thank you!
+          close-issue-message: >
+            This issue has been automatically closed due to inactivity. Please
+            feel free to reopen if you feel it is still relevant. Thank you!
+
+          days-before-pr-stale: 90
+          days-before-pr-close: 30
+          stale-pr-label: 'stale'
+          stale-pr-message: >
+            This pull request has been automatically marked as stale because it
+            has not had any activity within 90 days. It will be automatically
+            closed if no further activity occurs within 30 days. Leave a comment
+            if you feel this pull request should remain open. Thank you!
+          close-pr-message: >
+            This pull request has been automatically closed due to inactivity.
+            Please feel free to reopen if you intend to continue working on it.
+            Thank you!