sdg-hub 0.1.0a1__tar.gz

sdg_hub-0.1.0a1/.github/actionlint.yaml

@@ -0,0 +1,3 @@
+self-hosted-runner:
+  labels:
+  - ubuntu-gpu

sdg_hub-0.1.0a1/.github/actions/free-disk-space/action.yml

@@ -0,0 +1,19 @@
+name: 'Free Disk Space'
+description: 'Frees disk space on the runner'
+runs:
+  using: "composite"
+  steps:
+    - run: |
+        df -h
+        sudo docker rmi "$(docker image ls -aq)" >/dev/null 2>&1 || true
+        sudo rm -rf \
+          /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+          /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
+          /usr/lib/jvm || true
+        sudo apt install aptitude -y >/dev/null 2>&1
+        sudo aptitude purge '~n ^mysql' -f -y >/dev/null 2>&1
+        sudo aptitude purge '~n ^dotnet' -f -y >/dev/null 2>&1
+        sudo apt-get autoremove -y >/dev/null 2>&1
+        sudo apt-get autoclean -y >/dev/null 2>&1
+        df -h
+      shell: bash

sdg_hub-0.1.0a1/.github/dependabot.yml

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: Apache-2.0
+
+# GitHub Dependabot configuration file
+version: 2
+updates:
+
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+  - package-ecosystem: "docker"
+    directory: "/.github/workflows"
+    schedule:
+      interval: "daily"
+
+  # Maintain dependencies for Python scripts
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "daily"

sdg_hub-0.1.0a1/.github/mergify.yml

@@ -0,0 +1,75 @@
+pull_request_rules:
+
+- name: label-cicd
+  description: Automatically apply CI/CD label
+  conditions:
+    - or:
+      - files=.github/mergify.yml
+      - files=.github/workflows/**/*
+      - files=.pylintrc
+      - files=tox.ini
+      - files=.markdownlint-cli2.yaml
+  actions:
+    label:
+      add:
+        - CI/CD
+
+- name: label-documentation
+  description: Automatically apply documentation label
+  conditions:
+    - or:
+      - files~=.*\.md
+      - files~=docs/.*
+  actions:
+    label:
+      add:
+        - documentation
+
+- name: label-testing
+  description: Automatically apply testing label
+  conditions:
+    - or:
+      - files~=tests/.*
+      - files=tox.ini
+  actions:
+    label:
+      add:
+        - testing
+
+- name: ping author on conflicts and add 'needs-rebase' label
+  conditions:
+      - conflict
+      - -closed
+  actions:
+    label:
+      add:
+        - needs-rebase
+    comment:
+      message: |
+       This pull request has merge conflicts that must be resolved before it can be
+       merged. @{{author}} please rebase it. https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
+
+- name: remove 'needs-rebase' label when conflict is resolved
+  conditions:
+      - -conflict
+      - -closed
+  actions:
+    label:
+      remove:
+        - needs-rebase
+
+- name: Apply ci-failure label if any CI checks have failed
+  conditions:
+      - "#check-failure>0"
+  actions:
+    label:
+      add:
+        - ci-failure
+
+- name: Remove ci-failure label if no failures are present
+  conditions:
+      - "#check-failure=0"
+  actions:
+    label:
+      remove:
+        - ci-failure

sdg_hub-0.1.0a1/.github/workflows/actionlint.dockerfile

@@ -0,0 +1,3 @@
+# Since dependabot cannot update workflows using docker,
+# we use this indirection since dependabot can update this file.
+FROM rhysd/actionlint:1.7.1@sha256:435ecdb63b1169e80ca3e136290072548c07fc4d76a044cf5541021712f8f344

sdg_hub-0.1.0a1/.github/workflows/actionlint.yml

@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Lint GitHub Actions workflows
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - '.github/workflows/*.ya?ml'
+      - '.github/workflows/actionlint.*' # This workflow
+  pull_request:
+    branches:
+      - "main"
+    paths:
+      - '.github/workflows/*.ya?ml'
+      - '.github/workflows/actionlint.*' # This workflow
+
+env:
+  LC_ALL: en_US.UTF-8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  actionlint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Harden Runner"
+        uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: "Checkout"
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        with:
+          fetch-depth: 0
+
+      - name: "Download actionlint"
+        run: |
+          docker build --tag actionlint - < .github/workflows/actionlint.dockerfile
+
+      - name: "Check workflow files"
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/actionlint.json"
+          docker run --volume="${PWD}:/repo" --workdir=/repo actionlint -color

sdg_hub-0.1.0a1/.github/workflows/docs.yml

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Lint Markdown documents
+
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - '**/*.md'
+      - '.markdownlint-cli2.yaml'
+      - '.github/workflows/docs.yml' # This workflow
+  pull_request:
+    branches:
+      - "main"
+    paths:
+      - '**/*.md'
+      - '.markdownlint-cli2.yaml'
+      - '.github/workflows/docs.yml' # This workflow
+
+env:
+  LC_ALL: en_US.UTF-8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  markdown-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Harden Runner"
+        uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+      - name: "Checkout"
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+        with:
+          fetch-depth: 0
+      - name: "Check Markdown documents"
+        uses: DavidAnson/markdownlint-cli2-action@b4c9feab76d8025d1e83c653fa3990936df0e6c8 # v16.0.0
+        with:
+          globs: '**/*.md'

sdg_hub-0.1.0a1/.github/workflows/e2e.yml

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: E2E test
+
+on:
+  workflow_dispatch:
+  # TODO: e2e workflow set to manual dispatch until it's adjusted to use
+  # AWS GPU runners instead of ubuntu-gpu GitHub runners, since those
+  # GitHub runners are more expensive and not available in this org
+  #
+  # push:
+  #   branches:
+  #     - "main"
+  #     - "release-**"
+  #   paths:
+  #     - '**.py'
+  #     - 'pyproject.toml'
+  #     - 'requirements*.txt'
+  #     - '.github/workflows/e2e.yml'
+  # pull_request:
+  #   branches:
+  #     - "main"
+  #     - "release-**"
+  #   paths:
+  #     - '**.py'
+  #     - 'pyproject.toml'
+  #     - 'requirements*.txt'
+  #     - '.github/workflows/e2e.yml'
+
+jobs:
+  e2e:
+    runs-on: ubuntu-gpu
+
+    permissions:
+      pull-requests: write
+
+    steps:
+      # No step-security/harden-runner since this is a self-hosted runner
+      - name: Checkout instructlab/sdg
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          # https://github.com/actions/checkout/issues/249
+          fetch-depth: 0
+
+      - name: Checkout instructlab/instructlab
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          repository: "instructlab/instructlab"
+          path: "instructlab"
+          fetch-depth: 0
+
+      - name: Install Packages
+        run: |
+          sudo apt-get install -y cuda-toolkit git cmake build-essential virtualenv
+          nvidia-smi
+          sudo ls -l /dev/nvidia*
+
+      - name: Setup Python 3.11
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
+        with:
+          python-version: 3.11
+          cache: pip
+          cache-dependency-path: |
+            **/pyproject.toml
+            **/requirements*.txt
+
+      - name: Remove llama-cpp-python from cache
+        run: |
+          pip cache remove llama_cpp_python
+
+      - name: Cache huggingface
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: ~/.cache/huggingface
+          # config contains DEFAULT_MODEL
+          key: huggingface-${{ hashFiles('src/instructlab/configuration.py') }}
+
+      - name: Install instructlab and instructlab-sdg
+        run: |
+          export PATH="/home/runner/.local/bin:/usr/local/cuda/bin:$PATH"
+          python3 -m venv venv
+          . venv/bin/activate
+          cd instructlab
+          sed 's/\[.*\]//' requirements.txt > constraints.txt
+          python3 -m pip cache remove llama_cpp_python
+          CMAKE_ARGS="-DLLAMA_CUBLAS=on" python3 -m pip install --no-binary llama_cpp_python -c constraints.txt llama_cpp_python
+          # needed for --4-bit-quant option to ilab train
+          python3 -m pip install bitsandbytes
+          # install instructlab
+          python3 -m pip install .
+          cd ..
+          # Install instructlab-sdg
+          python3 -m pip install .
+
+      - name: Run e2e test
+        run: |
+          . venv/bin/activate
+          ./instructlab/scripts/basic-workflow-tests.sh -cm
+
+      - name: Remove llama-cpp-python from cache
+        if: always()
+        run: |
+          pip cache remove llama_cpp_python

sdg_hub-0.1.0a1/.github/workflows/lint.yml

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Lint, Format, and MyPy
+
+on:
+  push:
+    branches:
+      - "main-disabled"
+    paths:
+      - '**.py'
+      - 'pyproject.toml'
+      - 'requirements*.txt'
+      - 'tox.ini'
+      - 'scripts/*.sh'
+      - '.github/**'
+  pull_request:
+    branches:
+      - "main-disabled"
+    paths:
+      - '**.py'
+      - 'pyproject.toml'
+      - 'requirements*.txt'
+      - 'tox.ini'
+      - 'scripts/*.sh'
+      - '.github/**'
+
+env:
+  PYTHON_VERSION: 3.11
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # https://github.com/actions/checkout/issues/249
+          fetch-depth: 0
+          submodules: true
+
+      - name: Setup Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+          cache: pip
+          cache-dependency-path: |
+            **/pyproject.toml
+            **/requirements*.txt
+
+      - name: Install dependencies
+        id: deps
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox
+
+      - name: Run Ruff check
+        run: |
+          tox -e ruff -- check
+
+      - name: Run linting
+        if: ${{ !cancelled() && (steps.deps.outcome == 'success') }}
+        run: |
+          echo "::add-matcher::.github/workflows/matchers/pylint.json"
+          tox -e lint
+
+      - name: Run mypy type checks
+        if: ${{ !cancelled() && (steps.deps.outcome == 'success') }}
+        run: |
+          tox -e mypy

sdg_hub-0.1.0a1/.github/workflows/matchers/actionlint.json

@@ -0,0 +1,18 @@
+{
+    "problemMatcher": [
+      {
+        "owner": "actionlint",
+        "pattern": [
+          {
+            "regexp": "^(?:\\x1b\\[\\d+m)?(.+?)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*: (?:\\x1b\\[\\d+m)*(.+?)(?:\\x1b\\[\\d+m)* \\[(.+?)\\]$",
+            "file": 1,
+            "line": 2,
+            "column": 3,
+            "message": 4,
+            "code": 5
+          }
+        ]
+      }
+    ]
+  }
+  

sdg_hub-0.1.0a1/.github/workflows/matchers/pylint.json

@@ -0,0 +1,33 @@
+{
+    "problemMatcher": [
+      {
+        "owner": "pylint-error",
+        "severity": "error",
+        "pattern": [
+          {
+            "regexp": "^(.+):(\\d+):(\\d+):\\s(([EF]\\d{4}):\\s.+)$",
+            "file": 1,
+            "line": 2,
+            "column": 3,
+            "message": 4,
+            "code": 5
+          }
+        ]
+      },
+      {
+        "owner": "pylint-warning",
+        "severity": "warning",
+        "pattern": [
+          {
+            "regexp": "^(.+):(\\d+):(\\d+):\\s(([CRW]\\d{4}):\\s.+)$",
+            "file": 1,
+            "line": 2,
+            "column": 3,
+            "message": 4,
+            "code": 5
+          }
+        ]
+      }
+    ]
+  }
+  

sdg_hub-0.1.0a1/.github/workflows/pypi.yaml

@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Build, test, and upload PyPI package
+
+on:
+    push:
+        branches:
+            - "main"
+            - "release-**"
+        tags:
+            - "v*"
+    pull_request:
+        branches:
+            - "main"
+            - "release-**"
+    release:
+        types:
+            - published
+
+env:
+    LC_ALL: en_US.UTF-8
+
+defaults:
+    run:
+        shell: bash
+
+permissions:
+    contents: read
+
+jobs:
+    # Create and verify release artifacts
+    # - build source dist (tar ball) and wheel
+    # - validate artifacts with various tools
+    # - upload artifacts to GHA
+    build-package:
+        name: Build and check packages
+        runs-on: ubuntu-latest
+        steps:
+            - name: "Harden Runner"
+              uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1
+              with:
+                  egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+
+            - name: "Checkout"
+              uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+              with:
+                  # for setuptools-scm
+                  fetch-depth: 0
+
+            - name: "Build and Inspect"
+              uses: hynek/build-and-inspect-python-package@b5076c307dc91924a82ad150cdd1533b444d3310 # v2.12.0
+
+    # push to Test PyPI on
+    # - a new GitHub release is published
+    # - a PR is merged into main branch
+    publish-test-pypi:
+        name: Publish packages to test.pypi.org
+        # environment: publish-test-pypi
+        if: ${{ (github.repository_owner == 'Red-Hat-AI-Innovation-Team') && ((github.event.action == 'published') || ((github.event_name == 'push') && (github.ref == 'refs/heads/main'))) }}
+        permissions:
+            contents: read
+            # see https://docs.pypi.org/trusted-publishers/
+            id-token: write
+        runs-on: ubuntu-latest
+        needs: build-package
+
+        steps:
+            - name: "Harden Runner"
+              uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1
+              with:
+                  egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+            - name: "Download build artifacts"
+              uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
+              with:
+                  name: Packages
+                  path: dist
+
+            - name: "Upload to Test PyPI"
+              uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4
+              with:
+                  repository-url: https://test.pypi.org/legacy/
+
+    # push to Production PyPI on
+    # - a new GitHub release is published
+    publish-pypi:
+        name: Publish release to pypi.org
+        # environment: publish-pypi
+        if: ${{ (github.repository_owner == 'Red-Hat-AI-Innovation-Team') && (github.event.action == 'published') }}
+        permissions:
+            # see https://docs.pypi.org/trusted-publishers/
+            id-token: write
+            # allow gh release upload
+            contents: write
+
+        runs-on: ubuntu-latest
+        needs: build-package
+
+        steps:
+            - name: "Harden Runner"
+              uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1
+              with:
+                  egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+            - name: "Download build artifacts"
+              uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
+              with:
+                  name: Packages
+                  path: dist
+
+            - name: "Sigstore sign package"
+              uses: sigstore/gh-action-sigstore-python@f514d46b907ebcd5bedc05145c03b69c1edd8b46 # v3.0.0
+              with:
+                  inputs: |
+                      ./dist/*.tar.gz
+                      ./dist/*.whl
+                  release-signing-artifacts: false
+
+            - name: "Upload artifacts and signatures to GitHub release"
+              run: |
+                  gh release upload '${{ github.ref_name }}' dist/* --repo '${{ github.repository }}'
+              env:
+                  GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+            # PyPI does not accept .sigstore artifacts and
+            # gh-action-pypi-publish has no option to ignore them.
+            - name: "Remove sigstore signatures before uploading to PyPI"
+              run: |
+                  rm ./dist/*.sigstore.json
+
+            - name: "Upload to PyPI"
+              uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4

sdg_hub-0.1.0a1/.github/workflows/test.yml

@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: Apache-2.0
+
+name: Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      - '**.py'
+      - 'pyproject.toml'
+      - 'requirements*.txt'
+      - 'tox.ini'
+      - '.github/workflows/test.yml' # This workflow
+  pull_request:
+    branches:
+      - "main"
+      - "release-**"
+    paths:
+      - '**.py'
+      - 'pyproject.toml'
+      - 'requirements*.txt'
+      - 'tox.ini'
+      - '.github/workflows/test.yml' # This workflow
+
+env:
+  LC_ALL: en_US.UTF-8
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: "${{ matrix.python }} on ${{ matrix.platform }}"
+    runs-on: "${{ matrix.platform }}"
+    strategy:
+      matrix:
+        python:
+          - "3.10"
+          - "3.11"
+        platform:
+          - "ubuntu-latest"
+        include:
+          - python: "3.11"
+            platform: "macos-latest"
+    steps:
+      - name: "Harden Runner"
+        uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1
+        with:
+          egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          # https://github.com/actions/checkout/issues/249
+          fetch-depth: 0
+
+      - name: Free disk space
+        if: matrix.platform != 'macos-latest'
+        uses: ./.github/actions/free-disk-space
+
+      - name: Install the expect package
+        if: startsWith(matrix.platform, 'ubuntu')
+        run: |
+          sudo apt-get install -y expect
+
+      - name: Install tools on MacOS
+        if: startsWith(matrix.platform, 'macos')
+        run: |
+          brew install expect coreutils bash
+
+      - name: Setup Python ${{ matrix.python }}
+        uses: actions/setup-python@8d9ed9ac5c53483de85588cdf95a591a75ab9f55 # v5.5.0
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+          cache-dependency-path: |
+            **/pyproject.toml
+            **/requirements*.txt
+
+      - name: Remove llama-cpp-python from cache
+        run: |
+          pip cache remove llama_cpp_python
+
+      - name: Cache huggingface
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: ~/.cache/huggingface
+          # config contains DEFAULT_MODEL
+          key: huggingface-${{ hashFiles('src/instructlab/configuration.py') }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh>=1.2
+
+      - name: Run unit tests with tox
+        run: |
+          tox
+
+      - name: Remove llama-cpp-python from cache
+        if: always()
+        run: |
+          pip cache remove llama_cpp_python
+
+  test-workflow-complete:
+    needs: ["test"]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Test Workflow Complete
+        run: echo "Test Workflow Complete"