kerbside 0.2.5__tar.gz

kerbside-0.2.5/.github/workflows/functional-tests.yml

@@ -0,0 +1,475 @@
+name: Test cloud compatibility
+
+on:
+  workflow_dispatch:
+    inputs:
+      target:
+        description: Select target
+        type: choice
+        options:
+        - '["ovirt-44-rocky-8"]'
+        - '["ovirt-45-rocky-8"]'
+        - '["ovirt-45-rocky-9"]'
+        - '["kolla-ansible-master-debian-12"]'
+        default: '["kolla-ansible-master-debian-12"]'
+        required: true
+      retention:
+        description: 'How long to retain the environment after the workflow run in minutes'
+        required: true
+        default: '0'
+  pull_request:
+    branches:
+      - develop
+
+jobs:
+  sanity_checks:
+    runs-on: [self-hosted, vm, debian-12]
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-lint
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout code with two commits
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install additional dependencies not tracked by pip
+        timeout-minutes: 5
+        run: |
+          sudo apt-get install -y python3-venv
+          python3 -mvenv /tmp/bindep-venv
+          /tmp/bindep-venv/bin/pip3 install bindep
+          echo
+          /tmp/bindep-venv/bin/bindep || true
+          echo
+          sudo apt-get install -y $(/tmp/bindep-venv/bin/bindep --brief || true)
+
+      - name: Upgrade base test tools
+        timeout-minutes: 5
+        run: |
+          python3 -mvenv /tmp/venv-test
+          . /tmp/venv-test/bin/activate
+          pip3 install uv
+          uv pip install tox tox-uv
+
+      - name: Lint with flake8
+        timeout-minutes: 5
+        run: |
+          . /tmp/venv-test/bin/activate
+          tox -eflake8
+
+      - name: Attempt to install requirements
+        timeout-minutes: 5
+        run: |
+          python3 -mvenv /tmp/venv-reqs
+          . /tmp/venv-reqs/bin/activate
+          pip3 install uv
+          uv pip install -r pyproject.toml
+
+      - name: Run python3 unit tests
+        timeout-minutes: 10
+        run: |
+          . /tmp/venv-test/bin/activate
+          tox -epy3
+
+      - name: List failing tests
+        if: failure()
+        id: failures
+        run: |
+          echo "Gathering list of failed tests..."
+          touch /srv/ci/failed
+          . /tmp/venv-test/bin/activate
+          tox -efailing || true > /srv/ci/failed
+          echo
+
+          failed=$(cat /srv/ci/failed | egrep "^FAIL: " | \
+              sed -z 's/FAIL: shakenfist_ci\.//g;s/\n/, /g;s/, $/\n/')
+
+          echo "Failed tests:"
+          cat /srv/ci/failed
+          echo
+
+          echo "failures<<EOF" >> ${GITHUB_OUTPUT}
+          echo "${failed}" >> ${GITHUB_OUTPUT}
+          echo "EOF" >> ${GITHUB_OUTPUT}
+          echo "GITHUB_OUTPUT is:"
+          cat ${GITHUB_OUTPUT}
+
+      - name: Generate coverage report
+        timeout-minutes: 10
+        run: |
+          . /tmp/venv-test/bin/activate
+          tox -ecover
+          zip -r cover.zip cover
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: coverage
+          retention-days: 90
+          if-no-files-found: error
+          path: cover.zip
+
+  ovirt_matrix:
+    name: "${{ matrix.test.description }}"
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [
+          {
+            'name': 'ovirt-44-rocky-8',
+            'description': 'oVirt 4.4 on Rocky 8',
+            'environment_name': 'ovirt',
+            'ovirt_rpm': 'https://resources.ovirt.org/pub/yum-repo/ovirt-release44.rpm',
+            'base': 'rocky:8',
+            'baseuser': 'cloud-user',
+            'hostname': 'ovirt'
+          },
+          {
+            'name': 'ovirt-45-rocky-8',
+            'description': 'oVirt 4.5 on Rocky 8',
+            'environment_name': 'ovirt',
+            'ovirt_rpm': 'centos-release-ovirt45',
+            'base': 'rocky:8',
+            'baseuser': 'cloud-user',
+            'hostname': 'ovirt'
+          },
+          {
+            'name': 'ovirt-45-rocky-9',
+            'description': 'oVirt 4.5 on Rocky 9',
+            'environment_name': 'ovirt',
+            'ovirt_rpm': 'centos-release-ovirt45',
+            'base': 'rocky:9',
+            'baseuser': 'cloud-user',
+            'hostname': 'ovirt'
+          },
+        ]
+    runs-on: [self-hosted, vm]
+    needs: sanity_checks
+    timeout-minutes: 120
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.test.name }}
+      cancel-in-progress: true
+
+    # NOTE(mikal): git repos are checked out to /srv/github/_work/{org}/{repo}
+    # which is available as GITHUB_WORKSPACE. You can find other environment
+    # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables
+
+    steps:
+      - name: Filter workflow_dispatch runs
+        if: github.event_name == 'workflow_dispatch' && ! contains(inputs.target, matrix.test.name)
+        uses: actions/github-script@v6
+        with:
+          script: |
+              core.setFailed('Target skipped')
+
+      - name: Set environment variables
+        shell: bash
+        run: |
+          echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+          echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV
+
+      - name: Checkout the actions repository
+        uses: actions/checkout@v4
+        with:
+          repository: shakenfist/actions
+          path: actions
+          fetch-depth: 0
+
+      - name: Checkout kerbside
+        uses: actions/checkout@v4
+        with:
+          path: kerbside
+          fetch-depth: 0
+
+      - name: Build infrastructure
+        shell: bash
+        run: |
+          . $GITHUB_ENV
+          ansible-playbook -i /home/debian/ansible-hosts \
+              --extra-vars "identifier=${SHAKENFIST_NAMESPACE} base=${{ matrix.test.base }} environment_name=${{ matrix.test.environment_name }} hostname=${{ matrix.test.hostname }}" \
+              --tags all ${GITHUB_WORKSPACE}/actions/ansible/kerbside-single-node.yml
+
+      - name: Install base packages
+        run: |
+          . $GITHUB_ENV
+
+          script="export PS4='=======================\n+ '; set -x;"
+          script="${script} sudo dnf update -y;"
+          script="${script} sudo dnf install -y epel-release;"
+          script="${script} sudo crb enable;"
+          script="${script} sudo dnf update -y;"
+          script="${script} sudo dnf install -y vim patch yum-utils;"
+          script="${script} sudo chown ${{ matrix.test.baseuser }}:${{ matrix.test.baseuser }} /srv;"
+
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              "${script}"
+
+      - name: Install upstream oVirt release package
+        run: |
+          . $GITHUB_ENV
+
+          script="export PS4='=======================\n+ '; set -x;"
+          script="${script} sudo dnf install -y ${{ matrix.test.ovirt_rpm }};"
+
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              "${script}"
+
+      - name: Copy oVirt yum repos patch to target node
+        run: |
+          . $GITHUB_ENV
+
+          scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+            -o UserKnownHostsFile=/dev/null \
+            ${GITHUB_WORKSPACE}/kerbside/etc/${{ matrix.test.name }}-repos.patch \
+            ${{ matrix.test.baseuser }}@10.0.2.2:/srv/
+
+      - name: Apply repos patch
+        run: |
+          . $GITHUB_ENV
+
+          script="export PS4='=======================\n+ '; set -x;"
+          script="${script} cd /etc/yum.repos.d;"
+          script="${script} sudo patch -p1 < /srv/${{ matrix.test.name }}-repos.patch;"
+
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              "${script}"
+
+      - name: Further package configuration
+        run: |
+          . $GITHUB_ENV
+
+          script="export PS4='=======================\n+ '; set -x;"
+          script="${script} sudo dnf update -y;"
+          script="${script} sudo dnf module -y enable javapackages-tools;"
+          script="${script} sudo dnf module -y enable postgresql:12;"
+          script="${script} sudo dnf module -y enable mod_auth_openidc:2.3;"
+          script="${script} sudo dnf module -y enable nodejs:14;"
+          script="${script} sudo dnf module -y enable pki-deps;"
+          script="${script} sudo dnf distro-sync --nobest -y;"
+
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              "${script}"
+
+      - name: Install ovirt-engine
+        run: |
+          . $GITHUB_ENV
+
+          script="export PS4='=======================\n+ '; set -x;"
+          script="${script} sudo dnf install -y ovirt-engine;"
+
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              "${script}"
+
+      - name: Setup ovirt-engine
+        run: |
+          . $GITHUB_ENV
+
+          scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+            -o UserKnownHostsFile=/dev/null \
+            ${GITHUB_WORKSPACE}/kerbside/etc/${{ matrix.test.name }}-answers.conf \
+            ${{ matrix.test.baseuser }}@10.0.2.2:/srv/
+
+          script="export PS4='=======================\n+ '; set -x;"
+          script="${script} sudo engine-setup"
+              script="${script} --config-append=/srv/${{ matrix.test.name }}-answers.conf;"
+
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              "${script}"
+
+      - name: Gather artifacts
+        run: |
+          . $GITHUB_ENV
+
+          script="export PS4='=======================\n+ '; set -x;"
+
+          script="${script} sudo rpm -qa --qf "
+              script="${script} \"%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}\n\" "
+              script="${script} > /tmp/rpms.list;"
+
+          script="${script} for rpm in \$(cat /tmp/rpms.list); do"
+              script="${script} echo \"Caching URLs for \${rpm}\";"
+              script="${script} sudo yumdownloader --resolve --urls \${rpm} |"
+              script="${script} egrep -v '(metadata expiration|Waiting for)' "
+              script="${script} >> /tmp/rpms.urls;"
+              script="${script} done;"
+
+          script="${script} sudo zip -r /tmp/bundle.zip "
+              script="${script} /etc/yum.repos.d "
+              script="${script} /tmp/rpms.list "
+              script="${script} /tmp/rpms.urls "
+              script="${script} /var/lib/ovirt-engine/setup "
+              script="${script} || true;"
+          script="${script} sudo chmod ugo+r /tmp/bundle.zip;"
+          script="${script} ls -lrth /tmp/bundle.zip;"
+
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              "${script}"
+
+          mkdir -p /srv/github/artifacts/bundle/
+          scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2:/tmp/bundle.zip \
+              /srv/github/artifacts/bundle/
+
+          cd /srv/github/artifacts/bundle/
+          unzip bundle.zip
+          rm bundle.zip
+          cd ..
+          zip -r bundle.zip bundle
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: bundle-${{ matrix.test.name }}
+          retention-days: 90
+          if-no-files-found: error
+          path: /srv/github/artifacts/bundle.zip
+
+      - name: Retain the environment if requested
+        if: github.event_name == 'workflow_dispatch'
+        run: sleep $(( $MINUTES * 60 ))
+        env:
+          MINUTES: ${{ inputs.retention }}
+
+  openstack_matrix:
+    name: "${{ matrix.test.description }}"
+    strategy:
+      fail-fast: false
+      matrix:
+        test: [
+          {
+            'name': 'kolla-ansible-master-debian-12',
+            'environment_name': 'kolla',
+            'description': 'OpenStack via Kolla-Ansible master on Debian 12',
+            'openstack_release': 'master',
+            'base': 'debian:12',
+            'baseuser': 'debian'
+          },
+        ]
+    runs-on: [self-hosted, vm]
+    needs: sanity_checks
+    timeout-minutes: 120
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.test.name }}
+      cancel-in-progress: true
+
+    # NOTE(mikal): git repos are checked out to /srv/github/_work/{org}/{repo}
+    # which is available as GITHUB_WORKSPACE. You can find other environment
+    # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables
+
+    steps:
+      - name: Filter workflow_dispatch runs
+        if: github.event_name == 'workflow_dispatch' && ! contains(inputs.target, matrix.test.name)
+        uses: actions/github-script@v6
+        with:
+          script: |
+              core.setFailed('Target skipped')
+
+      - name: Setup test environment
+        uses: shakenfist/actions/setup-kerbside-environment@main
+        with:
+          base: ${{ matrix.test.base }}
+          base_user: ${{ matrix.test.baseuser }}
+          openstack_release: ${{ matrix.test.openstack_release }}
+          topology: all-in-one
+
+      - name: Build patched repositories
+        run: |
+          . $GITHUB_ENV
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              'cd /srv/shakenfist/kerbside-patches; ./_build/assemble-source.sh ${{ matrix.test.openstack_release }}'
+
+      - name: Build container images
+        run: |
+          . $GITHUB_ENV
+
+          script="cd /srv/shakenfist/kerbside-patches;"
+          script="${script} hash=\$(./_build/calculate-container-hash.sh kolla kolla-2* nova nova-2* requirements requirements-2*);"
+          script="${script} ./_build/build-containers.sh --use-ci-registry --build-targets ${{ matrix.test.openstack_release }} --build-images all --image-tag "\$hash-debian-bookworm";"
+
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 "${script}"
+
+      - name: Bootstrap Kolla-Ansible
+        run: |
+          . $GITHUB_ENV
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/bootstrap-kolla-ansible ${{ matrix.test.openstack_release }} all-in-one'
+
+      - name: Install Kolla-Ansible
+        run: |
+          . $GITHUB_ENV
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/install-kolla-ansible'
+
+      - name: Install patched OpenStack clients
+        run: |
+          . $GITHUB_ENV
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/install-openstack-clients'
+
+      - name: Post install Kolla-Ansible setup
+        run: |
+          . $GITHUB_ENV
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/postinstall-kolla-ansible'
+
+      - name: Attempt to start an instance with a console and connect to it
+        run: |
+          . $GITHUB_ENV
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/test-console'
+
+      - name: Collect logs
+        if: always()
+        run: |
+          . $GITHUB_ENV
+          ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2 \
+              'cd /srv/shakenfist/kerbside-patches; sudo ./tools/gather-logs'
+          scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \
+              -o UserKnownHostsFile=/dev/null \
+              ${{ matrix.test.baseuser }}@10.0.2.2:/tmp/bundle.zip /tmp/bundle.zip
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: bundle-${{ matrix.test.name }}
+          retention-days: 90
+          if-no-files-found: error
+          path: /tmp/bundle.zip
+
+      - name: Retain the environment if requested
+        if: github.event_name == 'workflow_dispatch'
+        run: sleep $(( $MINUTES * 60 ))
+        env:
+          MINUTES: ${{ inputs.retention }}

kerbside-0.2.5/.github/workflows/pin-indirect-dependencies.yml

@@ -0,0 +1,106 @@
+# NOTE(mikal): we need to pin indirect dependencies because some of our
+# dependencies don't pin things how we like to. If they're not pinned then when
+# something multiple layers away does a release and breaks us it can be super
+# confusing.
+
+name: Pin indirect dependencies
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/pin-indirect-dependencies.yml
+  workflow_dispatch:
+  schedule:
+    - cron: '00 00 * * *'        # utc
+
+jobs:
+  indirect-dependencies:
+    runs-on: [self-hosted, vm]
+    timeout-minutes: 60
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+
+    # NOTE(mikal): git repos are checked out to /srv/github/_work/{org}/{repo}
+    # which is available as GITHUB_WORKSPACE. You can find other environment
+    # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables
+
+    steps:
+      - name: Set environment variables
+        run: |
+          echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV
+
+      - name: Checkout kerbside
+        uses: actions/checkout@v4
+        with:
+          path: kerbside
+          fetch-depth: 0
+
+      - name: Install CI dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y pkg-config build-essential python3-dev \
+              default-libmysqlclient-dev
+
+      - name: Install the github command line
+        run: |
+          sudo apt update
+          sudo apt install -y curl
+
+          curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
+          sudo chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+
+          sudo apt update
+          sudo apt install -y gh
+
+      - name: Install all our dependencies in a venv
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -o DPkg::Lock::Timeout=-1 -o Dpkg::Options::="--force-confold" -y \
+            install git python3-cffi python3-dev python3-grpcio python3-pip python3-venv python3-wheel
+
+          cd ${GITHUB_WORKSPACE}/kerbside
+          python3 -m venv  --system-site-packages ${RUNNER_TEMP}/venv
+          ${RUNNER_TEMP}/venv/bin/pip install -r requirements.txt
+
+      - name: List all our dependencies
+        run: |
+          ${RUNNER_TEMP}/venv/bin/pip freeze --local
+
+      - name: Ensure all dependencies are pinned
+        env:
+          GITHUB_TOKEN: ${{ secrets.DEPENDENCIES_TOKEN }}
+        run: |
+          cd ${GITHUB_WORKSPACE}/kerbside
+
+          datestamp=$(date "+%Y%m%d")
+          git checkout -b pin-dependencies-${datestamp}
+
+          for depver in $(${RUNNER_TEMP}/venv/bin/pip freeze --local); do
+            dep=$(echo ${depver} | sed 's/==.*//')
+
+            if [ $(egrep -ic "^${dep}==" requirements.txt) -lt 1 ]; then
+              echo "${depver}" >> requirements.txt
+            fi
+          done
+
+          # Did we find something new?
+          if [ $(git diff | wc -l) -gt 0 ]; then
+            echo "New dependencies detected..."
+            echo
+            git diff
+
+            git config --global user.name "shakenfist-bot"
+            git config --global user.email "bot@shakenfist.com"
+            git commit -a -m "Update pinned dependencies."
+            git push -f origin pin-dependencies-${datestamp}
+            echo
+            gh pr create \
+                --assignee mikalstill \
+                --reviewer mikalstill \
+                --title "Update pinned dependencies." \
+                --body "New indirect dependencies were detected." \
+                --label dependencies
+            echo
+            echo "Pull request created."
+          fi