mlonmcu 0.8.dev0__tar.gz → 0.9.0__tar.gz

mlonmcu-0.9.0/.dockerignore

@@ -0,0 +1,8 @@
+deps/
+docker/
+out/
+results/
+logs/
+reports/
+workspace*/
+venv*/

mlonmcu-0.9.0/.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,15 @@
+* ML on MCU version:
+* Python version:
+* Operating System:
+
+### Description
+
+Describe what you were trying to get done.
+Tell us what happened, what went wrong, and what you expected to happen.
+
+### What I Did
+
+```
+Paste the command(s) you ran and the output.
+If there was a crash, please include the traceback here.
+```

mlonmcu-0.9.0/.github/workflows/bench.yml

@@ -0,0 +1,91 @@
+---
+##
+## Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+##
+## This file is part of MLonMCU.
+## See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+name: Run Benchmarks
+on:
+  workflow_dispatch:
+    inputs:
+      template:
+        description: MLonMCU Environment Template
+        required: true
+        default: default
+      benchmark:
+        description: Benchmark Name
+        required: true
+        default: ''
+jobs:
+  benchmark:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ['3.10']
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      # TODO: caching
+      - name: Install APT packages
+        run: |
+          sudo apt -qq install -y g++ libboost-all-dev libboost-system-dev libboost-filesystem-dev libboost-program-options-dev graphviz doxygen libtinfo-dev zlib1g-dev texinfo unzip device-tree-compiler rsync git-lfs libcurl4-openssl-dev
+          sudo apt -qq install -y ninja-build flex
+          sudo apt remove clang-11* llvm-11*
+      - name: Initialize Virtualenv
+        run: |
+          python -m pip install --upgrade pip
+          python -m venv .venv
+      - name: Install dependencies
+        run: |
+          source .venv/bin/activate
+          pip install -r requirements.txt
+      - name: Run package creation
+        run: |
+          source .venv/bin/activate
+          pip install .
+      - name: Initialize MLonMCU environment
+        run: |
+          source .venv/bin/activate
+          mlonmcu init home/ --non-interactive --template ${{ github.event.inputs.template }}
+      - name: Setup MLonMCU dependencies
+        run: |
+          source .venv/bin/activate
+          mlonmcu setup -H home/ -v
+      - name: Run Benchmark
+        run: |
+          source .venv/bin/activate
+          export MLONMCU_HOME=$(pwd)/home
+          cd scripts/
+          python bench.py ${{ github.event.inputs.benchmark }} out/
+      - name: Archive reports
+        uses: actions/upload-artifact@v4
+        with:
+          name: results
+          path: scripts/out/
+      - name: Get date
+        run: echo "timestamp=`date +%FT%T`" >> $GITHUB_ENV
+      - name: Upload results to a special branch
+        uses: JamesIves/github-pages-deploy-action@4.1.1
+        with:
+          folder: scripts/out/
+          branch: benchmark_results
+          target-folder: benchmarks/${{ github.event.inputs.benchmark }}/${{ env.timestamp
+            }}/

mlonmcu-0.9.0/.github/workflows/cicd.yml

@@ -0,0 +1,222 @@
+---
+##
+## Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+##
+## This file is part of MLonMCU.
+## See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+
+# GitHub CI build pipeline
+name: CI build
+on:
+  push:
+    branches: [main, develop]
+    tags: [v*]
+  pull_request:
+    branches: [main, develop]
+  workflow_dispatch:
+    inputs:
+      integration_tests:
+        description: Run slow integration tests in CI
+        required: true
+        default: 'false'
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/cache@v4
+        id: cache-venv  # name for referring later
+        with:
+          path: |
+            .venv/
+            dist/
+        # The cache key depends on requirements[_dev].txt
+          key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{
+            hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-
+      - name: Cleanup old artifacts
+        run: |
+          rm -rf .venv  # Clear venv because of a cache bug...
+          make clean
+      - name: Initialize Virtualenv
+        run: |
+          python -m pip install --upgrade pip virtualenv
+          python -m venv .venv
+      - name: Install dependencies
+        run: |
+          source .venv/bin/activate
+          pip install wheel
+          pip install -r requirements_dev.txt
+      - name: Run package creation
+        run: |
+          source .venv/bin/activate
+          make dist
+  release:
+    needs: build
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ['3.10']
+    if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/cache@v4
+        id: cache-venv  # name for referring later
+        with:
+          path: |
+            .venv/
+            dist/
+        # The cache key depends on requirements[_dev].txt
+          key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{
+            hashFiles('**/requirements*.txt') }}-${{ github.sha }}
+          restore-keys: |
+            build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}
+      - name: Archive package
+        uses: actions/upload-artifact@v4
+        with:
+          name: mlonmcu
+          path: dist/
+      - name: Publish package
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') &&
+          ${{ github.repository == 'tum-ei-eda/mlonmcu' }}
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          skip_existing: true
+  coverage:
+    needs: build
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ['3.10']
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/cache@v4
+        id: cache-venv  # name for referring later
+        with:
+          path: |
+            .venv/
+            dist/
+        # The cache key depends on requirements[_dev].txt
+          key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{
+            hashFiles('**/requirements*.txt') }}-${{ github.sha }}
+          restore-keys: |
+            build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}
+      - name: Initialize Virtualenv
+        run: |
+          python -m pip install --upgrade pip virtualenv
+          python -m venv .venv
+      - name: Install dependencies
+        run: |
+          source .venv/bin/activate
+          pip install -r requirements_dev.txt
+      - name: Install package into virtualenv
+        run: |
+          source .venv/bin/activate
+          pip install -e .
+      - name: Run Unit Tests and generate coverage report
+        if: ${{ github.event_name }} != "workflow_dispatch"
+        run: |
+          source .venv/bin/activate
+          make coverage
+      - name: Run Unit & Integration Tests and generate coverage report
+        if: ${{ github.event.inputs.integration_tests }} == "true"
+        run: |
+          source .venv/bin/activate
+          make coverage-full
+      - name: Archive code coverage html report
+        uses: actions/upload-artifact@v4
+        with:
+          name: code-coverage-report
+          path: htmlcov
+      - name: Get coverage percentage
+        id: report
+        run: |
+          source .venv/bin/activate
+          echo "name=percent::$(coverage report | awk '$1 == \"TOTAL\" {print $NF+0 \"%\"}')" >> $GITHUB_OUTPUT
+      - name: Test coverage Text
+        run: echo "|${{ steps.report.outputs.percent }}|"
+      - name: Create coverage badge
+        uses: RubbaBoy/BYOB@v1.2.1
+        with:
+          NAME: coverage
+          LABEL: Coverage
+          STATUS: ${{ steps.report.outputs.percent }}
+          COLOR: blue
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  docs:
+    needs: build
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: ['3.10']
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: actions/cache@v4
+        id: cache-venv  # name for referring later
+        with:
+          path: |
+            .venv/
+            dist/
+        # The cache key depends on requirements[_dev].txt
+          key: ${{ github.job }}-${{ runner.os }}-${{ matrix.python-version }}-venv-${{
+            hashFiles('**/requirements*.txt') }}-${{ github.sha }}
+          restore-keys: |
+            build-${{ runner.os }}-${{ matrix.python-version }}-venv-${{ hashFiles('**/requirements*.txt') }}
+      - name: Initialize Virtualenv
+        run: |
+          python -m pip install --upgrade pip virtualenv
+          python -m venv .venv
+      - name: Install dependencies
+        run: |  # TODO: get rid of if-else
+          source .venv/bin/activate
+          pip install -r requirements.txt
+          pip install -r docs/requirements.txt
+      - name: Install package into virtualenv
+        run: |
+          source .venv/bin/activate
+          make install
+      - name: Build docs
+        run: |
+          source .venv/bin/activate
+          make docs
+      - name: Deploy docs
+        uses: peaceiris/actions-gh-pages@v4
+        if: ${{ github.ref == 'refs/heads/main' }}
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/_build/html

mlonmcu-0.9.0/.github/workflows/container.yml

@@ -0,0 +1,279 @@
+---
+##
+## Copyright (c) 2022 TUM Department of Electrical and Computer Engineering.
+##
+## This file is part of MLonMCU.
+## See https://github.com/tum-ei-eda/mlonmcu.git for further info.
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+name: Build Docker Containers
+on:
+  workflow_dispatch:
+    inputs:
+      enable_cmake:
+        description: Install up-to-date CMake version
+        required: true
+        default: 'false'
+      conan_version:
+        description: Conan version (keep empty to skip conan install)
+        required: true
+        default: '1.59'  # legacy for old tgc-iss
+        # default: "2.20.1"  # new for dbt-rise-riscv
+      version:
+        description: Version Tag
+        required: true
+        default: latest
+      enable_bench:
+        description: Build very large image for benchmarking
+        required: true
+        default: 'false'
+      template:
+        description: Used environment.yml template for benchmarking
+        required: true
+        default: default
+jobs:
+  minimal:
+    name: Build minimal image for CI
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Remove unused software
+        run: |
+          echo "Available storage before:"
+          sudo df -h
+          echo
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --force
+          sudo docker builder prune
+          echo "Available storage after:"
+          sudo df -h
+          echo
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Update package version
+        run: |
+          ./scripts/update_version.sh
+      - name: Lowercase repository url
+        id: lowered
+        uses: ASzc/change-string-case-action@v6
+        with:
+          string: ${{ github.repository }}
+      - name: Build and push (CMake)
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: docker/Dockerfile
+          pull: true
+          push: true
+          target: cmake-install
+          build-args: |
+            ENABLE_CMAKE=${{ github.event.inputs.enable_cmake}}
+            MLONMCU_TEMPLATE=${{ github.event.inputs.template }}
+          cache-from: type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:${{
+            github.event.inputs.version }}
+          cache-to: type=inline
+          tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:${{ github.event.inputs.version
+            }}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: docker/Dockerfile
+          pull: true
+          push: true
+          target: mlonmcu-ci
+          build-args: |
+            ENABLE_CMAKE=${{ github.event.inputs.enable_cmake}}
+            MLONMCU_TEMPLATE=${{ github.event.inputs.template }}
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:${{ github.event.inputs.version }}
+            type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-ci:${{ github.event.inputs.version }}
+          cache-to: type=inline
+          tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}-ci:${{ github.event.inputs.version
+            }}
+  base:
+    name: Build base image for users
+    runs-on: ubuntu-22.04
+    needs: minimal
+    steps:
+      - name: Remove unused software
+        run: |
+          echo "Available storage before:"
+          sudo df -h
+          echo
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --force
+          sudo docker builder prune
+          echo "Available storage after:"
+          sudo df -h
+          echo
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Update package version
+        run: |
+          ./scripts/update_version.sh
+      - name: Lowercase repository url
+        id: lowered
+        uses: ASzc/change-string-case-action@v6
+        with:
+          string: ${{ github.repository }}
+      # - name: Build and push
+      #   uses: docker/build-push-action@v6
+      #   with:
+      #     context: .
+      #     file: docker/Dockerfile
+      #     pull: true
+      #     push: true
+      #     target: mlonmcu
+      #     cache-from: |
+      #       type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:${{ github.event.inputs.version }}
+      #       type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-ci:${{ github.event.inputs.version }}
+      #     cache-to: type=inline
+      #     tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}:${{ github.event.inputs.version }}
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }}
+        with:
+          registry: ${{ secrets.DOCKER_REGISTRY }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push to Docker Hub
+        uses: docker/build-push-action@v6
+        if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }}
+        with:
+          context: .
+          file: docker/Dockerfile
+          pull: true
+          push: true
+          target: mlonmcu
+          build-args: |
+            ENABLE_CMAKE=${{ github.event.inputs.enable_cmake}}
+            MLONMCU_TEMPLATE=${{ github.event.inputs.template }}
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:${{ github.event.inputs.version }}
+            type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-ci:${{ github.event.inputs.version }}
+          cache-to: type=inline
+          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ secrets.DOCKER_NAMESPACE }}/mlonmcu:${{
+            github.event.inputs.version }}
+  bench:
+    name: Build large image for benchmarking
+    runs-on: ubuntu-22.04
+    needs: base
+    if: ${{ github.event.inputs.enable_bench }} == "true"
+    steps:
+      - name: Remove unused software
+        run: |
+          echo "Available storage before:"
+          sudo df -h
+          echo
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --force
+          sudo docker builder prune
+          echo "Available storage after:"
+          sudo df -h
+          echo
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Lowercase repository url
+        id: lowered
+        uses: ASzc/change-string-case-action@v6
+        with:
+          string: ${{ github.repository }}
+      # - name: Build and push
+      #   uses: docker/build-push-action@v6
+      #   with:
+      #     context: .
+      #     file: docker/Dockerfile
+      #     pull: true
+      #  # push: true
+      #     target: mlonmcu-bench
+      #     build-args: |
+      #       MLONMCU_TEMPLATE=${{ github.event.inputs.template}}
+      #     cache-from: |
+      #       type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:${{ github.event.inputs.version }}
+      #       type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-ci:${{ github.event.inputs.version }}
+      #       type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}:${{ github.event.inputs.version }}
+      #     cache-to: type=inline
+      #     tags: ghcr.io/${{ steps.lowered.outputs.lowercase }}-bench:${{ github.event.inputs.version }}
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }}
+        with:
+          registry: ${{ secrets.DOCKER_REGISTRY }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push to Docker Hub
+        uses: docker/build-push-action@v6
+        if: ${{ github.repository == 'tum-ei-eda/mlonmcu' }}
+        with:
+          context: .
+          file: docker/Dockerfile
+          push: true
+          target: mlonmcu-bench
+          build-args: |
+            ENABLE_CMAKE=${{ github.event.inputs.enable_cmake}}
+            MLONMCU_TEMPLATE=${{ github.event.inputs.template }}
+            ENABLE_CONAN=true
+            CONAN_VERSION=${{ github.event.inputs.conan_version }}
+          cache-from: |
+            type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-cmake:${{ github.event.inputs.version }}
+            type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}-ci:${{ github.event.inputs.version }}
+            type=registry,ref=ghcr.io/${{ steps.lowered.outputs.lowercase }}:${{ github.event.inputs.version }}
+          cache-to: type=inline
+          tags: ${{ secrets.DOCKER_REGISTRY }}/${{ secrets.DOCKER_NAMESPACE }}/mlonmcu-bench:${{
+            github.event.inputs.template }}-${{ github.event.inputs.version }}