PySDM 3.0.0rc39__tar.gz → 3.0.0rc41__tar.gz

pysdm-3.0.0rc41/.github/workflows/docs_code_snippets.yml

@@ -0,0 +1,109 @@
+name: docs_code
+
+defaults:
+  run:
+    shell: bash
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  schedule:
+    - cron: '0 13 * * 4'
+
+jobs:
+  python:
+    strategy:
+      matrix:
+        platform: [ubuntu-latest, macos-14, windows-latest]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v4.1.6
+      - uses: actions/setup-python@v7.0.0
+        with:
+          python-version: "3.11"
+      - run: python -m pip install $PIP_INSTALL_ARGS -e .[CI_version_pins]
+      - run: python -m pip install $PIP_INSTALL_ARGS pytest-codeblocks pytest
+      - run: |
+          python -c "import os,pytest_codeblocks; code=pytest_codeblocks.extract_from_file('docs/markdown/pysdm_landing.md'); f=open('docs_code.py', 'w', encoding='utf-8'); f.write('# coding: utf-8'+os.linesep); f.writelines(block.code for block in code if block.syntax=='Python'); f.close()"
+      - run: cat -n docs_code.py
+      - name: run-on-cpu-backend
+        run: |
+          python -We docs_code.py
+      - name: run-on-gpu-backend
+        run: |
+          sed -i -e 's/CPU/GPU/g' docs_code.py
+          python -We docs_code.py
+      
+      - name: artefacts
+        if: github.ref == 'refs/heads/main' && matrix.platform == 'ubuntu-latest'
+        uses: eine/tip@master
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          files: |
+            readme.png
+      - name: run-with-jax-backend
+        run: |
+          sed -i -e 's/GPU/JAX/g' docs_code.py
+          sed -i -e 's/Coalescence(/Coalescence(adaptive=False,/g' docs_code.py
+          awk "{print}; /readme.png/ {exit}" docs_code.py > docs_code_jax.py
+          python -We docs_code_jax.py
+      - name: jax-artefacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: jax
+          if-no-files-found: error
+          path: readme.png
+
+  julia:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4.1.6
+      - uses: actions/setup-python@v7.0.0
+        with:
+          python-version: "3.11"
+      - run: pip install -e .[CI_version_pins]
+      - run: pip install pytest-codeblocks pytest
+      - run: python -c "import pytest_codeblocks; code=pytest_codeblocks.extract_from_file('docs/markdown/pysdm_landing.md'); f=open('docs_code.jl', 'w'); f.writelines(block.code for block in code if block.syntax=='Julia'); f.close()"
+      - uses: julia-actions/setup-julia@v3
+      - run: cat -n docs_code.jl
+      - run: echo PYTHON=`which python` >> $GITHUB_ENV
+      - run: julia docs_code.jl
+      - run: sed -i -e 's/CPU/GPU/g' docs_code.jl
+      - run: julia docs_code.jl
+
+  matlab:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.1.6
+      - uses: actions/setup-python@v7.0.0
+        with:
+          python-version: "3.12"
+
+      - run: echo NUMBA_THREADING_LAYER=workqueue >> $GITHUB_ENV
+
+      - run: pip install -e .[CI_version_pins]
+      - run: pip install pytest-codeblocks pytest
+      - run: python -c "import pytest_codeblocks; code=pytest_codeblocks.extract_from_file('docs/markdown/pysdm_landing.md'); f=open('docs_code.m', 'w'); f.writelines(block.code for block in code if block.syntax=='Matlab'); f.close()"
+      - run: cat -n docs_code.m
+      - uses: matlab-actions/setup-matlab@v3.1.0
+        with:
+          release: R2024b
+          cache: true
+      - uses: matlab-actions/run-command@v3.2
+        with:
+          command: docs_code
+      - run: sed -i -e 's/CPU/GPU/g' docs_code.m
+      - uses: matlab-actions/run-command@v3.2
+        with:
+          command: docs_code
+
+      - name: artefacts
+        if: github.ref == 'refs/heads/main'
+        uses: eine/tip@master
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          files: |
+            parcel.png
+

pysdm-3.0.0rc41/.github/workflows/tests.yml

@@ -0,0 +1,337 @@
+name: tests
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  # arbitrarily selected runs for generating animation
+  anim_python-version: '3.12'
+  anim_test-suite: 'multi-process_a'
+  # common options for pytest
+  pytest_options: "--durations=10 -v -s -We -p no:unraisableexception"
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore: ['docs/**', '**.md']
+  pull_request:
+    branches: [ main ]
+    paths-ignore: ['docs/**', '**.md']
+  schedule:
+    - cron: '0 13 * * 4'
+
+jobs:
+  env-unit-tests-setup:
+    strategy:
+      matrix:
+        platform: [ ubuntu-24.04, macos-14, windows-latest ]
+        python-version: [ "3.11", "3.12" ]
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v4.1.6
+        with:
+          submodules: recursive
+          fetch-depth: 0
+      - uses: ./.github/actions/env-setup
+        with:
+          python-version: ${{ matrix.python-version }}
+          setup-name: unit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
+
+  env-nonunit-tests-setup:
+    strategy:
+      matrix:
+        platform: [ubuntu-24.04, macos-14, windows-latest]
+        python-version: ["3.12"]
+        include: # for pylint
+          - platform: ubuntu-24.04
+            python-version: "3.11"
+    runs-on: ${{ matrix.platform }}
+    steps:
+      - uses: actions/checkout@v4.1.6
+        with:
+          submodules: recursive
+          fetch-depth: 0
+      - uses: ./.github/actions/env-setup
+        with:
+          python-version: ${{ matrix.python-version }}
+          setup-name: nonunit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
+
+  pylint:
+    needs: env-nonunit-tests-setup
+    strategy:
+      matrix:
+        platform: [ubuntu-24.04]
+        python-version: ["3.11"]
+    runs-on: ${{ matrix.platform }}
+    steps:
+    - uses: actions/checkout@v4.1.6
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    - uses: ./.github/actions/env-setup
+      with:
+        python-version: ${{ matrix.python-version }}
+        setup-name: nonunit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
+    - run: pip install pylint nbqa
+    - run: |
+        # TODO #682
+        pylint --unsafe-load-any-extension=y --disable=fixme,invalid-name,missing-function-docstring,missing-class-docstring,protected-access,duplicate-code $(git ls-files '*.py' | grep -v -e ^examples -e ^tutorials)
+    - run: |
+        # TODO #682
+        pylint --max-module-lines=550 --unsafe-load-any-extension=y --disable=fixme,too-many-function-args,unsubscriptable-object,consider-using-with,protected-access,too-many-statements,too-many-public-methods,too-many-branches,duplicate-code,invalid-name,missing-function-docstring,missing-module-docstring,missing-class-docstring,too-many-locals,too-many-instance-attributes,too-few-public-methods,too-many-arguments,c-extension-no-member $(git ls-files '*.py' | grep ^examples)
+    - run: |
+        # TODO #682
+        nbqa pylint --unsafe-load-any-extension=y --disable=fixme,duplicate-code,invalid-name,trailing-whitespace,line-too-long,missing-function-docstring,wrong-import-position,missing-module-docstring,wrong-import-order,ungrouped-imports,no-member,too-many-locals,unnecessary-lambda-assignment,multiple-imports $(git ls-files '*.ipynb')
+
+  unit-tests-without-jit-plus-codecov:
+    needs: env-unit-tests-setup
+    strategy:
+      matrix:
+        platform: [ ubuntu-24.04, macos-14, windows-latest ]
+        python-version: [ "3.11", "3.12" ]
+    runs-on: ${{ matrix.platform }}
+    env:
+      NUMBA_DISABLE_JIT: 1
+      PYTHON: ${{ matrix.python-version }}
+    steps:
+    - uses: actions/checkout@v4.1.6
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    - uses: ./.github/actions/env-setup
+      with:
+        python-version: ${{ matrix.python-version }}
+        setup-name: unit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
+    - name: Generate coverage report
+      run: |
+        python -m pip install pytest-cov
+        python -m pytest ${{ env.pytest_options }} tests/unit_tests --cov-report=xml --cov=PySDM
+    - uses: codecov/codecov-action@v7.1.1
+      with:
+        fail_ci_if_error: true
+        verbose: true
+        token: ${{ secrets.CODECOV_TOKEN }}
+        env_vars: PYTHON,RUNNER_OS
+
+  unit-tests-with-jit:
+    needs: [ pylint, unit-tests-without-jit-plus-codecov ]
+    strategy:
+      matrix:
+        platform: [ ubuntu-24.04, macos-14, windows-latest ]
+        python-version: [ "3.12" ]
+        test-suite: [ "unit_tests/!(dynamics)", "unit_tests/dynamics/!(condensation|collisions)", "unit_tests/dynamics/condensation", "unit_tests/dynamics/collisions" ]
+        exclude:
+          - platform: "macos-14"
+            test-suite: "unit_tests/!(dynamics)"
+      fail-fast: true
+    runs-on: ${{ matrix.platform }}
+    timeout-minutes: ${{ startsWith(matrix.platform, 'ubuntu-') && 30 || 40 }}
+    steps:
+      - uses: actions/checkout@v4.1.6
+        with:
+          submodules: recursive
+          fetch-depth: 0
+      - uses: ./.github/actions/env-setup
+        with:
+          python-version: ${{ matrix.python-version }}
+          setup-name: unit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
+      - name: Run pytest
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NUMBA_OPT: 1
+        run: |
+          shopt -s extglob
+          python -m pip install pytest-timeout
+          python -m pytest ${{ env.pytest_options }} --timeout=900 --timeout_method=thread tests/${{ matrix.test-suite }}
+
+  smoke-tests-with-jit:
+    needs: [ pylint, env-nonunit-tests-setup ]
+    strategy:
+      matrix:
+        platform: [ ubuntu-24.04, macos-14, windows-latest ]
+        python-version: [ "3.12" ]
+        test-suite:
+          - "tests/smoke_tests/no_env tests/smoke_tests/box tests/tutorials_tests"
+          - "tests/smoke_tests/parcel_a"
+          - "tests/smoke_tests/parcel_b"
+          - "tests/smoke_tests/parcel_c"
+          - "tests/smoke_tests/parcel_d"
+          - "tests/smoke_tests/kinematic_1d"
+          - "tests/smoke_tests/kinematic_2d"
+        include:
+          - platform: "macos-14"
+            python-version: "3.12"
+            test-suite: "tests/smoke_tests/parcel_b tests/smoke_tests/parcel_c tests/smoke_tests/parcel_d" # combined for macos
+          - platform: "macos-14"
+            python-version: "3.12"
+            test-suite: "tests/smoke_tests/no_env tests/tutorials_tests tests/smoke_tests/kinematic_2d tests/smoke_tests/kinematic_1d" # combined for macos
+        exclude:
+          # TODO #1406
+          - platform: "macos-14"
+            test-suite: "tests/smoke_tests/no_env tests/smoke_tests/box tests/tutorials_tests"
+          - platform: "macos-14"
+            test-suite: "tests/smoke_tests/parcel_a"
+          # test-suites already included in combined run
+          - platform: "macos-14"
+            test-suite: "tests/smoke_tests/kinematic_1d"
+          - platform: "macos-14"
+            test-suite: "tests/smoke_tests/kinematic_2d"
+          - platform: "macos-14"
+            test-suite: "tests/smoke_tests/parcel_b"
+          - platform: "macos-14"
+            test-suite: "tests/smoke_tests/parcel_c"
+          - platform: "macos-14"
+            test-suite: "tests/smoke_tests/parcel_d"
+      fail-fast: true
+    runs-on: ${{ matrix.platform }}
+    timeout-minutes: ${{ startsWith(matrix.platform, 'windows-') && 35 || 25 }}
+    steps:
+      - uses: actions/checkout@v4.1.6
+        with:
+          submodules: recursive
+          fetch-depth: 0
+      - uses: ./.github/actions/env-setup
+        with:
+          python-version: ${{ matrix.python-version }}
+          setup-name: nonunit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
+      - name: Run pytest
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          NUMBA_OPT: 1
+        run: |
+          shopt -s extglob
+          python -m pip install pytest-timeout
+          python -m pytest ${{ env.pytest_options }} --timeout=900 --timeout_method=thread ${{ matrix.test-suite }}
+
+  examples:
+    needs: [pylint, env-nonunit-tests-setup]
+    strategy:
+      matrix:
+        platform: [ubuntu-24.04, macos-14, windows-latest]
+        python-version: ["3.12"]
+        test-suite:
+          - "isotopes_chemistry_extraterrestrial"
+          - "homogeneous_freezing"
+          - "coagulation"
+          - "immersion_freezing"
+          - "condensation_a"
+          - "condensation_b"
+          - "condensation_c"
+          - "multi-process_a"
+          - "multi-process_b"
+          - "multi-process_c_breakup"
+          - "multi-process_d"
+          - "multi-process_e"
+        include:
+          - platform: "macos-14"
+            python-version: "3.12"
+            test-suite: "multi-process_a,multi-process_b"
+          - platform: "macos-14"
+            python-version: "3.12"
+            test-suite: "multi-process_d,multi-process_e"
+        exclude:
+          - platform: "macos-14"
+            test-suite: "multi-process_a"
+          - platform: "macos-14"
+            test-suite: "multi-process_b"
+          - platform: "macos-14"
+            test-suite: "multi-process_d"
+          - platform: "macos-14"
+            test-suite: "multi-process_e"
+          # TODO #1870
+          - platform: "ubuntu-24.04"
+            test-suite: "homogeneous_freezing"
+          - platform: "windows-latest"
+            test-suite: "homogeneous_freezing"
+          - platform: "windows-latest"
+            test-suite: "coagulation"
+      fail-fast: false
+    runs-on: ${{ matrix.platform }}
+    timeout-minutes: ${{ startsWith(matrix.platform, 'windows-') && 35 || 30 }}
+    steps:
+      - name: Debug contexts
+        run: |
+          echo "MATRIX:"
+          printf '%s\n' "${{ toJson(matrix) }}"
+          echo "ENV:"
+          printf '%s\n' "${{ toJson(env) }}"
+          echo "anim_test-suite=${{ env.anim_test_suite }}, anim_python-version=${{ env.anim_python_version }}"
+      - uses: actions/checkout@v4.1.6
+        with:
+          submodules: recursive
+          fetch-depth: 0
+      - uses: ./.github/actions/env-setup
+        with:
+          python-version: ${{ matrix.python-version }}
+          setup-name: nonunit-tests-${{ matrix.platform }}-${{ matrix.python-version }}
+          # https://github.com/numba/numba/issues/6350#issuecomment-728174860
+
+      - if: startsWith(matrix.platform, 'ubuntu-') && (contains(matrix.test-suite, 'multi-process_a') || contains(matrix.test-suite, 'multi-process_b'))
+        run: |
+          sudo apt-get update
+          sudo apt-get install libegl1 libopengl0 ghostscript ffmpeg
+          wget -nv "https://paraview.org/paraview-downloads/download.php?submit=Download&version=v5.13&type=binary&os=Linux&downloadFile=ParaView-5.13.1-egl-MPI-Linux-Python3.10-x86_64.tar.gz" -O paraview.tar.gz
+          tar xzf paraview.tar.gz
+          echo `pwd`/`dirname ParaView*/bin/pvpython` >> $GITHUB_PATH
+      - if: startsWith(matrix.platform, 'macos-') && (contains(matrix.test-suite, 'multi-process_a') || contains(matrix.test-suite, 'multi-process_b'))
+        run: |
+          export HOMEBREW_NO_AUTO_UPDATE=1
+          brew install --cask paraview
+          echo `dirname /Applications/ParaView-*.app/Contents/bin/pvpython | head -1` >> $GITHUB_PATH
+          brew install ghostscript ffmpeg
+      - if: startsWith(matrix.platform, 'windows-') && (contains(matrix.test-suite, 'multi-process_a') || contains(matrix.test-suite, 'multi-process_b'))
+        run: |
+          curl "https://www.paraview.org/paraview-downloads/download.php?submit=Download&version=v5.13&type=binary&os=Windows&downloadFile=ParaView-5.13.1-Windows-Python3.10-msvc2017-AMD64.msi" -o paraview.msi
+          msiexec //i paraview.msi //quiet //qn //norestart
+          echo "C:\\Program Files\\ParaView 5.13.1\\bin\\" >> $GITHUB_PATH
+      - if: contains(matrix.test-suite, 'multi-process_a') || contains(matrix.test-suite, 'multi-process_b')
+        run: |
+          pvpython --version
+          pvpython -c "from paraview import simple"
+
+      - if: startsWith(matrix.platform, 'windows-') && (contains(matrix.test-suite, 'multi-process_a') || contains(matrix.test-suite, 'multi-process_b'))
+        uses: pyvista/setup-headless-display-action@v5
+
+      - run: |
+          python -m pytest ${{ env.pytest_options }} tests/examples_tests/test_tests_completeness.py
+      - run: |
+          python -m pip install pytest-timeout
+          python -m pytest ${{ env.pytest_options }} --basetemp=/tmp/pytest --timeout_method=thread --timeout=${{ startsWith(matrix.platform, 'windows-') && 1200 || 900 }} tests/examples_tests/test_run* --suite ${{ matrix.test-suite }}
+
+      - if: ( ! startsWith(matrix.platform, 'windows-') ) && contains(matrix.test-suite, env.anim_test-suite) && matrix.python-version == env.anim_python-version
+        run: |
+          set -x
+          mkdir -p $HOME/work/_temp/_github_home/figures
+          rm /tmp/pytest/test_run_notebooks_*current
+          ls /tmp/pytest/test_run_notebooks_*/
+          ls /tmp/pytest/test_run_notebooks_*/output/
+          mv /tmp/pytest/test_run_notebooks_*/output/last_animation_frame.pdf $HOME/work/_temp/_github_home/figures/last_animation_frame_${{ matrix.platform }}.pdf
+          for anim_name in docs_intro_animation; do
+              ffmpeg -i /tmp/pytest/test_run_notebooks_*/output/${anim_name}.ogv -loop 0 -vf "fps=10,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" $HOME/work/_temp/_github_home/figures/${anim_name}_${{ matrix.platform }}.gif
+          done
+          for anim_name in parcel_animation; do
+              mv /tmp/pytest/test_run_notebooks_*/${anim_name}.gif $HOME/work/_temp/_github_home/figures/${anim_name}_${{ matrix.platform }}.gif
+          done
+          ls $HOME/work/_temp/_github_home/figures
+
+      - name: animation movie upload
+        if: ( ! startsWith(matrix.platform, 'windows-') ) && contains(matrix.test-suite, env.anim_test-suite) && matrix.python-version == env.anim_python-version
+        uses: actions/upload-artifact@v7
+        with:
+          name: animation-movie-${{ matrix.platform }}
+          if-no-files-found: error
+          path: ~/work/_temp/_github_home/figures/*.gif
+      - name: animation frame upload
+        if: ( ! startsWith(matrix.platform, 'windows-') ) && contains(matrix.test-suite, env.anim_test-suite) && matrix.python-version == env.anim_python-version
+        uses: actions/upload-artifact@v7
+        with:
+          name: animation-frame-${{ matrix.platform }}
+          if-no-files-found: error
+          path: ~/work/_temp/_github_home/figures/*.pdf
+      - name: tip release upload
+        if: github.ref == 'refs/heads/main' && ( startsWith(matrix.platform, 'ubuntu-') ) && contains(matrix.test-suite, env.anim_test-suite) && matrix.python-version == env.anim_python-version
+        uses: eine/tip@master
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          files: /github/home/figures/*.gif