ESMValCore 2.10.0__tar.gz → 2.11.0__tar.gz

esmvalcore-2.11.0/.circleci/config.yml

@@ -0,0 +1,225 @@
+---
+version: 2.1
+
+orbs:
+  coverage-reporter: codacy/coverage-reporter@13.16.5
+  codecov: codecov/codecov@3.2.5
+
+commands:
+  check_changes:
+    steps:
+      - run:
+          name: Check whether or not installation tests are needed
+          command: |
+            if (test "$CIRCLE_BRANCH" = main ||
+              git --no-pager diff --name-only origin/main... |
+              grep -q -E -f .circleci/install_triggers)
+            then
+              echo Running installation tests
+            else
+              echo Skipping installation tests
+              circleci step halt
+            fi
+  test_and_report:
+    steps:
+      - run:
+          name: Run tests
+          command: |
+            mkdir -p test-reports
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            flake8 -j 4
+            pytest -n 4 --junitxml=test-reports/report.xml
+            esmvaltool version
+      - store_test_results:
+          path: test-reports/report.xml
+      - store_artifacts:
+          path: /logs
+      - run:
+          name: Compress pytest artifacts
+          command: tar -cvzf pytest.tar.gz -C /tmp/pytest-of-root/pytest-0/ .
+      - store_artifacts:
+          path: pytest.tar.gz
+      - run:
+          name: Compress test-report artifacts
+          command: tar -cvzf test-reports.tar.gz test-reports/
+      - store_artifacts:
+          path: test-reports.tar.gz
+  test_installation_from_source:
+    parameters:
+      extra:
+        type: string
+      flags:
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: Install git+ssh
+          environment:
+            DEBIAN_FRONTEND: noninteractive  # needed to install tzdata
+          command: apt update && apt install -y git ssh
+      - checkout
+      - check_changes
+      - run:
+          name: Generate cache key
+          command: date '+%Y-%V' | tee cache_key.txt
+      - restore_cache:
+          key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - run:
+          name: Install dependencies
+          command: |
+            # Add additional requirements for running all tests
+            echo "
+              - r-base
+              - r-yaml
+              - ncl
+            " >> environment.yml
+            # Install
+            . /opt/conda/etc/profile.d/conda.sh
+            mkdir /logs
+            mamba env create >> /logs/conda.txt 2>&1
+            conda activate esmvaltool
+            pip install << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
+      - run:
+          name: Log versions
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            dpkg -l > /logs/versions.txt
+            conda env export > /logs/environment.yml
+            pip freeze > /logs/requirements.txt
+      - test_and_report
+      - save_cache:
+          key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          paths:
+            - /opt/conda/pkgs
+            - /root/.cache/pip
+            - .mypy_cache
+            - .pytest_cache
+
+jobs:
+  run_tests:
+    # Run tests
+    docker:
+      - image: esmvalgroup/esmvalcore:development
+    resource_class: large
+    steps:
+      - checkout
+      - run:
+          name: Generate cache key
+          command: date '+%Y-%V' | tee cache_key.txt
+      - restore_cache:
+          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - run:
+          name: Install dependencies
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            mkdir /logs
+            conda activate esmvaltool
+            pip install .[test] > /logs/install.txt 2>&1
+      - test_and_report
+      - save_cache:
+          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          paths:
+            - /root/.cache/pip
+            - .mypy_cache
+            - .pytest_cache
+      - run:
+          name: Install gpg (required by codecov orb)
+          command: apt update && apt install -y gpg
+      - codecov/upload:
+          when: always
+          file: 'test-reports/coverage.xml'
+      - coverage-reporter/send_report:
+          coverage-reports: 'test-reports/coverage.xml'
+          project-token: $CODACY_PROJECT_TOKEN
+          skip: true  # skip if project-token is not defined (i.e. on a fork)
+
+  test_installation_from_source_test_mode:
+    # Test installation from source
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: large
+    steps:
+      - test_installation_from_source:
+          extra: test
+
+  test_installation_from_source_develop_mode:
+    # Test development installation
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: large
+    steps:
+      - test_installation_from_source:
+          extra: develop
+          flags: "--editable"
+
+  test_installation_from_conda:
+    # Test conda package installation
+    working_directory: /esmvaltool
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: medium
+    steps:
+      - run:
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            set -x
+            # Install prerequisites
+            mkdir /logs
+            # conda update -y conda > /logs/conda.txt 2>&1
+            # Create and activate conda environment
+            mamba create -y --name esmvaltool 'python=3.11'
+            set +x; conda activate esmvaltool; set -x
+            # Install
+            mamba install -y esmvalcore
+            # Log versions
+            conda env export > /logs/environment.yml
+            # Test installation
+            esmvaltool version
+
+  build_documentation:
+    # Test building documentation
+    docker:
+      - image: condaforge/mambaforge
+    resource_class: medium
+    steps:
+      - checkout
+      - run:
+          command: |
+            mkdir /logs
+            . /opt/conda/etc/profile.d/conda.sh
+            # Install
+            mamba env create
+            conda activate esmvaltool
+            pip install .[doc]
+            # Log versions
+            dpkg -l > /logs/versions.txt
+            conda env export > /logs/environment.yml
+            pip freeze > /logs/requirements.txt
+            # Test building documentation
+            MPLBACKEND=Agg sphinx-build -W doc doc/build
+      - store_artifacts:
+          path: /logs
+
+workflows:
+  commit:
+    jobs:
+      - run_tests
+      - test_installation_from_source_test_mode
+      - test_installation_from_source_develop_mode
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - run_tests
+      - test_installation_from_source_test_mode
+      - build_documentation
+      - test_installation_from_source_develop_mode
+      - test_installation_from_conda

esmvalcore-2.11.0/.github/workflows/build-and-deploy-on-pypi.yml

@@ -0,0 +1,49 @@
+name: PyPi Build and Deploy 🐍📦
+
+on:
+  release:
+    types: [published]
+  # use this for testing
+  push:
+    branches:
+      - main
+
+jobs:
+  build-n-publish:
+    name: Build and publish ESMValCore on PyPi
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/ESMValCore/
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+      - name: Install pep517
+        run: >-
+          python -m
+          pip install
+          pep517
+          --user
+      - name: Build a binary wheel and a source tarball
+        run: >-
+          python -m
+          pep517.build
+          --source
+          --binary
+          --out-dir dist/
+          .
+      # - name: Publish distribution to Test PyPI
+      #   uses: pypa/gh-action-pypi-publish@release/v1
+      #   with:
+      #     repository-url: https://test.pypi.org/legacy/
+      - name: Publish distribution 📦 to PyPI
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1

esmvalcore-2.11.0/.github/workflows/create-condalock-file.yml

@@ -0,0 +1,99 @@
+name: Conda lock file creation
+
+on:
+  # Trigger on push on main or other branch for testing
+  # NOTE that push: main will create the file very often
+  # and hence lots of automated PRs
+  # push:
+  #   branches:
+  #     - main
+  schedule:
+    - cron: '0 4 */10 * *'
+
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  create-verify-lock-file:
+    name: Create and verify conda lock file for latest Python
+    runs-on: 'ubuntu-latest'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          activate-environment: esmvaltool-fromlock
+          python-version: "3.12"
+          miniforge-version: "latest"
+          miniforge-variant: Mambaforge
+          use-mamba: true
+      - name: Update and show conda config
+        run: |
+          conda update -n base -c conda-forge conda
+          conda info
+          conda list
+          conda config --show-sources
+          conda config --show
+          # setup-miniconda@v2 installs an old conda and mamba
+          # forcing a modern mamba updates both mamba and conda
+          conda install -c conda-forge "mamba>=1.4.8"
+          conda --version
+          mamba --version
+      - name: Gather Python info
+        run: |
+          which python
+          python --version
+      - name: Install conda-lock
+        run: mamba install -y -c conda-forge conda-lock
+      - name: Check version of conda-lock
+        run: conda-lock --version
+      - name: Create conda lock file for linux-64
+        run: conda-lock lock --platform linux-64 -f environment.yml --mamba --kind explicit
+      - name: Creating environment from lock file
+        run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock
+      - name: Installing pip
+        run: mamba install -y pip
+      - name: Gather pip info
+        run: pip --version
+      - name: Gather Python info again
+        run: |
+          which python
+          python --version
+      - name: Show environment contents
+        run: conda list
+      - name: Install esmvalcore
+        run: pip install -e .[develop]
+      - name: Run basic commands
+        run: |
+          esmvaltool --help
+          esmvaltool version
+      - name: Run flake8
+        run: flake8
+      - name: Run pytests
+        run: pytest -n 2 -m "not installation"
+      # Automated PR
+      # see https://github.com/marketplace/actions/create-pull-request
+      - name: Create Automated PR if conda lock file has changed
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: Updating Linux condalock file
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          signoff: false
+          branch: condalock-update
+          delete-branch: true
+          title: '[Condalock] Update Linux condalock file'
+          body: |
+            Update condalock file
+            Automatic Pull Request.
+          labels: |
+            testing
+            condalock
+            automatedPR
+          assignees: valeriupredoi
+          reviewers: valeriupredoi
+          draft: false

esmvalcore-2.11.0/.github/workflows/install-from-conda.yml

@@ -0,0 +1,107 @@
+# Install esmvalcore from PyPi on different OS's
+# and different Python version; test locally with
+# act: https://github.com/nektos/act
+# Example how to setup conda workflows:
+# https://github.com/marketplace/actions/setup-miniconda
+# Notes:
+#  - you can group commands with | delimiter (or &&) but those will be run
+#    in one single call; declaring the shell variable makes the action run each
+#    command separately (better for debugging);
+#  - can try multiple shells eg pwsh or cmd /C CALL {0} (but overkill for now!);
+# TODO: read the cron tasking documentation:
+# https://www.netiq.com/documentation/cloud-manager-2-5/ncm-reference/data/bexyssf.html
+
+name: Install from conda-forge
+
+# runs on a push on main and at the end of every day
+on:
+  # triggering on push without branch name will run tests every time
+  # there is a push on any branch
+  # turn it on only if needed
+  push:
+    branches:
+      - main
+  # run the test only if the PR is to main
+  # turn it on if required
+  # pull_request:
+  #   branches:
+  #     - main
+  schedule:
+    - cron: '0 4 * * *'
+
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  linux:
+    runs-on: "ubuntu-latest"
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+      # fail-fast set to False allows all other tests
+      # in the workflow to run regardless of any fail
+      fail-fast: false
+    name: Linux Python ${{ matrix.python-version }}
+    steps:
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          miniforge-version: "latest"
+          miniforge-variant: Mambaforge
+          use-mamba: true
+      - run: mkdir -p conda_install_linux_artifacts_python_${{ matrix.python-version }}
+      - name: Record versions
+        run: |
+          mamba --version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
+          which conda 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_path.txt
+          which mamba 2>&1 | tee -a conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_path.txt
+          python -V 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
+      - name: Install ESMValCore
+        run: mamba install esmvalcore 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/install.txt
+      - name: Verify installation
+        run: |
+          esmvaltool --help
+          esmvaltool version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
+      - name: Upload artifacts
+        if: ${{ always() }}  # upload artifacts even if fail
+        uses: actions/upload-artifact@v2
+        with:
+          name: Conda_Install_Linux_python_${{ matrix.python-version }}
+          path: conda_install_linux_artifacts_python_${{ matrix.python-version }}
+
+  osx:
+    runs-on: "macos-latest"
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
+      fail-fast: false
+    name: OSX Python ${{ matrix.python-version }}
+    steps:
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+          miniforge-version: "latest"
+          miniforge-variant: Mambaforge
+          use-mamba: true
+      - run: mkdir -p conda_install_osx_artifacts_python_${{ matrix.python-version }}
+      - name: Record versions
+        run: |
+          mamba --version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
+          which conda 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_path.txt
+          which mamba 2>&1 | tee -a conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_path.txt
+          python -V 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
+      - name: Install ESMValCore
+        run: mamba install esmvalcore 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/install.txt
+      - name: Verify installation
+        run: |
+          esmvaltool --help
+          esmvaltool version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
+      - name: Upload artifacts
+        if: ${{ always() }}  # upload artifacts even if fail
+        uses: actions/upload-artifact@v2
+        with:
+          name: Conda_Install_OSX_python_${{ matrix.python-version }}
+          path: conda_install_osx_artifacts_python_${{ matrix.python-version }}

esmvalcore-2.11.0/.github/workflows/install-from-condalock-file.yml

@@ -0,0 +1,61 @@
+# Install esmvalcore from a conda lock file
+# To build a conda lock file install conda-lock first then
+# run conda-lock lock --platform linux-64 -f environment.yml --mamba
+# (mamba activated for speed). Change platform for osx-64 or win-64.
+# Env creation then happens as per normal use with
+# mamba create --name esmvaltool-fromlock --file conda-linux-64.lock
+# note that pip and conda are NOT installed.
+
+name: Conda-lock Install
+on:
+  push:
+    branches:
+      - main
+  # run the test only if the PR is to main
+  # turn it on if required
+  #pull_request:
+  #  branches:
+  #  - main
+  schedule:
+    - cron: '0 0 * * *'
+
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  linux:
+    runs-on: "ubuntu-latest"
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+      fail-fast: false
+    name: Linux Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: esmvaltool-fromlock
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+      - run: mkdir -p source_install_linux_artifacts_python_${{ matrix.python-version }}
+      - run: conda --version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
+      - run: which python
+      - run: python -V 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
+      - run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock
+      - run: which python
+      - run: pip --version
+      - run: pip install -e .[develop]
+      - run: esmvaltool --help
+      - run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
+      - run: flake8
+      - run: pytest -n 2 -m "not installation"
+      - name: Upload artifacts
+        if: ${{ always() }}  # upload artifacts even if fail
+        uses: actions/upload-artifact@v2
+        with:
+          name: Source_Install_Linux_python_${{ matrix.python-version }}
+          path: source_install_linux_artifacts_python_${{ matrix.python-version }}

esmvalcore-2.11.0/.github/workflows/install-from-pypi.yml

@@ -0,0 +1,111 @@
+# Install esmvalcore from PyPi on different OS's
+# and different Python version; test locally with
+# act: https://github.com/nektos/act
+# Example how to setup conda workflows:
+# https://github.com/marketplace/actions/setup-miniconda
+# Notes:
+#  - you can group commands with | delimiter (or &&) but those will be run
+#    in one single call; declaring the shell variable makes the action run each
+#    command separately (better for debugging);
+#  - can try multiple shells eg pwsh or cmd /C CALL {0} (but overkill for now!);
+# TODO: read the cron tasking documentation:
+# https://www.netiq.com/documentation/cloud-manager-2-5/ncm-reference/data/bexyssf.html
+
+name: Install from PyPi
+
+# runs on a push on main and at the end of every day
+on:
+  # triggering on push without branch name will run tests every time
+  # there is a push on any branch
+  # turn it on only if needed
+  push:
+    branches:
+      - main
+  # run the test only if the PR is to main
+  # turn it on if required
+  #pull_request:
+  #  branches:
+  #  - main
+  schedule:
+    - cron: '0 0 * * *'
+
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  linux:
+    runs-on: "ubuntu-latest"
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+      # fail-fast set to False allows all other tests
+      # in the workflow to run regardless of any fail
+      fail-fast: false
+    name: Linux Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: esmvalcore
+          environment-file: environment.yml
+          python-version: ${{ matrix.python-version }}
+          miniforge-version: "latest"
+          miniforge-variant: Mambaforge
+          use-mamba: true
+      - run: mkdir -p pip_install_linux_artifacts_python_${{ matrix.python-version }}
+      - name: Record versions
+        run: |
+          mamba --version 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
+          python -V 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
+          pip -V 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/pip_version.txt
+      - name: Install
+        run: pip install esmvalcore 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/install.txt
+      - name: Verify installation
+        run: |
+          esmvaltool --help
+          esmvaltool version 2>&1 | tee pip_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
+      - name: Upload artifacts
+        if: ${{ always() }}  # upload artifacts even if fail
+        uses: actions/upload-artifact@v2
+        with:
+          name: PIP_Install_Linux_python_${{ matrix.python-version }}
+          path: pip_install_linux_artifacts_python_${{ matrix.python-version }}
+
+  osx:
+    runs-on: "macos-latest"
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        architecture: ["x64"]  # need to force Intel, arm64 builds have issues
+      fail-fast: false
+    name: OSX Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: esmvalcore
+          environment-file: environment.yml
+          python-version: ${{ matrix.python-version }}
+          miniforge-version: "latest"
+          miniforge-variant: Mambaforge
+          use-mamba: true
+      - run: mkdir -p pip_install_osx_artifacts_python_${{ matrix.python-version }}
+      - name: Record versions
+        run: |
+          mamba --version 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
+          python -V 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
+          pip -V 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/pip_version.txt
+      - name: Install
+        run: pip install esmvalcore 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/install.txt
+      - name: Verify installation
+        run: |
+          esmvaltool --help
+          esmvaltool version 2>&1 | tee pip_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
+      - name: Upload artifacts
+        if: ${{ always() }}  # upload artifacts even if fail
+        uses: actions/upload-artifact@v2
+        with:
+          name: PIP_Install_OSX_python_${{ matrix.python-version }}
+          path: pip_install_osx_artifacts_python_${{ matrix.python-version }}