ESMValCore 2.13.0rc4__tar.gz → 2.14.0__tar.gz

esmvalcore-2.14.0/.circleci/config.yml

@@ -0,0 +1,268 @@
+---
+version: 2.1
+
+orbs:
+  codecov: codecov/codecov@5.3.0
+
+commands:
+  check_changes:
+    steps:
+      - run:
+          name: Check whether or not installation tests are needed
+          command: |
+            if (test "$CIRCLE_BRANCH" = main ||
+              git --no-pager diff --name-only origin/main... |
+              grep -q -E -f .circleci/install_triggers)
+            then
+              echo Running installation tests
+            else
+              echo Skipping installation tests
+              circleci step halt
+            fi
+  test_and_report:
+    parameters:
+      args:
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: Run tests
+          command: |
+            mkdir -p test-reports
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            pytest -n 4 --junitxml=test-reports/report.xml << parameters.args >>
+            esmvaltool version
+            esmvaltool -- --help
+      - store_test_results:
+          path: test-reports/report.xml
+      - store_artifacts:
+          path: /logs
+      - run:
+          name: Compress pytest artifacts
+          command: tar -cvzf pytest.tar.gz -C /tmp/pytest-of-root/pytest-0/ .
+      - store_artifacts:
+          path: pytest.tar.gz
+      - run:
+          name: Compress test-report artifacts
+          command: tar -cvzf test-reports.tar.gz test-reports/
+      - store_artifacts:
+          path: test-reports.tar.gz
+  test_installation_from_source:
+    parameters:
+      extra:
+        description: pip "extra"s to install
+        type: string
+        default: "test"
+      flags:
+        description: pip install --no-deps flags
+        type: string
+        default: ""
+      upstream_packages:
+        description: List of packages that will be installed with pip.
+        type: string
+        default: ""
+    steps:
+      - run:
+          name: Install git+ssh
+          environment:
+            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
+          command: apt update && apt install -y git ssh
+      - checkout
+      - check_changes
+      - run:
+          name: Generate cache key
+          command: date '+%Y-%V' | tee cache_key.txt
+      - restore_cache:
+          key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - run:
+          name: Install dependencies
+          no_output_timeout: 30m
+          command: |
+            # Add additional requirements for running all tests
+            echo "
+              - r-base
+              - r-yaml
+              - ncl
+            " >> environment.yml
+            # Installation of development version of packages requires compilers
+            if [[ "<< parameters.upstream_packages >>" ]]; then
+                echo "  - compilers" >> environment.yml
+            fi
+            # Install
+            . /opt/conda/etc/profile.d/conda.sh
+            mkdir /logs
+            mamba env create -n esmvaltool -f environment.yml |& tee /logs/conda.txt
+            git stash  # Restore repository state to get clean version number.
+            conda activate esmvaltool
+            pip install --no-deps << parameters.flags >> ".[<<parameters.extra>>]" << parameters.upstream_packages >> |& tee /logs/install.txt
+      - run:
+          name: Log versions
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
+            dpkg -l | tee /logs/versions.txt
+            conda env export | tee /logs/environment.yml
+            pip freeze | tee /logs/requirements.txt
+      - run:
+          name: Check dependencies are compatible
+          command: |
+            if [[ "<< parameters.upstream_packages >>" = "" ]]; then
+                . /opt/conda/etc/profile.d/conda.sh
+                conda activate esmvaltool
+                pip check
+            else
+              echo "Skipping `pip check` due to upstream packages"
+            fi
+      - test_and_report
+      - save_cache:
+          key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          paths:
+            - /opt/conda/pkgs
+            - /root/.cache/pip
+            - .mypy_cache
+            - .pytest_cache
+
+jobs:
+  run_tests:
+    # Run tests
+    docker:
+      - image: esmvalgroup/esmvalcore:development
+    resource_class: large
+    steps:
+      - checkout
+      - run:
+          name: Generate cache key
+          command: date '+%Y-%V' | tee cache_key.txt
+      - restore_cache:
+          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - run:
+          name: Install dependencies
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            mkdir /logs
+            conda activate esmvaltool
+            pip install --no-deps .[test] > /logs/install.txt 2>&1
+            pip check
+      - test_and_report:
+          args: --cov
+      - save_cache:
+          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          paths:
+            - /root/.cache/pip
+            - .mypy_cache
+            - .pytest_cache
+      - run:
+          name: Install gpg (required by codecov orb)
+          command: apt update && apt install -y gpg
+      - codecov/upload:
+          files: "test-reports/coverage.xml"
+          disable_search: true
+
+  test_installation_from_source_test_mode:
+    # Test installation from source
+    docker:
+      - image: condaforge/miniforge3
+    resource_class: large
+    steps:
+      - test_installation_from_source
+
+  test_installation_from_source_develop_mode:
+    # Test development installation
+    docker:
+      - image: condaforge/miniforge3
+    resource_class: large
+    steps:
+      - test_installation_from_source:
+          extra: develop
+          flags: "--editable"
+
+  test_with_upstream_developments:
+    # Test with development versions of upstream packages
+    docker:
+      - image: condaforge/miniforge3
+    resource_class: large
+    steps:
+      - test_installation_from_source:
+          upstream_packages: >-
+            git+https://github.com/esgf/esgf-pyclient
+            git+https://github.com/euro-cordex/py-cordex
+            git+https://github.com/SciTools/cartopy
+            git+https://github.com/SciTools/cf-units
+            git+https://github.com/SciTools/iris
+            git+https://github.com/SciTools/iris-grib
+            git+https://github.com/SciTools/nc-time-axis
+            git+https://github.com/SciTools-incubator/iris-esmf-regrid
+            git+https://github.com/SciTools-incubator/python-stratify
+            git+https://github.com/Toblerity/Fiona
+
+  test_installation_from_conda:
+    # Test conda package installation
+    working_directory: /esmvaltool
+    docker:
+      - image: condaforge/miniforge3
+    resource_class: medium
+    steps:
+      - run:
+          command: |
+            . /opt/conda/etc/profile.d/conda.sh
+            set -x
+            # Install prerequisites
+            mkdir /logs
+            # Create and activate conda environment
+            mamba create -y --name esmvaltool 'python=3.12'
+            set +x; conda activate esmvaltool; set -x
+            # Install
+            mamba install -y esmvalcore
+            # Log versions
+            conda env export | tee /logs/environment.yml
+            # Test installation
+            esmvaltool version
+
+  build_documentation:
+    # Test building documentation
+    docker:
+      - image: condaforge/miniforge3
+    resource_class: medium
+    steps:
+      - checkout
+      - run:
+          command: |
+            mkdir /logs
+            . /opt/conda/etc/profile.d/conda.sh
+            # Install
+            mamba env create -n esmvaltool -f environment.yml
+            conda activate esmvaltool
+            pip install --no-deps .[doc]
+            pip check
+            # Log versions
+            dpkg -l | tee /logs/versions.txt
+            conda env export | tee /logs/environment.yml
+            pip freeze | tee /logs/requirements.txt
+            # Test building documentation
+            MPLBACKEND=Agg sphinx-build -W doc doc/build
+      - store_artifacts:
+          path: /logs
+
+workflows:
+  commit:
+    jobs:
+      - run_tests
+      - test_installation_from_source_develop_mode
+      - test_installation_from_source_test_mode
+
+  nightly:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
+          filters:
+            branches:
+              only:
+                - main
+    jobs:
+      - build_documentation
+      - run_tests
+      - test_installation_from_conda
+      - test_installation_from_source_develop_mode
+      - test_installation_from_source_test_mode
+      - test_with_upstream_developments

esmvalcore-2.14.0/.git-blame-ignore-revs

@@ -0,0 +1,7 @@
+# Use ruff formatter (#2524)
+436558caacda69d4966a5aff35959ce9188cac37
+# Enable more ruff rules (#2715)
+77c370314bac1e72400392ed98c8f2f75b1b5a98
+02583113c99304e4e99bd010a05856ea429259ea
+0e8e7164ef96c574f77367948f51c7ab822bb694
+e2f8cab2cc5a509452ca720e50f041f46f380ee2

esmvalcore-2.14.0/.github/dependabot.yml

@@ -0,0 +1,20 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  # Enable version updates for GitHub Actions
+  - package-ecosystem: "github-actions"
+    # Workflow files stored in the default location of `.github/workflows`
+    # You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    cooldown:
+      default-days: 7
+    labels:
+      - dependencies
+      - github_actions
+      - automatedPR

esmvalcore-2.14.0/.github/workflows/build-and-deploy-on-pypi.yml

@@ -0,0 +1,63 @@
+# Workflow from https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
+name: Build and publish Python 🐍 distribution 📦 to PyPI
+
+permissions: {}
+
+on:
+  release:
+    types: [published]
+  push:
+    # Use this for testing the package build.
+    branches:
+      - main
+      - v[0-9]+.[0-9]+.x # matches release branches like v2.12.x
+
+jobs:
+  build:
+    name: Build distribution 📦
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: "3.x"
+      - name: Install pypa/build
+        run: >-
+          python3 -m
+          pip install
+          build
+          --user
+      - name: Build a binary wheel and a source tarball
+        run: python3 -m build
+      - name: Store the distribution packages
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish-to-pypi:
+    name: >-
+      Publish Python 🐍 distribution 📦 to PyPI
+    if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/ESMValCore
+    permissions:
+      id-token: write # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download all the dists
+        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
+        with:
+          name: python-package-distributions
+          path: dist/
+      - name: Publish distribution 📦 to PyPI
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0

esmvalcore-2.14.0/.github/workflows/citation_file_validator.yml

@@ -0,0 +1,28 @@
+# workflow that performs a validity check of CITATION.cff file
+# author: Abel Siqueira, GH: @abelsiqueira
+
+name: CFF File Validator
+
+permissions: {}
+
+on:
+  push:
+    paths:
+      - CITATION.cff
+  schedule:
+    - cron: "0 0 1 * *"
+
+jobs:
+  validate:
+    name: "validate"
+    runs-on: ubuntu-latest
+    if: github.repository == 'ESMValGroup/ESMValCore' # avoid GAs in forks
+    steps:
+      - name: Check out a copy of the repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Check whether the citation metadata from CITATION.cff is valid
+        uses: citation-file-format/cffconvert-github-action@4cf11baa70a673bfdf9dad0acc7ee33b3f4b6084 # 2.0.0
+        with:
+          args: "--validate"

esmvalcore-2.14.0/.github/workflows/create-condalock-file.yml

@@ -0,0 +1,99 @@
+name: Conda lock file creation
+
+permissions: {}
+
+on:
+  # Trigger on push on main or other branch for testing
+  # NOTE that push: main will create the file very often
+  # and hence lots of automated PRs
+  # push:
+  #   branches:
+  #     - main
+  schedule:
+    - cron: "0 4 */10 * *"
+
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  create-verify-lock-file:
+    name: Create and verify conda lock file for latest Python
+    permissions:
+      contents: write
+      pull-requests: write
+    runs-on: "ubuntu-latest"
+    if: github.repository == 'ESMValGroup/ESMValCore' # avoid GAs in forks
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3.3.0
+        with:
+          auto-update-conda: true
+          activate-environment: esmvaltool-fromlock
+          python-version: "3.14"
+          miniforge-version: "latest"
+          use-mamba: true
+      - name: Update and show conda config
+        run: |
+          conda update -n base -c conda-forge conda
+          conda install -y -c conda-forge "mamba>=2.2"
+          conda --version
+          mamba --version
+      - name: Gather Python info
+        run: |
+          which python
+          python --version
+      - name: Install conda-lock
+        run: mamba install -y -c conda-forge conda-lock
+      - name: Check version of conda-lock
+        run: conda-lock --version
+      - name: Create conda lock file for linux-64
+        # --mamba flag no longer needed since conda uses mamba backend
+        run: conda-lock lock --platform linux-64 -f environment.yml --kind explicit
+      - name: Creating environment from lock file
+        run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock
+      - name: Installing pip
+        run: mamba install -y pip
+      - name: Gather pip info
+        run: pip --version
+      - name: Gather Python info again
+        run: |
+          which python
+          python --version
+      - name: Show environment contents
+        run: conda list
+      - name: Install esmvalcore
+        run: pip install --no-deps -e .[develop]
+      - name: Run basic commands
+        run: |
+          pip check
+          esmvaltool --help
+          esmvaltool version
+      - name: Run pytests
+        run: pytest -n 2 -m "not installation"
+      # Automated PR
+      # see https://github.com/marketplace/actions/create-pull-request
+      - name: Create Automated PR if conda lock file has changed
+        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: Updating Linux condalock file
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          signoff: false
+          branch: condalock-update
+          delete-branch: true
+          title: "[Condalock] Update Linux condalock file"
+          body: |
+            Update condalock file
+            Automatic Pull Request.
+          labels: |
+            testing
+            condalock
+            automatedPR
+          assignees: valeriupredoi
+          reviewers: valeriupredoi
+          draft: false

esmvalcore-2.14.0/.github/workflows/install-from-conda.yml

@@ -0,0 +1,110 @@
+# Install esmvalcore from PyPi on different OS's
+# and different Python version; test locally with
+# act: https://github.com/nektos/act
+# Example how to setup conda workflows:
+# https://github.com/marketplace/actions/setup-miniconda
+# Notes:
+#  - you can group commands with | delimiter (or &&) but those will be run
+#    in one single call; declaring the shell variable makes the action run each
+#    command separately (better for debugging);
+#  - can try multiple shells eg pwsh or cmd /C CALL {0} (but overkill for now!);
+# TODO: read the cron tasking documentation:
+# https://www.netiq.com/documentation/cloud-manager-2-5/ncm-reference/data/bexyssf.html
+
+name: Install from conda-forge
+
+permissions: {}
+
+# runs on a push on main and at the end of every day
+on:
+  # triggering on push without branch name will run tests every time
+  # there is a push on any branch
+  # turn it on only if needed
+  push:
+    branches:
+      - main
+  # run the test only if the PR is to main
+  # turn it on if required
+  # pull_request:
+  #   branches:
+  #     - main
+  schedule:
+    - cron: "0 4 * * *"
+
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  linux:
+    runs-on: "ubuntu-latest"
+    if: github.repository == 'ESMValGroup/ESMValCore' # avoid GAs in forks
+    strategy:
+      matrix:
+        python-version: ["3.12", "3.13", "3.14"]
+      # fail-fast set to False allows all other tests
+      # in the workflow to run regardless of any fail
+      fail-fast: false
+    name: Linux Python ${{ matrix.python-version }}
+    steps:
+      - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3.3.0
+        with:
+          python-version: ${{ matrix.python-version }}
+          miniforge-version: "latest"
+          use-mamba: true
+      - run: mkdir -p conda_install_linux_artifacts_python_${{ matrix.python-version }}
+      - name: Record versions
+        run: |
+          mamba --version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
+          which conda 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_path.txt
+          which mamba 2>&1 | tee -a conda_install_linux_artifacts_python_${{ matrix.python-version }}/conda_path.txt
+          python -V 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
+      - name: Install ESMValCore
+        run: mamba install esmvalcore 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/install.txt
+      - name: Verify installation
+        run: |
+          esmvaltool --help
+          esmvaltool version 2>&1 | tee conda_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
+      - name: Upload artifacts
+        if: ${{ always() }} # upload artifacts even if fail
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: Conda_Install_Linux_python_${{ matrix.python-version }}
+          path: conda_install_linux_artifacts_python_${{ matrix.python-version }}
+
+  osx:
+    runs-on: "macos-latest"
+    if: github.repository == 'ESMValGroup/ESMValCore' # avoid GAs in forks
+    strategy:
+      matrix:
+        python-version: ["3.12", "3.13", "3.14"]
+        architecture: ["x64"] # need to force Intel, arm64 builds have issues
+      fail-fast: false
+    name: OSX Python ${{ matrix.python-version }}
+    steps:
+      - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3.3.0
+        with:
+          architecture: ${{ matrix.architecture }}
+          python-version: ${{ matrix.python-version }}
+          miniforge-version: "latest"
+          use-mamba: true
+      - run: mkdir -p conda_install_osx_artifacts_python_${{ matrix.python-version }}
+      - name: Record versions
+        run: |
+          mamba --version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_version.txt
+          which conda 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_path.txt
+          which mamba 2>&1 | tee -a conda_install_osx_artifacts_python_${{ matrix.python-version }}/conda_path.txt
+          python -V 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/python_version.txt
+      - name: Install ESMValCore
+        run: mamba install esmvalcore 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/install.txt
+      - name: Verify installation
+        run: |
+          esmvaltool --help
+          esmvaltool version 2>&1 | tee conda_install_osx_artifacts_python_${{ matrix.python-version }}/version.txt
+      - name: Upload artifacts
+        if: ${{ always() }} # upload artifacts even if fail
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: Conda_Install_OSX_python_${{ matrix.python-version }}
+          path: conda_install_osx_artifacts_python_${{ matrix.python-version }}

esmvalcore-2.14.0/.github/workflows/install-from-condalock-file.yml

@@ -0,0 +1,66 @@
+# Install esmvalcore from a conda lock file
+# To build a conda lock file install conda-lock first then
+# run conda-lock lock --platform linux-64 -f environment.yml --mamba
+# (mamba activated for speed). Change platform for osx-64 or win-64.
+# Env creation then happens as per normal use with
+# mamba create --name esmvaltool-fromlock --file conda-linux-64.lock
+# note that pip and conda are NOT installed.
+
+name: Conda-lock Install
+
+permissions: {}
+
+on:
+  push:
+    branches:
+      - main
+  # run the test only if the PR is to main
+  # turn it on if required
+  #pull_request:
+  #  branches:
+  #  - main
+  schedule:
+    - cron: "0 0 * * *"
+
+# Required shell entrypoint to have properly configured bash shell
+defaults:
+  run:
+    shell: bash -l {0}
+
+jobs:
+  linux:
+    runs-on: "ubuntu-latest"
+    if: github.repository == 'ESMValGroup/ESMValCore' # avoid GAs in forks
+    strategy:
+      matrix:
+        python-version: ["3.12", "3.13", "3.14"]
+      fail-fast: false
+    name: Linux Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: conda-incubator/setup-miniconda@fc2d68f6413eb2d87b895e92f8584b5b94a10167 # v3.3.0
+        with:
+          activate-environment: esmvaltool-fromlock
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+      - run: mkdir -p source_install_linux_artifacts_python_${{ matrix.python-version }}
+      - run: conda --version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/conda_version.txt
+      - run: which python
+      - run: python -V 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/python_version.txt
+      - run: conda create --name esmvaltool-fromlock --file conda-linux-64.lock
+      - run: which python
+      - run: pip --version
+      - run: pip install --no-deps -e .[develop]
+      - run: pip check
+      - run: esmvaltool --help
+      - run: esmvaltool version 2>&1 | tee source_install_linux_artifacts_python_${{ matrix.python-version }}/version.txt
+      - run: pytest -n 2 -m "not installation"
+      - name: Upload artifacts
+        if: ${{ always() }} # upload artifacts even if fail
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: Source_Install_Linux_python_${{ matrix.python-version }}
+          path: source_install_linux_artifacts_python_${{ matrix.python-version }}