anemoi-utils 0.4.12__tar.gz → 0.4.13__tar.gz

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.github/dependabot.yml

@@ -4,7 +4,7 @@ updates:
   directory: "/"
   schedule:
     interval: "weekly"
-- package-ecosystem: "github-actions"
-  directory: "/"
-  schedule:
-    interval: "monthly"
+# - package-ecosystem: "github-actions"
+#   directory: "/"
+#   schedule:
+#     interval: "monthly"

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.github/labeler.yml

@@ -23,6 +23,7 @@ config:
 - changed-files:
   - any-glob-to-any-file:
     - "**/src/**/config/**/*"
+    - "**/src/anemoi/inference/config.py"
 
 CI/CD:
 - changed-files:

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.github/workflows/downstream-ci-hpc.yml

@@ -42,13 +42,19 @@ jobs:
     with:
       anemoi-utils: ecmwf/anemoi-utils@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
+      # Only run on fedora
+      skip_matrix_jobs: |
+        gnu@debian-11
+        gnu@rocky-8.6
+        clang@rocky-8.6
+        gnu@ubuntu-22.04
     secrets: inherit
 
-   # Build downstream packages on HPC
-  downstream-ci-hpc:
-    name: downstream-ci-hpc
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
-    with:
-      anemoi-utils: ecmwf/anemoi-utils@${{ github.event.pull_request.head.sha || github.sha }}
-    secrets: inherit
+  # # Build downstream packages on HPC
+  # downstream-ci-hpc:
+  #   name: downstream-ci-hpc
+  #   if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+  #   uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+  #   with:
+  #     anemoi-utils: ecmwf/anemoi-utils@${{ github.event.pull_request.head.sha || github.sha }}
+  #   secrets: inherit

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.github/workflows/pr-label-conventional-commits.yml

@@ -8,6 +8,9 @@ on:
     types:
       [opened, reopened, labeled, unlabeled]
 
+permissions:
+  pull-requests: write
+
 jobs:
   assign-labels:
     runs-on: ubuntu-latest
@@ -41,5 +44,5 @@ jobs:
               - type: 'config'
                 nouns: ['config', 'conf', 'configuration']
                 labels: ['config']
-          maintain-labels-not-matched: false
+          maintain-labels-not-matched: true
           apply-changes: true

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.github/workflows/pr-label-public.yml

@@ -7,4 +7,4 @@ on:
 
 jobs:
   label:
-    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2
+    uses: ecmwf/reusable-workflows/.github/workflows/label-pr.yml@v2

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.github/workflows/python-publish.yml

@@ -9,5 +9,5 @@ on:
 
 jobs:
   deploy:
-    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    uses: ecmwf/reusable-workflows/.github/workflows/cd-pypi.yml@v2
     secrets: inherit

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.github/workflows/python-pull-request.yml

@@ -12,7 +12,7 @@ on:
 
 jobs:
   quality:
-    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    uses: ecmwf/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
     with:
       skip-hooks: "no-commit-to-branch"
 
@@ -20,6 +20,6 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
-    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+    uses: ecmwf/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
     with:
       python-version: ${{ matrix.python-version }}

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.gitignore

@@ -78,6 +78,8 @@ dmypy.json
 
 # Data files
 *.grib
+*.grib1
+*.grib2
 *.onnx
 *.ckpt
 *.npy

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.pre-commit-config.yaml

@@ -58,7 +58,12 @@ repos:
   rev: v0.0.14
   hooks:
   - id: rstfmt
-    exclude: 'cli/.*' # Because we use argparse
+    exclude: '(cli|schemas)/.*' # Because we use argparse and pydantic sphinx directives
+- repo: https://github.com/b8raoult/pre-commit-docconvert
+  rev: "0.1.4"
+  hooks:
+  - id: docconvert
+    args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
   rev: "v2.5.0"
   hooks:
@@ -73,6 +78,7 @@ repos:
         - --check-overridden # Check overridden methods
         - --check-protected  # Check protected methods
         - --check-class      # Check class docstrings
-        - --disable=E113     # Disable empty docstrings
+        - --disable=SIG101,SIG102     # Disable empty docstrings
 ci:
   autoupdate_schedule: monthly
+  autoupdate_commit_msg: "chore(deps): pre-commit.ci autoupdate"

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.readthedocs.yaml

@@ -3,7 +3,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.10"
 
 sphinx:
   configuration: docs/conf.py

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/.release-please-config.json

@@ -4,10 +4,11 @@
   "bump-patch-for-minor-pre-major": true,
   "separate-pull-requests": true,
   "always-update": true,
-  "changelog-type": "github",
+  "changelog-type": "default",
   "include-component-in-tag": false,
   "include-v-in-tag": false,
-  "pull-request-title-pattern": "chore${scope}: Preparing Next Release for ${component} ${version}",
+  "draft-pull-request": true,
+  "pull-request-title-pattern": "chore${scope}: Release${component} ${version}",
   "pull-request-header": ":robot: Automated Release PR\n\nThis PR was created by `release-please` to prepare the next release. Once merged:\n\n1. A new version tag will be created\n2. A GitHub release will be published\n3. The changelog will be updated\n\nChanges to be included in the next release:",
   "pull-request-footer": "> [!IMPORTANT]\n> :warning: Merging this PR will:\n> - Create a new release\n> - Trigger deployment pipelines\n> - Update package versions\n\n **Before merging:**\n - Ensure all tests pass\n - Review the changelog carefully\n - Get required approvals\n\n [Release-please documentation](https://github.com/googleapis/release-please)",
   "packages": {

anemoi_utils-0.4.13/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.13"
+}

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/CHANGELOG.md

@@ -8,6 +8,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.4.13](https://github.com/ecmwf/anemoi-utils/compare/0.4.12...0.4.13) (2025-03-14)
+
+
+### Features
+
+* add robust requests ([#112](https://github.com/ecmwf/anemoi-utils/issues/112)) ([5d87227](https://github.com/ecmwf/anemoi-utils/commit/5d87227e6f0b39f087f8a34f238806a2f73480f1))
+* bugfix ([#100](https://github.com/ecmwf/anemoi-utils/issues/100)) ([c016cb4](https://github.com/ecmwf/anemoi-utils/commit/c016cb46c23b6a0575d9d843b06fd6b9f71b9f27))
+* keep yaml formating in error messages ([#108](https://github.com/ecmwf/anemoi-utils/issues/108)) ([3bd6682](https://github.com/ecmwf/anemoi-utils/commit/3bd66828cf19d8e3d7d3fbed27533161b6285828))
+* re-add default values in transfer function ([#101](https://github.com/ecmwf/anemoi-utils/issues/101)) ([6462205](https://github.com/ecmwf/anemoi-utils/commit/6462205ee25fa35a71af047b1fbb04bd3c4ca2c4))
+
+
+### Bug Fixes
+
+* add optional dependency. boto3 <= 1.36 ([#105](https://github.com/ecmwf/anemoi-utils/issues/105)) ([c8c8393](https://github.com/ecmwf/anemoi-utils/commit/c8c8393ab1e886289541d3aa47a614afe5cd379b))
+
+
+### Documentation
+
+* update logo ([#96](https://github.com/ecmwf/anemoi-utils/issues/96)) ([c297127](https://github.com/ecmwf/anemoi-utils/commit/c297127e066c92023ca065b3e7d36ac4ab62527e))
+
 ## 0.4.12 (2025-01-30)
 
 <!-- Release notes generated using configuration in .github/release.yml at main -->

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: anemoi-utils
-Version: 0.4.12
+Version: 0.4.13
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -226,13 +226,14 @@ Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: aniso8601
 Requires-Dist: importlib-metadata; python_version < "3.10"
+Requires-Dist: multiurl
 Requires-Dist: numpy
 Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
 Requires-Dist: tomli; python_version < "3.11"
 Requires-Dist: tqdm
 Provides-Extra: all
-Requires-Dist: anemoi-utils[grib,provenance,text]; extra == "all"
+Requires-Dist: anemoi-utils[grib,provenance,s3,text]; extra == "all"
 Provides-Extra: dev
 Requires-Dist: anemoi-utils[all,docs,tests]; extra == "dev"
 Provides-Extra: docs
@@ -240,7 +241,7 @@ Requires-Dist: nbsphinx; extra == "docs"
 Requires-Dist: pandoc; extra == "docs"
 Requires-Dist: requests; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
-Requires-Dist: sphinx-argparse<0.5; extra == "docs"
+Requires-Dist: sphinx-argparse; extra == "docs"
 Requires-Dist: sphinx-rtd-theme; extra == "docs"
 Requires-Dist: termcolor; extra == "docs"
 Provides-Extra: grib
@@ -248,6 +249,8 @@ Requires-Dist: requests; extra == "grib"
 Provides-Extra: provenance
 Requires-Dist: gitpython; extra == "provenance"
 Requires-Dist: nvsmi; extra == "provenance"
+Provides-Extra: s3
+Requires-Dist: boto3<1.36; extra == "s3"
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/pyproject.toml

@@ -42,6 +42,7 @@ dynamic = [ "version" ]
 dependencies = [
   "aniso8601",
   "importlib-metadata; python_version<'3.10'",
+  "multiurl",
   "numpy",
   "python-dateutil",
   "pyyaml",
@@ -49,7 +50,7 @@ dependencies = [
   "tqdm",
 ]
 
-optional-dependencies.all = [ "anemoi-utils[grib,provenance,text]" ]
+optional-dependencies.all = [ "anemoi-utils[grib,provenance,text,s3]" ]
 optional-dependencies.dev = [ "anemoi-utils[all,docs,tests]" ]
 
 optional-dependencies.docs = [
@@ -57,7 +58,7 @@ optional-dependencies.docs = [
   "pandoc",
   "requests",
   "sphinx",
-  "sphinx-argparse<0.5",
+  "sphinx-argparse",
   "sphinx-rtd-theme",
   "termcolor",
 ]
@@ -66,6 +67,10 @@ optional-dependencies.grib = [ "requests" ]
 
 optional-dependencies.provenance = [ "gitpython", "nvsmi" ]
 
+optional-dependencies.s3 = [
+  "boto3<1.36",
+]
+
 optional-dependencies.tests = [ "pytest" ]
 
 optional-dependencies.text = [ "termcolor", "wcwidth" ]
@@ -87,3 +92,14 @@ version_file = "src/anemoi/utils/_version.py"
 markers = [
   "skip_on_hpc: mark a test that should not be run on HPC",
 ]
+
+[tool.mypy]
+strict = false
+exclude = [
+  "docs/**",
+]
+
+[tool.pydocstringformatter]
+write = true
+exclude = "docs/**"
+style = "numpydoc"

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/src/anemoi/utils/__init__.py

@@ -7,6 +7,7 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+"""Anemoi Utils package."""
 
 try:
     # NOTE: the `_version.py` file must not be present in the git repository

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/src/anemoi/utils/__main__.py

@@ -7,6 +7,8 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+from typing import Any
+
 from anemoi.utils.cli import cli_main
 from anemoi.utils.cli import make_parser
 
@@ -15,11 +17,19 @@ from .commands import COMMANDS
 
 
 # For read-the-docs
-def create_parser():
+def create_parser() -> Any:
+    """Create the argument parser for the CLI.
+
+    Returns
+    -------
+    Any
+        The argument parser
+    """
     return make_parser(__doc__, COMMANDS)
 
 
-def main():
+def main() -> None:
+    """Main entry point for the CLI."""
     cli_main(__version__, __doc__, COMMANDS)
 
 

{anemoi_utils-0.4.12 → anemoi_utils-0.4.13}/src/anemoi/utils/_version.py

@@ -1,8 +1,13 @@
-# file generated by setuptools_scm
+# file generated by setuptools-scm
 # don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
 TYPE_CHECKING = False
 if TYPE_CHECKING:
-    from typing import Tuple, Union
+    from typing import Tuple
+    from typing import Union
+
     VERSION_TUPLE = Tuple[Union[int, str], ...]
 else:
     VERSION_TUPLE = object
@@ -12,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.12'
-__version_tuple__ = version_tuple = (0, 4, 12)
+__version__ = version = '0.4.13'
+__version_tuple__ = version_tuple = (0, 4, 13)

anemoi_utils-0.4.13/src/anemoi/utils/caching.py

@@ -0,0 +1,249 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import hashlib
+import json
+import os
+import time
+from threading import Lock
+from typing import Any
+from typing import Callable
+from typing import Optional
+
+import numpy as np
+
+LOCK = Lock()
+CACHE = {}
+
+
+def _get_cache_path(collection: str) -> str:
+    """Get the cache path for a collection.
+
+    Parameters
+    ----------
+    collection : str
+        The name of the collection
+
+    Returns
+    -------
+    str
+        The cache path
+    """
+    return os.path.join(os.path.expanduser("~"), ".cache", "anemoi", collection)
+
+
+def clean_cache(collection: str = "default") -> None:
+    """Clean the cache for a collection.
+
+    Parameters
+    ----------
+    collection : str, optional
+        The name of the collection, by default "default"
+    """
+    global CACHE
+    CACHE = {}
+    path = _get_cache_path(collection)
+    if not os.path.exists(path):
+        return
+    for filename in os.listdir(path):
+        os.remove(os.path.join(path, filename))
+
+
+class Cacher:
+    """This class implements a simple caching mechanism.
+    Private class, do not use directly.
+    """
+
+    def __init__(self, collection: str, expires: Optional[int]):
+        """Initialize the Cacher.
+
+        Parameters
+        ----------
+        collection : str
+            The name of the collection
+        expires : int, optional
+            The expiration time in seconds, or None for no expiration
+        """
+        self.collection = collection
+        self.expires = expires
+
+    def __call__(self, func: Callable) -> Callable:
+        """Wrap a function with caching.
+
+        Parameters
+        ----------
+        func : Callable
+            The function to wrap
+
+        Returns
+        -------
+        Callable
+            The wrapped function
+        """
+        full = f"{func.__module__}.{func.__name__}"
+
+        def wrapped(*args, **kwargs):
+            with LOCK:
+                return self.cache(
+                    (full, args, kwargs),
+                    lambda: func(*args, **kwargs),
+                )
+
+        return wrapped
+
+    def cache(self, key: tuple, proc: Callable) -> Any:
+        """Cache the result of a function.
+
+        Parameters
+        ----------
+        key : tuple
+            The cache key
+        proc : Callable
+            The function to call if the result is not cached
+
+        Returns
+        -------
+        Any
+            The cached result
+        """
+        key = json.dumps(key, sort_keys=True)
+        m = hashlib.md5()
+        m.update(key.encode("utf-8"))
+        m = m.hexdigest()
+
+        if m in CACHE:
+            return CACHE[m]
+
+        path = _get_cache_path(self.collection)
+
+        filename = os.path.join(path, m) + self.ext
+        if os.path.exists(filename):
+            data = self.load(filename)
+            if self.expires is None or data["expires"] > time.time():
+                if data["key"] == key:
+                    return data["value"]
+
+        value = proc()
+        data = {"key": key, "value": value}
+        if self.expires is not None:
+            data["expires"] = time.time() + self.expires
+
+        os.makedirs(path, exist_ok=True)
+        temp_filename = self.save(filename, data)
+        os.rename(temp_filename, filename)
+
+        CACHE[m] = value
+        return value
+
+
+class JsonCacher(Cacher):
+    """Cacher that uses JSON files."""
+
+    ext = ""
+
+    def save(self, path: str, data: dict) -> str:
+        """Save data to a JSON file.
+
+        Parameters
+        ----------
+        path : str
+            The path to the JSON file
+        data : dict
+            The data to save
+
+        Returns
+        -------
+        str
+            The temporary file path
+        """
+        temp_path = path + ".tmp"
+        with open(temp_path, "w") as f:
+            json.dump(data, f)
+        return temp_path
+
+    def load(self, path: str) -> dict:
+        """Load data from a JSON file.
+
+        Parameters
+        ----------
+        path : str
+            The path to the JSON file
+
+        Returns
+        -------
+        dict
+            The loaded data
+        """
+        with open(path, "r") as f:
+            return json.load(f)
+
+
+class NpzCacher(Cacher):
+    """Cacher that uses NPZ files."""
+
+    ext = ".npz"
+
+    def save(self, path: str, data: dict) -> str:
+        """Save data to an NPZ file.
+
+        Parameters
+        ----------
+        path : str
+            The path to the NPZ file
+        data : dict
+            The data to save
+
+        Returns
+        -------
+        str
+            The temporary file path
+        """
+        temp_path = path + ".tmp.npz"
+        np.savez(temp_path, **data)
+        return temp_path
+
+    def load(self, path: str) -> dict:
+        """Load data from an NPZ file.
+
+        Parameters
+        ----------
+        path : str
+            The path to the NPZ file
+
+        Returns
+        -------
+        dict
+            The loaded data
+        """
+        return np.load(path, allow_pickle=True)
+
+
+# This function is the main entry point for the caching mechanism for the other anemoi packages
+def cached(collection: str = "default", expires: Optional[int] = None, encoding: str = "json") -> Callable:
+    """Decorator to cache the result of a function.
+
+    Default is to use a json file to store the cache, but you can also use npz files
+    to cache dict of numpy arrays.
+
+    Parameters
+    ----------
+    collection : str, optional
+        The name of the collection, by default "default"
+    expires : int, optional
+        The expiration time in seconds, or None for no expiration, by default None
+    encoding : str, optional
+        The encoding type, either "json" or "npz", by default "json"
+
+    Returns
+    -------
+    Callable
+        The decorated function
+    """
+    return dict(json=JsonCacher, npz=NpzCacher)[encoding](collection, expires)