anemoi-utils 0.4.13__tar.gz → 0.4.15__tar.gz

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/.github/workflows/downstream-ci-hpc.yml

@@ -1,5 +1,5 @@
 # This workflow triggers tests on dependent packages.
-# The dependency tree itself is defined in ecmwf-actions/downstream-ci/
+# The dependency tree itself is defined in ecmwf/downstream-ci/
 name: Test downstream dependent packages on HPC
 
 on:
@@ -38,7 +38,7 @@ jobs:
   downstream-ci:
     name: downstream-ci
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@main
     with:
       anemoi-utils: ecmwf/anemoi-utils@${{ github.event.pull_request.head.sha || github.sha }}
       codecov_upload: true
@@ -54,7 +54,7 @@ jobs:
   # downstream-ci-hpc:
   #   name: downstream-ci-hpc
   #   if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-  #   uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+  #   uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
   #   with:
   #     anemoi-utils: ecmwf/anemoi-utils@${{ github.event.pull_request.head.sha || github.sha }}
   #   secrets: inherit

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/.gitignore

@@ -135,3 +135,4 @@ _version.py
 *.to_upload
 tempCodeRunnerFile.python
 Untitled-*.py
+.*cache/

anemoi_utils-0.4.15/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.15"
+}

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/CHANGELOG.md

@@ -8,6 +8,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.4.15](https://github.com/ecmwf/anemoi-utils/compare/0.4.14...0.4.15) (2025-03-21)
+
+
+### Features
+
+* accept hyphens in factory names ([#116](https://github.com/ecmwf/anemoi-utils/issues/116)) ([ada96e9](https://github.com/ecmwf/anemoi-utils/commit/ada96e911b592ff9d95d3a93fff5a6aa21cdebbe))
+
+## [0.4.14](https://github.com/ecmwf/anemoi-utils/compare/0.4.13...0.4.14) (2025-03-21)
+
+
+### Bug Fixes
+
+* plugin support ([#110](https://github.com/ecmwf/anemoi-utils/issues/110)) ([329395a](https://github.com/ecmwf/anemoi-utils/commit/329395a5870cbf59bacb39cb5afea6b91c465b07))
+
 ## [0.4.13](https://github.com/ecmwf/anemoi-utils/compare/0.4.12...0.4.13) (2025-03-14)
 
 

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.13
+Version: 0.4.15
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -256,3 +256,4 @@ Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text
 Requires-Dist: termcolor; extra == "text"
 Requires-Dist: wcwidth; extra == "text"
+Dynamic: license-file

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/src/anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.13'
-__version_tuple__ = version_tuple = (0, 4, 13)
+__version__ = version = '0.4.15'
+__version_tuple__ = version_tuple = (0, 4, 15)

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/src/anemoi/utils/devtools.py

@@ -81,13 +81,11 @@ def plot_values(
     ax.add_feature(cfeature.BORDERS, linestyle=":")
 
     missing_values = np.isnan(values)
-
     if missing_value is None:
-        values = values[~missing_values]
-        longitudes = longitudes[~missing_values]
-        latitudes = latitudes[~missing_values]
-    else:
-        values = np.where(missing_values, missing_value, values)
+        min = np.nanmin(values)
+        missing_value = min - np.abs(min) * 0.001
+
+    values = np.where(missing_values, missing_value, values)
 
     if max_value is not None:
         values = np.where(values > max_value, max_value, values)

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/src/anemoi/utils/humanize.py

@@ -48,7 +48,7 @@ def bytes_to_human(n: float) -> str:
     """
     if n < 0:
         sign = "-"
-        n -= 0
+        n = -n
     else:
         sign = ""
 
@@ -411,15 +411,9 @@ def when(
         if years > 1:
             return _("%d years" % (years,))
 
-        month = then.month
-        if now.year != then.year:
-            month -= 12
-
-        d = abs(now.month - month)
-        if d >= 12:
-            return _("a year")
-        else:
-            return _("%d month%s" % (d, _plural(d)))
+        delta = abs(now - then)
+        if delta.days > 1 and delta.days < 30:
+            return _("%d days" % (delta.days,))
 
     return "on %s %d %s %d" % (
         DOW[then.weekday()],

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/src/anemoi/utils/provenance.py

@@ -10,10 +10,10 @@
 
 """Collect information about the current environment, like:
 
- - The Python version
- - The versions of the modules which are currently loaded
- - The git information for the modules which are currently loaded from a git repository
- - ...
+- The Python version
+- The versions of the modules which are currently loaded
+- The git information for the modules which are currently loaded from a git repository
+- ...
 """
 
 import datetime

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/src/anemoi/utils/registry.py

@@ -12,9 +12,12 @@ import importlib
 import logging
 import os
 import sys
+import warnings
+from functools import cached_property
 from typing import Any
 from typing import Callable
 from typing import Dict
+from typing import List
 from typing import Optional
 from typing import Union
 
@@ -22,6 +25,8 @@ import entrypoints
 
 LOG = logging.getLogger(__name__)
 
+DEBUG_ANEMOI_REGISTRY = int(os.environ.get("DEBUG_ANEMOI_REGISTRY", "0"))
+
 
 class Wrapper:
     """A wrapper for the registry.
@@ -55,6 +60,22 @@ class Wrapper:
         return factory
 
 
+class Error:
+    """An error class. Used in place of a plugin that failed to load.
+
+    Parameters
+    ----------
+    error : Exception
+        The error.
+    """
+
+    def __init__(self, error: Exception):
+        self.error = error
+
+    def __call__(self, *args, **kwargs):
+        raise self.error
+
+
 _BY_KIND = {}
 
 
@@ -67,13 +88,17 @@ class Registry:
         The package name.
     key : str, optional
         The key to use for the registry, by default "_type".
+    api_version : str, optional
+        The API version, by default '1.0.0'.
     """
 
-    def __init__(self, package: str, key: str = "_type"):
+    def __init__(self, package: str, key: str = "_type", api_version: str = "1.0.0"):
         self.package = package
-        self.registered = {}
+        self.__registered = {}
+        self._sources = {}
         self.kind = package.split(".")[-1]
         self.key = key
+        self.api_version = api_version
         _BY_KIND[self.kind] = self
 
     @classmethod
@@ -92,7 +117,9 @@ class Registry:
         """
         return _BY_KIND.get(kind)
 
-    def register(self, name: str, factory: Optional[Callable] = None) -> Optional[Wrapper]:
+    def register(
+        self, name: str, factory: Optional[Callable] = None, source: Optional[Any] = None
+    ) -> Optional[Wrapper]:
         """Register a factory with the registry.
 
         Parameters
@@ -101,19 +128,31 @@ class Registry:
             The name of the factory.
         factory : Callable, optional
             The factory to register, by default None.
+        source : Any, optional
+            The source of the factory, by default None.
 
         Returns
         -------
         Wrapper, optional
             A wrapper if the factory is None, otherwise None.
         """
+
+        name = name.replace("_", "-")
+
         if factory is None:
+            # This happens when the @register decorator is used
             return Wrapper(name, self)
 
-        self.registered[name] = factory
+        if source is None:
+            source = getattr(factory, "_source") if hasattr(factory, "_source") else factory
+
+        if name in self.__registered:
+            warnings.warn(f"Factory '{name}' is already registered in {self.package}")
+            warnings.warn(f"Existing: {self._sources[name]}")
+            warnings.warn(f"New: {source}")
 
-    # def registered(self, name: str):
-    #     return name in self.registered
+        self.__registered[name] = factory
+        self._sources[name] = source
 
     def _load(self, file: str) -> None:
         """Load a module from a file.
@@ -126,8 +165,33 @@ class Registry:
         name, _ = os.path.splitext(file)
         try:
             importlib.import_module(f".{name}", package=self.package)
-        except Exception:
-            LOG.warning(f"Error loading filter '{self.package}.{name}'", exc_info=True)
+        except Exception as e:
+            if DEBUG_ANEMOI_REGISTRY:
+                raise
+            self._registered[name] = Error(e)
+
+    def is_registered(self, name: str) -> bool:
+        """Check if a factory is registered.
+
+        Parameters
+        ----------
+        name : str
+            The name of the factory.
+
+        Returns
+        -------
+        bool
+            Whether the factory is registered.
+        """
+
+        name = name.replace("_", "-")
+
+        ok = name in self.factories
+        if not ok:
+            LOG.error(f"Cannot find '{name}' in {self.package}")
+            for e in self.factories:
+                LOG.info(f"Registered: {e} ({self._sources.get(e)})")
+        return ok
 
     def lookup(self, name: str, *, return_none: bool = False) -> Optional[Callable]:
         """Lookup a factory by name.
@@ -144,9 +208,25 @@ class Registry:
         Callable, optional
             The factory if found, otherwise None.
         """
-        # print('✅✅✅✅✅✅✅✅✅✅✅✅✅', name, self.registered)
-        if name in self.registered:
-            return self.registered[name]
+
+        name = name.replace("_", "-")
+
+        if return_none:
+            return self.factories.get(name)
+
+        factory = self.factories.get(name)
+        if factory is None:
+
+            LOG.error(f"Cannot find '{name}' in {self.package}")
+            for e in self.factories:
+                LOG.info(f"Registered: {e} ({self._sources.get(e)})")
+
+            raise ValueError(f"Cannot find '{name}' in {self.package}")
+
+        return factory
+
+    @cached_property
+    def factories(self) -> Dict[str, Callable]:
 
         directory = sys.modules[self.package].__path__[0]
 
@@ -167,25 +247,41 @@ class Registry:
             if file.endswith(".py"):
                 self._load(file)
 
-        entrypoint_group = f"anemoi.{self.kind}"
-        for entry_point in entrypoints.get_group_all(entrypoint_group):
-            if entry_point.name == name:
-                if name in self.registered:
-                    LOG.warning(
-                        f"Overwriting builtin '{name}' from {self.package} with plugin '{entry_point.module_name}'"
-                    )
-                self.registered[name] = entry_point.load()
+        bits = self.package.split(".")
+        # We assume a name like anemoi.datasets.create.sources, with kind = sources
+        assert bits[-1] == self.kind, (self.package, self.kind)
+        assert len(bits) > 1, self.package
+
+        groups = []
+        middle = bits[1:-1]
+        while True:
+            group = ".".join([bits[0], *middle, bits[-1]])
+            groups.append(group)
+            if len(middle) == 0:
+                break
+            middle.pop()
+
+        groups.reverse()
 
-        if name not in self.registered:
-            if return_none:
-                return None
+        LOG.debug("Loading plugins from %s", groups)
 
-            for e in self.registered:
-                LOG.info(f"Registered: {e}")
+        for entrypoint_group in groups:
+            for entry_point in entrypoints.get_group_all(entrypoint_group):
+                source = entry_point.distro
+                try:
+                    self.register(entry_point.name, entry_point.load(), source=source)
+                except Exception as e:
+                    if DEBUG_ANEMOI_REGISTRY:
+                        raise
+                    self.register(entry_point.name, Error(e), source=source)
 
-            raise ValueError(f"Cannot load '{name}' from {self.package}")
+        return self.__registered
 
-        return self.registered[name]
+    @property
+    def registered(self) -> List[str]:
+        """Get the registered factories."""
+
+        return sorted(self.factories.keys())
 
     def create(self, name: str, *args: Any, **kwargs: Any) -> Any:
         """Create an instance using a factory.
@@ -204,12 +300,12 @@ class Registry:
         Any
             The created instance.
         """
+
+        name = name.replace("_", "-")
+
         factory = self.lookup(name)
         return factory(*args, **kwargs)
 
-    # def __call__(self, name: str, *args, **kwargs):
-    #     return self.create(name, *args, **kwargs)
-
     def from_config(self, config: Union[str, Dict[str, Any]], *args: Any, **kwargs: Any) -> Any:
         """Create an instance from a configuration.
 

anemoi_utils-0.4.15/src/anemoi/utils/testing.py

@@ -0,0 +1,182 @@
+# (C) Copyright 2025- Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import atexit
+import logging
+import os
+import shutil
+import tempfile
+import threading
+
+from multiurl import download
+
+LOG = logging.getLogger(__name__)
+
+TEST_DATA_URL = "https://object-store.os-api.cci1.ecmwf.int/ml-tests/test-data/samples/"
+
+lock = threading.RLock()
+TEMPORARY_DIRECTORY = None
+
+
+def _temporary_directory() -> str:
+    """Return a temporary directory in which to download test data.
+
+    Returns
+    -------
+    str
+        The path to the temporary directory.
+    """
+    global TEMPORARY_DIRECTORY
+    with lock:
+        if TEMPORARY_DIRECTORY is not None:
+            return TEMPORARY_DIRECTORY
+
+        TEMPORARY_DIRECTORY = tempfile.mkdtemp()
+
+        # Register a cleanup function to remove the directory at exit
+        atexit.register(shutil.rmtree, TEMPORARY_DIRECTORY)
+
+        return TEMPORARY_DIRECTORY
+
+
+def _check_path(path: str) -> None:
+    """Check if the given path is normalized, not absolute, and does not start with a dot.
+
+    Parameters
+    ----------
+    path : str
+        The path to check.
+
+    Raises
+    ------
+    AssertionError
+        If the path is not normalized, is absolute, or starts with a dot.
+    """
+    assert os.path.normpath(path) == path, f"Path '{path}' should be normalized"
+    assert not os.path.isabs(path), f"Path '{path}' should not be absolute"
+    assert not path.startswith("."), f"Path '{path}' should not start with '.'"
+
+
+def url_for_test_data(path: str) -> str:
+    """Generate the URL for the test data based on the given path.
+
+    Parameters
+    ----------
+    path : str
+        The relative path to the test data.
+
+    Returns
+    -------
+    str
+        The full URL to the test data.
+    """
+    _check_path(path)
+
+    return f"{TEST_DATA_URL}{path}"
+
+
+def get_test_data(path: str, gzipped=False) -> str:
+    """Download the test data to a temporary directory and return the local path.
+
+    Parameters
+    ----------
+    path : str
+        The relative path to the test data.
+    gzipped : bool, optional
+        Flag indicating if the remote file is gzipped, by default False. The local file will be gunzipped.
+
+    Returns
+    -------
+    str
+        The local path to the downloaded test data.
+    """
+    _check_path(path)
+
+    target = os.path.normpath(os.path.join(_temporary_directory(), path))
+    with lock:
+        if os.path.exists(target):
+            return target
+
+        os.makedirs(os.path.dirname(target), exist_ok=True)
+        url = url_for_test_data(path)
+
+        if gzipped:
+            url += ".gz"
+            target += ".gz"
+
+        LOG.info(f"Downloading test data from {url} to {target}")
+
+        download(url, target)
+
+        if gzipped:
+            import gzip
+
+            with gzip.open(target, "rb") as f_in:
+                with open(target[:-3], "wb") as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+            os.remove(target)
+            target = target[:-3]
+
+        return target
+
+
+def get_test_archive(path: str, extension=".extracted") -> str:
+    """Download an archive file (.zip, .tar, .tar.gz, .tar.bz2, .tar.xz) to a temporary directory
+    unpack it, and return the local path to the directory containing the extracted files.
+
+    Parameters
+    ----------
+    path : str
+        The relative path to the test data.
+    extension : str, optional
+        The extension to add to the extracted directory, by default '.extracted'
+
+    Returns
+    -------
+    str
+        The local path to the downloaded test data.
+    """
+
+    with lock:
+
+        archive = get_test_data(path)
+        target = archive + extension
+
+        shutil.unpack_archive(archive, os.path.dirname(target) + ".tmp")
+        os.rename(os.path.dirname(target) + ".tmp", target)
+
+        return target
+
+
+def packages_installed(*names) -> bool:
+    """Check if all the given packages are installed.
+
+    Use this function to check if the required packages are installed before running tests.
+
+    >>> @pytest.mark.skipif(not packages_installed("foo", "bar"), reason="Packages 'foo' and 'bar' are not installed")
+    >>> def test_foo_bar() -> None:
+    >>>    ...
+
+    Parameters
+    ----------
+    names : str
+        The names of the packages to check.
+
+    Returns
+    -------
+    bool:
+        Flag indicating if all the packages are installed."
+    """
+
+    for name in names:
+        try:
+            __import__(name)
+        except ImportError:
+            return False
+    return True

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/src/anemoi_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.13
+Version: 0.4.15
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -256,3 +256,4 @@ Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text
 Requires-Dist: termcolor; extra == "text"
 Requires-Dist: wcwidth; extra == "text"
+Dynamic: license-file

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/src/anemoi_utils.egg-info/SOURCES.txt

@@ -59,6 +59,7 @@ src/anemoi/utils/registry.py
 src/anemoi/utils/s3.py
 src/anemoi/utils/sanitise.py
 src/anemoi/utils/sanitize.py
+src/anemoi/utils/testing.py
 src/anemoi/utils/text.py
 src/anemoi/utils/timer.py
 src/anemoi/utils/commands/__init__.py

{anemoi_utils-0.4.13 → anemoi_utils-0.4.15}/tests/test_remote.py

@@ -7,12 +7,14 @@
 
 import os
 import shutil
+import sys
 
 import pytest
 
 from anemoi.utils.remote import TransferMethodNotImplementedError
 from anemoi.utils.remote import _find_transfer_class
 from anemoi.utils.remote import transfer
+from anemoi.utils.testing import packages_installed
 
 IN_CI = (os.environ.get("GITHUB_WORKFLOW") is not None) or (os.environ.get("IN_CI_HPC") is not None)
 
@@ -112,6 +114,7 @@ def test_transfer_find_none(source: str, target: str) -> None:
 
 
 @pytest.mark.skipif(IN_CI, reason="Test requires access to S3")
+@pytest.mark.skipif(not packages_installed("boto3"), reason="boto3 is not installed")
 def test_transfer_zarr_s3_to_local(tmpdir: pytest.TempPathFactory) -> None:
     """Test transferring a Zarr file from S3 to local.
 
@@ -132,6 +135,7 @@ def test_transfer_zarr_s3_to_local(tmpdir: pytest.TempPathFactory) -> None:
 
 
 @pytest.mark.skipif(IN_CI, reason="Test requires access to S3")
+@pytest.mark.skipif(not packages_installed("boto3"), reason="boto3 is not installed")
 def test_transfer_zarr_local_to_s3(tmpdir: pytest.TempPathFactory) -> None:
     """Test transferring a Zarr file from local to S3.
 
@@ -193,6 +197,7 @@ def compare(local1: str, local2: str) -> None:
 
 
 @pytest.mark.skipif(IN_CI, reason="Test requires access to S3")
+@pytest.mark.skipif(not packages_installed("boto3"), reason="boto3 is not installed")
 @pytest.mark.parametrize("path", ["directory/", "file"])
 def test_transfer_local_to_s3_to_local(path: str) -> None:
     """Test transferring a file or directory from local to S3 and back to local.
@@ -224,6 +229,7 @@ def test_transfer_local_to_s3_to_local(path: str) -> None:
 
 
 @pytest.mark.skipif(IN_CI, reason="Test requires ssh access to localhost")
+@pytest.mark.skipif(sys.platform == "darwin", reason="Does not work on MacOS")
 @pytest.mark.parametrize("path", ["directory", "file"])
 @pytest.mark.parametrize("temporary_target", [True, False])
 def test_transfer_local_to_ssh(path: str, temporary_target: bool) -> None:

anemoi_utils-0.4.13/.release-please-manifest.json

@@ -1,3 +0,0 @@
-{
-    ".": "0.4.13"
-}