anemoi-utils 0.4.19__py3-none-any.whl → 0.4.21__py3-none-any.whl

anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.19'
-__version_tuple__ = version_tuple = (0, 4, 19)
+__version__ = version = '0.4.21'
+__version_tuple__ = version_tuple = (0, 4, 21)

anemoi/utils/config.py

@@ -10,6 +10,7 @@
 
 from __future__ import annotations
 
+import contextlib
 import json
 import logging
 import os
@@ -239,6 +240,7 @@ CONFIG = {}
 CHECKED = {}
 CONFIG_LOCK = threading.RLock()
 QUIET = False
+CONFIG_PATCH = None
 
 
 def _find(config: Union[dict, list], what: str, result: list = None) -> list:
@@ -550,7 +552,10 @@ def load_config(
     """
 
     with CONFIG_LOCK:
-        return _load_config(name, secrets, defaults)
+        config = _load_config(name, secrets, defaults)
+        if CONFIG_PATCH is not None:
+            config = CONFIG_PATCH(config)
+        return config
 
 
 def load_raw_config(name: str, default: Any = None) -> Union[DotDict, str]:
@@ -668,3 +673,21 @@ def merge_configs(*configs: dict) -> dict:
         _merge_dicts(result, config)
 
     return result
+
+
+@contextlib.contextmanager
+def temporary_config(tmp: dict) -> None:
+
+    global CONFIG_PATCH
+
+    def patch_config(config: dict) -> dict:
+        return merge_configs(config, tmp)
+
+    with CONFIG_LOCK:
+
+        CONFIG_PATCH = patch_config
+
+        try:
+            yield
+        finally:
+            CONFIG_PATCH = None

anemoi/utils/grids.py

@@ -13,6 +13,9 @@
 import logging
 import os
 from io import BytesIO
+from typing import List
+from typing import Tuple
+from typing import Union
 
 import numpy as np
 import requests
@@ -121,7 +124,7 @@ def nearest_grid_points(
 
 
 @cached(collection="grids", encoding="npz")
-def _grids(name: str) -> bytes:
+def _grids(name: Union[str, List[float], Tuple[float, ...]]) -> bytes:
     """Get grid data by name.
 
     Parameters
@@ -136,6 +139,15 @@ def _grids(name: str) -> bytes:
     """
     from anemoi.utils.config import load_config
 
+    if isinstance(name, (tuple, list)):
+        assert len(name) == 2, "Grid name must be a list or a tuple of length 2"
+        assert all(isinstance(i, (int, float)) for i in name), "Grid name must be a list or a tuple of numbers"
+        if name[0] == name[1]:
+            name = str(float(name[0]))
+        else:
+            name = str(float(name[0])) + "x" + str(float(name[1]))
+        name = name.replace(".", "p")
+
     user_path = load_config().get("utils", {}).get("grids_path")
     if user_path:
         path = os.path.expanduser(os.path.join(user_path, f"grid-{name}.npz"))
@@ -146,6 +158,10 @@ def _grids(name: str) -> bytes:
         else:
             LOG.warning("Custom user path %s does not exist", path)
 
+    # To add a grid
+    # anemoi-transform get-grid --source mars grid=o400,levtype=sfc,param=2t grid-o400.npz
+    # nexus-cli -u xxxx -p yyyy -s GET_INSTANCE --repository anemoi upload --remote-path grids --local-path grid-o400.npz
+
     url = GRIDS_URL_PATTERN.format(name=name.lower())
     LOG.warning("Downloading grids from %s", url)
     response = requests.get(url)
@@ -153,7 +169,7 @@ def _grids(name: str) -> bytes:
     return response.content
 
 
-def grids(name: str) -> dict:
+def grids(name: Union[str, List[float], Tuple[float, ...]]) -> dict:
     """Load grid data by name.
 
     Parameters
@@ -166,7 +182,7 @@ def grids(name: str) -> dict:
     dict
         The grid data
     """
-    if name.endswith(".npz"):
+    if isinstance(name, str) and name.endswith(".npz"):
         return dict(np.load(name))
 
     data = _grids(name)

anemoi/utils/rules.py

@@ -51,10 +51,12 @@ class Rule:
 
     @property
     def result(self) -> Any:
+        """The result associated with the rule."""
         return self._result
 
     @property
     def condition(self) -> Dict[str, Any]:
+        """The conditions that define the rule."""
         return self._match
 
 
@@ -216,3 +218,28 @@ class RuleSet:
             An iterator over the Rule objects in the RuleSet.
         """
         return iter(self.rules)
+
+    def __len__(self) -> int:
+        """Return the number of rules in the RuleSet.
+
+        Returns
+        -------
+        int
+            The number of rules in the RuleSet.
+        """
+        return len(self.rules)
+
+    def __getitem__(self, index: int) -> Rule:
+        """Retrieve a rule by its index.
+
+        Parameters
+        ----------
+        index : int
+            The index of the rule to retrieve.
+
+        Returns
+        -------
+        Rule
+            The rule at the specified index.
+        """
+        return self.rules[index]

anemoi/utils/schemas/__init__.py

@@ -0,0 +1,20 @@
+# (C) Copyright 2025 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+from pydantic import BaseModel as PydanticBaseModel
+
+
+class BaseModel(PydanticBaseModel):
+    class Config:
+        """Pydantic BaseModel configuration."""
+
+        use_attribute_docstrings = True
+        use_enum_values = True
+        validate_assignment = True
+        validate_default = True
+        extra = "forbid"

anemoi/utils/schemas/errors.py

@@ -0,0 +1,54 @@
+# (C) Copyright 2025 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+from collections.abc import Iterator
+from typing import Any
+
+from pydantic import BaseModel as PydanticBaseModel
+from pydantic import ValidationError
+from pydantic_core import ErrorDetails
+
+CUSTOM_MESSAGES = {
+    "missing": "A config entry seems to be missing. If not please check for any typos.",
+    "extra_forbidden": "Extra entries in the config are forebidden. Please check for typos.",
+}
+
+
+def convert_errors(e: ValidationError, custom_messages: dict[str, str]) -> list[ErrorDetails]:
+    new_errors: list[ErrorDetails] = []
+    for error in e.errors():
+        custom_message = custom_messages.get(error["type"])
+
+        if custom_message:
+
+            ctx = error.get("ctx")
+            error["msg"] = custom_message.format(**ctx) if ctx else custom_message
+        new_errors.append(error)
+    return new_errors
+
+
+class ValidationError(Exception):
+    pass
+
+
+def allowed_values(v: Any, values: list[Any]) -> Any:
+    if v not in values:
+        msg = {f"Value {v} not in {values}"}
+        raise ValidationError(msg)
+    return v
+
+
+def required_fields(model: type[PydanticBaseModel], recursive: bool = False) -> Iterator[str]:
+    for name, field in model.model_fields.items():
+        if not field.is_required():
+            continue
+        t = field.annotation
+        if recursive and isinstance(t, type) and issubclass(t, PydanticBaseModel):
+            yield from required_fields(t, recursive=True)
+        else:
+            yield name

anemoi/utils/testing.py

@@ -19,6 +19,8 @@ from functools import lru_cache
 import pytest
 from multiurl import download
 
+from anemoi.utils.humanize import list_to_human
+
 LOG = logging.getLogger(__name__)
 
 TEST_DATA_URL = "https://object-store.os-api.cci1.ecmwf.int/ml-tests/test-data/samples/"
@@ -160,7 +162,7 @@ def get_test_archive(path: str, extension=".extracted") -> str:
         return target
 
 
-def packages_installed(*names) -> bool:
+def packages_installed(*names: str) -> bool:
     """Check if all the given packages are installed.
 
     Use this function to check if the required packages are installed before running tests.
@@ -194,7 +196,7 @@ def packages_installed(*names) -> bool:
     return True
 
 
-def _missing_packages(*names) -> list[str]:
+def _missing_packages(*names: str) -> list[str]:
     """Check if the given packages are missing.
 
     Use this function to check if the required packages are missing before running tests.
@@ -252,12 +254,26 @@ skip_if_offline = pytest.mark.skipif(_offline(), reason="No internet connection"
 skip_slow_tests = pytest.mark.skipif(not _run_slow_tests(), reason="Skipping slow tests")
 
 
-def skip_missing_packages(*names):
-    missing = _missing_packages(*names)
+def skip_missing_packages(*names: str) -> callable:
+    """Skip a test if any of the specified packages are missing.
+
+    Parameters
+    ----------
+    names : str
+        The names of the packages to check.
+
+    Returns
+    -------
+    Callable
+        A decorator that skips the test if any of the specified packages are missing.
+    """
+
+    missing = [f"'{p}'" for p in _missing_packages(*names)]
+
     if len(missing) == 0:
         return lambda f: f
 
     if len(missing) == 1:
         return pytest.mark.skipif(True, reason=f"Package {missing[0]} is not installed")
 
-    return pytest.mark.skipif(True, reason=f"Packages {', '.join(missing)} are not installed")
+    return pytest.mark.skipif(True, reason=f"Packages {list_to_human(missing)} are not installed")

{anemoi_utils-0.4.19.dist-info → anemoi_utils-0.4.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.19
+Version: 0.4.21
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -228,6 +228,7 @@ Requires-Dist: aniso8601
 Requires-Dist: importlib-metadata; python_version < "3.10"
 Requires-Dist: multiurl
 Requires-Dist: numpy
+Requires-Dist: pydantic>=2.9
 Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
 Requires-Dist: tomli; python_version < "3.11"

{anemoi_utils-0.4.19.dist-info → anemoi_utils-0.4.21.dist-info}/RECORD

@@ -1,25 +1,25 @@
 anemoi/utils/__init__.py,sha256=uVhpF-VjIl_4mMywOVtgTutgsdIsqz-xdkwxeMhzuag,730
 anemoi/utils/__main__.py,sha256=6LlE4MYrPvqqrykxXh7XMi50UZteUY59NeM8P9Zs2dU,910
-anemoi/utils/_version.py,sha256=ckoIBbD6SbpUXLLR-S_0NN-a9fgpE7KOAJuuoR3ua9s,513
+anemoi/utils/_version.py,sha256=jhdlV71JQ5bBRT2wFnZ7TRbXO71DWPBUoBiZyZnIPYQ,513
 anemoi/utils/caching.py,sha256=rXbeAmpBcMbbfN4EVblaHWKicsrtx1otER84FEBtz98,6183
 anemoi/utils/checkpoints.py,sha256=N4WpAZXa4etrpSEKhHqUUtG2-x9w3FJMHcLO-dDAXPY,9600
 anemoi/utils/cli.py,sha256=IyZfnSw0u0yYnrjOrzvm2RuuKvDk4cVb8pf8BkaChgA,6209
 anemoi/utils/compatibility.py,sha256=wRBRMmxQP88rNcWiP5gqXliwYQbBv1iCAsDjcCRi5UY,2234
-anemoi/utils/config.py,sha256=UtxlkSMhqZR0LAKi19aG6jaaNiSmsXCzE6v2AWHhx5E,16861
+anemoi/utils/config.py,sha256=EEfcSxW2CD6fFOzDtqz_uYlMKuYq4X5QinJW_8GBYj4,17325
 anemoi/utils/dates.py,sha256=CnY6JOdpk0T-waPEowMRTkcMzxcN0GcjPVtLkwH_byw,17196
 anemoi/utils/devtools.py,sha256=W3OBu96MkXRIl7Qh1SE5Zd6aB1R0QlnmlrlpBYM0fVY,3527
 anemoi/utils/grib.py,sha256=201WcxjjAl92Y2HX2kZ2S8Qr5dN-oG7nV-vQLaybzP4,3610
-anemoi/utils/grids.py,sha256=edTrMK8hpE9ZBzSfwcRftgk0jljNAK3i8CraadILQoM,4427
+anemoi/utils/grids.py,sha256=uYgkU_KIg8FTUiuKV0Pho2swMMeXcSQ9CQe0MFlRr_I,5262
 anemoi/utils/hindcasts.py,sha256=iYVIxSNFL2HJcc_k1abCFLkpJFGHT8WKRIR4wcAwA3s,2144
 anemoi/utils/humanize.py,sha256=pjnFJAKHbEAOfcvn8c48kt-8eFy6FGW_U2ruJvfamrA,25189
 anemoi/utils/logs.py,sha256=naTgrmPwWHD4eekFttXftS4gtcAGYHpCqG4iwYprNDA,1804
 anemoi/utils/provenance.py,sha256=xC6mTstF7f_asqtPSrulC7c34xjOSuAxWhkwc3yKhHg,14629
 anemoi/utils/registry.py,sha256=e3nOIRyMYQ-mpEvaHAv5tuvMYNbkJ5yz94ns7BnvkjM,9717
-anemoi/utils/rules.py,sha256=xYCiUV_HXTGFe93diqMLQsMJCWGi5umd_bWEeYP8XFY,6318
+anemoi/utils/rules.py,sha256=VspUoPmw7tijrs6l_wl4vDjr_zVQsFjx9ITiBSvxgc8,6972
 anemoi/utils/s3.py,sha256=xMT48kbcelcjjqsaU567WI3oZ5eqo88Rlgyx5ECszAU,4074
 anemoi/utils/sanitise.py,sha256=ZYGdSX6qihQANr3pHZjbKnoapnzP1KcrWdW1Ul1mOGk,3668
 anemoi/utils/sanitize.py,sha256=43ZKDcfVpeXSsJ9TFEc9aZnD6oe2cUh151XnDspM98M,462
-anemoi/utils/testing.py,sha256=2H9yxsZriCF_juVX0zYsUCBwA9XC20OfSRZvFkRH7jY,6873
+anemoi/utils/testing.py,sha256=RJJGlIriQode3eWQ3k1I30ZQe0yXjsO8fZGvMuRAjYM,7263
 anemoi/utils/text.py,sha256=HkzIvi24obDceFLpJEwBJ9PmPrJUkQN2TrElJ-A87gU,14441
 anemoi/utils/timer.py,sha256=_leKMYza2faM7JKlGE7LCNy13rbdPnwaCF7PSrI_NmI,3895
 anemoi/utils/commands/__init__.py,sha256=5u_6EwdqYczIAgJfCwRSyQAYFEqh2ZuHHT57g9g7sdI,808
@@ -31,9 +31,11 @@ anemoi/utils/mars/requests.py,sha256=VFMHBVAAl0_2lOcMBa1lvaKHctN0lDJsI6_U4BucGew
 anemoi/utils/remote/__init__.py,sha256=swPWHQoh-B6Xq9R489tPw0FykMue7f-bJ8enneFYSYE,20776
 anemoi/utils/remote/s3.py,sha256=spQ8l0rwQjLZh9dZu5cOsYIvNwKihQfCJ6YsFYegeqI,17339
 anemoi/utils/remote/ssh.py,sha256=xNtsawh8okytCKRehkRCVExbHZj-CRUQNormEHglfuw,8088
-anemoi_utils-0.4.19.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
-anemoi_utils-0.4.19.dist-info/METADATA,sha256=IeSKCPc77w2aDSaRudxLPmy70LE9-bMwqNq15lZyT24,15410
-anemoi_utils-0.4.19.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-anemoi_utils-0.4.19.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.4.19.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.4.19.dist-info/RECORD,,
+anemoi/utils/schemas/__init__.py,sha256=nkinKlsPLPXEjfTYQT1mpKC4cvs-14w_zBkDRxakwxw,698
+anemoi/utils/schemas/errors.py,sha256=lgOXzVTYzAE0qWQf3OZ42vCWixv8lilSqLLhzARBmvI,1831
+anemoi_utils-0.4.21.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
+anemoi_utils-0.4.21.dist-info/METADATA,sha256=AWNLa8J5pQTvIgxWOPOOo8skiBLCbbCR_DPuCxAGeAI,15439
+anemoi_utils-0.4.21.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+anemoi_utils-0.4.21.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.4.21.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.4.21.dist-info/RECORD,,