anemoi-utils 0.4.16__tar.gz → 0.4.18__tar.gz

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/.gitignore

@@ -42,6 +42,7 @@ cover/
 
 # Documentation
 docs/_build/
+docs/_api/
 /site
 *.mo
 *.pot
@@ -119,7 +120,7 @@ tmp/
 temp/
 logs/
 _dev/
-outputs/
+outputs
 *tmp_data/
 
 # Project specific
@@ -130,9 +131,7 @@ bar
 ~$images.pptx
 test.py
 test.ipynb
-cutout.png
 _version.py
 *.to_upload
 tempCodeRunnerFile.python
 Untitled-*.py
-.*cache/

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/.pre-commit-config.yaml

@@ -27,12 +27,12 @@ repos:
     -   id: python-check-blanket-noqa # Check for # noqa: all
     -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.10.0
+  rev: 25.1.0
   hooks:
   - id: black
     args: [--line-length=120]
 - repo: https://github.com/pycqa/isort
-  rev: 5.13.2
+  rev: 6.0.1
   hooks:
   - id: isort
     args:
@@ -40,7 +40,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.6
+  rev: v0.9.9
   hooks:
   - id: ruff
     args:
@@ -60,16 +60,16 @@ repos:
   - id: rstfmt
     exclude: '(cli|schemas)/.*' # Because we use argparse and pydantic sphinx directives
 - repo: https://github.com/b8raoult/pre-commit-docconvert
-  rev: "0.1.4"
+  rev: "0.1.5"
   hooks:
   - id: docconvert
     args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "v2.5.0"
+  rev: "v2.5.1"
   hooks:
   - id: pyproject-fmt
 -   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
-    rev: v0.66.1
+    rev: v0.69.1
     hooks:
     -   id: docsig
         args:

anemoi_utils-0.4.18/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.18"
+}

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/CHANGELOG.md

@@ -8,6 +8,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.4.18](https://github.com/ecmwf/anemoi-utils/compare/0.4.17...0.4.18) (2025-03-31)
+
+
+### Features
+
+* add matching rules ([#132](https://github.com/ecmwf/anemoi-utils/issues/132)) ([2382980](https://github.com/ecmwf/anemoi-utils/commit/2382980f4f53909a73fa0a5c8cfab108625f3c55))
+
+## [0.4.17](https://github.com/ecmwf/anemoi-utils/compare/0.4.16...0.4.17) (2025-03-27)
+
+
+### Features
+
+* add generic env variables to override anemoi user config ([#128](https://github.com/ecmwf/anemoi-utils/issues/128)) ([fdc7248](https://github.com/ecmwf/anemoi-utils/commit/fdc72485616a0c092356a9ffa4cdca838a0c1a9d))
+
+
+### Bug Fixes
+
+* Iterate over copy of sys.modules. ([#127](https://github.com/ecmwf/anemoi-utils/issues/127)) ([7b0e7d0](https://github.com/ecmwf/anemoi-utils/commit/7b0e7d08264f7eb4c92fdcd744ff8c46eac82fb7))
+* plugin name on error ([#120](https://github.com/ecmwf/anemoi-utils/issues/120)) ([a747f63](https://github.com/ecmwf/anemoi-utils/commit/a747f63d74bf1b108d913694915df59ffc4640c1))
+
+
+### Documentation
+
+* add links to GitHub  ([#123](https://github.com/ecmwf/anemoi-utils/issues/123)) ([cfe1ea2](https://github.com/ecmwf/anemoi-utils/commit/cfe1ea281e03a56b9a02108b6787c6c05b9518b0))
+* Docathon ([#121](https://github.com/ecmwf/anemoi-utils/issues/121)) ([e1c9292](https://github.com/ecmwf/anemoi-utils/commit/e1c9292d65b1ffc8c9ce8eed41c7ffbe81f865a3))
+* fix comment ([#125](https://github.com/ecmwf/anemoi-utils/issues/125)) ([ad3ed12](https://github.com/ecmwf/anemoi-utils/commit/ad3ed126f9a507dde7ce19064f1d32dae2cee6a3))
+
 ## [0.4.16](https://github.com/ecmwf/anemoi-utils/compare/0.4.15...0.4.16) (2025-03-22)
 
 

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.16
+Version: 0.4.18
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/docs/conf.py

@@ -105,6 +105,10 @@ intersphinx_mapping = {
         "https://anemoi-transform.readthedocs.io/en/latest/",
         ("../../anemoi-transform/docs/_build/html/objects.inv", None),
     ),
+    "anemoi-plugins": (
+        "https://anemoi-plugins.readthedocs.io/en/latest/",
+        ("../../anemoi-plugins/docs/_build/html/objects.inv", None),
+    ),
 }
 
 # -- Options for HTML output -------------------------------------------------
@@ -124,3 +128,10 @@ html_css_files = ["style.css"]
 todo_include_todos = not read_the_docs_build
 
 autodoc_member_order = "bysource"  # Keep file order
+
+html_context = {
+    "display_github": True,
+    "github_user": "ecmwf",
+    "github_repo": "anemoi-utils",
+    "github_version": "main/docs/",
+}

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/docs/index.rst

@@ -40,9 +40,9 @@ of the *Anemoi* packages.
 
    modules/*
 
-*****************
- Anemoi packages
-*****************
+***********************
+ Other Anemoi packages
+***********************
 
 -  :ref:`anemoi-utils <anemoi-utils:index-page>`
 -  :ref:`anemoi-transform <anemoi-transform:index-page>`
@@ -52,6 +52,7 @@ of the *Anemoi* packages.
 -  :ref:`anemoi-training <anemoi-training:index-page>`
 -  :ref:`anemoi-inference <anemoi-inference:index-page>`
 -  :ref:`anemoi-registry <anemoi-registry:index-page>`
+-  :ref:`anemoi-plugins <anemoi-plugins:index-page>`
 
 *********
  License

anemoi_utils-0.4.18/docs/modules/testing.rst

@@ -0,0 +1,8 @@
+#########
+ testing
+#########
+
+.. automodule:: anemoi.utils.testing
+   :members:
+   :no-undoc-members:
+   :show-inheritance:

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/src/anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.16'
-__version_tuple__ = version_tuple = (0, 4, 16)
+__version__ = version = '0.4.18'
+__version_tuple__ = version_tuple = (0, 4, 18)

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/src/anemoi/utils/config.py

@@ -459,6 +459,24 @@ def _load_config(
         secret_config = _load_config(secret_name)
         _merge_dicts(config, secret_config)
 
+    for env, value in os.environ.items():
+
+        if not env.startswith("ANEMOI_CONFIG_"):
+            continue
+        rest = env[len("ANEMOI_CONFIG_") :]
+
+        package = rest.split("_")[0]
+        sub = rest[len(package) + 1 :]
+
+        package = package.lower()
+        sub = sub.lower()
+
+        LOG.info(f"Using environment variable {env} to override the anemoi config key '{package}.{sub}'")
+
+        if package not in config:
+            config[package] = {}
+        config[package][sub] = value
+
     CONFIG[key] = DotDict(config)
     return CONFIG[key]
 

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/src/anemoi/utils/provenance.py

@@ -199,12 +199,12 @@ def _module_versions(full: bool) -> Tuple[Dict[str, Any], set]:
 
     versions = {}
     namespaces = set()
-    for k, v in sorted(sys.modules.items()):
+    for k, v in sorted(sys.modules.copy().items()):
         if "." not in k:
             version(versions, k, v, roots, namespaces, paths, full)
 
     # Catter for modules like "earthkit.meteo"
-    for k, v in sorted(sys.modules.items()):
+    for k, v in sorted(sys.modules.copy().items()):
         bits = k.split(".")
         if len(bits) == 2 and bits[0] in namespaces:
             version(versions, k, v, roots, namespaces, paths, full)

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/src/anemoi/utils/registry.py

@@ -162,12 +162,16 @@ class Registry:
         file : str
             The file to load.
         """
+
         name, _ = os.path.splitext(file)
+
         try:
             importlib.import_module(f".{name}", package=self.package)
         except Exception as e:
             if DEBUG_ANEMOI_REGISTRY:
                 raise
+
+            name = name.replace("_", "-")
             self.__registered[name] = Error(e)
 
     def is_registered(self, name: str) -> bool:

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/src/anemoi/utils/remote/__init__.py

@@ -627,7 +627,7 @@ def _find_transfer_class(source: str, target: str) -> type:
     raise TransferMethodNotImplementedError(f"Transfer from {source} to {target} is not implemented")
 
 
-# this is the public API
+# This function is the main entry point for the transfer mechanism for the other anemoi packages
 def transfer(
     source, target, *, overwrite=False, resume=False, verbosity=1, progress=None, threads=1, temporary_target=False
 ) -> Loader:

anemoi_utils-0.4.18/src/anemoi/utils/rules.py

@@ -0,0 +1,218 @@
+# (C) Copyright 2025 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Mapping
+from typing import Optional
+from typing import Union
+
+
+class Rule:
+
+    def __init__(self, match: Dict[str, Any], result: Any):
+        """Initialize a Rule object.
+
+        Parameters
+        ----------
+        match : Dict[str, Any]
+            A dictionary defining the conditions for the rule to match.
+        result : Any
+            The result to return if the rule matches.
+        """
+        self._match = match
+        self._result = result
+
+    def match(self, obj: Mapping[str, Any]) -> bool:
+        """Check if the rule matches the given object.
+
+        Parameters
+        ----------
+        obj : Mapping[str, Any]
+            The object to check against the rule's conditions.
+
+        Returns
+        -------
+        bool
+            True if the rule matches, False otherwise.
+        """
+        for key, value in self._match.items():
+            if key not in obj or obj[key] != value:
+                return False
+
+        return True
+
+    @property
+    def result(self) -> Any:
+        return self._result
+
+    @property
+    def condition(self) -> Dict[str, Any]:
+        return self._match
+
+
+class RuleSet:
+
+    def __init__(self, rules: List[Union[Rule, Dict[str, Any], List[Any]]]):
+        """Initialize a RuleSet object.
+
+        Parameters
+        ----------
+        rules : List[Union[Rule, Dict[str, Any], List[Any]]]
+            A list of rules, where each rule can be a Rule object, a dictionary with
+            'match' and 'result' keys, or a list with two elements (match and result).
+        """
+        assert isinstance(rules, list), "rules must be a list"
+
+        self.rules: List[Rule] = []
+
+        for rule in rules:
+            if isinstance(rule, Rule):
+                self.rules.append(rule)
+                continue
+
+            if isinstance(rule, dict):
+
+                assert len(rule) == 2, "Rule dictionary must contain exactly two key-value pair."
+
+                match = rule.get("match")
+                if match is None:
+                    raise ValueError("Rule dictionary must contain a 'match' key.")
+
+                result = rule.get("result")
+                if result is None:
+                    raise ValueError("Rule dictionary must contain a 'result' key.")
+
+                self.rules.append(Rule(match, result))
+                continue
+
+            if isinstance(rule, list):
+                assert len(rule) == 2, "Rule list must contain exactly two elements."
+                match = rule[0]
+                result = rule[1]
+                self.rules.append(Rule(match, result))
+                continue
+
+            raise ValueError(
+                "Rule must be either a Rule object, a dictionary with 'match' and 'result' keys, or a list with two elements."
+            )
+
+    @classmethod
+    def from_list(cls, rules: List[Any]) -> "RuleSet":
+        """Create a RuleSet from a list of rules.
+
+        Parameters
+        ----------
+        rules : List[Any]
+            A list of rules to initialize the RuleSet.
+
+        Returns
+        -------
+        RuleSet
+            A new RuleSet object.
+        """
+        return cls(rules)
+
+    @classmethod
+    def from_files(cls, path: str) -> "RuleSet":
+        """Create a RuleSet from a file.
+
+        Parameters
+        ----------
+        path : str
+            The path to the file containing the rules. Supported formats are .json and .yaml/.yml.
+
+        Returns
+        -------
+        RuleSet
+            A new RuleSet object.
+
+        Raises
+        ------
+        ValueError
+            If the file format is unsupported.
+        """
+        if path.endswith(".json"):
+            import json
+
+            with open(path, "r") as f:
+                return cls.from_list(json.load(f))
+
+        if path.endswith(".yaml") or path.endswith(".yml"):
+            import yaml
+
+            with open(path, "r") as f:
+                return cls.from_list(yaml.safe_load(f))
+
+        raise ValueError("Unsupported file format. Supported formats are .json and .yaml/.yml.")
+
+    @classmethod
+    def from_any(cls, rules: Union[str, List[Any]]) -> "RuleSet":
+        """Create a RuleSet from a list or a file path.
+
+        Parameters
+        ----------
+        rules : Union[str, List[Any]]
+            The rules to initialize the RuleSet, either as a list or a file path.
+
+        Returns
+        -------
+        RuleSet
+            A new RuleSet object.
+
+        Raises
+        ------
+        ValueError
+            If the rules format is unsupported.
+        """
+        if isinstance(rules, str):
+            return cls.from_files(rules)
+
+        if isinstance(rules, list):
+            return cls.from_list(rules)
+
+        raise ValueError("Unsupported rules format. Must be a list or a file path.")
+
+    def match(self, obj: Mapping[str, Any], strategy: str = "first-match") -> Optional[Rule]:
+        """Match an object against the rules in the RuleSet.
+
+        Parameters
+        ----------
+        obj : Mapping[str, Any]
+            The object to match against the rules.
+        strategy : str, optional
+            The matching strategy to use. Currently, only 'first-match' is supported.
+
+        Returns
+        -------
+        Optional[Rule]
+            The first matching rule, or None if no match is found.
+
+        Raises
+        ------
+        AssertionError
+            If an unsupported strategy is provided.
+        """
+        assert strategy == "first-match", "Only 'first-match' strategy is supported for now."
+        for rule in self.rules:
+            if rule.match(obj):
+                return rule
+
+        return None
+
+    def __iter__(self) -> iter:
+        """Return an iterator over the rules in the RuleSet.
+
+        Returns
+        -------
+        iter
+            An iterator over the Rule objects in the RuleSet.
+        """
+        return iter(self.rules)

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/src/anemoi_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.16
+Version: 0.4.18
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.4.16 → anemoi_utils-0.4.18}/src/anemoi_utils.egg-info/SOURCES.txt

@@ -38,6 +38,7 @@ docs/modules/grib.rst
 docs/modules/humanize.rst
 docs/modules/provenance.rst
 docs/modules/s3.rst
+docs/modules/testing.rst
 docs/modules/text.rst
 src/anemoi/utils/__init__.py
 src/anemoi/utils/__main__.py
@@ -56,6 +57,7 @@ src/anemoi/utils/humanize.py
 src/anemoi/utils/logs.py
 src/anemoi/utils/provenance.py
 src/anemoi/utils/registry.py
+src/anemoi/utils/rules.py
 src/anemoi/utils/s3.py
 src/anemoi/utils/sanitise.py
 src/anemoi/utils/sanitize.py

anemoi_utils-0.4.16/.release-please-manifest.json

@@ -1,3 +0,0 @@
-{
-    ".": "0.4.16"
-}