anemoi-utils 0.4.24__tar.gz → 0.4.26__tar.gz

anemoi_utils-0.4.26/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.26"
+}

{anemoi_utils-0.4.24 → anemoi_utils-0.4.26}/CHANGELOG.md

@@ -8,6 +8,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.4.26](https://github.com/ecmwf/anemoi-utils/compare/0.4.25...0.4.26) (2025-06-25)
+
+
+### Features
+
+* Fixtures for temp dir handling for test data ([#166](https://github.com/ecmwf/anemoi-utils/issues/166)) ([2b9677f](https://github.com/ecmwf/anemoi-utils/commit/2b9677fffc5eba84876f974001b87b73c7e542af))
+* Move anemoi-inference metadata command to this package, add metadata removal options ([#167](https://github.com/ecmwf/anemoi-utils/issues/167)) ([cabb989](https://github.com/ecmwf/anemoi-utils/commit/cabb989bdd4154a0476acf48e1ac44099c91c6db))
+
+## [0.4.25](https://github.com/ecmwf/anemoi-utils/compare/0.4.24...0.4.25) (2025-06-24)
+
+
+### Features
+
+* Add a CLI to transfer data ([#164](https://github.com/ecmwf/anemoi-utils/issues/164)) ([3a845ca](https://github.com/ecmwf/anemoi-utils/commit/3a845ca0c31d115e6b3d0496d862a3eaee5fb236))
+* Add function to test cli ([#168](https://github.com/ecmwf/anemoi-utils/issues/168)) ([9ac9b06](https://github.com/ecmwf/anemoi-utils/commit/9ac9b06b8fd0a62cad33ea5de6a6b482f0a13656))
+
 ## [0.4.24](https://github.com/ecmwf/anemoi-utils/compare/0.4.23...0.4.24) (2025-06-06)
 
 

{anemoi_utils-0.4.24 → anemoi_utils-0.4.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.24
+Version: 0.4.26
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -225,12 +225,14 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: aniso8601
+Requires-Dist: deprecation
 Requires-Dist: importlib-metadata; python_version < "3.10"
 Requires-Dist: multiurl
 Requires-Dist: numpy
 Requires-Dist: pydantic>=2.9
 Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
+Requires-Dist: rich
 Requires-Dist: tomli; python_version < "3.11"
 Requires-Dist: tqdm
 Provides-Extra: all

{anemoi_utils-0.4.24 → anemoi_utils-0.4.26}/pyproject.toml

@@ -41,12 +41,14 @@ classifiers = [
 dynamic = [ "version" ]
 dependencies = [
   "aniso8601",
+  "deprecation",
   "importlib-metadata; python_version<'3.10'",
   "multiurl",
   "numpy",
   "pydantic>=2.9",
   "python-dateutil",
   "pyyaml",
+  "rich",
   "tomli; python_version<'3.11'",
   "tqdm",
 ]

{anemoi_utils-0.4.24 → anemoi_utils-0.4.26}/src/anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.24'
-__version_tuple__ = version_tuple = (0, 4, 24)
+__version__ = version = '0.4.26'
+__version_tuple__ = version_tuple = (0, 4, 26)

{anemoi_utils-0.4.24 → anemoi_utils-0.4.26}/src/anemoi/utils/checkpoints.py

@@ -308,7 +308,6 @@ def remove_metadata(path: str, *, name: str = DEFAULT_NAME) -> None:
     name : str, optional
         The name of the metadata file in the zip archive
     """
-    LOG.info("Removing metadata '%s' from %s", name, path)
 
     def callback(full):
         os.remove(full)

{anemoi_utils-0.4.24 → anemoi_utils-0.4.26}/src/anemoi/utils/cli.py

@@ -15,6 +15,7 @@ import os
 import sys
 import traceback
 from typing import Callable
+from typing import Optional
 
 try:
     import argcomplete
@@ -185,7 +186,9 @@ def register_commands(here: str, package: str, select: Callable, fail: Callable
     return result
 
 
-def cli_main(version: str, description: str, commands: dict[str, Command]) -> None:
+def cli_main(
+    version: str, description: str, commands: dict[str, Command], test_arguments: Optional[list[str]] = None
+) -> None:
     """Main entry point for the CLI.
 
     Parameters
@@ -196,9 +199,11 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
         The description of the CLI
     commands : dict[str, Command]
         A dictionary of command names to Command instances
+    test_arguments : list[str], optional
+        The command line arguments to parse, used for testing purposes, by default None
     """
     parser = make_parser(description, commands)
-    args, unknown = parser.parse_known_args()
+    args, unknown = parser.parse_known_args(test_arguments)
     if argcomplete:
         argcomplete.autocomplete(parser)
 
@@ -220,7 +225,7 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
 
     if unknown and not cmd.accept_unknown_args:
         # This should trigger an error
-        parser.parse_args()
+        parser.parse_args(test_arguments)
 
     try:
         if unknown:
@@ -228,9 +233,14 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
         else:
             cmd.run(args)
     except ValueError as e:
+
+        if test_arguments:
+            raise
+
         traceback.print_exc()
         LOG.error("\n💣 %s", str(e).lstrip())
         LOG.error("💣 Exiting")
         sys.exit(1)
 
-    sys.exit(0)
+    if not test_arguments:
+        sys.exit(0)

anemoi_utils-0.4.26/src/anemoi/utils/commands/metadata.py

@@ -0,0 +1,391 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import json
+import logging
+import os
+import shutil
+import subprocess
+from argparse import ArgumentParser
+from argparse import Namespace
+from tempfile import TemporaryDirectory
+from typing import Any
+from typing import Dict
+
+import yaml
+
+from . import Command
+
+LOG = logging.getLogger(__name__)
+
+EDITOR_OPTIONS = {"code": ["--wait"]}
+
+
+class Metadata(Command):
+    """Edit, remove, dump or load metadata from a checkpoint file."""
+
+    def add_arguments(self, command_parser: ArgumentParser) -> None:
+        """Add command line arguments to the parser.
+
+        Parameters
+        ----------
+        command_parser : ArgumentParser
+            The argument parser to which the arguments will be added.
+        """
+        from anemoi.utils.checkpoints import DEFAULT_NAME
+
+        command_parser.add_argument("path", help="Path to the checkpoint.")
+
+        group = command_parser.add_mutually_exclusive_group(required=True)
+
+        group.add_argument(
+            "--dump",
+            action="store_true",
+            help=(
+                "Extract the metadata from the checkpoint and print it to the standard output"
+                " or the file specified by ``--output``, in JSON or YAML format."
+            ),
+        )
+        group.add_argument(
+            "--load",
+            action="store_true",
+            help=(
+                "Set the metadata in the checkpoint from the content"
+                " of a file specified by the ``--input`` argument."
+            ),
+        )
+
+        group.add_argument(
+            "--edit",
+            action="store_true",
+            help=(
+                "Edit the metadata in place, using the specified editor."
+                " See the ``--editor`` argument for more information."
+            ),
+        )
+
+        group.add_argument(
+            "--view",
+            action="store_true",
+            help=(
+                "View the metadata in place, using the specified pager."
+                " See the ``--pager`` argument for more information."
+            ),
+        )
+
+        group.add_argument(
+            "--remove",
+            action="store_true",
+            help="Remove the metadata from the checkpoint.",
+        )
+
+        group.add_argument(
+            "--supporting-arrays",
+            action="store_true",
+            help="Print the supporting arrays.",
+        )
+
+        group.add_argument(
+            "--get",
+            help="Navigate the metadata via dot-separated path.",
+        )
+
+        group.add_argument(
+            "--pytest",
+            action="store_true",
+            help=("Extract the metadata from the checkpoint so it can be added to the test suite."),
+        )
+
+        command_parser.add_argument(
+            "--name",
+            default=DEFAULT_NAME,
+            help="Name of metadata record to be used with the actions above.",
+        )
+
+        command_parser.add_argument(
+            "--input",
+            help="The output file name to be used by the ``--load`` option.",
+        )
+
+        command_parser.add_argument(
+            "--output",
+            help="The output file name to be used by the ``--dump`` option.",
+        )
+
+        command_parser.add_argument(
+            "--inplace",
+            action="store_true",
+            help="If set, update the source file in place instead of writing to a separate target.",
+        )
+
+        command_parser.add_argument(
+            "--editor",
+            help="Editor to use for the ``--edit`` option. Default to ``$EDITOR`` if defined, else ``vi``.",
+            default=os.environ.get("EDITOR", "vi"),
+        )
+
+        command_parser.add_argument(
+            "--pager",
+            help="Editor to use for the ``--view`` option. Default to ``$PAGER`` if defined, else ``less``.",
+            default=os.environ.get("PAGER", "less"),
+        )
+
+        command_parser.add_argument(
+            "--json",
+            action="store_true",
+            help="Use the JSON format with ``--dump``, ``--view`` and ``--edit``.",
+        )
+
+        command_parser.add_argument(
+            "--yaml",
+            action="store_true",
+            help="Use the YAML format with ``--dump``, ``--view`` and ``--edit``.",
+        )
+
+    def run(self, args: Namespace) -> None:
+        """Execute the command based on the provided arguments.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        if args.edit:
+            return self.edit(args)
+
+        if args.view:
+            return self.view(args)
+
+        if args.get:
+            return self.get(args)
+
+        if args.remove:
+            return self.remove(args)
+
+        if args.dump or args.pytest:
+            return self.dump(args)
+
+        if args.load:
+            return self.load(args)
+
+        if args.supporting_arrays:
+            return self.supporting_arrays(args)
+
+    def edit(self, args: Namespace) -> None:
+        """Edit the metadata in place using the specified editor.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        return self._edit(args, view=False, cmd=args.editor)
+
+    def view(self, args: Namespace) -> None:
+        """View the metadata in place using the specified pager.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        return self._edit(args, view=True, cmd=args.pager)
+
+    def _edit(self, args: Namespace, view: bool, cmd: str) -> None:
+        """Internal method to edit or view the metadata.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        view : bool
+            If True, view the metadata; otherwise, edit it.
+        cmd : str
+            The command to use for editing or viewing.
+        """
+        from anemoi.utils.checkpoints import load_metadata
+        from anemoi.utils.checkpoints import replace_metadata
+
+        kwargs: Dict[str, Any] = {}
+
+        if args.json:
+            ext = "json"
+            dump = json.dump
+            load = json.load
+            if args.test:
+                kwargs = {"sort_keys": True}
+            else:
+                kwargs = {"indent": 4, "sort_keys": True}
+        else:
+            ext = "yaml"
+            dump = yaml.dump
+            load = yaml.safe_load
+            kwargs = {"default_flow_style": False}
+
+        with TemporaryDirectory() as temp_dir:
+
+            path = os.path.join(temp_dir, f"checkpoint.{ext}")
+            metadata = load_metadata(args.path)
+
+            with open(path, "w") as f:
+                dump(metadata, f, **kwargs)
+
+            subprocess.check_call([cmd, *EDITOR_OPTIONS.get(cmd, []), path])
+
+            if not view:
+                with open(path) as f:
+                    edited = load(f)
+
+                if edited != metadata:
+                    replace_metadata(args.path, edited)
+                else:
+                    LOG.info("No changes made.")
+
+    def remove(self, args: Namespace) -> None:
+        """Remove the metadata from the checkpoint.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        from anemoi.utils.checkpoints import remove_metadata
+
+        if args.inplace and args.output:
+            raise ValueError("Only choose one of --inplace and --output")
+
+        LOG.info("Removing metadata from %s", args.path)
+
+        if args.inplace:
+            output = args.path
+        else:
+            if not args.output:
+                raise ValueError("Argument --output is required unless --inplace is set")
+
+            shutil.copy2(args.path, args.output)
+            output = args.output
+
+        LOG.info("Writing checkpoint at %s", output)
+        remove_metadata(output)
+
+    def dump(self, args: Namespace) -> None:
+        """Dump the metadata from the checkpoint to a file or standard output.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        from anemoi.utils.checkpoints import load_metadata
+
+        if args.output:
+            file = open(args.output, "w")
+        else:
+            file = None
+
+        metadata = load_metadata(args.path)
+        if args.pytest:
+            from anemoi.inference.testing.mock_checkpoint import minimum_mock_checkpoint
+
+            # We remove all unessential metadata for testing purposes
+            metadata = minimum_mock_checkpoint(metadata)
+
+        if args.yaml:
+            print(yaml.dump(metadata, indent=2, sort_keys=True), file=file)
+            return
+
+        if args.json or True:
+            if args.pytest:
+                print(json.dumps(metadata, sort_keys=True), file=file)
+            else:
+                print(json.dumps(metadata, indent=4, sort_keys=True), file=file)
+            return
+
+    def get(self, args: Namespace) -> None:
+        """Navigate and print the metadata via a dot-separated path.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        from pprint import pprint
+
+        from anemoi.utils.checkpoints import load_metadata
+
+        metadata = load_metadata(args.path, name=args.name)
+
+        if args.get == ".":
+            print("Metadata from root: ", list(metadata.keys()))
+            return
+
+        for key in args.get.split("."):
+            if key == "":
+                keys = list(metadata.keys())
+                print(f"Metadata keys from {args.get[:-1]}: ", keys)
+                return
+            else:
+                metadata = metadata[key]
+
+        print(f"Metadata values for {args.get}: ", end="\n" if isinstance(metadata, (dict, list)) else "")
+        if isinstance(metadata, dict):
+            pprint(metadata, indent=2, compact=True)
+        else:
+            print(metadata)
+
+    def load(self, args: Namespace) -> None:
+        """Load metadata into the checkpoint from a specified file.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        from anemoi.utils.checkpoints import has_metadata
+        from anemoi.utils.checkpoints import replace_metadata
+        from anemoi.utils.checkpoints import save_metadata
+
+        if args.input is None:
+            raise ValueError("Please specify a value for --input")
+
+        _, ext = os.path.splitext(args.input)
+        if ext == ".json" or args.json:
+            with open(args.input) as f:
+                metadata = json.load(f)
+
+        elif ext in (".yaml", ".yml") or args.yaml:
+            with open(args.input) as f:
+                metadata = yaml.safe_load(f)
+
+        else:
+            raise ValueError(f"Unknown file extension {ext}. Please specify --json or --yaml")
+
+        if has_metadata(args.path, name=args.name):
+            replace_metadata(args.path, metadata)
+        else:
+            save_metadata(args.path, metadata, name=args.name)
+
+    def supporting_arrays(self, args: Namespace) -> None:
+        """Print the supporting arrays from the metadata.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        from anemoi.utils.checkpoints import load_metadata
+
+        _, supporting_arrays = load_metadata(args.path, supporting_arrays=True)
+
+        for name, array in supporting_arrays.items():
+            print(f"{name}: shape={array.shape} dtype={array.dtype}")
+
+
+command = Metadata

anemoi_utils-0.4.26/src/anemoi/utils/commands/transfer.py

@@ -0,0 +1,74 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+from argparse import ArgumentParser
+from argparse import Namespace
+
+from anemoi.utils.remote import transfer
+
+from . import Command
+
+
+class Transfer(Command):
+    """Transfer files or folders from the source to the target location."""
+
+    def add_arguments(self, command_parser: ArgumentParser) -> None:
+        """Add arguments to the command parser.
+
+        Parameters
+        ----------
+        command_parser : ArgumentParser
+            The argument parser to which the arguments will be added.
+        """
+        command_parser.add_argument(
+            "--source", help="A path to a local file or folder or a URL to a file or a folder on S3."
+        )
+        command_parser.add_argument(
+            "--target", help="A path to a local file or folder or a URL to a file or a folder on S3 or a remote folder."
+        )
+        command_parser.add_argument(
+            "--overwrite",
+            action="store_true",
+            help="If the data is already on in the target location it will be overwritten..",
+        )
+        command_parser.add_argument(
+            "--resume",
+            action="store_true",
+            help="If the data is already on S3 it will not be uploaded, unless the remote file has a different size.",
+        )
+        command_parser.add_argument("--verbosity", default=1, help="The level of verbosity, by default 1.")
+        command_parser.add_argument(
+            "--progress", default=None, help="A callable that will be called with the number of files."
+        )
+        command_parser.add_argument(
+            "--threads", default=1, help="The number of threads to use when uploading a directory, by default 1."
+        )
+
+    def run(self, args: Namespace) -> None:
+        """Execute the command with the provided arguments.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        transfer(
+            source=args.source,
+            target=args.target,
+            overwrite=args.overwrite,
+            resume=args.resume,
+            verbosity=args.verbosity,
+            progress=args.progress,
+            threads=args.threads,
+            temporary_target=False,
+        )
+
+
+command = Transfer

{anemoi_utils-0.4.24 → anemoi_utils-0.4.26}/src/anemoi/utils/grids.py

@@ -17,9 +17,12 @@ from typing import List
 from typing import Tuple
 from typing import Union
 
+import deprecation
 import numpy as np
 import requests
 
+from anemoi.utils._version import __version__
+
 from .caching import cached
 
 LOG = logging.getLogger(__name__)
@@ -28,6 +31,12 @@ LOG = logging.getLogger(__name__)
 GRIDS_URL_PATTERN = "https://get.ecmwf.int/repository/anemoi/grids/grid-{name}.npz"
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.xyz_to_latlon instead.",
+)
 def xyz_to_latlon(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
     """Convert Cartesian coordinates to latitude and longitude.
 
@@ -51,6 +60,12 @@ def xyz_to_latlon(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> tuple[np.ndarr
     )
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.xyz_to_latlon instead.",
+)
 def latlon_to_xyz(lat: np.ndarray, lon: np.ndarray, radius: float = 1.0) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
     """Convert latitude and longitude to Cartesian coordinates.
 
@@ -87,6 +102,12 @@ def latlon_to_xyz(lat: np.ndarray, lon: np.ndarray, radius: float = 1.0) -> tupl
     return x, y, z
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.nearest_grid_points instead.",
+)
 def nearest_grid_points(
     source_latitudes: np.ndarray,
     source_longitudes: np.ndarray,
@@ -169,6 +190,12 @@ def _grids(name: Union[str, List[float], Tuple[float, ...]]) -> bytes:
     return response.content
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.grids.named.lookup instead.",
+)
 def grids(name: Union[str, List[float], Tuple[float, ...]]) -> dict:
     """Load grid data by name.