anemoi-utils 0.4.24__tar.gz → 0.4.25__tar.gz

anemoi_utils-0.4.25/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.25"
+}

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/CHANGELOG.md

@@ -8,6 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.4.25](https://github.com/ecmwf/anemoi-utils/compare/0.4.24...0.4.25) (2025-06-24)
+
+
+### Features
+
+* Add a CLI to transfer data ([#164](https://github.com/ecmwf/anemoi-utils/issues/164)) ([3a845ca](https://github.com/ecmwf/anemoi-utils/commit/3a845ca0c31d115e6b3d0496d862a3eaee5fb236))
+* Add function to test cli ([#168](https://github.com/ecmwf/anemoi-utils/issues/168)) ([9ac9b06](https://github.com/ecmwf/anemoi-utils/commit/9ac9b06b8fd0a62cad33ea5de6a6b482f0a13656))
+
 ## [0.4.24](https://github.com/ecmwf/anemoi-utils/compare/0.4.23...0.4.24) (2025-06-06)
 
 

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.24
+Version: 0.4.25
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -225,12 +225,14 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: aniso8601
+Requires-Dist: deprecation
 Requires-Dist: importlib-metadata; python_version < "3.10"
 Requires-Dist: multiurl
 Requires-Dist: numpy
 Requires-Dist: pydantic>=2.9
 Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
+Requires-Dist: rich
 Requires-Dist: tomli; python_version < "3.11"
 Requires-Dist: tqdm
 Provides-Extra: all

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/pyproject.toml

@@ -41,12 +41,14 @@ classifiers = [
 dynamic = [ "version" ]
 dependencies = [
   "aniso8601",
+  "deprecation",
   "importlib-metadata; python_version<'3.10'",
   "multiurl",
   "numpy",
   "pydantic>=2.9",
   "python-dateutil",
   "pyyaml",
+  "rich",
   "tomli; python_version<'3.11'",
   "tqdm",
 ]

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/src/anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.24'
-__version_tuple__ = version_tuple = (0, 4, 24)
+__version__ = version = '0.4.25'
+__version_tuple__ = version_tuple = (0, 4, 25)

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/src/anemoi/utils/cli.py

@@ -15,6 +15,7 @@ import os
 import sys
 import traceback
 from typing import Callable
+from typing import Optional
 
 try:
     import argcomplete
@@ -185,7 +186,9 @@ def register_commands(here: str, package: str, select: Callable, fail: Callable
     return result
 
 
-def cli_main(version: str, description: str, commands: dict[str, Command]) -> None:
+def cli_main(
+    version: str, description: str, commands: dict[str, Command], test_arguments: Optional[list[str]] = None
+) -> None:
     """Main entry point for the CLI.
 
     Parameters
@@ -196,9 +199,11 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
         The description of the CLI
     commands : dict[str, Command]
         A dictionary of command names to Command instances
+    test_arguments : list[str], optional
+        The command line arguments to parse, used for testing purposes, by default None
     """
     parser = make_parser(description, commands)
-    args, unknown = parser.parse_known_args()
+    args, unknown = parser.parse_known_args(test_arguments)
     if argcomplete:
         argcomplete.autocomplete(parser)
 
@@ -220,7 +225,7 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
 
     if unknown and not cmd.accept_unknown_args:
         # This should trigger an error
-        parser.parse_args()
+        parser.parse_args(test_arguments)
 
     try:
         if unknown:
@@ -228,9 +233,14 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
         else:
             cmd.run(args)
     except ValueError as e:
+
+        if test_arguments:
+            raise
+
         traceback.print_exc()
         LOG.error("\n💣 %s", str(e).lstrip())
         LOG.error("💣 Exiting")
         sys.exit(1)
 
-    sys.exit(0)
+    if not test_arguments:
+        sys.exit(0)

anemoi_utils-0.4.25/src/anemoi/utils/commands/transfer.py

@@ -0,0 +1,74 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+from argparse import ArgumentParser
+from argparse import Namespace
+
+from anemoi.utils.remote import transfer
+
+from . import Command
+
+
+class Transfer(Command):
+    """Handle configuration related commands."""
+
+    def add_arguments(self, command_parser: ArgumentParser) -> None:
+        """Add arguments to the command parser.
+
+        Parameters
+        ----------
+        command_parser : ArgumentParser
+            The argument parser to which the arguments will be added.
+        """
+        command_parser.add_argument(
+            "--source", help="A path to a local file or folder or a URL to a file or a folder on S3."
+        )
+        command_parser.add_argument(
+            "--target", help="A path to a local file or folder or a URL to a file or a folder on S3 or a remote folder."
+        )
+        command_parser.add_argument(
+            "--overwrite",
+            action="store_true",
+            help="If the data is already on in the target location it will be overwritten..",
+        )
+        command_parser.add_argument(
+            "--resume",
+            action="store_true",
+            help="If the data is already on S3 it will not be uploaded, unless the remote file has a different size.",
+        )
+        command_parser.add_argument("--verbosity", default=1, help="The level of verbosity, by default 1.")
+        command_parser.add_argument(
+            "--progress", default=None, help="A callable that will be called with the number of files."
+        )
+        command_parser.add_argument(
+            "--threads", default=1, help="The number of threads to use when uploading a directory, by default 1."
+        )
+
+    def run(self, args: Namespace) -> None:
+        """Execute the command with the provided arguments.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        transfer(
+            source=args.source,
+            target=args.target,
+            overwrite=args.overwrite,
+            resume=args.resume,
+            verbosity=args.verbosity,
+            progress=args.progress,
+            threads=args.threads,
+            temporary_target=False,
+        )
+
+
+command = Transfer

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/src/anemoi/utils/grids.py

@@ -17,9 +17,12 @@ from typing import List
 from typing import Tuple
 from typing import Union
 
+import deprecation
 import numpy as np
 import requests
 
+from anemoi.utils._version import __version__
+
 from .caching import cached
 
 LOG = logging.getLogger(__name__)
@@ -28,6 +31,12 @@ LOG = logging.getLogger(__name__)
 GRIDS_URL_PATTERN = "https://get.ecmwf.int/repository/anemoi/grids/grid-{name}.npz"
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.xyz_to_latlon instead.",
+)
 def xyz_to_latlon(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
     """Convert Cartesian coordinates to latitude and longitude.
 
@@ -51,6 +60,12 @@ def xyz_to_latlon(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> tuple[np.ndarr
     )
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.xyz_to_latlon instead.",
+)
 def latlon_to_xyz(lat: np.ndarray, lon: np.ndarray, radius: float = 1.0) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
     """Convert latitude and longitude to Cartesian coordinates.
 
@@ -87,6 +102,12 @@ def latlon_to_xyz(lat: np.ndarray, lon: np.ndarray, radius: float = 1.0) -> tupl
     return x, y, z
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.nearest_grid_points instead.",
+)
 def nearest_grid_points(
     source_latitudes: np.ndarray,
     source_longitudes: np.ndarray,
@@ -169,6 +190,12 @@ def _grids(name: Union[str, List[float], Tuple[float, ...]]) -> bytes:
     return response.content
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.grids.named.lookup instead.",
+)
 def grids(name: Union[str, List[float], Tuple[float, ...]]) -> dict:
     """Load grid data by name.
 

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/src/anemoi/utils/remote/__init__.py

@@ -371,7 +371,12 @@ class BaseDownload(Loader):
             The target path.
         """
         if os.path.exists(target):
-            shutil.rmtree(target)
+            if os.path.isfile(target):
+                LOGGER.info(f"Deleting file {target}")
+                os.remove(target)
+            else:
+                LOGGER.info(f"Deleting directory {target}")
+                shutil.rmtree(target)
 
 
 class BaseUpload(Loader):
@@ -642,10 +647,10 @@ def transfer(
         A path to a local file or folder or a URL to a file or a folder on S3 or a remote folder.
         The url should start with 's3://' or 'ssh://'.
     overwrite : bool, optional
-        If the data is alreay on in the target location it will be overwritten.
+        If the data is already on in the target location it will be overwritten.
         By default False
     resume : bool, optional
-        If the data is alreay on S3 it will not be uploaded, unless the remote file has a different size
+        If the data is already on S3 it will not be uploaded, unless the remote file has a different size
         Ignored if the target is an SSH remote folder (ssh://).
         By default False
     verbosity : int, optional

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/src/anemoi/utils/remote/s3.py

@@ -146,7 +146,7 @@ def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
 
         return options
 
-    LOG.info(f"Using S3 options: {_(options)}")
+    LOG.debug(f"Using S3 options: {_(options)}")
 
     thread_local.s3_clients[key] = boto3.client(service, **options)
 

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/src/anemoi/utils/testing.py

@@ -299,3 +299,104 @@ def skip_missing_packages(*names: str) -> callable:
         return pytest.mark.skipif(True, reason=f"Package {missing[0]} is not installed")
 
     return pytest.mark.skipif(True, reason=f"Packages {list_to_human(missing)} are not installed")
+
+
+def skip_if_missing_command(cmd: str) -> callable:
+    """Skip a test if the specified command is not available.
+
+    Parameters
+    ----------
+    cmd : str
+        The name of the command to check.
+
+    Returns
+    -------
+    Callable
+        A decorator that skips the test if the specified command is not available.
+    """
+
+    import shutil
+
+    if shutil.which(cmd):
+        return lambda f: f
+
+    return pytest.mark.skipif(True, reason=f"Command '{cmd}' is not available")
+
+
+def cli_testing(package: str, cmd: str, *args: list[str]) -> None:
+    """Run a CLI command for testing purposes.
+
+    Parameters
+    ----------
+    package : str
+        The name of the package containing the CLI commands.
+        Can be 'anemoi-datasets' or 'anemoi.datasets'.
+    cmd : str
+        The command to run.
+    *args : list[str]
+        Additional arguments to pass to the command.
+    """
+
+    package = package.replace("-", ".")
+    COMMANDS = getattr(__import__(f"{package}.commands", fromlist=["COMMANDS"]), "COMMANDS")
+    version = getattr(__import__(f"{package}._version", fromlist=["__version__"]), "__version__", "0.1.0")
+
+    from anemoi.utils.cli import cli_main
+
+    cli_main(
+        version=version,
+        description=f"Testing the '{cmd}' CLI command from the '{package}' package.",
+        commands=COMMANDS,
+        test_arguments=(cmd,) + args,
+    )
+
+
+def run_tests(globals: dict[str, callable]) -> None:
+    """Run all test functions that start with 'test_'.
+
+    Parameters
+    ----------
+    globals : dict[str, callable]
+        The global namespace containing the test functions.
+
+    Example
+    -------
+
+    Call from a test file to run all tests in that file:
+
+    ```python
+    if __name__ == "__main__":
+        from anemoi.utils.testing import run_tests
+        run_tests(globals())
+    ```
+
+    Useful for debugging or running tests in an interactive environment.
+
+    """
+    import logging
+
+    import rich
+
+    logging.basicConfig(level=logging.INFO)
+
+    for name, obj in list(globals.items()):
+        if name.startswith("test_") and callable(obj):
+
+            pytestmark = getattr(obj, "pytestmark", None)
+            if pytestmark is not None:
+                if not isinstance(pytestmark, list):
+                    pytestmark = [pytestmark]
+
+                skip = False
+                for m in pytestmark:
+                    if m.name == "skipif" and m.args == (True,):
+                        skip = True
+                        rich.print(
+                            f"[red]Skipping [bold]{name}[/bold] due to skipif condition [bold]{m.kwargs['reason']}[/bold].[/red]"
+                        )
+                        break
+                if skip:
+                    continue
+
+            rich.print(f"[green]Running [bold]{name}[/bold]...[/green]")
+            obj()

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/src/anemoi_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.24
+Version: 0.4.25
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -225,12 +225,14 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: aniso8601
+Requires-Dist: deprecation
 Requires-Dist: importlib-metadata; python_version < "3.10"
 Requires-Dist: multiurl
 Requires-Dist: numpy
 Requires-Dist: pydantic>=2.9
 Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
+Requires-Dist: rich
 Requires-Dist: tomli; python_version < "3.11"
 Requires-Dist: tqdm
 Provides-Extra: all

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/src/anemoi_utils.egg-info/SOURCES.txt

@@ -68,6 +68,7 @@ src/anemoi/utils/timer.py
 src/anemoi/utils/commands/__init__.py
 src/anemoi/utils/commands/config.py
 src/anemoi/utils/commands/requests.py
+src/anemoi/utils/commands/transfer.py
 src/anemoi/utils/mars/__init__.py
 src/anemoi/utils/mars/mars.yaml
 src/anemoi/utils/mars/requests.py
@@ -86,7 +87,6 @@ tests/test_caching.py
 tests/test_compatibility.py
 tests/test_dates.py
 tests/test_frequency.py
-tests/test_grids.py
 tests/test_provenance.py
 tests/test_remote.py
 tests/test_sanetise.py

{anemoi_utils-0.4.24 → anemoi_utils-0.4.25}/src/anemoi_utils.egg-info/requires.txt

@@ -1,9 +1,11 @@
 aniso8601
+deprecation
 multiurl
 numpy
 pydantic>=2.9
 python-dateutil
 pyyaml
+rich
 tqdm
 
 [:python_version < "3.10"]

anemoi_utils-0.4.24/.release-please-manifest.json

@@ -1,3 +0,0 @@
-{
-    ".": "0.4.24"
-}

anemoi_utils-0.4.24/tests/test_grids.py

@@ -1,29 +0,0 @@
-# (C) Copyright 2025 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-from anemoi.utils.grids import grids
-
-
-def test_o96() -> None:
-    """Test the grids function for the 'o96' grid."""
-    x = grids("o96")
-    assert x["latitudes"].mean() == 0.0
-    assert x["longitudes"].mean() == 179.14285714285714
-    assert x["latitudes"].shape == (40320,)
-    assert x["longitudes"].shape == (40320,)
-    assert x["latitudes"][31415] == -31.324557701757268
-    assert x["longitudes"][31415] == 224.32835820895522
-
-
-if __name__ == "__main__":
-    for name, obj in list(globals().items()):
-        if name.startswith("test_") and callable(obj):
-            print(f"Running {name}...")
-            obj()