anemoi-utils 0.4.24__py3-none-any.whl → 0.4.25__py3-none-any.whl

anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.24'
-__version_tuple__ = version_tuple = (0, 4, 24)
+__version__ = version = '0.4.25'
+__version_tuple__ = version_tuple = (0, 4, 25)

anemoi/utils/cli.py

@@ -15,6 +15,7 @@ import os
 import sys
 import traceback
 from typing import Callable
+from typing import Optional
 
 try:
     import argcomplete
@@ -185,7 +186,9 @@ def register_commands(here: str, package: str, select: Callable, fail: Callable
     return result
 
 
-def cli_main(version: str, description: str, commands: dict[str, Command]) -> None:
+def cli_main(
+    version: str, description: str, commands: dict[str, Command], test_arguments: Optional[list[str]] = None
+) -> None:
     """Main entry point for the CLI.
 
     Parameters
@@ -196,9 +199,11 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
         The description of the CLI
     commands : dict[str, Command]
         A dictionary of command names to Command instances
+    test_arguments : list[str], optional
+        The command line arguments to parse, used for testing purposes, by default None
     """
     parser = make_parser(description, commands)
-    args, unknown = parser.parse_known_args()
+    args, unknown = parser.parse_known_args(test_arguments)
     if argcomplete:
         argcomplete.autocomplete(parser)
 
@@ -220,7 +225,7 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
 
     if unknown and not cmd.accept_unknown_args:
         # This should trigger an error
-        parser.parse_args()
+        parser.parse_args(test_arguments)
 
     try:
         if unknown:
@@ -228,9 +233,14 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
         else:
             cmd.run(args)
     except ValueError as e:
+
+        if test_arguments:
+            raise
+
         traceback.print_exc()
         LOG.error("\n💣 %s", str(e).lstrip())
         LOG.error("💣 Exiting")
         sys.exit(1)
 
-    sys.exit(0)
+    if not test_arguments:
+        sys.exit(0)

anemoi/utils/commands/transfer.py

@@ -0,0 +1,74 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+from argparse import ArgumentParser
+from argparse import Namespace
+
+from anemoi.utils.remote import transfer
+
+from . import Command
+
+
+class Transfer(Command):
+    """Handle configuration related commands."""
+
+    def add_arguments(self, command_parser: ArgumentParser) -> None:
+        """Add arguments to the command parser.
+
+        Parameters
+        ----------
+        command_parser : ArgumentParser
+            The argument parser to which the arguments will be added.
+        """
+        command_parser.add_argument(
+            "--source", help="A path to a local file or folder or a URL to a file or a folder on S3."
+        )
+        command_parser.add_argument(
+            "--target", help="A path to a local file or folder or a URL to a file or a folder on S3 or a remote folder."
+        )
+        command_parser.add_argument(
+            "--overwrite",
+            action="store_true",
+            help="If the data is already on in the target location it will be overwritten..",
+        )
+        command_parser.add_argument(
+            "--resume",
+            action="store_true",
+            help="If the data is already on S3 it will not be uploaded, unless the remote file has a different size.",
+        )
+        command_parser.add_argument("--verbosity", default=1, help="The level of verbosity, by default 1.")
+        command_parser.add_argument(
+            "--progress", default=None, help="A callable that will be called with the number of files."
+        )
+        command_parser.add_argument(
+            "--threads", default=1, help="The number of threads to use when uploading a directory, by default 1."
+        )
+
+    def run(self, args: Namespace) -> None:
+        """Execute the command with the provided arguments.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        transfer(
+            source=args.source,
+            target=args.target,
+            overwrite=args.overwrite,
+            resume=args.resume,
+            verbosity=args.verbosity,
+            progress=args.progress,
+            threads=args.threads,
+            temporary_target=False,
+        )
+
+
+command = Transfer

anemoi/utils/grids.py

@@ -17,9 +17,12 @@ from typing import List
 from typing import Tuple
 from typing import Union
 
+import deprecation
 import numpy as np
 import requests
 
+from anemoi.utils._version import __version__
+
 from .caching import cached
 
 LOG = logging.getLogger(__name__)
@@ -28,6 +31,12 @@ LOG = logging.getLogger(__name__)
 GRIDS_URL_PATTERN = "https://get.ecmwf.int/repository/anemoi/grids/grid-{name}.npz"
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.xyz_to_latlon instead.",
+)
 def xyz_to_latlon(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
     """Convert Cartesian coordinates to latitude and longitude.
 
@@ -51,6 +60,12 @@ def xyz_to_latlon(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> tuple[np.ndarr
     )
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.xyz_to_latlon instead.",
+)
 def latlon_to_xyz(lat: np.ndarray, lon: np.ndarray, radius: float = 1.0) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
     """Convert latitude and longitude to Cartesian coordinates.
 
@@ -87,6 +102,12 @@ def latlon_to_xyz(lat: np.ndarray, lon: np.ndarray, radius: float = 1.0) -> tupl
     return x, y, z
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.nearest_grid_points instead.",
+)
 def nearest_grid_points(
     source_latitudes: np.ndarray,
     source_longitudes: np.ndarray,
@@ -169,6 +190,12 @@ def _grids(name: Union[str, List[float], Tuple[float, ...]]) -> bytes:
     return response.content
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.grids.named.lookup instead.",
+)
 def grids(name: Union[str, List[float], Tuple[float, ...]]) -> dict:
     """Load grid data by name.
 

anemoi/utils/remote/__init__.py

@@ -371,7 +371,12 @@ class BaseDownload(Loader):
             The target path.
         """
         if os.path.exists(target):
-            shutil.rmtree(target)
+            if os.path.isfile(target):
+                LOGGER.info(f"Deleting file {target}")
+                os.remove(target)
+            else:
+                LOGGER.info(f"Deleting directory {target}")
+                shutil.rmtree(target)
 
 
 class BaseUpload(Loader):
@@ -642,10 +647,10 @@ def transfer(
         A path to a local file or folder or a URL to a file or a folder on S3 or a remote folder.
         The url should start with 's3://' or 'ssh://'.
     overwrite : bool, optional
-        If the data is alreay on in the target location it will be overwritten.
+        If the data is already on in the target location it will be overwritten.
         By default False
     resume : bool, optional
-        If the data is alreay on S3 it will not be uploaded, unless the remote file has a different size
+        If the data is already on S3 it will not be uploaded, unless the remote file has a different size
         Ignored if the target is an SSH remote folder (ssh://).
         By default False
     verbosity : int, optional

anemoi/utils/remote/s3.py

@@ -146,7 +146,7 @@ def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
 
         return options
 
-    LOG.info(f"Using S3 options: {_(options)}")
+    LOG.debug(f"Using S3 options: {_(options)}")
 
     thread_local.s3_clients[key] = boto3.client(service, **options)
 

anemoi/utils/testing.py

@@ -299,3 +299,104 @@ def skip_missing_packages(*names: str) -> callable:
         return pytest.mark.skipif(True, reason=f"Package {missing[0]} is not installed")
 
     return pytest.mark.skipif(True, reason=f"Packages {list_to_human(missing)} are not installed")
+
+
+def skip_if_missing_command(cmd: str) -> callable:
+    """Skip a test if the specified command is not available.
+
+    Parameters
+    ----------
+    cmd : str
+        The name of the command to check.
+
+    Returns
+    -------
+    Callable
+        A decorator that skips the test if the specified command is not available.
+    """
+
+    import shutil
+
+    if shutil.which(cmd):
+        return lambda f: f
+
+    return pytest.mark.skipif(True, reason=f"Command '{cmd}' is not available")
+
+
+def cli_testing(package: str, cmd: str, *args: list[str]) -> None:
+    """Run a CLI command for testing purposes.
+
+    Parameters
+    ----------
+    package : str
+        The name of the package containing the CLI commands.
+        Can be 'anemoi-datasets' or 'anemoi.datasets'.
+    cmd : str
+        The command to run.
+    *args : list[str]
+        Additional arguments to pass to the command.
+    """
+
+    package = package.replace("-", ".")
+    COMMANDS = getattr(__import__(f"{package}.commands", fromlist=["COMMANDS"]), "COMMANDS")
+    version = getattr(__import__(f"{package}._version", fromlist=["__version__"]), "__version__", "0.1.0")
+
+    from anemoi.utils.cli import cli_main
+
+    cli_main(
+        version=version,
+        description=f"Testing the '{cmd}' CLI command from the '{package}' package.",
+        commands=COMMANDS,
+        test_arguments=(cmd,) + args,
+    )
+
+
+def run_tests(globals: dict[str, callable]) -> None:
+    """Run all test functions that start with 'test_'.
+
+    Parameters
+    ----------
+    globals : dict[str, callable]
+        The global namespace containing the test functions.
+
+    Example
+    -------
+
+    Call from a test file to run all tests in that file:
+
+    ```python
+    if __name__ == "__main__":
+        from anemoi.utils.testing import run_tests
+        run_tests(globals())
+    ```
+
+    Useful for debugging or running tests in an interactive environment.
+
+    """
+    import logging
+
+    import rich
+
+    logging.basicConfig(level=logging.INFO)
+
+    for name, obj in list(globals.items()):
+        if name.startswith("test_") and callable(obj):
+
+            pytestmark = getattr(obj, "pytestmark", None)
+            if pytestmark is not None:
+                if not isinstance(pytestmark, list):
+                    pytestmark = [pytestmark]
+
+                skip = False
+                for m in pytestmark:
+                    if m.name == "skipif" and m.args == (True,):
+                        skip = True
+                        rich.print(
+                            f"[red]Skipping [bold]{name}[/bold] due to skipif condition [bold]{m.kwargs['reason']}[/bold].[/red]"
+                        )
+                        break
+                if skip:
+                    continue
+
+            rich.print(f"[green]Running [bold]{name}[/bold]...[/green]")
+            obj()

{anemoi_utils-0.4.24.dist-info → anemoi_utils-0.4.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.24
+Version: 0.4.25
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -225,12 +225,14 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: aniso8601
+Requires-Dist: deprecation
 Requires-Dist: importlib-metadata; python_version < "3.10"
 Requires-Dist: multiurl
 Requires-Dist: numpy
 Requires-Dist: pydantic>=2.9
 Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
+Requires-Dist: rich
 Requires-Dist: tomli; python_version < "3.11"
 Requires-Dist: tqdm
 Provides-Extra: all

{anemoi_utils-0.4.24.dist-info → anemoi_utils-0.4.25.dist-info}/RECORD

@@ -1,15 +1,15 @@
 anemoi/utils/__init__.py,sha256=uVhpF-VjIl_4mMywOVtgTutgsdIsqz-xdkwxeMhzuag,730
 anemoi/utils/__main__.py,sha256=6LlE4MYrPvqqrykxXh7XMi50UZteUY59NeM8P9Zs2dU,910
-anemoi/utils/_version.py,sha256=eqPTx1mit5QVY_CIZPeIYTsnPPxmLZEDtDv2l58DdBE,513
+anemoi/utils/_version.py,sha256=3hQuMRH6SNq4Gy0jS57VxD61lFAwagLT8wP_tfZa6fA,513
 anemoi/utils/caching.py,sha256=rXbeAmpBcMbbfN4EVblaHWKicsrtx1otER84FEBtz98,6183
 anemoi/utils/checkpoints.py,sha256=N4WpAZXa4etrpSEKhHqUUtG2-x9w3FJMHcLO-dDAXPY,9600
-anemoi/utils/cli.py,sha256=IyZfnSw0u0yYnrjOrzvm2RuuKvDk4cVb8pf8BkaChgA,6209
+anemoi/utils/cli.py,sha256=Cd3ESQkNWecbGnJjkR_SKHdFPETJWFrHqg5ovtANDKs,6522
 anemoi/utils/compatibility.py,sha256=wRBRMmxQP88rNcWiP5gqXliwYQbBv1iCAsDjcCRi5UY,2234
 anemoi/utils/config.py,sha256=EEfcSxW2CD6fFOzDtqz_uYlMKuYq4X5QinJW_8GBYj4,17325
 anemoi/utils/dates.py,sha256=CnY6JOdpk0T-waPEowMRTkcMzxcN0GcjPVtLkwH_byw,17196
 anemoi/utils/devtools.py,sha256=W3OBu96MkXRIl7Qh1SE5Zd6aB1R0QlnmlrlpBYM0fVY,3527
 anemoi/utils/grib.py,sha256=201WcxjjAl92Y2HX2kZ2S8Qr5dN-oG7nV-vQLaybzP4,3610
-anemoi/utils/grids.py,sha256=uYgkU_KIg8FTUiuKV0Pho2swMMeXcSQ9CQe0MFlRr_I,5262
+anemoi/utils/grids.py,sha256=VlhuN8MZDBu-G50QFI1XHV9IXLway2LpvI9X8sDjb5s,6047
 anemoi/utils/hindcasts.py,sha256=iYVIxSNFL2HJcc_k1abCFLkpJFGHT8WKRIR4wcAwA3s,2144
 anemoi/utils/humanize.py,sha256=pjnFJAKHbEAOfcvn8c48kt-8eFy6FGW_U2ruJvfamrA,25189
 anemoi/utils/logs.py,sha256=naTgrmPwWHD4eekFttXftS4gtcAGYHpCqG4iwYprNDA,1804
@@ -19,23 +19,24 @@ anemoi/utils/rules.py,sha256=VspUoPmw7tijrs6l_wl4vDjr_zVQsFjx9ITiBSvxgc8,6972
 anemoi/utils/s3.py,sha256=xMT48kbcelcjjqsaU567WI3oZ5eqo88Rlgyx5ECszAU,4074
 anemoi/utils/sanitise.py,sha256=ZYGdSX6qihQANr3pHZjbKnoapnzP1KcrWdW1Ul1mOGk,3668
 anemoi/utils/sanitize.py,sha256=43ZKDcfVpeXSsJ9TFEc9aZnD6oe2cUh151XnDspM98M,462
-anemoi/utils/testing.py,sha256=kwgAgLh3exYOTZSaX4xcPFjiMOyQDz-vcAlPcJqMiZk,7784
+anemoi/utils/testing.py,sha256=1qisyMA71HMT2mQUpLOsh01F9KR6vIOLid5WVyniaiw,10605
 anemoi/utils/text.py,sha256=HkzIvi24obDceFLpJEwBJ9PmPrJUkQN2TrElJ-A87gU,14441
 anemoi/utils/timer.py,sha256=_leKMYza2faM7JKlGE7LCNy13rbdPnwaCF7PSrI_NmI,3895
 anemoi/utils/commands/__init__.py,sha256=5u_6EwdqYczIAgJfCwRSyQAYFEqh2ZuHHT57g9g7sdI,808
 anemoi/utils/commands/config.py,sha256=nYvYbjcum9uumRa3gzPfmQCjNZKURXMA2XOUwz9b7ls,1369
 anemoi/utils/commands/requests.py,sha256=AEbssF1OlpbmSwrV5Lj6amCCn0w_-nbajBWTwYV15vA,2059
+anemoi/utils/commands/transfer.py,sha256=05GT5BAY9xt6vx86--xfJ8bkPCUkIRz0d265hDcJLCE,2600
 anemoi/utils/mars/__init__.py,sha256=b-Lc3L1TAQd9ODs0Z1YSJzgZCO1K_M3DSgx_yd2qXvM,2724
 anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
 anemoi/utils/mars/requests.py,sha256=VFMHBVAAl0_2lOcMBa1lvaKHctN0lDJsI6_U4BucGew,1142
-anemoi/utils/remote/__init__.py,sha256=swPWHQoh-B6Xq9R489tPw0FykMue7f-bJ8enneFYSYE,20776
-anemoi/utils/remote/s3.py,sha256=lhLxwBU-AKERzYURXJ-cOlaccEYmjoXqlvX3zYu5h-8,19977
+anemoi/utils/remote/__init__.py,sha256=7nHu-LRspYW2Fx9GNLjsxpytAUIvhIbOjb0Xmxb-33s,20988
+anemoi/utils/remote/s3.py,sha256=VgxNuJpJBnfxwkkV9TyrSkf7fjIuHqy1YG9vRdj9QRc,19978
 anemoi/utils/remote/ssh.py,sha256=xNtsawh8okytCKRehkRCVExbHZj-CRUQNormEHglfuw,8088
 anemoi/utils/schemas/__init__.py,sha256=nkinKlsPLPXEjfTYQT1mpKC4cvs-14w_zBkDRxakwxw,698
 anemoi/utils/schemas/errors.py,sha256=lgOXzVTYzAE0qWQf3OZ42vCWixv8lilSqLLhzARBmvI,1831
-anemoi_utils-0.4.24.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
-anemoi_utils-0.4.24.dist-info/METADATA,sha256=EsIP40bMdXsSEAp1v4iFkKMrzdVaE4Cw892SLNlzuWs,15439
-anemoi_utils-0.4.24.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-anemoi_utils-0.4.24.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.4.24.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.4.24.dist-info/RECORD,,
+anemoi_utils-0.4.25.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
+anemoi_utils-0.4.25.dist-info/METADATA,sha256=PILyf6_K2AZs161wasDnRvfhMdQjm4Cub1YzXkdKFmM,15486
+anemoi_utils-0.4.25.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+anemoi_utils-0.4.25.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.4.25.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.4.25.dist-info/RECORD,,