anemoi-utils 0.4.23__tar.gz → 0.4.25__tar.gz

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.github/CODEOWNERS

@@ -1,4 +1,3 @@
-
 # Workflows
 /.github/ @ecmwf/AnemoiSecurity
 

anemoi_utils-0.4.25/.github/pull_request_template.md

@@ -0,0 +1,13 @@
+## Description
+<!-- What issue or task does this change relate to? -->
+
+## What problem does this change solve?
+<!-- Describe if it's a bugfix, new feature, doc update, or breaking change -->
+
+## What issue or task does this change relate to?
+<!-- link to Issue Number -->
+
+##  Additional notes ##
+<!-- Include any additional information, caveats, or considerations that the reviewer should be aware of. -->
+
+***As a contributor to the Anemoi framework, please ensure that your changes include unit tests, updates to any affected dependencies and documentation, and have been tested in a parallel setting  (i.e., with multiple GPUs). As a reviewer, you are also responsible for verifying these aspects and requesting changes if they are not adequately addressed. For guidelines about those please refer to https://anemoi.readthedocs.io/en/latest/***

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.github/workflows/downstream-ci-hpc.yml

@@ -1,6 +1,6 @@
 # This workflow triggers tests on dependent packages.
 # The dependency tree itself is defined in ecmwf/downstream-ci/
-name: Test downstream dependent packages on HPC
+name: Test downstream dependent packages
 
 on:
   # Trigger the workflow on push to main or develop, except tag creation

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.github/workflows/pr-conventional-commit.yml

@@ -1,5 +1,5 @@
 # This workflow ensures that the PR title follows the Conventional Commit format.
-name: "[Pull Request] Ensure Conventional Commit in PR title"
+name: "[PR] Ensure Conventional Commit Title"
 
 on:
   pull_request_target:

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.github/workflows/pr-label-conventional-commits.yml

@@ -1,6 +1,6 @@
 # This workflow assigns labels to a pull request based on the Conventional Commits format.
 # This is necessary for release-please to work properly.
-name: "[Pull Request] Label Conventional Commits"
+name: "[PR] Label Conventional Commits"
 
 on:
   pull_request:

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.github/workflows/pr-label-file-based.yml

@@ -1,6 +1,6 @@
 # This workflow assigns labels to a pull request based on the files changed in the PR.
 # The labels are defined in the `.github/labels.yml` file.
-name: "[Pull Request] Label File-based"
+name: "[PR] Label File-based"
 on:
   pull_request_target:
     types: [opened, synchronize]

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.github/workflows/pr-label-public.yml

@@ -1,5 +1,5 @@
 # Manage labels of pull requests that originate from forks
-name: "[Pull Request] Label PRs from public forks"
+name: "[PR] Label Forks"
 
 on:
   pull_request_target:

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.github/workflows/python-pull-request.yml

@@ -1,5 +1,5 @@
 # This workflow runs pre-commit checks and pytest tests against multiple platforms and Python versions.
-name: Code Quality checks and Testing
+name: Code Quality and Testing
 
 on:
   pull_request:

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.github/workflows/readthedocs-pr-update.yml

@@ -1,6 +1,6 @@
 # This workflow adds a link to the experimental documentation build to the PR.
 # This does NOT trigger a build of the documentation, this is handled through webhooks.
-name: Read the Docs PR Preview
+name: "[PR] Read the Docs Preview"
 on:
   pull_request_target:
     types:

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.gitignore

@@ -94,6 +94,11 @@ dmypy.json
 *.csv
 *.xlsx
 *.xls
+*.json
+*.txt
+*.zip
+*.db
+*.tgz
 
 # ML artifacts
 wandb/
@@ -120,7 +125,8 @@ tmp/
 temp/
 logs/
 _dev/
-outputs
+_api/
+./outputs
 *tmp_data/
 
 # Project specific

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.pre-commit-config.yaml

@@ -39,8 +39,9 @@ repos:
     - -l 120
     - --force-single-line-imports
     - --profile black
+    - --project anemoi
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.11.8
+  rev: v0.11.12
   hooks:
   - id: ruff
     args:
@@ -64,7 +65,7 @@ repos:
   - id: docconvert
     args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "v2.5.1"
+  rev: "v2.6.0"
   hooks:
   - id: pyproject-fmt
 -   repo: https://github.com/jshwi/docsig # Check docstrings against function sig

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/.release-please-config.json

@@ -10,11 +10,16 @@
   "draft-pull-request": true,
   "pull-request-title-pattern": "chore${scope}: Release${component} ${version}",
   "pull-request-header": ":robot: Automated Release PR\n\nThis PR was created by `release-please` to prepare the next release. Once merged:\n\n1. A new version tag will be created\n2. A GitHub release will be published\n3. The changelog will be updated\n\nChanges to be included in the next release:",
-  "pull-request-footer": "> [!IMPORTANT]\n> :warning: Merging this PR will:\n> - Create a new release\n> - Trigger deployment pipelines\n> - Update package versions\n\n **Before merging:**\n - Ensure all tests pass\n - Review the changelog carefully\n - Get required approvals\n\n [Release-please documentation](https://github.com/googleapis/release-please)",
+  "pull-request-footer": "> [!IMPORTANT]\n> Please do not change the PR title, manifest file, or any other automatically generated content in this PR unless you understand the implications. Changes here can break the release process.\n> :warning: Merging this PR will:\n> - Create a new release\n> - Trigger deployment pipelines\n> - Update package versions\n\n **Before merging:**\n - Ensure all tests pass\n - Review the changelog carefully\n - Get required approvals\n\n [Release-please documentation](https://github.com/googleapis/release-please)",
   "packages": {
     ".": {
         "package-name": "anemoi-utils"
     }
   },
+  "plugins": [
+    {
+      "type": "sentence-case"
+    }
+  ],
   "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json"
 }

anemoi_utils-0.4.25/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.25"
+}

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/CHANGELOG.md

@@ -8,6 +8,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.4.25](https://github.com/ecmwf/anemoi-utils/compare/0.4.24...0.4.25) (2025-06-24)
+
+
+### Features
+
+* Add a CLI to transfer data ([#164](https://github.com/ecmwf/anemoi-utils/issues/164)) ([3a845ca](https://github.com/ecmwf/anemoi-utils/commit/3a845ca0c31d115e6b3d0496d862a3eaee5fb236))
+* Add function to test cli ([#168](https://github.com/ecmwf/anemoi-utils/issues/168)) ([9ac9b06](https://github.com/ecmwf/anemoi-utils/commit/9ac9b06b8fd0a62cad33ea5de6a6b482f0a13656))
+
+## [0.4.24](https://github.com/ecmwf/anemoi-utils/compare/0.4.23...0.4.24) (2025-06-06)
+
+
+### Features
+
+* Add s3.object_exists() function ([#157](https://github.com/ecmwf/anemoi-utils/issues/157)) ([d898811](https://github.com/ecmwf/anemoi-utils/commit/d8988116320265dc6dfe467c57e0b6f29f76a2c1))
+* Allow wildcard in config for matching s3 buckets to end points ([#160](https://github.com/ecmwf/anemoi-utils/issues/160)) ([ab20da7](https://github.com/ecmwf/anemoi-utils/commit/ab20da7e9497435a7183705b02dcbb7317d2700b))
+
 ## [0.4.23](https://github.com/ecmwf/anemoi-utils/compare/0.4.22...0.4.23) (2025-05-20)
 
 

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.23
+Version: 0.4.25
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -225,12 +225,14 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: aniso8601
+Requires-Dist: deprecation
 Requires-Dist: importlib-metadata; python_version < "3.10"
 Requires-Dist: multiurl
 Requires-Dist: numpy
 Requires-Dist: pydantic>=2.9
 Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
+Requires-Dist: rich
 Requires-Dist: tomli; python_version < "3.11"
 Requires-Dist: tqdm
 Provides-Extra: all

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/pyproject.toml

@@ -41,12 +41,14 @@ classifiers = [
 dynamic = [ "version" ]
 dependencies = [
   "aniso8601",
+  "deprecation",
   "importlib-metadata; python_version<'3.10'",
   "multiurl",
   "numpy",
   "pydantic>=2.9",
   "python-dateutil",
   "pyyaml",
+  "rich",
   "tomli; python_version<'3.11'",
   "tqdm",
 ]

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/src/anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.23'
-__version_tuple__ = version_tuple = (0, 4, 23)
+__version__ = version = '0.4.25'
+__version_tuple__ = version_tuple = (0, 4, 25)

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/src/anemoi/utils/cli.py

@@ -15,6 +15,7 @@ import os
 import sys
 import traceback
 from typing import Callable
+from typing import Optional
 
 try:
     import argcomplete
@@ -185,7 +186,9 @@ def register_commands(here: str, package: str, select: Callable, fail: Callable
     return result
 
 
-def cli_main(version: str, description: str, commands: dict[str, Command]) -> None:
+def cli_main(
+    version: str, description: str, commands: dict[str, Command], test_arguments: Optional[list[str]] = None
+) -> None:
     """Main entry point for the CLI.
 
     Parameters
@@ -196,9 +199,11 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
         The description of the CLI
     commands : dict[str, Command]
         A dictionary of command names to Command instances
+    test_arguments : list[str], optional
+        The command line arguments to parse, used for testing purposes, by default None
     """
     parser = make_parser(description, commands)
-    args, unknown = parser.parse_known_args()
+    args, unknown = parser.parse_known_args(test_arguments)
     if argcomplete:
         argcomplete.autocomplete(parser)
 
@@ -220,7 +225,7 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
 
     if unknown and not cmd.accept_unknown_args:
         # This should trigger an error
-        parser.parse_args()
+        parser.parse_args(test_arguments)
 
     try:
         if unknown:
@@ -228,9 +233,14 @@ def cli_main(version: str, description: str, commands: dict[str, Command]) -> No
         else:
             cmd.run(args)
     except ValueError as e:
+
+        if test_arguments:
+            raise
+
         traceback.print_exc()
         LOG.error("\n💣 %s", str(e).lstrip())
         LOG.error("💣 Exiting")
         sys.exit(1)
 
-    sys.exit(0)
+    if not test_arguments:
+        sys.exit(0)

anemoi_utils-0.4.25/src/anemoi/utils/commands/transfer.py

@@ -0,0 +1,74 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+from argparse import ArgumentParser
+from argparse import Namespace
+
+from anemoi.utils.remote import transfer
+
+from . import Command
+
+
+class Transfer(Command):
+    """Handle configuration related commands."""
+
+    def add_arguments(self, command_parser: ArgumentParser) -> None:
+        """Add arguments to the command parser.
+
+        Parameters
+        ----------
+        command_parser : ArgumentParser
+            The argument parser to which the arguments will be added.
+        """
+        command_parser.add_argument(
+            "--source", help="A path to a local file or folder or a URL to a file or a folder on S3."
+        )
+        command_parser.add_argument(
+            "--target", help="A path to a local file or folder or a URL to a file or a folder on S3 or a remote folder."
+        )
+        command_parser.add_argument(
+            "--overwrite",
+            action="store_true",
+            help="If the data is already on in the target location it will be overwritten..",
+        )
+        command_parser.add_argument(
+            "--resume",
+            action="store_true",
+            help="If the data is already on S3 it will not be uploaded, unless the remote file has a different size.",
+        )
+        command_parser.add_argument("--verbosity", default=1, help="The level of verbosity, by default 1.")
+        command_parser.add_argument(
+            "--progress", default=None, help="A callable that will be called with the number of files."
+        )
+        command_parser.add_argument(
+            "--threads", default=1, help="The number of threads to use when uploading a directory, by default 1."
+        )
+
+    def run(self, args: Namespace) -> None:
+        """Execute the command with the provided arguments.
+
+        Parameters
+        ----------
+        args : Namespace
+            The arguments passed to the command.
+        """
+        transfer(
+            source=args.source,
+            target=args.target,
+            overwrite=args.overwrite,
+            resume=args.resume,
+            verbosity=args.verbosity,
+            progress=args.progress,
+            threads=args.threads,
+            temporary_target=False,
+        )
+
+
+command = Transfer

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/src/anemoi/utils/grids.py

@@ -17,9 +17,12 @@ from typing import List
 from typing import Tuple
 from typing import Union
 
+import deprecation
 import numpy as np
 import requests
 
+from anemoi.utils._version import __version__
+
 from .caching import cached
 
 LOG = logging.getLogger(__name__)
@@ -28,6 +31,12 @@ LOG = logging.getLogger(__name__)
 GRIDS_URL_PATTERN = "https://get.ecmwf.int/repository/anemoi/grids/grid-{name}.npz"
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.xyz_to_latlon instead.",
+)
 def xyz_to_latlon(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
     """Convert Cartesian coordinates to latitude and longitude.
 
@@ -51,6 +60,12 @@ def xyz_to_latlon(x: np.ndarray, y: np.ndarray, z: np.ndarray) -> tuple[np.ndarr
     )
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.xyz_to_latlon instead.",
+)
 def latlon_to_xyz(lat: np.ndarray, lon: np.ndarray, radius: float = 1.0) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
     """Convert latitude and longitude to Cartesian coordinates.
 
@@ -87,6 +102,12 @@ def latlon_to_xyz(lat: np.ndarray, lon: np.ndarray, radius: float = 1.0) -> tupl
     return x, y, z
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.spatial.nearest_grid_points instead.",
+)
 def nearest_grid_points(
     source_latitudes: np.ndarray,
     source_longitudes: np.ndarray,
@@ -169,6 +190,12 @@ def _grids(name: Union[str, List[float], Tuple[float, ...]]) -> bytes:
     return response.content
 
 
+@deprecation.deprecated(
+    deprecated_in="0.4.25",
+    removed_in="0.5.0",
+    current_version=__version__,
+    details="Use anemoi.transform.grids.named.lookup instead.",
+)
 def grids(name: Union[str, List[float], Tuple[float, ...]]) -> dict:
     """Load grid data by name.
 

{anemoi_utils-0.4.23 → anemoi_utils-0.4.25}/src/anemoi/utils/remote/__init__.py

@@ -371,7 +371,12 @@ class BaseDownload(Loader):
             The target path.
         """
         if os.path.exists(target):
-            shutil.rmtree(target)
+            if os.path.isfile(target):
+                LOGGER.info(f"Deleting file {target}")
+                os.remove(target)
+            else:
+                LOGGER.info(f"Deleting directory {target}")
+                shutil.rmtree(target)
 
 
 class BaseUpload(Loader):
@@ -642,10 +647,10 @@ def transfer(
         A path to a local file or folder or a URL to a file or a folder on S3 or a remote folder.
         The url should start with 's3://' or 'ssh://'.
     overwrite : bool, optional
-        If the data is alreay on in the target location it will be overwritten.
+        If the data is already on in the target location it will be overwritten.
         By default False
     resume : bool, optional
-        If the data is alreay on S3 it will not be uploaded, unless the remote file has a different size
+        If the data is already on S3 it will not be uploaded, unless the remote file has a different size
         Ignored if the target is an SSH remote folder (ssh://).
         By default False
     verbosity : int, optional