anemoi-utils 0.4.23__tar.gz → 0.4.24__tar.gz

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.github/CODEOWNERS

@@ -1,4 +1,3 @@
-
 # Workflows
 /.github/ @ecmwf/AnemoiSecurity
 

anemoi_utils-0.4.24/.github/pull_request_template.md

@@ -0,0 +1,13 @@
+## Description
+<!-- What issue or task does this change relate to? -->
+
+## What problem does this change solve?
+<!-- Describe if it's a bugfix, new feature, doc update, or breaking change -->
+
+## What issue or task does this change relate to?
+<!-- link to Issue Number -->
+
+##  Additional notes ##
+<!-- Include any additional information, caveats, or considerations that the reviewer should be aware of. -->
+
+***As a contributor to the Anemoi framework, please ensure that your changes include unit tests, updates to any affected dependencies and documentation, and have been tested in a parallel setting  (i.e., with multiple GPUs). As a reviewer, you are also responsible for verifying these aspects and requesting changes if they are not adequately addressed. For guidelines about those please refer to https://anemoi.readthedocs.io/en/latest/***

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.github/workflows/downstream-ci-hpc.yml

@@ -1,6 +1,6 @@
 # This workflow triggers tests on dependent packages.
 # The dependency tree itself is defined in ecmwf/downstream-ci/
-name: Test downstream dependent packages on HPC
+name: Test downstream dependent packages
 
 on:
   # Trigger the workflow on push to main or develop, except tag creation

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.github/workflows/pr-conventional-commit.yml

@@ -1,5 +1,5 @@
 # This workflow ensures that the PR title follows the Conventional Commit format.
-name: "[Pull Request] Ensure Conventional Commit in PR title"
+name: "[PR] Ensure Conventional Commit Title"
 
 on:
   pull_request_target:

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.github/workflows/pr-label-conventional-commits.yml

@@ -1,6 +1,6 @@
 # This workflow assigns labels to a pull request based on the Conventional Commits format.
 # This is necessary for release-please to work properly.
-name: "[Pull Request] Label Conventional Commits"
+name: "[PR] Label Conventional Commits"
 
 on:
   pull_request:

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.github/workflows/pr-label-file-based.yml

@@ -1,6 +1,6 @@
 # This workflow assigns labels to a pull request based on the files changed in the PR.
 # The labels are defined in the `.github/labels.yml` file.
-name: "[Pull Request] Label File-based"
+name: "[PR] Label File-based"
 on:
   pull_request_target:
     types: [opened, synchronize]

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.github/workflows/pr-label-public.yml

@@ -1,5 +1,5 @@
 # Manage labels of pull requests that originate from forks
-name: "[Pull Request] Label PRs from public forks"
+name: "[PR] Label Forks"
 
 on:
   pull_request_target:

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.github/workflows/python-pull-request.yml

@@ -1,5 +1,5 @@
 # This workflow runs pre-commit checks and pytest tests against multiple platforms and Python versions.
-name: Code Quality checks and Testing
+name: Code Quality and Testing
 
 on:
   pull_request:

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.github/workflows/readthedocs-pr-update.yml

@@ -1,6 +1,6 @@
 # This workflow adds a link to the experimental documentation build to the PR.
 # This does NOT trigger a build of the documentation, this is handled through webhooks.
-name: Read the Docs PR Preview
+name: "[PR] Read the Docs Preview"
 on:
   pull_request_target:
     types:

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.gitignore

@@ -94,6 +94,11 @@ dmypy.json
 *.csv
 *.xlsx
 *.xls
+*.json
+*.txt
+*.zip
+*.db
+*.tgz
 
 # ML artifacts
 wandb/
@@ -120,7 +125,8 @@ tmp/
 temp/
 logs/
 _dev/
-outputs
+_api/
+./outputs
 *tmp_data/
 
 # Project specific

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.pre-commit-config.yaml

@@ -39,8 +39,9 @@ repos:
     - -l 120
     - --force-single-line-imports
     - --profile black
+    - --project anemoi
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.11.8
+  rev: v0.11.12
   hooks:
   - id: ruff
     args:
@@ -64,7 +65,7 @@ repos:
   - id: docconvert
     args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "v2.5.1"
+  rev: "v2.6.0"
   hooks:
   - id: pyproject-fmt
 -   repo: https://github.com/jshwi/docsig # Check docstrings against function sig

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/.release-please-config.json

@@ -10,11 +10,16 @@
   "draft-pull-request": true,
   "pull-request-title-pattern": "chore${scope}: Release${component} ${version}",
   "pull-request-header": ":robot: Automated Release PR\n\nThis PR was created by `release-please` to prepare the next release. Once merged:\n\n1. A new version tag will be created\n2. A GitHub release will be published\n3. The changelog will be updated\n\nChanges to be included in the next release:",
-  "pull-request-footer": "> [!IMPORTANT]\n> :warning: Merging this PR will:\n> - Create a new release\n> - Trigger deployment pipelines\n> - Update package versions\n\n **Before merging:**\n - Ensure all tests pass\n - Review the changelog carefully\n - Get required approvals\n\n [Release-please documentation](https://github.com/googleapis/release-please)",
+  "pull-request-footer": "> [!IMPORTANT]\n> Please do not change the PR title, manifest file, or any other automatically generated content in this PR unless you understand the implications. Changes here can break the release process.\n> :warning: Merging this PR will:\n> - Create a new release\n> - Trigger deployment pipelines\n> - Update package versions\n\n **Before merging:**\n - Ensure all tests pass\n - Review the changelog carefully\n - Get required approvals\n\n [Release-please documentation](https://github.com/googleapis/release-please)",
   "packages": {
     ".": {
         "package-name": "anemoi-utils"
     }
   },
+  "plugins": [
+    {
+      "type": "sentence-case"
+    }
+  ],
   "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json"
 }

anemoi_utils-0.4.24/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.24"
+}

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/CHANGELOG.md

@@ -8,6 +8,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.4.24](https://github.com/ecmwf/anemoi-utils/compare/0.4.23...0.4.24) (2025-06-06)
+
+
+### Features
+
+* Add s3.object_exists() function ([#157](https://github.com/ecmwf/anemoi-utils/issues/157)) ([d898811](https://github.com/ecmwf/anemoi-utils/commit/d8988116320265dc6dfe467c57e0b6f29f76a2c1))
+* Allow wildcard in config for matching s3 buckets to end points ([#160](https://github.com/ecmwf/anemoi-utils/issues/160)) ([ab20da7](https://github.com/ecmwf/anemoi-utils/commit/ab20da7e9497435a7183705b02dcbb7317d2700b))
+
 ## [0.4.23](https://github.com/ecmwf/anemoi-utils/compare/0.4.22...0.4.23) (2025-05-20)
 
 

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.23
+Version: 0.4.24
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/src/anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.23'
-__version_tuple__ = version_tuple = (0, 4, 23)
+__version__ = version = '0.4.24'
+__version_tuple__ = version_tuple = (0, 4, 24)

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/src/anemoi/utils/remote/s3.py

@@ -24,6 +24,7 @@ the `~/.config/anemoi/settings.toml`
 or `~/.config/anemoi/settings-secrets.toml` files.
 """
 
+import fnmatch
 import logging
 import os
 import threading
@@ -38,15 +39,15 @@ from ..humanize import bytes_to_human
 from . import BaseDownload
 from . import BaseUpload
 
-LOGGER = logging.getLogger(__name__)
-
+LOG = logging.getLogger(__name__)
+SECRETS = ["aws_access_key_id", "aws_secret_access_key"]
 
 # s3_clients are not thread-safe, so we need to create a new client for each thread
 
 thread_local = threading.local()
 
 
-def s3_client(bucket: str, region: str = None) -> Any:
+def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
     """Get an S3 client for the specified bucket and region.
 
     Parameters
@@ -55,6 +56,8 @@ def s3_client(bucket: str, region: str = None) -> Any:
         The name of the S3 bucket.
     region : str, optional
         The AWS region of the S3 bucket.
+    service : str, optional
+        The AWS service to use, default is "s3".
 
     Returns
     -------
@@ -68,7 +71,7 @@ def s3_client(bucket: str, region: str = None) -> Any:
     if not hasattr(thread_local, "s3_clients"):
         thread_local.s3_clients = {}
 
-    key = f"{bucket}-{region}"
+    key = f"{bucket}-{region}-{service}"
 
     if key in thread_local.s3_clients:
         return thread_local.s3_clients[key]
@@ -96,17 +99,27 @@ def s3_client(bucket: str, region: str = None) -> Any:
         # We may be accessing a different S3 compatible service
         # Use anemoi.config to get the configuration
 
-        options = {}
-        config = load_config(secrets=["aws_access_key_id", "aws_secret_access_key"])
+        region = "unknown-region"
+
+        options = {"region_name": region}
+        config = load_config(secrets=SECRETS)
 
         cfg = config.get("object-storage", {})
+        candidate = None
         for k, v in cfg.items():
             if isinstance(v, (str, int, float, bool)):
                 options[k] = v
 
-        for k, v in cfg.get(bucket, {}).items():
-            if isinstance(v, (str, int, float, bool)):
-                options[k] = v
+            if isinstance(v, dict):
+                if fnmatch.fnmatch(bucket, k):
+                    if candidate is not None:
+                        raise ValueError(f"Multiple object storage configurations match {bucket}: {candidate} and {k}")
+                    candidate = k
+
+        if candidate is not None:
+            for k, v in cfg.get(candidate, {}).items():
+                if isinstance(v, (str, int, float, bool)):
+                    options[k] = v
 
         type = options.pop("type", "s3")
         if type != "s3":
@@ -115,11 +128,27 @@ def s3_client(bucket: str, region: str = None) -> Any:
         if "config" in options:
             boto3_config.update(options["config"])
             del options["config"]
-            from botocore.client import Config
 
     options["config"] = Config(**boto3_config)
 
-    thread_local.s3_clients[key] = boto3.client("s3", **options)
+    def _(options):
+
+        def __(k, v):
+            if k in SECRETS:
+                return "***"
+            return v
+
+        if isinstance(options, dict):
+            return {k: __(k, v) for k, v in options.items()}
+
+        if isinstance(options, list):
+            return [_(o) for o in options]
+
+        return options
+
+    LOG.info(f"Using S3 options: {_(options)}")
+
+    thread_local.s3_clients[key] = boto3.client(service, **options)
 
     return thread_local.s3_clients[key]
 
@@ -215,7 +244,7 @@ class S3Upload(BaseUpload):
         size = os.path.getsize(source)
 
         if verbosity > 0:
-            LOGGER.info(f"{self.action} {source} to {target} ({bytes_to_human(size)})")
+            LOG.info(f"{self.action} {source} to {target} ({bytes_to_human(size)})")
 
         try:
             results = s3.head_object(Bucket=bucket, Key=key)
@@ -227,7 +256,7 @@ class S3Upload(BaseUpload):
 
         if remote_size is not None:
             if remote_size != size:
-                LOGGER.warning(
+                LOG.warning(
                     f"{target} already exists, but with different size, re-uploading (remote={remote_size}, local={size})"
                 )
             elif resume:
@@ -400,7 +429,7 @@ class S3Download(BaseDownload):
         size = int(response["ContentLength"])
 
         if verbosity > 0:
-            LOGGER.info(f"{self.action} {source} to {target} ({bytes_to_human(size)})")
+            LOG.info(f"{self.action} {source} to {target} ({bytes_to_human(size)})")
 
         if overwrite:
             resume = False
@@ -409,7 +438,7 @@ class S3Download(BaseDownload):
             if os.path.exists(target):
                 local_size = os.path.getsize(target)
                 if local_size != size:
-                    LOGGER.warning(
+                    LOG.warning(
                         f"{target} already with different size, re-downloading (remote={size}, local={local_size})"
                     )
                 else:
@@ -464,7 +493,7 @@ def _list_objects(target: str, batch: bool = False) -> Iterable:
                 yield from objects
 
 
-def _delete_folder(target: str) -> None:
+def delete_folder(target: str) -> None:
     """Delete a folder from S3.
 
     Parameters
@@ -477,13 +506,13 @@ def _delete_folder(target: str) -> None:
 
     total = 0
     for batch in _list_objects(target, batch=True):
-        LOGGER.info(f"Deleting {len(batch):,} objects from {target}")
+        LOG.info(f"Deleting {len(batch):,} objects from {target}")
         s3.delete_objects(Bucket=bucket, Delete={"Objects": [{"Key": o["Key"]} for o in batch]})
         total += len(batch)
-        LOGGER.info(f"Deleted {len(batch):,} objects (total={total:,})")
+        LOG.info(f"Deleted {len(batch):,} objects (total={total:,})")
 
 
-def _delete_file(target: str) -> None:
+def delete_file(target: str) -> None:
     """Delete a file from S3.
 
     Parameters
@@ -505,12 +534,12 @@ def _delete_file(target: str) -> None:
         exits = False
 
     if not exits:
-        LOGGER.warning(f"{target} does not exist. Did you mean to delete a folder? Then add a trailing '/'")
+        LOG.warning(f"{target} does not exist. Did you mean to delete a folder? Then add a trailing '/'")
         return
 
-    LOGGER.info(f"Deleting {target}")
+    LOG.info(f"Deleting {target}")
     s3.delete_object(Bucket=bucket, Key=key)
-    LOGGER.info(f"{target} is deleted")
+    LOG.info(f"{target} is deleted")
 
 
 def delete(target: str) -> None:
@@ -525,9 +554,9 @@ def delete(target: str) -> None:
     assert target.startswith("s3://")
 
     if target.endswith("/"):
-        _delete_folder(target)
+        delete_folder(target)
     else:
-        _delete_file(target)
+        delete_file(target)
 
 
 def list_folder(folder: str) -> Iterable:
@@ -581,7 +610,33 @@ def object_info(target: str) -> dict:
         return s3.head_object(Bucket=bucket, Key=key)
     except s3.exceptions.ClientError as e:
         if e.response["Error"]["Code"] == "404":
-            raise ValueError(f"{target} does not exist")
+            raise FileNotFoundError(f"{target} does not exist")
+        raise
+
+
+def object_exists(target: str) -> bool:
+    """Check if an object exists.
+
+    Parameters
+    ----------
+    target : str
+        The URL of a file or a folder on S3. The URL should start with 's3://'.
+
+    Returns
+    -------
+    bool
+        True if the object exists, False otherwise.
+    """
+
+    _, _, bucket, key = target.split("/", 3)
+    s3 = s3_client(bucket)
+
+    try:
+        s3.head_object(Bucket=bucket, Key=key)
+        return True
+    except s3.exceptions.ClientError as e:
+        if e.response["Error"]["Code"] == "404":
+            return False
         raise
 
 
@@ -600,7 +655,7 @@ def object_acl(target: str) -> dict:
     """
 
     _, _, bucket, key = target.split("/", 3)
-    s3 = s3_client()
+    s3 = s3_client(bucket)
 
     return s3.get_object_acl(Bucket=bucket, Key=key)
 
@@ -643,3 +698,29 @@ def upload(source: str, target: str, *args, **kwargs) -> None:
 
     assert target.startswith("s3://"), f"target {target} should start with 's3://'"
     return transfer(source, target, *args, **kwargs)
+
+
+def quotas(target: str) -> dict:
+    """Get the quotas for an S3 bucket.
+
+    Parameters
+    ----------
+    target : str
+        The URL of a file or a folder on S3. The URL should start with 's3://'.
+
+    Returns
+    -------
+    dict
+        A dictionary with the quotas for the bucket.
+    """
+    from botocore.exceptions import ClientError
+
+    _, _, bucket, _ = target.split("/", 3)
+    s3 = s3_client(bucket, service="service-quotas")
+
+    try:
+        return s3.list_service_quotas(ServiceCode="ec2")
+    except ClientError as e:
+        if e.response["Error"]["Code"] == "404":
+            raise ValueError(f"{target} does not exist")
+        raise

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/src/anemoi_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.23
+Version: 0.4.24
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.4.23 → anemoi_utils-0.4.24}/src/anemoi_utils.egg-info/SOURCES.txt

@@ -14,7 +14,6 @@ pyproject.toml
 .github/dependabot.yml
 .github/labeler.yml
 .github/pull_request_template.md
-.github/release.yml
 .github/workflows/downstream-ci-hpc.yml
 .github/workflows/pr-conventional-commit.yml
 .github/workflows/pr-label-conventional-commits.yml

anemoi_utils-0.4.23/.github/pull_request_template.md

@@ -1,46 +0,0 @@
-## Description
-
-<!-- Provide a brief summary of the changes introduced in this pull request. -->
-
-## Type of Change
-
--   [ ] Bug fix (non-breaking change which fixes an issue)
--   [ ] New feature (non-breaking change which adds functionality)
--   [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
--   [ ] Documentation update
-
-## Issue Number
-
-<!-- Link the Issue number this change addresses, ideally in one of the "magic format" such as Closes #XYZ -->
-
-<!-- Alternatively, explain the motivation behind the changes and the context in which they are being made. -->
-
-## Code Compatibility
-
--   [ ] I have performed a self-review of my code
-
-### Code Performance and Testing
-
--   [ ] I have added tests that prove my fix is effective or that my feature works
--   [ ] I ran the [complete Pytest test](https://anemoi.readthedocs.io/projects/training/en/latest/dev/testing.html) suite locally, and they pass
-
-<!-- In case this affects the model sharding or other specific components please describe these here. -->
-
-### Dependencies
-
--   [ ] I have ensured that the code is still pip-installable after the changes and runs
--   [ ] I have tested that new dependencies themselves are pip-installable.
-
-<!-- List any new dependencies that are required for this change and the justification to add them. -->
-
-### Documentation
-
--   [ ] My code follows the style guidelines of this project
--   [ ] I have updated the documentation and docstrings to reflect the changes
--   [ ] I have added comments to my code, particularly in hard-to-understand areas
-
-<!-- Describe any major updates to the documentation -->
-
-## Additional Notes
-
-<!-- Include any additional information, caveats, or considerations that the reviewer should be aware of. -->

anemoi_utils-0.4.23/.github/release.yml

@@ -1,23 +0,0 @@
-# .github/release.yml
-# https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes
-
-changelog:
-  exclude:
-    labels:
-      - ignore-for-release
-      - no-changelog
-    authors:
-      - pre-commit-ci
-  categories:
-    - title: Breaking Changes 🛠
-      labels:
-        - "breaking change"
-    - title: Exciting New Features 🎉
-      labels:
-        - enhancement
-    - title: Config Changes 📑
-      labels:
-        - config
-    - title: Other Changes 🔗
-      labels:
-        - "*"

anemoi_utils-0.4.23/.release-please-manifest.json

@@ -1,3 +0,0 @@
-{
-    ".": "0.4.23"
-}