anemoi-utils 0.4.22__tar.gz → 0.4.23__tar.gz

{anemoi_utils-0.4.22 → anemoi_utils-0.4.23}/.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.11.4
+  rev: v0.11.8
   hooks:
   - id: ruff
     args:

anemoi_utils-0.4.23/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.23"
+}

{anemoi_utils-0.4.22 → anemoi_utils-0.4.23}/CHANGELOG.md

@@ -8,6 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.4.23](https://github.com/ecmwf/anemoi-utils/compare/0.4.22...0.4.23) (2025-05-20)
+
+
+### Bug Fixes
+
+* fix list_folder on s3 ([#154](https://github.com/ecmwf/anemoi-utils/issues/154)) ([3ceb42c](https://github.com/ecmwf/anemoi-utils/commit/3ceb42c5185290d4c12e3fe90c3c331e3d8c7a5f))
+* Remove the requirment to have git installed ([#149](https://github.com/ecmwf/anemoi-utils/issues/149)) ([88846e8](https://github.com/ecmwf/anemoi-utils/commit/88846e80be2927050a879ff953a78aecf39c3ac5))
+* Use urllib to make _offline() aware of HTTP(s) proxies. ([#150](https://github.com/ecmwf/anemoi-utils/issues/150)) ([5c4d06f](https://github.com/ecmwf/anemoi-utils/commit/5c4d06f931590cc360eb4ffeeb8753a5d3d72bcb))
+
 ## [0.4.22](https://github.com/ecmwf/anemoi-utils/compare/0.4.21...0.4.22) (2025-04-10)
 
 

{anemoi_utils-0.4.22 → anemoi_utils-0.4.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.22
+Version: 0.4.23
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -252,7 +252,7 @@ Provides-Extra: provenance
 Requires-Dist: gitpython; extra == "provenance"
 Requires-Dist: nvsmi; extra == "provenance"
 Provides-Extra: s3
-Requires-Dist: boto3<1.36; extra == "s3"
+Requires-Dist: boto3>1.36; extra == "s3"
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text

{anemoi_utils-0.4.22 → anemoi_utils-0.4.23}/pyproject.toml

@@ -70,7 +70,7 @@ optional-dependencies.grib = [ "requests" ]
 optional-dependencies.provenance = [ "gitpython", "nvsmi" ]
 
 optional-dependencies.s3 = [
-  "boto3<1.36",
+  "boto3>1.36",
 ]
 
 optional-dependencies.tests = [ "pytest" ]

{anemoi_utils-0.4.22 → anemoi_utils-0.4.23}/src/anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.22'
-__version_tuple__ = version_tuple = (0, 4, 22)
+__version__ = version = '0.4.23'
+__version_tuple__ = version_tuple = (0, 4, 23)

{anemoi_utils-0.4.22 → anemoi_utils-0.4.23}/src/anemoi/utils/provenance.py

@@ -47,8 +47,11 @@ def lookup_git_repo(path: str) -> Optional[Any]:
     Repo, optional
         The git repository if found, otherwise None.
     """
-    from git import InvalidGitRepositoryError
-    from git import Repo
+    try:
+        from git import InvalidGitRepositoryError
+        from git import Repo
+    except ImportError:
+        return None
 
     while path != "/":
         try:

{anemoi_utils-0.4.22 → anemoi_utils-0.4.23}/src/anemoi/utils/remote/s3.py

@@ -1,10 +1,13 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024-2025 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+
 """This module provides functions to upload, download, list and delete files and folders on S3.
 The functions of this package expect that the AWS credentials are set up in the environment
 typicaly by setting the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables or
@@ -67,12 +70,14 @@ def s3_client(bucket: str, region: str = None) -> Any:
 
     key = f"{bucket}-{region}"
 
-    boto3_config = dict(max_pool_connections=25)
-
     if key in thread_local.s3_clients:
         return thread_local.s3_clients[key]
 
-    boto3_config = dict(max_pool_connections=25)
+    boto3_config = dict(
+        max_pool_connections=25,
+        request_checksum_calculation="when_required",
+        response_checksum_validation="when_required",
+    )
 
     if region:
         # This is using AWS
@@ -162,7 +167,14 @@ class S3Upload(BaseUpload):
         # delete(target)
 
     def _transfer_file(
-        self, source: str, target: str, overwrite: bool, resume: bool, verbosity: int, threads: int, config: dict = None
+        self,
+        source: str,
+        target: str,
+        overwrite: bool,
+        resume: bool,
+        verbosity: int,
+        threads: int,
+        config: dict = None,
     ) -> int:
         """Transfer a file to S3.
 
@@ -227,7 +239,13 @@ class S3Upload(BaseUpload):
 
         if verbosity > 0:
             with tqdm.tqdm(total=size, unit="B", unit_scale=True, unit_divisor=1024, leave=False) as pbar:
-                s3.upload_file(source, bucket, key, Callback=lambda x: pbar.update(x), Config=config)
+                s3.upload_file(
+                    source,
+                    bucket,
+                    key,
+                    Callback=lambda x: pbar.update(x),
+                    Config=config,
+                )
         else:
             s3.upload_file(source, bucket, key, Config=config)
 
@@ -326,7 +344,14 @@ class S3Download(BaseDownload):
         return s3_object["Size"]
 
     def _transfer_file(
-        self, source: str, target: str, overwrite: bool, resume: bool, verbosity: int, threads: int, config: dict = None
+        self,
+        source: str,
+        target: str,
+        overwrite: bool,
+        resume: bool,
+        verbosity: int,
+        threads: int,
+        config: dict = None,
     ) -> int:
         """Transfer a file from S3 to the local filesystem.
 
@@ -397,7 +422,13 @@ class S3Download(BaseDownload):
 
         if verbosity > 0:
             with tqdm.tqdm(total=size, unit="B", unit_scale=True, unit_divisor=1024, leave=False) as pbar:
-                s3.download_file(bucket, key, target, Callback=lambda x: pbar.update(x), Config=config)
+                s3.download_file(
+                    bucket,
+                    key,
+                    target,
+                    Callback=lambda x: pbar.update(x),
+                    Config=config,
+                )
         else:
             s3.download_file(bucket, key, target, Config=config)
 
@@ -524,7 +555,9 @@ def list_folder(folder: str) -> Iterable:
 
     for page in paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter="/"):
         if "CommonPrefixes" in page:
-            yield from [folder + _["Prefix"] for _ in page.get("CommonPrefixes")]
+            yield from [folder + _["Prefix"] for _ in page.get("CommonPrefixes") if _["Prefix"] != "/"]
+        if "Contents" in page:
+            yield from [folder + _["Key"] for _ in page.get("Contents")]
 
 
 def object_info(target: str) -> dict:

{anemoi_utils-0.4.22 → anemoi_utils-0.4.23}/src/anemoi/utils/testing.py

@@ -261,12 +261,12 @@ def _run_slow_tests() -> bool:
 @lru_cache(maxsize=None)
 def _offline() -> bool:
     """Check if we are offline."""
-
-    import socket
+    from urllib import request
 
     try:
-        socket.create_connection(("anemoi.ecmwf.int", 443), timeout=5)
-    except OSError:
+        request.urlopen("https://anemoi.ecmwf.int", timeout=1)
+        return False
+    except request.URLError:
         return True
 
     return False

{anemoi_utils-0.4.22 → anemoi_utils-0.4.23}/src/anemoi_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.22
+Version: 0.4.23
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -252,7 +252,7 @@ Provides-Extra: provenance
 Requires-Dist: gitpython; extra == "provenance"
 Requires-Dist: nvsmi; extra == "provenance"
 Provides-Extra: s3
-Requires-Dist: boto3<1.36; extra == "s3"
+Requires-Dist: boto3>1.36; extra == "s3"
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text

{anemoi_utils-0.4.22 → anemoi_utils-0.4.23}/src/anemoi_utils.egg-info/requires.txt

@@ -36,7 +36,7 @@ gitpython
 nvsmi
 
 [s3]
-boto3<1.36
+boto3>1.36
 
 [tests]
 pytest

anemoi_utils-0.4.22/.release-please-manifest.json

@@ -1,3 +0,0 @@
-{
-    ".": "0.4.22"
-}