anemoi-utils 0.4.21__tar.gz → 0.4.23__tar.gz

anemoi_utils-0.4.23/.github/CODEOWNERS

@@ -0,0 +1,8 @@
+
+# Workflows
+/.github/ @ecmwf/AnemoiSecurity
+
+# Project configs
+/pyproject.toml @ecmwf/AnemoiSecurity
+/.pre-commit-config.yaml @ecmwf/AnemoiSecurity
+/.release-please-config.json @ecmwf/AnemoiSecurity

{anemoi_utils-0.4.21 → anemoi_utils-0.4.23}/.pre-commit-config.yaml

@@ -40,14 +40,13 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.9.9
+  rev: v0.11.8
   hooks:
   - id: ruff
     args:
     - --line-length=120
     - --fix
     - --exit-non-zero-on-fix
-    - --preview
     - --exclude=docs/**/*_.py
 - repo: https://github.com/sphinx-contrib/sphinx-lint
   rev: v1.0.0
@@ -69,7 +68,7 @@ repos:
   hooks:
   - id: pyproject-fmt
 -   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
-    rev: v0.69.1
+    rev: v0.69.3
     hooks:
     -   id: docsig
         args:

anemoi_utils-0.4.23/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.23"
+}

{anemoi_utils-0.4.21 → anemoi_utils-0.4.23}/CHANGELOG.md

@@ -8,6 +8,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.4.23](https://github.com/ecmwf/anemoi-utils/compare/0.4.22...0.4.23) (2025-05-20)
+
+
+### Bug Fixes
+
+* fix list_folder on s3 ([#154](https://github.com/ecmwf/anemoi-utils/issues/154)) ([3ceb42c](https://github.com/ecmwf/anemoi-utils/commit/3ceb42c5185290d4c12e3fe90c3c331e3d8c7a5f))
+* Remove the requirment to have git installed ([#149](https://github.com/ecmwf/anemoi-utils/issues/149)) ([88846e8](https://github.com/ecmwf/anemoi-utils/commit/88846e80be2927050a879ff953a78aecf39c3ac5))
+* Use urllib to make _offline() aware of HTTP(s) proxies. ([#150](https://github.com/ecmwf/anemoi-utils/issues/150)) ([5c4d06f](https://github.com/ecmwf/anemoi-utils/commit/5c4d06f931590cc360eb4ffeeb8753a5d3d72bcb))
+
+## [0.4.22](https://github.com/ecmwf/anemoi-utils/compare/0.4.21...0.4.22) (2025-04-10)
+
+
+### Bug Fixes
+
+* do not write to existing dir ([#148](https://github.com/ecmwf/anemoi-utils/issues/148)) ([38c6db6](https://github.com/ecmwf/anemoi-utils/commit/38c6db62c113e093d11c49b0fc398587ee89946c))
+* remove archive file after unpacking ([#145](https://github.com/ecmwf/anemoi-utils/issues/145)) ([790e2a3](https://github.com/ecmwf/anemoi-utils/commit/790e2a3370db3d5c275f95b920926d5a01f894a7))
+
 ## [0.4.21](https://github.com/ecmwf/anemoi-utils/compare/0.4.20...0.4.21) (2025-04-07)
 
 

{anemoi_utils-0.4.21 → anemoi_utils-0.4.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.21
+Version: 0.4.23
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -252,7 +252,7 @@ Provides-Extra: provenance
 Requires-Dist: gitpython; extra == "provenance"
 Requires-Dist: nvsmi; extra == "provenance"
 Provides-Extra: s3
-Requires-Dist: boto3<1.36; extra == "s3"
+Requires-Dist: boto3>1.36; extra == "s3"
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text

{anemoi_utils-0.4.21 → anemoi_utils-0.4.23}/pyproject.toml

@@ -70,7 +70,7 @@ optional-dependencies.grib = [ "requests" ]
 optional-dependencies.provenance = [ "gitpython", "nvsmi" ]
 
 optional-dependencies.s3 = [
-  "boto3<1.36",
+  "boto3>1.36",
 ]
 
 optional-dependencies.tests = [ "pytest" ]

{anemoi_utils-0.4.21 → anemoi_utils-0.4.23}/src/anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.21'
-__version_tuple__ = version_tuple = (0, 4, 21)
+__version__ = version = '0.4.23'
+__version_tuple__ = version_tuple = (0, 4, 23)

{anemoi_utils-0.4.21 → anemoi_utils-0.4.23}/src/anemoi/utils/provenance.py

@@ -47,8 +47,11 @@ def lookup_git_repo(path: str) -> Optional[Any]:
     Repo, optional
         The git repository if found, otherwise None.
     """
-    from git import InvalidGitRepositoryError
-    from git import Repo
+    try:
+        from git import InvalidGitRepositoryError
+        from git import Repo
+    except ImportError:
+        return None
 
     while path != "/":
         try:

{anemoi_utils-0.4.21 → anemoi_utils-0.4.23}/src/anemoi/utils/remote/s3.py

@@ -1,10 +1,13 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024-2025 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+
 """This module provides functions to upload, download, list and delete files and folders on S3.
 The functions of this package expect that the AWS credentials are set up in the environment
 typicaly by setting the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables or
@@ -67,12 +70,14 @@ def s3_client(bucket: str, region: str = None) -> Any:
 
     key = f"{bucket}-{region}"
 
-    boto3_config = dict(max_pool_connections=25)
-
     if key in thread_local.s3_clients:
         return thread_local.s3_clients[key]
 
-    boto3_config = dict(max_pool_connections=25)
+    boto3_config = dict(
+        max_pool_connections=25,
+        request_checksum_calculation="when_required",
+        response_checksum_validation="when_required",
+    )
 
     if region:
         # This is using AWS
@@ -162,7 +167,14 @@ class S3Upload(BaseUpload):
         # delete(target)
 
     def _transfer_file(
-        self, source: str, target: str, overwrite: bool, resume: bool, verbosity: int, threads: int, config: dict = None
+        self,
+        source: str,
+        target: str,
+        overwrite: bool,
+        resume: bool,
+        verbosity: int,
+        threads: int,
+        config: dict = None,
     ) -> int:
         """Transfer a file to S3.
 
@@ -227,7 +239,13 @@ class S3Upload(BaseUpload):
 
         if verbosity > 0:
             with tqdm.tqdm(total=size, unit="B", unit_scale=True, unit_divisor=1024, leave=False) as pbar:
-                s3.upload_file(source, bucket, key, Callback=lambda x: pbar.update(x), Config=config)
+                s3.upload_file(
+                    source,
+                    bucket,
+                    key,
+                    Callback=lambda x: pbar.update(x),
+                    Config=config,
+                )
         else:
             s3.upload_file(source, bucket, key, Config=config)
 
@@ -326,7 +344,14 @@ class S3Download(BaseDownload):
         return s3_object["Size"]
 
     def _transfer_file(
-        self, source: str, target: str, overwrite: bool, resume: bool, verbosity: int, threads: int, config: dict = None
+        self,
+        source: str,
+        target: str,
+        overwrite: bool,
+        resume: bool,
+        verbosity: int,
+        threads: int,
+        config: dict = None,
     ) -> int:
         """Transfer a file from S3 to the local filesystem.
 
@@ -397,7 +422,13 @@ class S3Download(BaseDownload):
 
         if verbosity > 0:
             with tqdm.tqdm(total=size, unit="B", unit_scale=True, unit_divisor=1024, leave=False) as pbar:
-                s3.download_file(bucket, key, target, Callback=lambda x: pbar.update(x), Config=config)
+                s3.download_file(
+                    bucket,
+                    key,
+                    target,
+                    Callback=lambda x: pbar.update(x),
+                    Config=config,
+                )
         else:
             s3.download_file(bucket, key, target, Config=config)
 
@@ -524,7 +555,9 @@ def list_folder(folder: str) -> Iterable:
 
     for page in paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter="/"):
         if "CommonPrefixes" in page:
-            yield from [folder + _["Prefix"] for _ in page.get("CommonPrefixes")]
+            yield from [folder + _["Prefix"] for _ in page.get("CommonPrefixes") if _["Prefix"] != "/"]
+        if "Contents" in page:
+            yield from [folder + _["Key"] for _ in page.get("Contents")]
 
 
 def object_info(target: str) -> dict:

{anemoi_utils-0.4.21 → anemoi_utils-0.4.23}/src/anemoi/utils/testing.py

@@ -68,6 +68,23 @@ def _check_path(path: str) -> None:
     assert not path.startswith("."), f"Path '{path}' should not start with '.'"
 
 
+def _temporary_directory_for_test_data(path: str) -> str:
+    """Get the temporary directory for a test dataset.
+
+    Parameters
+    ----------
+    path : str
+        The relative path to the test data in the object store.
+
+    Returns
+    -------
+    str
+        The path to the temporary directory.
+    """
+    _check_path(path)
+    return os.path.normpath(os.path.join(_temporary_directory(), path))
+
+
 def url_for_test_data(path: str) -> str:
     """Generate the URL for the test data based on the given path.
 
@@ -101,12 +118,11 @@ def get_test_data(path: str, gzipped=False) -> str:
     str
         The local path to the downloaded test data.
     """
-    _check_path(path)
 
     if _offline():
         raise RuntimeError("Offline mode: cannot download test data, add @pytest.mark.skipif(not offline(),...)")
 
-    target = os.path.normpath(os.path.join(_temporary_directory(), path))
+    target = _temporary_directory_for_test_data(path)
     with lock:
         if os.path.exists(target):
             return target
@@ -153,12 +169,18 @@ def get_test_archive(path: str, extension=".extracted") -> str:
 
     with lock:
 
+        target = _temporary_directory_for_test_data(path) + extension
+
+        if os.path.exists(target):
+            return target
+
         archive = get_test_data(path)
-        target = archive + extension
 
         shutil.unpack_archive(archive, os.path.dirname(target) + ".tmp")
         os.rename(os.path.dirname(target) + ".tmp", target)
 
+        os.remove(archive)
+
         return target
 
 
@@ -239,12 +261,12 @@ def _run_slow_tests() -> bool:
 @lru_cache(maxsize=None)
 def _offline() -> bool:
     """Check if we are offline."""
-
-    import socket
+    from urllib import request
 
     try:
-        socket.create_connection(("anemoi.ecmwf.int", 443), timeout=5)
-    except OSError:
+        request.urlopen("https://anemoi.ecmwf.int", timeout=1)
+        return False
+    except request.URLError:
         return True
 
     return False

{anemoi_utils-0.4.21 → anemoi_utils-0.4.23}/src/anemoi_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.21
+Version: 0.4.23
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -252,7 +252,7 @@ Provides-Extra: provenance
 Requires-Dist: gitpython; extra == "provenance"
 Requires-Dist: nvsmi; extra == "provenance"
 Provides-Extra: s3
-Requires-Dist: boto3<1.36; extra == "s3"
+Requires-Dist: boto3>1.36; extra == "s3"
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text

{anemoi_utils-0.4.21 → anemoi_utils-0.4.23}/src/anemoi_utils.egg-info/requires.txt

@@ -36,7 +36,7 @@ gitpython
 nvsmi
 
 [s3]
-boto3<1.36
+boto3>1.36
 
 [tests]
 pytest

anemoi_utils-0.4.21/.github/CODEOWNERS

@@ -1,6 +0,0 @@
-# CODEOWNERS file
-
-# Protect workflow files
-/.github/ @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb @anaprietonem @HCookie @JPXKQX @mchantry
-/.pre-commit-config.yaml @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb @anaprietonem @HCookie @JPXKQX @mchantry
-/pyproject.toml @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb @anaprietonem @HCookie @JPXKQX @mchantry

anemoi_utils-0.4.21/.release-please-manifest.json

@@ -1,3 +0,0 @@
-{
-    ".": "0.4.21"
-}