anemoi-utils 0.4.21__py3-none-any.whl → 0.4.23__py3-none-any.whl

anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.21'
-__version_tuple__ = version_tuple = (0, 4, 21)
+__version__ = version = '0.4.23'
+__version_tuple__ = version_tuple = (0, 4, 23)

anemoi/utils/provenance.py

@@ -47,8 +47,11 @@ def lookup_git_repo(path: str) -> Optional[Any]:
     Repo, optional
         The git repository if found, otherwise None.
     """
-    from git import InvalidGitRepositoryError
-    from git import Repo
+    try:
+        from git import InvalidGitRepositoryError
+        from git import Repo
+    except ImportError:
+        return None
 
     while path != "/":
         try:

anemoi/utils/remote/s3.py

@@ -1,10 +1,13 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# (C) Copyright 2024-2025 Anemoi contributors.
+#
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
 # In applying this licence, ECMWF does not waive the privileges and immunities
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+
 """This module provides functions to upload, download, list and delete files and folders on S3.
 The functions of this package expect that the AWS credentials are set up in the environment
 typicaly by setting the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables or
@@ -67,12 +70,14 @@ def s3_client(bucket: str, region: str = None) -> Any:
 
     key = f"{bucket}-{region}"
 
-    boto3_config = dict(max_pool_connections=25)
-
     if key in thread_local.s3_clients:
         return thread_local.s3_clients[key]
 
-    boto3_config = dict(max_pool_connections=25)
+    boto3_config = dict(
+        max_pool_connections=25,
+        request_checksum_calculation="when_required",
+        response_checksum_validation="when_required",
+    )
 
     if region:
         # This is using AWS
@@ -162,7 +167,14 @@ class S3Upload(BaseUpload):
         # delete(target)
 
     def _transfer_file(
-        self, source: str, target: str, overwrite: bool, resume: bool, verbosity: int, threads: int, config: dict = None
+        self,
+        source: str,
+        target: str,
+        overwrite: bool,
+        resume: bool,
+        verbosity: int,
+        threads: int,
+        config: dict = None,
     ) -> int:
         """Transfer a file to S3.
 
@@ -227,7 +239,13 @@ class S3Upload(BaseUpload):
 
         if verbosity > 0:
             with tqdm.tqdm(total=size, unit="B", unit_scale=True, unit_divisor=1024, leave=False) as pbar:
-                s3.upload_file(source, bucket, key, Callback=lambda x: pbar.update(x), Config=config)
+                s3.upload_file(
+                    source,
+                    bucket,
+                    key,
+                    Callback=lambda x: pbar.update(x),
+                    Config=config,
+                )
         else:
             s3.upload_file(source, bucket, key, Config=config)
 
@@ -326,7 +344,14 @@ class S3Download(BaseDownload):
         return s3_object["Size"]
 
     def _transfer_file(
-        self, source: str, target: str, overwrite: bool, resume: bool, verbosity: int, threads: int, config: dict = None
+        self,
+        source: str,
+        target: str,
+        overwrite: bool,
+        resume: bool,
+        verbosity: int,
+        threads: int,
+        config: dict = None,
     ) -> int:
         """Transfer a file from S3 to the local filesystem.
 
@@ -397,7 +422,13 @@ class S3Download(BaseDownload):
 
         if verbosity > 0:
             with tqdm.tqdm(total=size, unit="B", unit_scale=True, unit_divisor=1024, leave=False) as pbar:
-                s3.download_file(bucket, key, target, Callback=lambda x: pbar.update(x), Config=config)
+                s3.download_file(
+                    bucket,
+                    key,
+                    target,
+                    Callback=lambda x: pbar.update(x),
+                    Config=config,
+                )
         else:
             s3.download_file(bucket, key, target, Config=config)
 
@@ -524,7 +555,9 @@ def list_folder(folder: str) -> Iterable:
 
     for page in paginator.paginate(Bucket=bucket, Prefix=prefix, Delimiter="/"):
         if "CommonPrefixes" in page:
-            yield from [folder + _["Prefix"] for _ in page.get("CommonPrefixes")]
+            yield from [folder + _["Prefix"] for _ in page.get("CommonPrefixes") if _["Prefix"] != "/"]
+        if "Contents" in page:
+            yield from [folder + _["Key"] for _ in page.get("Contents")]
 
 
 def object_info(target: str) -> dict:

anemoi/utils/testing.py

@@ -68,6 +68,23 @@ def _check_path(path: str) -> None:
     assert not path.startswith("."), f"Path '{path}' should not start with '.'"
 
 
+def _temporary_directory_for_test_data(path: str) -> str:
+    """Get the temporary directory for a test dataset.
+
+    Parameters
+    ----------
+    path : str
+        The relative path to the test data in the object store.
+
+    Returns
+    -------
+    str
+        The path to the temporary directory.
+    """
+    _check_path(path)
+    return os.path.normpath(os.path.join(_temporary_directory(), path))
+
+
 def url_for_test_data(path: str) -> str:
     """Generate the URL for the test data based on the given path.
 
@@ -101,12 +118,11 @@ def get_test_data(path: str, gzipped=False) -> str:
     str
         The local path to the downloaded test data.
     """
-    _check_path(path)
 
     if _offline():
         raise RuntimeError("Offline mode: cannot download test data, add @pytest.mark.skipif(not offline(),...)")
 
-    target = os.path.normpath(os.path.join(_temporary_directory(), path))
+    target = _temporary_directory_for_test_data(path)
     with lock:
         if os.path.exists(target):
             return target
@@ -153,12 +169,18 @@ def get_test_archive(path: str, extension=".extracted") -> str:
 
     with lock:
 
+        target = _temporary_directory_for_test_data(path) + extension
+
+        if os.path.exists(target):
+            return target
+
         archive = get_test_data(path)
-        target = archive + extension
 
         shutil.unpack_archive(archive, os.path.dirname(target) + ".tmp")
         os.rename(os.path.dirname(target) + ".tmp", target)
 
+        os.remove(archive)
+
         return target
 
 
@@ -239,12 +261,12 @@ def _run_slow_tests() -> bool:
 @lru_cache(maxsize=None)
 def _offline() -> bool:
     """Check if we are offline."""
-
-    import socket
+    from urllib import request
 
     try:
-        socket.create_connection(("anemoi.ecmwf.int", 443), timeout=5)
-    except OSError:
+        request.urlopen("https://anemoi.ecmwf.int", timeout=1)
+        return False
+    except request.URLError:
         return True
 
     return False

{anemoi_utils-0.4.21.dist-info → anemoi_utils-0.4.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.21
+Version: 0.4.23
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -252,7 +252,7 @@ Provides-Extra: provenance
 Requires-Dist: gitpython; extra == "provenance"
 Requires-Dist: nvsmi; extra == "provenance"
 Provides-Extra: s3
-Requires-Dist: boto3<1.36; extra == "s3"
+Requires-Dist: boto3>1.36; extra == "s3"
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text

{anemoi_utils-0.4.21.dist-info → anemoi_utils-0.4.23.dist-info}/RECORD

@@ -1,6 +1,6 @@
 anemoi/utils/__init__.py,sha256=uVhpF-VjIl_4mMywOVtgTutgsdIsqz-xdkwxeMhzuag,730
 anemoi/utils/__main__.py,sha256=6LlE4MYrPvqqrykxXh7XMi50UZteUY59NeM8P9Zs2dU,910
-anemoi/utils/_version.py,sha256=jhdlV71JQ5bBRT2wFnZ7TRbXO71DWPBUoBiZyZnIPYQ,513
+anemoi/utils/_version.py,sha256=AnB7uXIf9bJtYv1LZYemC9XOrzt9AeZnOcUDxAsfacA,513
 anemoi/utils/caching.py,sha256=rXbeAmpBcMbbfN4EVblaHWKicsrtx1otER84FEBtz98,6183
 anemoi/utils/checkpoints.py,sha256=N4WpAZXa4etrpSEKhHqUUtG2-x9w3FJMHcLO-dDAXPY,9600
 anemoi/utils/cli.py,sha256=IyZfnSw0u0yYnrjOrzvm2RuuKvDk4cVb8pf8BkaChgA,6209
@@ -13,13 +13,13 @@ anemoi/utils/grids.py,sha256=uYgkU_KIg8FTUiuKV0Pho2swMMeXcSQ9CQe0MFlRr_I,5262
 anemoi/utils/hindcasts.py,sha256=iYVIxSNFL2HJcc_k1abCFLkpJFGHT8WKRIR4wcAwA3s,2144
 anemoi/utils/humanize.py,sha256=pjnFJAKHbEAOfcvn8c48kt-8eFy6FGW_U2ruJvfamrA,25189
 anemoi/utils/logs.py,sha256=naTgrmPwWHD4eekFttXftS4gtcAGYHpCqG4iwYprNDA,1804
-anemoi/utils/provenance.py,sha256=xC6mTstF7f_asqtPSrulC7c34xjOSuAxWhkwc3yKhHg,14629
+anemoi/utils/provenance.py,sha256=iTsn4r-VPq2D8tSHPSuAIqG077_opkqMT42G03DRWJg,14690
 anemoi/utils/registry.py,sha256=e3nOIRyMYQ-mpEvaHAv5tuvMYNbkJ5yz94ns7BnvkjM,9717
 anemoi/utils/rules.py,sha256=VspUoPmw7tijrs6l_wl4vDjr_zVQsFjx9ITiBSvxgc8,6972
 anemoi/utils/s3.py,sha256=xMT48kbcelcjjqsaU567WI3oZ5eqo88Rlgyx5ECszAU,4074
 anemoi/utils/sanitise.py,sha256=ZYGdSX6qihQANr3pHZjbKnoapnzP1KcrWdW1Ul1mOGk,3668
 anemoi/utils/sanitize.py,sha256=43ZKDcfVpeXSsJ9TFEc9aZnD6oe2cUh151XnDspM98M,462
-anemoi/utils/testing.py,sha256=RJJGlIriQode3eWQ3k1I30ZQe0yXjsO8fZGvMuRAjYM,7263
+anemoi/utils/testing.py,sha256=kwgAgLh3exYOTZSaX4xcPFjiMOyQDz-vcAlPcJqMiZk,7784
 anemoi/utils/text.py,sha256=HkzIvi24obDceFLpJEwBJ9PmPrJUkQN2TrElJ-A87gU,14441
 anemoi/utils/timer.py,sha256=_leKMYza2faM7JKlGE7LCNy13rbdPnwaCF7PSrI_NmI,3895
 anemoi/utils/commands/__init__.py,sha256=5u_6EwdqYczIAgJfCwRSyQAYFEqh2ZuHHT57g9g7sdI,808
@@ -29,13 +29,13 @@ anemoi/utils/mars/__init__.py,sha256=b-Lc3L1TAQd9ODs0Z1YSJzgZCO1K_M3DSgx_yd2qXvM
 anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
 anemoi/utils/mars/requests.py,sha256=VFMHBVAAl0_2lOcMBa1lvaKHctN0lDJsI6_U4BucGew,1142
 anemoi/utils/remote/__init__.py,sha256=swPWHQoh-B6Xq9R489tPw0FykMue7f-bJ8enneFYSYE,20776
-anemoi/utils/remote/s3.py,sha256=spQ8l0rwQjLZh9dZu5cOsYIvNwKihQfCJ6YsFYegeqI,17339
+anemoi/utils/remote/s3.py,sha256=dcXcgNddlgxwJ_OpgqOff8EWk-LT2mz20m7FcHHqz7w,17869
 anemoi/utils/remote/ssh.py,sha256=xNtsawh8okytCKRehkRCVExbHZj-CRUQNormEHglfuw,8088
 anemoi/utils/schemas/__init__.py,sha256=nkinKlsPLPXEjfTYQT1mpKC4cvs-14w_zBkDRxakwxw,698
 anemoi/utils/schemas/errors.py,sha256=lgOXzVTYzAE0qWQf3OZ42vCWixv8lilSqLLhzARBmvI,1831
-anemoi_utils-0.4.21.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
-anemoi_utils-0.4.21.dist-info/METADATA,sha256=AWNLa8J5pQTvIgxWOPOOo8skiBLCbbCR_DPuCxAGeAI,15439
-anemoi_utils-0.4.21.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-anemoi_utils-0.4.21.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.4.21.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.4.21.dist-info/RECORD,,
+anemoi_utils-0.4.23.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
+anemoi_utils-0.4.23.dist-info/METADATA,sha256=jxTYBIvx6wzO0g8ftu2MoSPfeViRSTUrE38Zy3HUvlw,15439
+anemoi_utils-0.4.23.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+anemoi_utils-0.4.23.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.4.23.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.4.23.dist-info/RECORD,,

{anemoi_utils-0.4.21.dist-info → anemoi_utils-0.4.23.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.7.1)
 Root-Is-Purelib: true
 Tag: py3-none-any