anemoi-utils 0.4.26__tar.gz → 0.4.27__tar.gz

anemoi_utils-0.4.27/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.27"
+}

{anemoi_utils-0.4.26 → anemoi_utils-0.4.27}/CHANGELOG.md

@@ -8,6 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.4.27](https://github.com/ecmwf/anemoi-utils/compare/0.4.26...0.4.27) (2025-06-27)
+
+
+### Features
+
+* Split s3 config from s3 client code ([#170](https://github.com/ecmwf/anemoi-utils/issues/170)) ([56dacb1](https://github.com/ecmwf/anemoi-utils/commit/56dacb19efa0979acd72edb72a95f058b69d612a))
+
 ## [0.4.26](https://github.com/ecmwf/anemoi-utils/compare/0.4.25...0.4.26) (2025-06-25)
 
 

{anemoi_utils-0.4.26/src/anemoi_utils.egg-info → anemoi_utils-0.4.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.26
+Version: 0.4.27
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.4.26 → anemoi_utils-0.4.27}/src/anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.26'
-__version_tuple__ = version_tuple = (0, 4, 26)
+__version__ = version = '0.4.27'
+__version_tuple__ = version_tuple = (0, 4, 27)

{anemoi_utils-0.4.26 → anemoi_utils-0.4.27}/src/anemoi/utils/remote/s3.py

@@ -47,8 +47,8 @@ SECRETS = ["aws_access_key_id", "aws_secret_access_key"]
 thread_local = threading.local()
 
 
-def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
-    """Get an S3 client for the specified bucket and region.
+def _s3_config(bucket: str, *, region: str = None) -> Any:
+    """Get an S3 client config for the specified bucket and region.
 
     Parameters
     ----------
@@ -56,31 +56,15 @@ def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
         The name of the S3 bucket.
     region : str, optional
         The AWS region of the S3 bucket.
-    service : str, optional
-        The AWS service to use, default is "s3".
 
     Returns
     -------
     Any
         The S3 client.
     """
-    import boto3
     from botocore import UNSIGNED
-    from botocore.client import Config
 
-    if not hasattr(thread_local, "s3_clients"):
-        thread_local.s3_clients = {}
-
-    key = f"{bucket}-{region}-{service}"
-
-    if key in thread_local.s3_clients:
-        return thread_local.s3_clients[key]
-
-    boto3_config = dict(
-        max_pool_connections=25,
-        request_checksum_calculation="when_required",
-        response_checksum_validation="when_required",
-    )
+    boto3_config = {}
 
     if region:
         # This is using AWS
@@ -129,6 +113,85 @@ def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
             boto3_config.update(options["config"])
             del options["config"]
 
+    def _(options):
+
+        def __(k, v):
+            if k in SECRETS:
+                return "***"
+            return v
+
+        if isinstance(options, dict):
+            return {k: __(k, v) for k, v in options.items()}
+
+        if isinstance(options, list):
+            return [_(o) for o in options]
+
+        return options
+
+    LOG.debug(f"Using S3 options: {_(options)}")
+
+    return boto3_config, options
+
+
+def s3_options(bucket: str, *, region: str = None, service: str = "s3") -> dict:
+    """Get the S3 configuration for the specified bucket and region.
+
+    Parameters
+    ----------
+    bucket : str
+        The name of the S3 bucket.
+    region : str, optional
+        The AWS region of the S3 bucket.
+    service : str, optional
+        The AWS service to use, default is "s3".
+
+    Returns
+    -------
+    dict
+        The S3 configuration.
+    """
+    _, options = _s3_config(bucket, region=region)
+    return options
+
+
+def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
+    """Get an S3 client for the specified bucket and region.
+
+    Parameters
+    ----------
+    bucket : str
+        The name of the S3 bucket.
+    region : str, optional
+        The AWS region of the S3 bucket.
+    service : str, optional
+        The AWS service to use, default is "s3".
+
+    Returns
+    -------
+    Any
+        The S3 client.
+    """
+    import boto3
+    from botocore.client import Config
+
+    if not hasattr(thread_local, "s3_clients"):
+        thread_local.s3_clients = {}
+
+    key = f"{bucket}-{region}-{service}"
+
+    if key in thread_local.s3_clients:
+        return thread_local.s3_clients[key]
+
+    boto3_config, options = _s3_config(bucket, region=region)
+
+    boto3_config.update(
+        dict(
+            max_pool_connections=25,
+            request_checksum_calculation="when_required",
+            response_checksum_validation="when_required",
+        )
+    )
+
     options["config"] = Config(**boto3_config)
 
     def _(options):

{anemoi_utils-0.4.26 → anemoi_utils-0.4.27/src/anemoi_utils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.26
+Version: 0.4.27
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

anemoi_utils-0.4.26/.release-please-manifest.json

@@ -1,3 +0,0 @@
-{
-    ".": "0.4.26"
-}