anemoi-utils 0.4.26__py3-none-any.whl → 0.4.27__py3-none-any.whl

anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.26'
-__version_tuple__ = version_tuple = (0, 4, 26)
+__version__ = version = '0.4.27'
+__version_tuple__ = version_tuple = (0, 4, 27)

anemoi/utils/remote/s3.py

@@ -47,8 +47,8 @@ SECRETS = ["aws_access_key_id", "aws_secret_access_key"]
 thread_local = threading.local()
 
 
-def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
-    """Get an S3 client for the specified bucket and region.
+def _s3_config(bucket: str, *, region: str = None) -> Any:
+    """Get an S3 client config for the specified bucket and region.
 
     Parameters
     ----------
@@ -56,31 +56,15 @@ def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
         The name of the S3 bucket.
     region : str, optional
         The AWS region of the S3 bucket.
-    service : str, optional
-        The AWS service to use, default is "s3".
 
     Returns
     -------
     Any
         The S3 client.
     """
-    import boto3
     from botocore import UNSIGNED
-    from botocore.client import Config
 
-    if not hasattr(thread_local, "s3_clients"):
-        thread_local.s3_clients = {}
-
-    key = f"{bucket}-{region}-{service}"
-
-    if key in thread_local.s3_clients:
-        return thread_local.s3_clients[key]
-
-    boto3_config = dict(
-        max_pool_connections=25,
-        request_checksum_calculation="when_required",
-        response_checksum_validation="when_required",
-    )
+    boto3_config = {}
 
     if region:
         # This is using AWS
@@ -129,6 +113,85 @@ def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
             boto3_config.update(options["config"])
             del options["config"]
 
+    def _(options):
+
+        def __(k, v):
+            if k in SECRETS:
+                return "***"
+            return v
+
+        if isinstance(options, dict):
+            return {k: __(k, v) for k, v in options.items()}
+
+        if isinstance(options, list):
+            return [_(o) for o in options]
+
+        return options
+
+    LOG.debug(f"Using S3 options: {_(options)}")
+
+    return boto3_config, options
+
+
+def s3_options(bucket: str, *, region: str = None, service: str = "s3") -> dict:
+    """Get the S3 configuration for the specified bucket and region.
+
+    Parameters
+    ----------
+    bucket : str
+        The name of the S3 bucket.
+    region : str, optional
+        The AWS region of the S3 bucket.
+    service : str, optional
+        The AWS service to use, default is "s3".
+
+    Returns
+    -------
+    dict
+        The S3 configuration.
+    """
+    _, options = _s3_config(bucket, region=region)
+    return options
+
+
+def s3_client(bucket: str, *, region: str = None, service: str = "s3") -> Any:
+    """Get an S3 client for the specified bucket and region.
+
+    Parameters
+    ----------
+    bucket : str
+        The name of the S3 bucket.
+    region : str, optional
+        The AWS region of the S3 bucket.
+    service : str, optional
+        The AWS service to use, default is "s3".
+
+    Returns
+    -------
+    Any
+        The S3 client.
+    """
+    import boto3
+    from botocore.client import Config
+
+    if not hasattr(thread_local, "s3_clients"):
+        thread_local.s3_clients = {}
+
+    key = f"{bucket}-{region}-{service}"
+
+    if key in thread_local.s3_clients:
+        return thread_local.s3_clients[key]
+
+    boto3_config, options = _s3_config(bucket, region=region)
+
+    boto3_config.update(
+        dict(
+            max_pool_connections=25,
+            request_checksum_calculation="when_required",
+            response_checksum_validation="when_required",
+        )
+    )
+
     options["config"] = Config(**boto3_config)
 
     def _(options):

{anemoi_utils-0.4.26.dist-info → anemoi_utils-0.4.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.26
+Version: 0.4.27
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.4.26.dist-info → anemoi_utils-0.4.27.dist-info}/RECORD

@@ -1,6 +1,6 @@
 anemoi/utils/__init__.py,sha256=uVhpF-VjIl_4mMywOVtgTutgsdIsqz-xdkwxeMhzuag,730
 anemoi/utils/__main__.py,sha256=6LlE4MYrPvqqrykxXh7XMi50UZteUY59NeM8P9Zs2dU,910
-anemoi/utils/_version.py,sha256=ggjVlIoxYrdpYCbFUc7ixm914S2kJVBvieeqeCBnvjY,513
+anemoi/utils/_version.py,sha256=l2RpdEWLXHjIi56KlS-u-66k8wBi9yvFH_NPLD7QkmQ,513
 anemoi/utils/caching.py,sha256=rXbeAmpBcMbbfN4EVblaHWKicsrtx1otER84FEBtz98,6183
 anemoi/utils/checkpoints.py,sha256=PydBqA-wI8_05zU-3yT-ZP53GnvuqB7vPXl-w6-9XX8,9541
 anemoi/utils/cli.py,sha256=Cd3ESQkNWecbGnJjkR_SKHdFPETJWFrHqg5ovtANDKs,6522
@@ -31,13 +31,13 @@ anemoi/utils/mars/__init__.py,sha256=b-Lc3L1TAQd9ODs0Z1YSJzgZCO1K_M3DSgx_yd2qXvM
 anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
 anemoi/utils/mars/requests.py,sha256=VFMHBVAAl0_2lOcMBa1lvaKHctN0lDJsI6_U4BucGew,1142
 anemoi/utils/remote/__init__.py,sha256=7nHu-LRspYW2Fx9GNLjsxpytAUIvhIbOjb0Xmxb-33s,20988
-anemoi/utils/remote/s3.py,sha256=VgxNuJpJBnfxwkkV9TyrSkf7fjIuHqy1YG9vRdj9QRc,19978
+anemoi/utils/remote/s3.py,sha256=DxO_TjmetX_r3ZvGaHjpz40oqvcQYP3Vd_A4ojMGlSA,21379
 anemoi/utils/remote/ssh.py,sha256=xNtsawh8okytCKRehkRCVExbHZj-CRUQNormEHglfuw,8088
 anemoi/utils/schemas/__init__.py,sha256=nkinKlsPLPXEjfTYQT1mpKC4cvs-14w_zBkDRxakwxw,698
 anemoi/utils/schemas/errors.py,sha256=lgOXzVTYzAE0qWQf3OZ42vCWixv8lilSqLLhzARBmvI,1831
-anemoi_utils-0.4.26.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
-anemoi_utils-0.4.26.dist-info/METADATA,sha256=tx5q6lKfqmvh5eq9ty0Pw1e8LtzX4wysYzm_lVNeIQQ,15486
-anemoi_utils-0.4.26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-anemoi_utils-0.4.26.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.4.26.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.4.26.dist-info/RECORD,,
+anemoi_utils-0.4.27.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
+anemoi_utils-0.4.27.dist-info/METADATA,sha256=zqysYlsP5gsTcGN93z7KW5rSIehOpVjg85arxug6MZE,15486
+anemoi_utils-0.4.27.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+anemoi_utils-0.4.27.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.4.27.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.4.27.dist-info/RECORD,,