anemoi-utils 0.3.1__py3-none-any.whl → 0.3.2__py3-none-any.whl

anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.1'
-__version_tuple__ = version_tuple = (0, 3, 1)
+__version__ = version = '0.3.2'
+__version_tuple__ = version_tuple = (0, 3, 2)

anemoi/utils/cli.py

@@ -41,7 +41,7 @@ def make_parser(description, commands):
 
     subparsers = parser.add_subparsers(help="commands:", dest="command")
     for name, command in commands.items():
-        command_parser = subparsers.add_parser(name, help=command.__doc__)
+        command_parser = subparsers.add_parser(name, description=command.__doc__, help=command.__doc__)
         command.add_arguments(command_parser)
 
     return parser

anemoi/utils/s3.py

@@ -0,0 +1,57 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+import logging
+import os
+from contextlib import closing
+
+import boto3
+import tqdm
+
+LOG = logging.getLogger(__name__)
+
+
+def upload(source, target, overwrite=False, ignore_existing=False):
+    # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html
+    assert target.startswith("s3://")
+
+    _, _, bucket, key = target.split("/", 3)
+
+    LOG.info(f"Uploading {source} to {target}")
+    s3_client = boto3.client("s3")
+
+    if not overwrite:
+        results = s3_client.list_objects(Bucket=bucket, Prefix=key)
+        if results.get("Contents"):
+            if ignore_existing:
+                LOG.info(f"{target} already exists, skipping")
+                return
+            else:
+                raise ValueError(f"{target} already exists, use --overwrite to replace")
+
+    size = os.path.getsize(source)
+    with closing(tqdm.tqdm(total=size, unit="B", unit_scale=True)) as t:
+        s3_client.upload_file(source, bucket, key, Callback=lambda x: t.update(x))
+
+    LOG.info(f"{target} is ready")
+
+
+def download(source, target, overwrite=False):
+    assert source.startswith("s3://")
+
+    _, _, bucket, key = source.split("/", 3)
+
+    s3 = boto3.client("s3")
+    response = s3.head_object(Bucket=bucket, Key=key)
+    size = response["ContentLength"]
+
+    if not overwrite:
+        if os.path.exists(source) and os.path.getsize(source) == size:
+            LOG.info(f"{source} already exists, skipping")
+            return
+
+    with closing(tqdm.tqdm(total=size, unit="B", unit_scale=True)) as t:
+        s3.download_file(bucket, key, target, Callback=lambda x: t.update(x))

{anemoi_utils-0.3.1.dist-info → anemoi_utils-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.1
+Version: 0.3.2
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License: Apache License

{anemoi_utils-0.3.1.dist-info → anemoi_utils-0.3.2.dist-info}/RECORD

@@ -1,23 +1,24 @@
 anemoi/utils/__init__.py,sha256=zZZpbKIoGWwdCOuo6YSruLR7C0GzvzI1Wzhyqaa0K7M,456
 anemoi/utils/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
-anemoi/utils/_version.py,sha256=HzPz9rq3s1AiZXregKlqKaJJ2wGMtvH_a3V9la9CnpM,411
+anemoi/utils/_version.py,sha256=9jP8Fo8egXoMs_T3DFqSuJYg4n9o9mnwYubl_hnut4k,411
 anemoi/utils/caching.py,sha256=HrC9aFHlcCTaM2Z5u0ivGIXz7eFu35UQQhUuwwuG2pk,1743
 anemoi/utils/checkpoints.py,sha256=1_3mg4B-ykTVfIvIUEv7IxGyREx_ZcilVbB3U-V6O6I,5165
-anemoi/utils/cli.py,sha256=39THXs5SVtJ3L4ugemgbRIlK4ivEGraSsu5ns9KUn68,3271
+anemoi/utils/cli.py,sha256=d3TT9WSm6TDkbaJ9fL74wWG91Y27a9Uh17fPM4SomTs,3300
 anemoi/utils/config.py,sha256=XEesqODvkuE3ZA7dnEnZ-ooBRtU6ecPmkfP65FtialA,2147
 anemoi/utils/dates.py,sha256=Ot9OTY1uFvHxW1EU4DPv3oUqmzvkXTwKuwhlfVlY788,8426
 anemoi/utils/grib.py,sha256=gVfo4KYQv31iRyoqRDwk5tiqZDUgOIvhag_kO0qjYD0,3067
 anemoi/utils/humanize.py,sha256=LD6dGnqChxA5j3tMhSybsAGRQzi33d_qS9pUoUHubkc,10330
 anemoi/utils/provenance.py,sha256=v54L9jF1JgYcclOhg3iojRl1v3ajbiWz_oc289xTgO4,9574
+anemoi/utils/s3.py,sha256=HG3YulSwBvutTOZe0vY9oDkXExtUCcr26U9WObzl2I4,2024
 anemoi/utils/text.py,sha256=pGWtDvRFoDxAnSuZJiA-GOGJOJLHsw2dAm0tfVvPKno,8599
 anemoi/utils/timer.py,sha256=JKOgFkpJxmVRn57DEBolmTGwr25P-ePTWASBd8CLeqM,972
 anemoi/utils/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
 anemoi/utils/commands/checkpoint.py,sha256=SEnAizU3WklqMXUjmIh4eNrgBVwmheKG9gEBS90zwYU,1741
 anemoi/utils/mars/__init__.py,sha256=RAeY8gJ7ZvsPlcIvrQ4fy9xVHs3SphTAPw_XJDtNIKo,1750
 anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
-anemoi_utils-0.3.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_utils-0.3.1.dist-info/METADATA,sha256=Byi0PexhF8x5mUW-Pc0uChEfxMvtQctOHg9czUVfN0g,15513
-anemoi_utils-0.3.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-anemoi_utils-0.3.1.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.3.1.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.3.1.dist-info/RECORD,,
+anemoi_utils-0.3.2.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_utils-0.3.2.dist-info/METADATA,sha256=hZoPO-WGvJS7uOsRUxTd_RvyG27KvL6uXHEL638vnZg,15513
+anemoi_utils-0.3.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+anemoi_utils-0.3.2.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.3.2.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.3.2.dist-info/RECORD,,