anemoi-utils 0.3.1__tar.gz → 0.3.2__tar.gz

{anemoi_utils-0.3.1 → anemoi_utils-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.1
+Version: 0.3.2
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.3.1 → anemoi_utils-0.3.2}/src/anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.1'
-__version_tuple__ = version_tuple = (0, 3, 1)
+__version__ = version = '0.3.2'
+__version_tuple__ = version_tuple = (0, 3, 2)

{anemoi_utils-0.3.1 → anemoi_utils-0.3.2}/src/anemoi/utils/cli.py

@@ -41,7 +41,7 @@ def make_parser(description, commands):
 
     subparsers = parser.add_subparsers(help="commands:", dest="command")
     for name, command in commands.items():
-        command_parser = subparsers.add_parser(name, help=command.__doc__)
+        command_parser = subparsers.add_parser(name, description=command.__doc__, help=command.__doc__)
         command.add_arguments(command_parser)
 
     return parser

anemoi_utils-0.3.2/src/anemoi/utils/s3.py

@@ -0,0 +1,57 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+import logging
+import os
+from contextlib import closing
+
+import boto3
+import tqdm
+
+LOG = logging.getLogger(__name__)
+
+
+def upload(source, target, overwrite=False, ignore_existing=False):
+    # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html
+    assert target.startswith("s3://")
+
+    _, _, bucket, key = target.split("/", 3)
+
+    LOG.info(f"Uploading {source} to {target}")
+    s3_client = boto3.client("s3")
+
+    if not overwrite:
+        results = s3_client.list_objects(Bucket=bucket, Prefix=key)
+        if results.get("Contents"):
+            if ignore_existing:
+                LOG.info(f"{target} already exists, skipping")
+                return
+            else:
+                raise ValueError(f"{target} already exists, use --overwrite to replace")
+
+    size = os.path.getsize(source)
+    with closing(tqdm.tqdm(total=size, unit="B", unit_scale=True)) as t:
+        s3_client.upload_file(source, bucket, key, Callback=lambda x: t.update(x))
+
+    LOG.info(f"{target} is ready")
+
+
+def download(source, target, overwrite=False):
+    assert source.startswith("s3://")
+
+    _, _, bucket, key = source.split("/", 3)
+
+    s3 = boto3.client("s3")
+    response = s3.head_object(Bucket=bucket, Key=key)
+    size = response["ContentLength"]
+
+    if not overwrite:
+        if os.path.exists(source) and os.path.getsize(source) == size:
+            LOG.info(f"{source} already exists, skipping")
+            return
+
+    with closing(tqdm.tqdm(total=size, unit="B", unit_scale=True)) as t:
+        s3.download_file(bucket, key, target, Callback=lambda x: t.update(x))

{anemoi_utils-0.3.1 → anemoi_utils-0.3.2}/src/anemoi_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.1
+Version: 0.3.2
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.3.1 → anemoi_utils-0.3.2}/src/anemoi_utils.egg-info/SOURCES.txt

@@ -31,6 +31,7 @@ src/anemoi/utils/dates.py
 src/anemoi/utils/grib.py
 src/anemoi/utils/humanize.py
 src/anemoi/utils/provenance.py
+src/anemoi/utils/s3.py
 src/anemoi/utils/text.py
 src/anemoi/utils/timer.py
 src/anemoi/utils/commands/__init__.py