anemoi-utils 0.3.1__tar.gz → 0.3.3__tar.gz

{anemoi_utils-0.3.1 → anemoi_utils-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.1
+Version: 0.3.3
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.3.1 → anemoi_utils-0.3.3}/src/anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.1'
-__version_tuple__ = version_tuple = (0, 3, 1)
+__version__ = version = '0.3.3'
+__version_tuple__ = version_tuple = (0, 3, 3)

{anemoi_utils-0.3.1 → anemoi_utils-0.3.3}/src/anemoi/utils/cli.py

@@ -41,7 +41,7 @@ def make_parser(description, commands):
 
     subparsers = parser.add_subparsers(help="commands:", dest="command")
     for name, command in commands.items():
-        command_parser = subparsers.add_parser(name, help=command.__doc__)
+        command_parser = subparsers.add_parser(name, description=command.__doc__, help=command.__doc__)
         command.add_arguments(command_parser)
 
     return parser

anemoi_utils-0.3.3/src/anemoi/utils/s3.py

@@ -0,0 +1,57 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+import logging
+import os
+from contextlib import closing
+
+import boto3
+import tqdm
+
+LOG = logging.getLogger(__name__)
+
+
+def upload(source, target, overwrite=False, ignore_existing=False):
+    # https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html
+    assert target.startswith("s3://")
+
+    _, _, bucket, key = target.split("/", 3)
+
+    LOG.info(f"Uploading {source} to {target}")
+    s3_client = boto3.client("s3")
+
+    if not overwrite:
+        results = s3_client.list_objects(Bucket=bucket, Prefix=key)
+        if results.get("Contents"):
+            if ignore_existing:
+                LOG.info(f"{target} already exists, skipping")
+                return
+            else:
+                raise ValueError(f"{target} already exists, use --overwrite to replace")
+
+    size = os.path.getsize(source)
+    with closing(tqdm.tqdm(total=size, unit="B", unit_scale=True)) as t:
+        s3_client.upload_file(source, bucket, key, Callback=lambda x: t.update(x))
+
+    LOG.info(f"{target} is ready")
+
+
+def download(source, target, overwrite=False):
+    assert source.startswith("s3://")
+
+    _, _, bucket, key = source.split("/", 3)
+
+    s3 = boto3.client("s3")
+    response = s3.head_object(Bucket=bucket, Key=key)
+    size = response["ContentLength"]
+
+    if not overwrite:
+        if os.path.exists(source) and os.path.getsize(source) == size:
+            LOG.info(f"{source} already exists, skipping")
+            return
+
+    with closing(tqdm.tqdm(total=size, unit="B", unit_scale=True)) as t:
+        s3.download_file(bucket, key, target, Callback=lambda x: t.update(x))

{anemoi_utils-0.3.1 → anemoi_utils-0.3.3}/src/anemoi/utils/text.py

@@ -7,7 +7,6 @@
 
 """Text utilities"""
 
-import sys
 from collections import defaultdict
 
 # https://en.wikipedia.org/wiki/Box-drawing_character
@@ -143,13 +142,13 @@ class Tree:
             self._kids.append(node)
         return node
 
-    def print(self, file=sys.stdout):
+    def print(self):
         padding = []
 
         while self._factorise():
             pass
 
-        self._print(padding, file=file)
+        self._print(padding)
 
     def _leaves(self, result):
         if self.is_leaf:
@@ -206,21 +205,21 @@ class Tree:
 
         return result
 
-    def _print(self, padding, file=sys.stdout):
+    def _print(self, padding):
         for i, p in enumerate(padding[:-1]):
             if p == " └":
                 padding[i] = "  "
             if p == " ├":
                 padding[i] = " │"
         if padding:
-            print(f"{''.join(padding)}─{self._text}", file=file)
+            print(f"{''.join(padding)}─{self._text}")
         else:
-            print(self._text, file=file)
+            print(self._text)
         padding.append(" ")
         for i, k in enumerate(self._kids):
             sep = " ├" if i < len(self._kids) - 1 else " └"
             padding[-1] = sep
-            k._print(padding, file=file)
+            k._print(padding)
 
         padding.pop()
 

{anemoi_utils-0.3.1 → anemoi_utils-0.3.3}/src/anemoi_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.1
+Version: 0.3.3
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.3.1 → anemoi_utils-0.3.3}/src/anemoi_utils.egg-info/SOURCES.txt

@@ -31,6 +31,7 @@ src/anemoi/utils/dates.py
 src/anemoi/utils/grib.py
 src/anemoi/utils/humanize.py
 src/anemoi/utils/provenance.py
+src/anemoi/utils/s3.py
 src/anemoi/utils/text.py
 src/anemoi/utils/timer.py
 src/anemoi/utils/commands/__init__.py