anemoi-utils 0.3.9__tar.gz → 0.3.11__tar.gz

{anemoi_utils-0.3.9/src/anemoi_utils.egg-info → anemoi_utils-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.9
+Version: 0.3.11
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.3.9 → anemoi_utils-0.3.11}/docs/installing.rst

@@ -22,7 +22,7 @@ The options are:
 
 .. code:: bash
 
-   git clone ...
+   git clone git@github.com:ecmwf/anemoi-registry.git
    cd anemoi-utils
    pip install .[dev]
    pip install -r docs/requirements.txt

{anemoi_utils-0.3.9 → anemoi_utils-0.3.11}/src/anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.9'
-__version_tuple__ = version_tuple = (0, 3, 9)
+__version__ = version = '0.3.11'
+__version_tuple__ = version_tuple = (0, 3, 11)

{anemoi_utils-0.3.9 → anemoi_utils-0.3.11}/src/anemoi/utils/caching.py

@@ -9,17 +9,26 @@ import hashlib
 import json
 import os
 import time
+from threading import Lock
+
+LOCK = Lock()
+CACHE = {}
 
 
 def cache(key, proc, collection="default", expires=None):
-    path = os.path.join(os.path.expanduser("~"), ".cache", "anemoi", collection)
-    os.makedirs(path, exist_ok=True)
 
     key = json.dumps(key, sort_keys=True)
     m = hashlib.md5()
     m.update(key.encode("utf-8"))
+    m = m.hexdigest()
+
+    if m in CACHE:
+        return CACHE[m]
+
+    path = os.path.join(os.path.expanduser("~"), ".cache", "anemoi", collection)
+    os.makedirs(path, exist_ok=True)
 
-    filename = os.path.join(path, m.hexdigest())
+    filename = os.path.join(path, m)
     if os.path.exists(filename):
         with open(filename, "r") as f:
             data = json.load(f)
@@ -35,6 +44,7 @@ def cache(key, proc, collection="default", expires=None):
     with open(filename, "w") as f:
         json.dump(data, f)
 
+    CACHE[m] = value
     return value
 
 
@@ -49,11 +59,12 @@ class cached:
         full = f"{func.__module__}.{func.__name__}"
 
         def wrapped(*args, **kwargs):
-            return cache(
-                (full, args, kwargs),
-                lambda: func(*args, **kwargs),
-                self.collection,
-                self.expires,
-            )
+            with LOCK:
+                return cache(
+                    (full, args, kwargs),
+                    lambda: func(*args, **kwargs),
+                    self.collection,
+                    self.expires,
+                )
 
         return wrapped

{anemoi_utils-0.3.9 → anemoi_utils-0.3.11}/src/anemoi/utils/commands/config.py

@@ -11,7 +11,7 @@
 
 import json
 
-from ..config import _config_path
+from ..config import config_path
 from ..config import load_config
 from . import Command
 
@@ -23,7 +23,7 @@ class Config(Command):
 
     def run(self, args):
         if args.path:
-            print(_config_path())
+            print(config_path())
         else:
             print(json.dumps(load_config(), indent=4))
 

{anemoi_utils-0.3.9 → anemoi_utils-0.3.11}/src/anemoi/utils/config.py

@@ -143,7 +143,7 @@ def _set_defaults(a, b):
             a.setdefault(k, v)
 
 
-def _config_path(name="settings.toml"):
+def config_path(name="settings.toml"):
     global QUIET
 
     if name.startswith("/") or name.startswith("."):
@@ -205,10 +205,11 @@ def load_any_dict_format(path):
 
 def _load_config(name="settings.toml", secrets=None, defaults=None):
 
-    if name in CONFIG:
-        return CONFIG[name]
+    key = json.dumps((name, secrets, defaults), sort_keys=True, default=str)
+    if key in CONFIG:
+        return CONFIG[key]
 
-    path = _config_path(name)
+    path = config_path(name)
     if os.path.exists(path):
         config = load_any_dict_format(path)
     else:
@@ -238,14 +239,14 @@ def _load_config(name="settings.toml", secrets=None, defaults=None):
         secret_config = _load_config(secret_name)
         _merge_dicts(config, secret_config)
 
-    CONFIG[name] = DotDict(config)
-    return CONFIG[name]
+    CONFIG[key] = DotDict(config)
+    return CONFIG[key]
 
 
 def _save_config(name, data):
     CONFIG.pop(name, None)
 
-    conf = _config_path(name)
+    conf = config_path(name)
 
     if conf.endswith(".json"):
         with open(conf, "w") as f:
@@ -300,7 +301,7 @@ def load_config(name="settings.toml", secrets=None, defaults=None):
 
 def load_raw_config(name, default=None):
 
-    path = _config_path(name)
+    path = config_path(name)
     if os.path.exists(path):
         return load_any_dict_format(path)
 
@@ -324,13 +325,13 @@ def check_config_mode(name="settings.toml", secrets_name=None, secrets=None):
         if name in CHECKED:
             return
 
-        conf = _config_path(name)
+        conf = config_path(name)
         if not os.path.exists(conf):
             return
         mode = os.stat(conf).st_mode
         if mode & 0o777 != 0o600:
             if secrets_name:
-                secret_path = _config_path(secrets_name)
+                secret_path = config_path(secrets_name)
                 raise SystemError(
                     f"Configuration file {conf} should not hold entries {secrets}.\n"
                     f"Please move them to {secret_path}."

{anemoi_utils-0.3.9 → anemoi_utils-0.3.11}/src/anemoi/utils/humanize.py

@@ -12,10 +12,11 @@
 import datetime
 import json
 import re
+import warnings
 from collections import defaultdict
 
 
-def bytes(n: float) -> str:
+def bytes_to_human(n: float) -> str:
     """Convert a number of bytes to a human readable string
 
     >>> bytes(4096)
@@ -55,7 +56,16 @@ def bytes(n: float) -> str:
     return "%s%g%s" % (sign, int(n * 10 + 0.5) / 10.0, u[i])
 
 
-def base2(n) -> str:
+def bytes(n: float) -> str:
+    warnings.warn(
+        "Function bytes is deprecated and will be removed in a future version. Use bytes_to_human instead.",
+        category=DeprecationWarning,
+        stacklevel=2,
+    )
+    return bytes_to_human(n)
+
+
+def base2_to_human(n) -> str:
 
     u = ["", "K", "M", "G", "T", " P", "E", "Z", "Y"]
     i = 0
@@ -65,6 +75,16 @@ def base2(n) -> str:
     return "%g%s" % (int(n * 10 + 0.5) / 10.0, u[i])
 
 
+def base2(n) -> str:
+
+    warnings.warn(
+        "Function base2 is deprecated and will be removed in a future version. Use base2_to_human instead.",
+        category=DeprecationWarning,
+        stacklevel=2,
+    )
+    return base2_to_human(n)
+
+
 PERIODS = (
     (7 * 24 * 60 * 60, "week"),
     (24 * 60 * 60, "day"),
@@ -81,7 +101,7 @@ def _plural(count):
         return ""
 
 
-def seconds(seconds: float) -> str:
+def seconds_to_human(seconds: float) -> str:
     """Convert a number of seconds to a human readable string
 
     >>> seconds(4000)
@@ -140,12 +160,17 @@ def seconds(seconds: float) -> str:
     return " ".join(s)
 
 
-def number(value):
-    return f"{value:,}"
+def seconds(seconds: float) -> str:
+    warnings.warn(
+        "Function seconds is deprecated and will be removed in a future version. Use seconds_to_human instead.",
+        category=DeprecationWarning,
+        stacklevel=2,
+    )
+    return seconds_to_human(seconds)
 
 
 def plural(value, what):
-    return f"{number(value)} {what}{_plural(value)}"
+    return f"{value:,} {what}{_plural(value)}"
 
 
 DOW = [
@@ -395,7 +420,7 @@ def list_to_human(lst, conjunction="and"):
     return f" {conjunction} ".join(lst)
 
 
-def as_number(value, name, units, none_ok):
+def human_to_number(value, name, units, none_ok):
     if value is None and none_ok:
         return None
 
@@ -414,17 +439,44 @@ def as_number(value, name, units, none_ok):
     return value * units[unit]
 
 
-def as_seconds(value, name=None, none_ok=False):
+def as_number(value, name=None, units=None, none_ok=False):
+    warnings.warn(
+        "Function as_number is deprecated and will be removed in a future version. Use human_to_number instead.",
+        category=DeprecationWarning,
+        stacklevel=2,
+    )
+    return human_to_number(value, name, units, none_ok)
+
+
+def human_seconds(value, name=None, none_ok=False):
     units = dict(s=1, m=60, h=3600, d=86400, w=86400 * 7)
-    return as_number(value, name, units, none_ok)
+    return human_to_number(value, name, units, none_ok)
 
 
-def as_percent(value, name=None, none_ok=False):
+def as_seconds(value, name=None, none_ok=False):
+    warnings.warn(
+        "Function as_seconds is deprecated and will be removed in a future version. Use human_seconds instead.",
+        category=DeprecationWarning,
+        stacklevel=2,
+    )
+    return human_seconds(value, name, none_ok)
+
+
+def human_to_percent(value, name=None, none_ok=False):
     units = {"%": 1}
-    return as_number(value, name, units, none_ok)
+    return human_to_number(value, name, units, none_ok)
 
 
-def as_bytes(value, name=None, none_ok=False):
+def as_percent(value, name=None, none_ok=False):
+    warnings.warn(
+        "Function as_percent is deprecated and will be removed in a future version. Use human_to_percent instead.",
+        category=DeprecationWarning,
+        stacklevel=2,
+    )
+    return human_to_percent(value, name, none_ok)
+
+
+def human_to_bytes(value, name=None, none_ok=False):
     units = {}
     n = 1
     for u in "KMGTP":
@@ -432,10 +484,19 @@ def as_bytes(value, name=None, none_ok=False):
         units[u] = n
         units[u.lower()] = n
 
-    return as_number(value, name, units, none_ok)
+    return human_to_number(value, name, units, none_ok)
 
 
-def as_timedelta(value, name=None, none_ok=False):
+def as_bytes(value, name=None, none_ok=False):
+    warnings.warn(
+        "Function as_bytes is deprecated and will be removed in a future version. Use human_to_bytes instead.",
+        category=DeprecationWarning,
+        stacklevel=2,
+    )
+    return human_to_bytes(value, name, none_ok)
+
+
+def human_to_timedelta(value, name=None, none_ok=False):
     if value is None and none_ok:
         return None
 
@@ -471,6 +532,15 @@ def as_timedelta(value, name=None, none_ok=False):
     )
 
 
+def as_timedelta(value, name=None, none_ok=False):
+    warnings.warn(
+        "Function as_timedelta is deprecated and will be removed in a future version. Use human_to_timedelta instead.",
+        category=DeprecationWarning,
+        stacklevel=2,
+    )
+    return human_to_timedelta(value, name, none_ok)
+
+
 def rounded_datetime(d):
     if float(d.microsecond) / 1000.0 / 1000.0 >= 0.5:
         d = d + datetime.timedelta(seconds=1)
@@ -530,3 +600,28 @@ def json_pretty_dump(obj, max_line_length=120, default=str):
             return json.dumps(obj, default=default)
 
     return _format_json(obj)
+
+
+def shorten_list(lst, max_length=5):
+    """Shorten a list to a maximum length.
+
+    Parameters
+    ----------
+    lst
+        The list to be shortened.
+    max_length
+        Maximum length of the shortened list.
+
+    Returns
+    -------
+    list
+        Shortened list.
+    """
+    if len(lst) <= max_length:
+        return lst
+    else:
+        half = max_length // 2
+        result = list(lst[:half]) + ["..."] + list(lst[max_length - half :])
+        if isinstance(lst, tuple):
+            return tuple(result)
+        return result

{anemoi_utils-0.3.9 → anemoi_utils-0.3.11}/src/anemoi/utils/provenance.py

@@ -86,7 +86,7 @@ def version(versions, name, module, roots, namespaces, paths, full):
                 paths.add((name, path))
 
     try:
-        versions[name] = module.__version__
+        versions[name] = str(module.__version__)
         return
     except AttributeError:
         pass

{anemoi_utils-0.3.9 → anemoi_utils-0.3.11}/src/anemoi/utils/s3.py

@@ -16,6 +16,10 @@ to use a different S3 compatible service::
     aws_access_key_id = xxxxxxxxxxxxxxxxxxxxxxxx
     aws_secret_access_key = xxxxxxxxxxxxxxxxxxxxxxxx
 
+Alternatively, the `endpoint_url`, and keys can be set in one of
+the `~/.config/anemoi/settings.toml`
+or `~/.config/anemoi/settings-secrets.toml` files.
+
 """
 
 import concurrent.futures
@@ -27,7 +31,7 @@ from copy import deepcopy
 import tqdm
 
 from .config import load_config
-from .humanize import bytes
+from .humanize import bytes_to_human
 
 LOGGER = logging.getLogger(__name__)
 
@@ -37,17 +41,43 @@ LOGGER = logging.getLogger(__name__)
 thread_local = threading.local()
 
 
-def s3_client(bucket):
+def s3_client(bucket, region=None):
     import boto3
-
-    config = load_config(secrets=["aws_access_key_id", "aws_secret_access_key"])
+    from botocore import UNSIGNED
+    from botocore.client import Config
 
     if not hasattr(thread_local, "s3_clients"):
         thread_local.s3_clients = {}
 
-    if bucket not in thread_local.s3_clients:
+    key = f"{bucket}-{region}"
+
+    boto3_config = dict(max_pool_connections=25)
+
+    if key in thread_local.s3_clients:
+        return thread_local.s3_clients[key]
+
+    boto3_config = dict(max_pool_connections=25)
+
+    if region:
+        # This is using AWS
+
+        options = {"region_name": region}
+
+        # Anonymous access
+        if not (
+            os.path.exists(os.path.expanduser("~/.aws/credentials"))
+            or ("AWS_ACCESS_KEY_ID" in os.environ and "AWS_SECRET_ACCESS_KEY" in os.environ)
+        ):
+            boto3_config["signature_version"] = UNSIGNED
+
+    else:
+
+        # We may be accessing a different S3 compatible service
+        # Use anemoi.config to get the configuration
 
         options = {}
+        config = load_config(secrets=["aws_access_key_id", "aws_secret_access_key"])
+
         cfg = config.get("object-storage", {})
         for k, v in cfg.items():
             if isinstance(v, (str, int, float, bool)):
@@ -62,21 +92,29 @@ def s3_client(bucket):
             raise ValueError(f"Unsupported object storage type {type}")
 
         if "config" in options:
+            boto3_config.update(options["config"])
+            del options["config"]
             from botocore.client import Config
 
-            options["config"] = Config(**options["config"])
-            del options["config"]
+    options["config"] = Config(**boto3_config)
+
+    thread_local.s3_clients[key] = boto3.client("s3", **options)
+
+    return thread_local.s3_clients[key]
 
-        thread_local.s3_clients[bucket] = boto3.client("s3", **options)
 
-    return thread_local.s3_clients[bucket]
+def _ignore(number_of_files, total_size, total_transferred, transfering):
+    pass
 
 
 class Transfer:
 
-    def transfer_folder(self, *, source, target, overwrite=False, resume=False, verbosity=1, threads=1):
+    def transfer_folder(self, *, source, target, overwrite=False, resume=False, verbosity=1, threads=1, progress=None):
         assert verbosity == 1, verbosity
 
+        if progress is None:
+            progress = _ignore
+
         # from boto3.s3.transfer import TransferConfig
         # config = TransferConfig(use_threads=False)
         config = None
@@ -85,7 +123,8 @@ class Transfer:
                 if verbosity > 0:
                     LOGGER.info(f"{self.action} {source} to {target}")
 
-                total = 0
+                total_size = 0
+                total_transferred = 0
 
                 futures = []
                 for name in self.list_source(source):
@@ -101,11 +140,14 @@ class Transfer:
                             config=config,
                         )
                     )
-                    total += self.source_size(name)
+                    total_size += self.source_size(name)
 
                     if len(futures) % 10000 == 0:
+
+                        progress(len(futures), total_size, 0, False)
+
                         if verbosity > 0:
-                            LOGGER.info(f"Preparing transfer, {len(futures):,} files... ({bytes(total)})")
+                            LOGGER.info(f"Preparing transfer, {len(futures):,} files... ({bytes_to_human(total_size)})")
                         done, _ = concurrent.futures.wait(
                             futures,
                             timeout=0.001,
@@ -115,14 +157,22 @@ class Transfer:
                         for future in done:
                             future.result()
 
+                number_of_files = len(futures)
+                progress(number_of_files, total_size, 0, True)
+
                 if verbosity > 0:
-                    LOGGER.info(f"{self.action} {len(futures):,} files ({bytes(total)})")
-                    with tqdm.tqdm(total=total, unit="B", unit_scale=True, unit_divisor=1024) as pbar:
-                        for future in futures:
-                            pbar.update(future.result())
+                    LOGGER.info(f"{self.action} {number_of_files:,} files ({bytes_to_human(total_size)})")
+                    with tqdm.tqdm(total=total_size, unit="B", unit_scale=True, unit_divisor=1024) as pbar:
+                        for future in concurrent.futures.as_completed(futures):
+                            size = future.result()
+                            pbar.update(size)
+                            total_transferred += size
+                            progress(number_of_files, total_size, total_transferred, True)
                 else:
-                    for future in futures:
-                        future.result()
+                    for future in concurrent.futures.as_completed(futures):
+                        size = future.result()
+                        total_transferred += size
+                        progress(number_of_files, total_size, total_transferred, True)
 
             except Exception:
                 executor.shutdown(wait=False, cancel_futures=True)
@@ -168,7 +218,7 @@ class Upload(Transfer):
         size = os.path.getsize(source)
 
         if verbosity > 0:
-            LOGGER.info(f"{self.action} {source} to {target} ({bytes(size)})")
+            LOGGER.info(f"{self.action} {source} to {target} ({bytes_to_human(size)})")
 
         try:
             results = s3.head_object(Bucket=bucket, Key=key)
@@ -235,7 +285,6 @@ class Download(Transfer):
         try:
             response = s3.head_object(Bucket=bucket, Key=key)
         except s3.exceptions.ClientError as e:
-            print(e.response["Error"]["Code"], e.response["Error"]["Message"], bucket, key)
             if e.response["Error"]["Code"] == "404":
                 raise ValueError(f"{source} does not exist ({bucket}, {key})")
             raise
@@ -243,7 +292,7 @@ class Download(Transfer):
         size = int(response["ContentLength"])
 
         if verbosity > 0:
-            LOGGER.info(f"Downloading {source} to {target} ({bytes(size)})")
+            LOGGER.info(f"Downloading {source} to {target} ({bytes_to_human(size)})")
 
         if overwrite:
             resume = False
@@ -272,7 +321,7 @@ class Download(Transfer):
         return size
 
 
-def upload(source, target, *, overwrite=False, resume=False, verbosity=1, threads=1):
+def upload(source, target, *, overwrite=False, resume=False, verbosity=1, progress=None, threads=1):
     """Upload a file or a folder to S3.
 
     Parameters
@@ -286,6 +335,9 @@ def upload(source, target, *, overwrite=False, resume=False, verbosity=1, thread
     resume : bool, optional
         If the data is alreay on S3 it will not be uploaded, unless the remote file
         has a different size, by default False
+    progress: callable, optional
+        A callable that will be called with the number of files, the total size of the files, the total size
+        transferred and a boolean indicating if the transfer has started. By default None
     threads : int, optional
         The number of threads to use when uploading a directory, by default 1
     """
@@ -293,13 +345,26 @@ def upload(source, target, *, overwrite=False, resume=False, verbosity=1, thread
     uploader = Upload()
     if os.path.isdir(source):
         uploader.transfer_folder(
-            source=source, target=target, overwrite=overwrite, resume=resume, verbosity=verbosity, threads=threads
+            source=source,
+            target=target,
+            overwrite=overwrite,
+            resume=resume,
+            verbosity=verbosity,
+            progress=progress,
+            threads=threads,
         )
     else:
-        uploader.transfer_file(source=source, target=target, overwrite=overwrite, resume=resume, verbosity=verbosity)
+        uploader.transfer_file(
+            source=source,
+            target=target,
+            overwrite=overwrite,
+            resume=resume,
+            verbosity=verbosity,
+            progress=progress,
+        )
 
 
-def download(source, target, *, overwrite=False, resume=False, verbosity=1, threads=1):
+def download(source, target, *, overwrite=False, resume=False, verbosity=1, progress=None, threads=1):
     """Download a file or a folder from S3.
 
     Parameters
@@ -315,6 +380,9 @@ def download(source, target, *, overwrite=False, resume=False, verbosity=1, thre
     resume : bool, optional
         If the data is alreay on local it will not be downloaded, unless the remote file
         has a different size, by default False
+    progress: callable, optional
+        A callable that will be called with the number of files, the total size of the files, the total size
+        transferred and a boolean indicating if the transfer has started. By default None
     threads : int, optional
         The number of threads to use when downloading a directory, by default 1
     """
@@ -329,6 +397,7 @@ def download(source, target, *, overwrite=False, resume=False, verbosity=1, thre
             overwrite=overwrite,
             resume=resume,
             verbosity=verbosity,
+            progress=progress,
             threads=threads,
         )
     else:
@@ -338,6 +407,7 @@ def download(source, target, *, overwrite=False, resume=False, verbosity=1, thre
             overwrite=overwrite,
             resume=resume,
             verbosity=verbosity,
+            progress=progress,
         )
 
 
@@ -387,7 +457,7 @@ def _delete_file(target):
         return
 
     LOGGER.info(f"Deleting {target}")
-    print(s3.delete_object(Bucket=bucket, Key=key))
+    s3.delete_object(Bucket=bucket, Key=key)
     LOGGER.info(f"{target} is deleted")
 
 
@@ -423,13 +493,11 @@ def list_folder(folder):
         A list of the subfolders names in the folder.
     """
 
-    print(folder)
     assert folder.startswith("s3://")
     if not folder.endswith("/"):
         folder += "/"
 
     _, _, bucket, prefix = folder.split("/", 3)
-    print(bucket, prefix)
 
     s3 = s3_client(bucket)
     paginator = s3.get_paginator("list_objects_v2")

{anemoi_utils-0.3.9 → anemoi_utils-0.3.11/src/anemoi_utils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.9
+Version: 0.3.11
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License