anemoi-utils 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl

anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.5'
-__version_tuple__ = version_tuple = (0, 3, 5)
+__version__ = version = '0.3.7'
+__version_tuple__ = version_tuple = (0, 3, 7)

anemoi/utils/commands/config.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# (C) Copyright 2024 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+
+import json
+
+from ..config import config_path
+from ..config import load_config
+from . import Command
+
+
+class Config(Command):
+
+    def add_arguments(self, command_parser):
+        command_parser.add_argument("--path", help="Print path to config file")
+
+    def run(self, args):
+        if args.path:
+            print(config_path())
+        else:
+            print(json.dumps(load_config(), indent=4))
+
+
+command = Config

anemoi/utils/config.py

@@ -6,10 +6,13 @@
 # nor does it submit to any jurisdiction.
 
 
+import json
 import logging
 import os
 import threading
 
+import yaml
+
 try:
     import tomllib  # Only available since 3.11
 except ImportError:
@@ -41,10 +44,49 @@ class DotDict(dict):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+
         for k, v in self.items():
             if isinstance(v, dict):
                 self[k] = DotDict(v)
 
+            if isinstance(v, list):
+                self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
+
+            if isinstance(v, tuple):
+                self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
+
+    @classmethod
+    def from_file(cls, path: str):
+        _, ext = os.path.splitext(path)
+        if ext == ".yaml" or ext == ".yml":
+            return cls.from_yaml_file(path)
+        elif ext == ".json":
+            return cls.from_json_file(path)
+        elif ext == ".toml":
+            return cls.from_toml_file(path)
+        else:
+            raise ValueError(f"Unknown file extension {ext}")
+
+    @classmethod
+    def from_yaml_file(cls, path: str):
+        with open(path, "r") as file:
+            data = yaml.safe_load(file)
+
+        return cls(data)
+
+    @classmethod
+    def from_json_file(cls, path: str):
+        with open(path, "r") as file:
+            data = json.load(file)
+
+        return cls(data)
+
+    @classmethod
+    def from_toml_file(cls, path: str):
+        with open(path, "r") as file:
+            data = tomllib.load(file)
+        return cls(data)
+
     def __getattr__(self, attr):
         try:
             return self[attr]
@@ -60,41 +102,146 @@ class DotDict(dict):
         return f"DotDict({super().__repr__()})"
 
 
-CONFIG = None
+CONFIG = {}
+CHECKED = {}
 CONFIG_LOCK = threading.Lock()
+QUIET = False
+
+
+def config_path(name="settings.toml"):
+    global QUIET
+    full = os.path.join(os.path.expanduser("~"), ".config", "anemoi", name)
+    os.makedirs(os.path.dirname(full), exist_ok=True)
+
+    if name == "settings.toml":
+        old = os.path.join(os.path.expanduser("~"), ".anemoi.toml")
+        if not os.path.exists(full) and os.path.exists(old):
+            if not QUIET:
+                LOG.warning(
+                    "Configuration file found at ~/.anemoi.toml. Please move it to ~/.config/anemoi/settings.toml"
+                )
+                QUIET = True
+            return old
+        else:
+            if os.path.exists(old):
+                if not QUIET:
+                    LOG.warning(
+                        "Configuration file found at ~/.anemoi.toml and ~/.config/anemoi/settings.toml, ignoring the former"
+                    )
+                    QUIET = True
+
+    return full
+
+
+def _load(path):
+    if path.endswith(".json"):
+        with open(path, "rb") as f:
+            return json.load(f)
+
+    if path.endswith(".yaml") or path.endswith(".yml"):
+        with open(path, "rb") as f:
+            return yaml.safe_load(f)
+
+    if path.endswith(".toml"):
+        with open(path, "rb") as f:
+            return tomllib.load(f)
+
+    return open(path).read()
+
 
+def _load_config(name="settings.toml"):
 
-def _load_config():
-    global CONFIG
-    if CONFIG is not None:
-        return CONFIG
+    if name in CONFIG:
+        return CONFIG[name]
 
-    conf = os.path.expanduser("~/.anemoi.toml")
+    conf = config_path(name)
 
     if os.path.exists(conf):
+        config = _load(conf)
+    else:
+        config = {}
 
-        with open(conf, "rb") as f:
-            CONFIG = tomllib.load(f)
+    if isinstance(config, dict):
+        CONFIG[name] = DotDict(config)
     else:
-        CONFIG = {}
+        CONFIG[name] = config
+
+    return CONFIG[name]
+
+
+def _save_config(name, data):
+    CONFIG.pop(name, None)
+
+    conf = config_path(name)
+
+    if conf.endswith(".json"):
+        with open(conf, "w") as f:
+            json.dump(data, f, indent=4)
+        return
+
+    if conf.endswith(".yaml") or conf.endswith(".yml"):
+        with open(conf, "w") as f:
+            yaml.dump(data, f)
+        return
+
+    if conf.endswith(".toml"):
+        raise NotImplementedError("Saving to TOML is not implemented yet")
+
+    with open(conf, "w") as f:
+        f.write(data)
+
 
-    return DotDict(CONFIG)
+def save_config(name, data):
+    """Save a configuration file.
 
+    Parameters
+    ----------
+    name : str
+        The name of the configuration file to save.
 
-def load_config():
-    """Load the configuration from `~/.anemoi.toml`.
+    data : Any
+        The data to save.
+
+    """
+    with CONFIG_LOCK:
+        _save_config(name, data)
+
+
+def load_config(name="settings.toml"):
+    """Read a configuration file.
+
+    Parameters
+    ----------
+    name : str, optional
+        The name of the config file to read, by default "settings.toml"
 
     Returns
     -------
-    DotDict
-        The configuration
+    DotDict or str
+        Return DotDict if it is a dictionary, otherwise the raw data
     """
     with CONFIG_LOCK:
-        return _load_config()
+        return _load_config(name)
+
+
+def check_config_mode(name="settings.toml"):
+    """Check that a configuration file is secure.
 
+    Parameters
+    ----------
+    name : str, optional
+        The name of the configuration file, by default "settings.toml"
 
-def check_config_mode():
-    conf = os.path.expanduser("~/.anemoi.toml")
-    mode = os.stat(conf).st_mode
-    if mode & 0o777 != 0o600:
-        raise SystemError(f"Configuration file {conf} is not secure. " "Please run `chmod 600 ~/.anemoi.toml`.")
+    Raises
+    ------
+    SystemError
+        If the configuration file is not secure.
+    """
+    with CONFIG_LOCK:
+        if name in CHECKED:
+            return
+        conf = config_path(name)
+        mode = os.stat(conf).st_mode
+        if mode & 0o777 != 0o600:
+            raise SystemError(f"Configuration file {conf} is not secure. " "Please run `chmod 600 {conf}`.")
+        CHECKED[name] = True

anemoi/utils/s3.py

@@ -146,6 +146,14 @@ class Upload(Transfer):
         return os.path.getsize(local_path)
 
     def transfer_file(self, source, target, overwrite, resume, verbosity, config=None):
+        try:
+            return self._transfer_file(source, target, overwrite, resume, verbosity, config=config)
+        except Exception as e:
+            LOGGER.exception(f"Error transferring {source} to {target}")
+            LOGGER.error(e)
+            raise
+
+    def _transfer_file(self, source, target, overwrite, resume, verbosity, config=None):
 
         from botocore.exceptions import ClientError
 
@@ -208,6 +216,14 @@ class Download(Transfer):
         return s3_object["Size"]
 
     def transfer_file(self, source, target, overwrite, resume, verbosity, config=None):
+        try:
+            return self._transfer_file(source, target, overwrite, resume, verbosity, config=config)
+        except Exception as e:
+            LOGGER.exception(f"Error transferring {source} to {target}")
+            LOGGER.error(e)
+            raise
+
+    def _transfer_file(self, source, target, overwrite, resume, verbosity, config=None):
         # from boto3.s3.transfer import TransferConfig
 
         _, _, bucket, key = source.split("/", 3)

{anemoi_utils-0.3.5.dist-info → anemoi_utils-0.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.5
+Version: 0.3.7
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License: Apache License

{anemoi_utils-0.3.5.dist-info → anemoi_utils-0.3.7.dist-info}/RECORD

@@ -1,24 +1,24 @@
 anemoi/utils/__init__.py,sha256=zZZpbKIoGWwdCOuo6YSruLR7C0GzvzI1Wzhyqaa0K7M,456
 anemoi/utils/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
-anemoi/utils/_version.py,sha256=3eLsZPTwWh0zKBhR6n3c4iAL9geCCdJGMojIL6dF0IA,411
+anemoi/utils/_version.py,sha256=vVN20516E2VTC9JNgtvqrQNlj5XptaB_a5z2XL8NFxg,411
 anemoi/utils/caching.py,sha256=HrC9aFHlcCTaM2Z5u0ivGIXz7eFu35UQQhUuwwuG2pk,1743
 anemoi/utils/checkpoints.py,sha256=1_3mg4B-ykTVfIvIUEv7IxGyREx_ZcilVbB3U-V6O6I,5165
 anemoi/utils/cli.py,sha256=w6YVYfJV-50Zm9FrO0KNrrIWDdgj5hPjxJvgAh391NY,3308
-anemoi/utils/config.py,sha256=WD3EZ2Ylt-HLvJnOPXKEGMepR2VVNYbJKC10fPJu45w,2333
+anemoi/utils/config.py,sha256=BKH-ZOIVZWhuvKxmhK33JVpIr3q7Tdx-tI8kwd4SvgA,6175
 anemoi/utils/dates.py,sha256=Ot9OTY1uFvHxW1EU4DPv3oUqmzvkXTwKuwhlfVlY788,8426
 anemoi/utils/grib.py,sha256=gVfo4KYQv31iRyoqRDwk5tiqZDUgOIvhag_kO0qjYD0,3067
 anemoi/utils/humanize.py,sha256=LD6dGnqChxA5j3tMhSybsAGRQzi33d_qS9pUoUHubkc,10330
 anemoi/utils/provenance.py,sha256=v54L9jF1JgYcclOhg3iojRl1v3ajbiWz_oc289xTgO4,9574
-anemoi/utils/s3.py,sha256=OEZGm85gzWfZ29OuCx-A2dczC00XKM_SYnkGv9vgs14,15204
+anemoi/utils/s3.py,sha256=icYeOlCnb0vfKDMIR_Rq6Machl-3KB67sAFSi8OmftI,15910
 anemoi/utils/text.py,sha256=4Zlc4r9dzRjkKL9xqp2vuQsoJY15bJ3y_Xv3YW_XsmU,8510
 anemoi/utils/timer.py,sha256=JKOgFkpJxmVRn57DEBolmTGwr25P-ePTWASBd8CLeqM,972
 anemoi/utils/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
-anemoi/utils/commands/checkpoint.py,sha256=SEnAizU3WklqMXUjmIh4eNrgBVwmheKG9gEBS90zwYU,1741
+anemoi/utils/commands/config.py,sha256=KEffXZh0ZQfn8t6LXresfd94kDY0gEyulx9Wto5ttW0,824
 anemoi/utils/mars/__init__.py,sha256=RAeY8gJ7ZvsPlcIvrQ4fy9xVHs3SphTAPw_XJDtNIKo,1750
 anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
-anemoi_utils-0.3.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_utils-0.3.5.dist-info/METADATA,sha256=N9sPN7e-Zb6y0sxhw6N124iHNOoWWh7Gc2RWjIAD0UY,15513
-anemoi_utils-0.3.5.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-anemoi_utils-0.3.5.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.3.5.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.3.5.dist-info/RECORD,,
+anemoi_utils-0.3.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_utils-0.3.7.dist-info/METADATA,sha256=TR-5CWm6hwvgDbTyu-P4cIoobZOBJmWWdrrdfQSu3eY,15513
+anemoi_utils-0.3.7.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
+anemoi_utils-0.3.7.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.3.7.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.3.7.dist-info/RECORD,,

{anemoi_utils-0.3.5.dist-info → anemoi_utils-0.3.7.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: setuptools (70.1.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

anemoi/utils/commands/checkpoint.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-# (C) Copyright 2024 ECMWF.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-#
-
-
-import json
-
-from . import Command
-
-
-def visit(x, path, name, value):
-    if isinstance(x, dict):
-        for k, v in x.items():
-            if k == name:
-                print(".".join(path), k, v)
-
-            if v == value:
-                print(".".join(path), k, v)
-
-            path.append(k)
-            visit(v, path, name, value)
-            path.pop()
-
-    if isinstance(x, list):
-        for i, v in enumerate(x):
-            path.append(str(i))
-            visit(v, path, name, value)
-            path.pop()
-
-
-class Checkpoint(Command):
-
-    def add_arguments(self, command_parser):
-        command_parser.add_argument("path", help="Path to the checkpoint.")
-        command_parser.add_argument("--name", help="Search for a specific name.")
-        command_parser.add_argument("--value", help="Search for a specific value.")
-
-    def run(self, args):
-        from anemoi.utils.checkpoints import load_metadata
-
-        checkpoint = load_metadata(args.path, "*.json")
-
-        if args.name or args.value:
-            visit(
-                checkpoint,
-                [],
-                args.name if args.name is not None else object(),
-                args.value if args.value is not None else object(),
-            )
-            return
-
-        print(json.dumps(checkpoint, sort_keys=True, indent=4))
-
-
-command = Checkpoint