anemoi-utils 0.3.6__tar.gz → 0.3.8__tar.gz

{anemoi_utils-0.3.6/src/anemoi_utils.egg-info → anemoi_utils-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.6
+Version: 0.3.8
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.3.6 → anemoi_utils-0.3.8}/pyproject.toml

@@ -104,6 +104,7 @@ urls.Documentation = "https://anemoi-utils.readthedocs.io/"
 urls.Homepage = "https://github.com/ecmwf/anemoi-utils/"
 urls.Issues = "https://github.com/ecmwf/anemoi-utils/issues"
 urls.Repository = "https://github.com/ecmwf/anemoi-utils/"
+
 scripts.anemoi-utils = "anemoi.utils.__main__:main"
 
 [tool.setuptools.package-data]

{anemoi_utils-0.3.6 → anemoi_utils-0.3.8}/src/anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.6'
-__version_tuple__ = version_tuple = (0, 3, 6)
+__version__ = version = '0.3.8'
+__version_tuple__ = version_tuple = (0, 3, 8)

{anemoi_utils-0.3.6 → anemoi_utils-0.3.8}/src/anemoi/utils/cli.py

@@ -97,7 +97,7 @@ def register_commands(here, package, select, fail=None):
 
 def cli_main(version, description, commands):
     parser = make_parser(description, commands)
-    args = parser.parse_args()
+    args, unknown = parser.parse_known_args()
 
     if args.version:
         print(version)
@@ -115,8 +115,15 @@ def cli_main(version, description, commands):
         level=logging.DEBUG if args.debug else logging.INFO,
     )
 
+    if unknown and not cmd.accept_unknown_args:
+        # This should trigger an error
+        parser.parse_args()
+
     try:
-        cmd.run(args)
+        if unknown:
+            cmd.run(args, unknown)
+        else:
+            cmd.run(args)
     except ValueError as e:
         traceback.print_exc()
         LOG.error("\n💣 %s", str(e).lstrip())

anemoi_utils-0.3.8/src/anemoi/utils/commands/config.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# (C) Copyright 2024 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+
+import json
+
+from ..config import config_path
+from ..config import load_config
+from . import Command
+
+
+class Config(Command):
+
+    def add_arguments(self, command_parser):
+        command_parser.add_argument("--path", help="Print path to config file")
+
+    def run(self, args):
+        if args.path:
+            print(config_path())
+        else:
+            print(json.dumps(load_config(), indent=4))
+
+
+command = Config

anemoi_utils-0.3.8/src/anemoi/utils/config.py

@@ -0,0 +1,260 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import json
+import logging
+import os
+import threading
+
+import yaml
+
+try:
+    import tomllib  # Only available since 3.11
+except ImportError:
+    import tomli as tomllib
+
+
+LOG = logging.getLogger(__name__)
+
+
+class DotDict(dict):
+    """A dictionary that allows access to its keys as attributes.
+
+    >>> d = DotDict({"a": 1, "b": {"c": 2}})
+    >>> d.a
+    1
+    >>> d.b.c
+    2
+    >>> d.b = 3
+    >>> d.b
+    3
+
+    The class is recursive, so nested dictionaries are also DotDicts.
+
+    The DotDict class has the same constructor as the dict class.
+
+    >>> d = DotDict(a=1, b=2)
+
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        for k, v in self.items():
+            if isinstance(v, dict):
+                self[k] = DotDict(v)
+
+            if isinstance(v, list):
+                self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
+
+            if isinstance(v, tuple):
+                self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
+
+    @classmethod
+    def from_file(cls, path: str):
+        _, ext = os.path.splitext(path)
+        if ext == ".yaml" or ext == ".yml":
+            return cls.from_yaml_file(path)
+        elif ext == ".json":
+            return cls.from_json_file(path)
+        elif ext == ".toml":
+            return cls.from_toml_file(path)
+        else:
+            raise ValueError(f"Unknown file extension {ext}")
+
+    @classmethod
+    def from_yaml_file(cls, path: str):
+        with open(path, "r") as file:
+            data = yaml.safe_load(file)
+
+        return cls(data)
+
+    @classmethod
+    def from_json_file(cls, path: str):
+        with open(path, "r") as file:
+            data = json.load(file)
+
+        return cls(data)
+
+    @classmethod
+    def from_toml_file(cls, path: str):
+        with open(path, "r") as file:
+            data = tomllib.load(file)
+        return cls(data)
+
+    def __getattr__(self, attr):
+        try:
+            return self[attr]
+        except KeyError:
+            raise AttributeError(attr)
+
+    def __setattr__(self, attr, value):
+        if isinstance(value, dict):
+            value = DotDict(value)
+        self[attr] = value
+
+    def __repr__(self) -> str:
+        return f"DotDict({super().__repr__()})"
+
+
+CONFIG = {}
+CHECKED = {}
+CONFIG_LOCK = threading.Lock()
+QUIET = False
+
+
+def config_path(name="settings.toml"):
+    global QUIET
+    full = os.path.join(os.path.expanduser("~"), ".config", "anemoi", name)
+    os.makedirs(os.path.dirname(full), exist_ok=True)
+
+    if name == "settings.toml":
+        old = os.path.join(os.path.expanduser("~"), ".anemoi.toml")
+        if not os.path.exists(full) and os.path.exists(old):
+            if not QUIET:
+                LOG.warning(
+                    "Configuration file found at ~/.anemoi.toml. Please move it to ~/.config/anemoi/settings.toml"
+                )
+                QUIET = True
+            return old
+        else:
+            if os.path.exists(old):
+                if not QUIET:
+                    LOG.warning(
+                        "Configuration file found at ~/.anemoi.toml and ~/.config/anemoi/settings.toml, ignoring the former"
+                    )
+                    QUIET = True
+
+    return full
+
+
+def _load(path):
+    try:
+        if path.endswith(".json"):
+            with open(path, "rb") as f:
+                return json.load(f)
+
+        if path.endswith(".yaml") or path.endswith(".yml"):
+            with open(path, "rb") as f:
+                return yaml.safe_load(f)
+
+        if path.endswith(".toml"):
+            with open(path, "rb") as f:
+                return tomllib.load(f)
+    except (json.JSONDecodeError, yaml.YAMLError, tomllib.TOMLDecodeError) as e:
+        LOG.warning(f"Failed to parse config file {path}", exc_info=e)
+        return ValueError(f"Failed to parse config file {path} [{e}]")
+
+    return open(path).read()
+
+
+def _load_config(name="settings.toml"):
+
+    if name in CONFIG:
+        return CONFIG[name]
+
+    conf = config_path(name)
+
+    if os.path.exists(conf):
+        config = _load(conf)
+    else:
+        config = {}
+
+    if isinstance(config, dict):
+        CONFIG[name] = DotDict(config)
+    else:
+        CONFIG[name] = config
+
+    return CONFIG[name]
+
+
+def _save_config(name, data):
+    CONFIG.pop(name, None)
+
+    conf = config_path(name)
+
+    if conf.endswith(".json"):
+        with open(conf, "w") as f:
+            json.dump(data, f, indent=4)
+        return
+
+    if conf.endswith(".yaml") or conf.endswith(".yml"):
+        with open(conf, "w") as f:
+            yaml.dump(data, f)
+        return
+
+    if conf.endswith(".toml"):
+        raise NotImplementedError("Saving to TOML is not implemented yet")
+
+    with open(conf, "w") as f:
+        f.write(data)
+
+
+def save_config(name, data):
+    """Save a configuration file.
+
+    Parameters
+    ----------
+    name : str
+        The name of the configuration file to save.
+
+    data : Any
+        The data to save.
+
+    """
+    with CONFIG_LOCK:
+        _save_config(name, data)
+
+
+def load_config(name="settings.toml"):
+    """Read a configuration file.
+
+    Parameters
+    ----------
+    name : str, optional
+        The name of the config file to read, by default "settings.toml"
+
+    Returns
+    -------
+    DotDict or str
+        Return DotDict if it is a dictionary, otherwise the raw data
+    """
+    with CONFIG_LOCK:
+        return _load_config(name)
+
+
+def load_raw_config(name, default=None):
+
+    path = config_path(name)
+    if os.path.exists(path):
+        return _load(path)
+
+    return default
+
+
+def check_config_mode(name="settings.toml"):
+    """Check that a configuration file is secure.
+
+    Parameters
+    ----------
+    name : str, optional
+        The name of the configuration file, by default "settings.toml"
+
+    Raises
+    ------
+    SystemError
+        If the configuration file is not secure.
+    """
+    with CONFIG_LOCK:
+        if name in CHECKED:
+            return
+        conf = config_path(name)
+        mode = os.stat(conf).st_mode
+        if mode & 0o777 != 0o600:
+            raise SystemError(f"Configuration file {conf} is not secure. " "Please run `chmod 600 {conf}`.")
+        CHECKED[name] = True

{anemoi_utils-0.3.6 → anemoi_utils-0.3.8}/src/anemoi/utils/dates.py

@@ -9,6 +9,8 @@
 import calendar
 import datetime
 
+from .hindcasts import HindcastDatesTimes
+
 
 def normalise_frequency(frequency):
     if isinstance(frequency, int):
@@ -158,37 +160,6 @@ class DateTimes:
             date += self.increment
 
 
-class HindcastDatesTimes:
-    """The HindcastDatesTimes class is an iterator that generates datetime objects within a given range."""
-
-    def __init__(self, reference_dates, years=20):
-        """_summary_
-
-        Parameters
-        ----------
-        reference_dates : _type_
-            _description_
-        years : int, optional
-            _description_, by default 20
-        """
-
-        self.reference_dates = reference_dates
-
-        if isinstance(years, list):
-            self.years = years
-        else:
-            self.years = range(1, years + 1)
-
-    def __iter__(self):
-        for reference_date in self.reference_dates:
-            for year in self.years:
-                if reference_date.month == 2 and reference_date.day == 29:
-                    date = datetime.datetime(reference_date.year - year, 2, 28)
-                else:
-                    date = datetime.datetime(reference_date.year - year, reference_date.month, reference_date.day)
-                yield (date, reference_date)
-
-
 class Year(DateTimes):
     """Year is defined as the months of January to December."""
 

anemoi_utils-0.3.8/src/anemoi/utils/hindcasts.py

@@ -0,0 +1,40 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import datetime
+
+
+class HindcastDatesTimes:
+    """The HindcastDatesTimes class is an iterator that generates datetime objects within a given range."""
+
+    def __init__(self, reference_dates, years=20):
+        """_summary_
+
+        Parameters
+        ----------
+        reference_dates : _type_
+            _description_
+        years : int, optional
+            _description_, by default 20
+        """
+
+        self.reference_dates = reference_dates
+
+        if isinstance(years, list):
+            self.years = years
+        else:
+            self.years = range(1, years + 1)
+
+    def __iter__(self):
+        for reference_date in self.reference_dates:
+            for year in self.years:
+                if reference_date.month == 2 and reference_date.day == 29:
+                    date = datetime.datetime(reference_date.year - year, 2, 28)
+                else:
+                    date = datetime.datetime(reference_date.year - year, reference_date.month, reference_date.day)
+                yield (date, reference_date)

{anemoi_utils-0.3.6 → anemoi_utils-0.3.8}/src/anemoi/utils/humanize.py

@@ -10,6 +10,7 @@
 """Generate human readable strings"""
 
 import datetime
+import json
 import re
 from collections import defaultdict
 
@@ -472,3 +473,57 @@ def rounded_datetime(d):
         d = d + datetime.timedelta(seconds=1)
     d = d.replace(microsecond=0)
     return d
+
+
+def json_pretty_dump(obj, max_line_length=120, default=str):
+    """Custom JSON dump function that keeps dicts and lists on one line if they are short enough.
+
+    Parameters
+    ----------
+    obj
+        The object to be dumped as JSON.
+    max_line_length
+        Maximum allowed line length for pretty-printing.
+
+    Returns
+    -------
+    unknown
+        JSON string.
+    """
+
+    def _format_json(obj, indent_level=0):
+        """Helper function to format JSON objects with custom pretty-print rules.
+
+        Parameters
+        ----------
+        obj
+            The object to format.
+        indent_level
+            Current indentation level.
+
+        Returns
+        -------
+        unknown
+            Formatted JSON string.
+        """
+        indent = " " * 4 * indent_level
+        if isinstance(obj, dict):
+            items = []
+            for key, value in obj.items():
+                items.append(f'"{key}": {_format_json(value, indent_level + 1)}')
+            line = "{" + ", ".join(items) + "}"
+            if len(line) <= max_line_length:
+                return line
+            else:
+                return "{\n" + ",\n".join([f"{indent}    {item}" for item in items]) + "\n" + indent + "}"
+        elif isinstance(obj, list):
+            items = [_format_json(item, indent_level + 1) for item in obj]
+            line = "[" + ", ".join(items) + "]"
+            if len(line) <= max_line_length:
+                return line
+            else:
+                return "[\n" + ",\n".join([f"{indent}    {item}" for item in items]) + "\n" + indent + "]"
+        else:
+            return json.dumps(obj, default=default)
+
+    return _format_json(obj)

{anemoi_utils-0.3.6 → anemoi_utils-0.3.8}/src/anemoi/utils/s3.py

@@ -146,6 +146,14 @@ class Upload(Transfer):
         return os.path.getsize(local_path)
 
     def transfer_file(self, source, target, overwrite, resume, verbosity, config=None):
+        try:
+            return self._transfer_file(source, target, overwrite, resume, verbosity, config=config)
+        except Exception as e:
+            LOGGER.exception(f"Error transferring {source} to {target}")
+            LOGGER.error(e)
+            raise
+
+    def _transfer_file(self, source, target, overwrite, resume, verbosity, config=None):
 
         from botocore.exceptions import ClientError
 
@@ -208,6 +216,14 @@ class Download(Transfer):
         return s3_object["Size"]
 
     def transfer_file(self, source, target, overwrite, resume, verbosity, config=None):
+        try:
+            return self._transfer_file(source, target, overwrite, resume, verbosity, config=config)
+        except Exception as e:
+            LOGGER.exception(f"Error transferring {source} to {target}")
+            LOGGER.error(e)
+            raise
+
+    def _transfer_file(self, source, target, overwrite, resume, verbosity, config=None):
         # from boto3.s3.transfer import TransferConfig
 
         _, _, bucket, key = source.split("/", 3)

{anemoi_utils-0.3.6 → anemoi_utils-0.3.8/src/anemoi_utils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.6
+Version: 0.3.8
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.3.6 → anemoi_utils-0.3.8}/src/anemoi_utils.egg-info/SOURCES.txt

@@ -30,13 +30,14 @@ src/anemoi/utils/cli.py
 src/anemoi/utils/config.py
 src/anemoi/utils/dates.py
 src/anemoi/utils/grib.py
+src/anemoi/utils/hindcasts.py
 src/anemoi/utils/humanize.py
 src/anemoi/utils/provenance.py
 src/anemoi/utils/s3.py
 src/anemoi/utils/text.py
 src/anemoi/utils/timer.py
 src/anemoi/utils/commands/__init__.py
-src/anemoi/utils/commands/checkpoint.py
+src/anemoi/utils/commands/config.py
 src/anemoi/utils/mars/__init__.py
 src/anemoi/utils/mars/mars.yaml
 src/anemoi_utils.egg-info/PKG-INFO

anemoi_utils-0.3.6/src/anemoi/utils/commands/checkpoint.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-# (C) Copyright 2024 ECMWF.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-#
-
-
-import json
-
-from . import Command
-
-
-def visit(x, path, name, value):
-    if isinstance(x, dict):
-        for k, v in x.items():
-            if k == name:
-                print(".".join(path), k, v)
-
-            if v == value:
-                print(".".join(path), k, v)
-
-            path.append(k)
-            visit(v, path, name, value)
-            path.pop()
-
-    if isinstance(x, list):
-        for i, v in enumerate(x):
-            path.append(str(i))
-            visit(v, path, name, value)
-            path.pop()
-
-
-class Checkpoint(Command):
-
-    def add_arguments(self, command_parser):
-        command_parser.add_argument("path", help="Path to the checkpoint.")
-        command_parser.add_argument("--name", help="Search for a specific name.")
-        command_parser.add_argument("--value", help="Search for a specific value.")
-
-    def run(self, args):
-        from anemoi.utils.checkpoints import load_metadata
-
-        checkpoint = load_metadata(args.path, "*.json")
-
-        if args.name or args.value:
-            visit(
-                checkpoint,
-                [],
-                args.name if args.name is not None else object(),
-                args.value if args.value is not None else object(),
-            )
-            return
-
-        print(json.dumps(checkpoint, sort_keys=True, indent=4))
-
-
-command = Checkpoint

anemoi_utils-0.3.6/src/anemoi/utils/config.py

@@ -1,142 +0,0 @@
-# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-import json
-import logging
-import os
-import threading
-
-import yaml
-
-try:
-    import tomllib  # Only available since 3.11
-except ImportError:
-    import tomli as tomllib
-
-
-LOG = logging.getLogger(__name__)
-
-
-class DotDict(dict):
-    """A dictionary that allows access to its keys as attributes.
-
-    >>> d = DotDict({"a": 1, "b": {"c": 2}})
-    >>> d.a
-    1
-    >>> d.b.c
-    2
-    >>> d.b = 3
-    >>> d.b
-    3
-
-    The class is recursive, so nested dictionaries are also DotDicts.
-
-    The DotDict class has the same constructor as the dict class.
-
-    >>> d = DotDict(a=1, b=2)
-
-    """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-        for k, v in self.items():
-            if isinstance(v, dict):
-                self[k] = DotDict(v)
-
-            if isinstance(v, list):
-                self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
-
-            if isinstance(v, tuple):
-                self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
-
-    @classmethod
-    def from_file(cls, path: str):
-        _, ext = os.path.splitext(path)
-        if ext == ".yaml" or ext == ".yml":
-            return cls.from_yaml_file(path)
-        elif ext == ".json":
-            return cls.from_json_file(path)
-        elif ext == ".toml":
-            return cls.from_toml_file(path)
-        else:
-            raise ValueError(f"Unknown file extension {ext}")
-
-    @classmethod
-    def from_yaml_file(cls, path: str):
-        with open(path, "r") as file:
-            data = yaml.safe_load(file)
-
-        return cls(data)
-
-    @classmethod
-    def from_json_file(cls, path: str):
-        with open(path, "r") as file:
-            data = json.load(file)
-
-        return cls(data)
-
-    @classmethod
-    def from_toml_file(cls, path: str):
-        with open(path, "r") as file:
-            data = tomllib.load(file)
-        return cls(data)
-
-    def __getattr__(self, attr):
-        try:
-            return self[attr]
-        except KeyError:
-            raise AttributeError(attr)
-
-    def __setattr__(self, attr, value):
-        if isinstance(value, dict):
-            value = DotDict(value)
-        self[attr] = value
-
-    def __repr__(self) -> str:
-        return f"DotDict({super().__repr__()})"
-
-
-CONFIG = None
-CONFIG_LOCK = threading.Lock()
-
-
-def _load_config():
-    global CONFIG
-    if CONFIG is not None:
-        return CONFIG
-
-    conf = os.path.expanduser("~/.anemoi.toml")
-
-    if os.path.exists(conf):
-
-        with open(conf, "rb") as f:
-            CONFIG = tomllib.load(f)
-    else:
-        CONFIG = {}
-
-    return DotDict(CONFIG)
-
-
-def load_config():
-    """Load the configuration from `~/.anemoi.toml`.
-
-    Returns
-    -------
-    DotDict
-        The configuration
-    """
-    with CONFIG_LOCK:
-        return _load_config()
-
-
-def check_config_mode():
-    conf = os.path.expanduser("~/.anemoi.toml")
-    mode = os.stat(conf).st_mode
-    if mode & 0o777 != 0o600:
-        raise SystemError(f"Configuration file {conf} is not secure. " "Please run `chmod 600 ~/.anemoi.toml`.")