anemoi-utils 0.3.7__py3-none-any.whl → 0.3.9__py3-none-any.whl

anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.7'
-__version_tuple__ = version_tuple = (0, 3, 7)
+__version__ = version = '0.3.9'
+__version_tuple__ = version_tuple = (0, 3, 9)

anemoi/utils/cli.py

@@ -16,6 +16,8 @@ LOG = logging.getLogger(__name__)
 
 
 class Command:
+    accept_unknown_args = False
+
     def run(self, args):
         raise NotImplementedError(f"Command not implemented: {args.command}")
 
@@ -97,7 +99,7 @@ def register_commands(here, package, select, fail=None):
 
 def cli_main(version, description, commands):
     parser = make_parser(description, commands)
-    args = parser.parse_args()
+    args, unknown = parser.parse_known_args()
 
     if args.version:
         print(version)
@@ -115,8 +117,15 @@ def cli_main(version, description, commands):
         level=logging.DEBUG if args.debug else logging.INFO,
     )
 
+    if unknown and not cmd.accept_unknown_args:
+        # This should trigger an error
+        parser.parse_args()
+
     try:
-        cmd.run(args)
+        if unknown:
+            cmd.run(args, unknown)
+        else:
+            cmd.run(args)
     except ValueError as e:
         traceback.print_exc()
         LOG.error("\n💣 %s", str(e).lstrip())

anemoi/utils/commands/config.py

@@ -11,7 +11,7 @@
 
 import json
 
-from ..config import config_path
+from ..config import _config_path
 from ..config import load_config
 from . import Command
 
@@ -23,7 +23,7 @@ class Config(Command):
 
     def run(self, args):
         if args.path:
-            print(config_path())
+            print(_config_path())
         else:
             print(json.dumps(load_config(), indent=4))
 

anemoi/utils/config.py

@@ -104,12 +104,54 @@ class DotDict(dict):
 
 CONFIG = {}
 CHECKED = {}
-CONFIG_LOCK = threading.Lock()
+CONFIG_LOCK = threading.RLock()
 QUIET = False
 
 
-def config_path(name="settings.toml"):
+def _find(config, what, result=None):
+    if result is None:
+        result = []
+
+    if isinstance(config, list):
+        for i in config:
+            _find(i, what, result)
+        return result
+
+    if isinstance(config, dict):
+        if what in config:
+            result.append(config[what])
+
+        for k, v in config.items():
+            _find(v, what, result)
+
+    return result
+
+
+def _merge_dicts(a, b):
+    for k, v in b.items():
+        if k in a and isinstance(a[k], dict) and isinstance(v, dict):
+            _merge_dicts(a[k], v)
+        else:
+            a[k] = v
+
+
+def _set_defaults(a, b):
+    for k, v in b.items():
+        if k in a and isinstance(a[k], dict) and isinstance(v, dict):
+            _set_defaults(a[k], v)
+        else:
+            a.setdefault(k, v)
+
+
+def _config_path(name="settings.toml"):
     global QUIET
+
+    if name.startswith("/") or name.startswith("."):
+        return name
+
+    if name.startswith("~"):
+        return os.path.expanduser(name)
+
     full = os.path.join(os.path.expanduser("~"), ".config", "anemoi", name)
     os.makedirs(os.path.dirname(full), exist_ok=True)
 
@@ -133,46 +175,77 @@ def config_path(name="settings.toml"):
     return full
 
 
-def _load(path):
-    if path.endswith(".json"):
-        with open(path, "rb") as f:
-            return json.load(f)
+def load_any_dict_format(path):
+    """Load a configuration file in any supported format: JSON, YAML and TOML.
+
+    Returns
+    -------
+    dict
+        The decoded configuration file.
+    """
+
+    try:
+        if path.endswith(".json"):
+            with open(path, "rb") as f:
+                return json.load(f)
 
-    if path.endswith(".yaml") or path.endswith(".yml"):
-        with open(path, "rb") as f:
-            return yaml.safe_load(f)
+        if path.endswith(".yaml") or path.endswith(".yml"):
+            with open(path, "rb") as f:
+                return yaml.safe_load(f)
 
-    if path.endswith(".toml"):
-        with open(path, "rb") as f:
-            return tomllib.load(f)
+        if path.endswith(".toml"):
+            with open(path, "rb") as f:
+                return tomllib.load(f)
+    except (json.JSONDecodeError, yaml.YAMLError, tomllib.TOMLDecodeError) as e:
+        LOG.warning(f"Failed to parse config file {path}", exc_info=e)
+        return ValueError(f"Failed to parse config file {path} [{e}]")
 
     return open(path).read()
 
 
-def _load_config(name="settings.toml"):
+def _load_config(name="settings.toml", secrets=None, defaults=None):
 
     if name in CONFIG:
         return CONFIG[name]
 
-    conf = config_path(name)
-
-    if os.path.exists(conf):
-        config = _load(conf)
+    path = _config_path(name)
+    if os.path.exists(path):
+        config = load_any_dict_format(path)
     else:
         config = {}
 
-    if isinstance(config, dict):
-        CONFIG[name] = DotDict(config)
-    else:
-        CONFIG[name] = config
+    if defaults is not None:
+        if isinstance(defaults, str):
+            defaults = load_raw_config(defaults)
+        _set_defaults(config, defaults)
+
+    if secrets is not None:
+        if isinstance(secrets, str):
+            secrets = [secrets]
+
+        base, ext = os.path.splitext(path)
+        secret_name = base + ".secrets" + ext
+
+        found = set()
+        for secret in secrets:
+            if _find(config, secret):
+                found.add(secret)
+
+        if found:
+            check_config_mode(name, secret_name, found)
+
+        check_config_mode(secret_name, None)
+        secret_config = _load_config(secret_name)
+        _merge_dicts(config, secret_config)
 
+    CONFIG[name] = DotDict(config)
     return CONFIG[name]
 
 
 def _save_config(name, data):
     CONFIG.pop(name, None)
 
-    conf = config_path(name)
+    conf = _config_path(name)
 
     if conf.endswith(".json"):
         with open(conf, "w") as f:
@@ -207,7 +280,7 @@ def save_config(name, data):
         _save_config(name, data)
 
 
-def load_config(name="settings.toml"):
+def load_config(name="settings.toml", secrets=None, defaults=None):
     """Read a configuration file.
 
     Parameters
@@ -220,11 +293,21 @@ def load_config(name="settings.toml"):
     DotDict or str
         Return DotDict if it is a dictionary, otherwise the raw data
     """
+
     with CONFIG_LOCK:
-        return _load_config(name)
+        return _load_config(name, secrets, defaults)
+
+
+def load_raw_config(name, default=None):
+
+    path = _config_path(name)
+    if os.path.exists(path):
+        return load_any_dict_format(path)
+
+    return default
 
 
-def check_config_mode(name="settings.toml"):
+def check_config_mode(name="settings.toml", secrets_name=None, secrets=None):
     """Check that a configuration file is secure.
 
     Parameters
@@ -240,8 +323,18 @@ def check_config_mode(name="settings.toml"):
     with CONFIG_LOCK:
         if name in CHECKED:
             return
-        conf = config_path(name)
+
+        conf = _config_path(name)
+        if not os.path.exists(conf):
+            return
         mode = os.stat(conf).st_mode
         if mode & 0o777 != 0o600:
-            raise SystemError(f"Configuration file {conf} is not secure. " "Please run `chmod 600 {conf}`.")
+            if secrets_name:
+                secret_path = _config_path(secrets_name)
+                raise SystemError(
+                    f"Configuration file {conf} should not hold entries {secrets}.\n"
+                    f"Please move them to {secret_path}."
+                )
+            raise SystemError(f"Configuration file {conf} is not secure.\n" f"Please run `chmod 600 {conf}`.")
+
         CHECKED[name] = True

anemoi/utils/dates.py

@@ -9,6 +9,8 @@
 import calendar
 import datetime
 
+from .hindcasts import HindcastDatesTimes
+
 
 def normalise_frequency(frequency):
     if isinstance(frequency, int):
@@ -158,37 +160,6 @@ class DateTimes:
             date += self.increment
 
 
-class HindcastDatesTimes:
-    """The HindcastDatesTimes class is an iterator that generates datetime objects within a given range."""
-
-    def __init__(self, reference_dates, years=20):
-        """_summary_
-
-        Parameters
-        ----------
-        reference_dates : _type_
-            _description_
-        years : int, optional
-            _description_, by default 20
-        """
-
-        self.reference_dates = reference_dates
-
-        if isinstance(years, list):
-            self.years = years
-        else:
-            self.years = range(1, years + 1)
-
-    def __iter__(self):
-        for reference_date in self.reference_dates:
-            for year in self.years:
-                if reference_date.month == 2 and reference_date.day == 29:
-                    date = datetime.datetime(reference_date.year - year, 2, 28)
-                else:
-                    date = datetime.datetime(reference_date.year - year, reference_date.month, reference_date.day)
-                yield (date, reference_date)
-
-
 class Year(DateTimes):
     """Year is defined as the months of January to December."""
 

anemoi/utils/hindcasts.py

@@ -0,0 +1,40 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import datetime
+
+
+class HindcastDatesTimes:
+    """The HindcastDatesTimes class is an iterator that generates datetime objects within a given range."""
+
+    def __init__(self, reference_dates, years=20):
+        """_summary_
+
+        Parameters
+        ----------
+        reference_dates : _type_
+            _description_
+        years : int, optional
+            _description_, by default 20
+        """
+
+        self.reference_dates = reference_dates
+
+        if isinstance(years, list):
+            self.years = years
+        else:
+            self.years = range(1, years + 1)
+
+    def __iter__(self):
+        for reference_date in self.reference_dates:
+            for year in self.years:
+                if reference_date.month == 2 and reference_date.day == 29:
+                    date = datetime.datetime(reference_date.year - year, 2, 28)
+                else:
+                    date = datetime.datetime(reference_date.year - year, reference_date.month, reference_date.day)
+                yield (date, reference_date)

anemoi/utils/humanize.py

@@ -10,6 +10,7 @@
 """Generate human readable strings"""
 
 import datetime
+import json
 import re
 from collections import defaultdict
 
@@ -189,7 +190,7 @@ def __(n):
     return "th"
 
 
-def when(then, now=None, short=True):
+def when(then, now=None, short=True, use_utc=False):
     """Generate a human readable string for a date, relative to now
 
     >>> when(datetime.datetime.now() - datetime.timedelta(hours=2))
@@ -225,7 +226,10 @@ def when(then, now=None, short=True):
     last = "last"
 
     if now is None:
-        now = datetime.datetime.now()
+        if use_utc:
+            now = datetime.datetime.utcnow()
+        else:
+            now = datetime.datetime.now()
 
     diff = (now - then).total_seconds()
 
@@ -472,3 +476,57 @@ def rounded_datetime(d):
         d = d + datetime.timedelta(seconds=1)
     d = d.replace(microsecond=0)
     return d
+
+
+def json_pretty_dump(obj, max_line_length=120, default=str):
+    """Custom JSON dump function that keeps dicts and lists on one line if they are short enough.
+
+    Parameters
+    ----------
+    obj
+        The object to be dumped as JSON.
+    max_line_length
+        Maximum allowed line length for pretty-printing.
+
+    Returns
+    -------
+    unknown
+        JSON string.
+    """
+
+    def _format_json(obj, indent_level=0):
+        """Helper function to format JSON objects with custom pretty-print rules.
+
+        Parameters
+        ----------
+        obj
+            The object to format.
+        indent_level
+            Current indentation level.
+
+        Returns
+        -------
+        unknown
+            Formatted JSON string.
+        """
+        indent = " " * 4 * indent_level
+        if isinstance(obj, dict):
+            items = []
+            for key, value in obj.items():
+                items.append(f'"{key}": {_format_json(value, indent_level + 1)}')
+            line = "{" + ", ".join(items) + "}"
+            if len(line) <= max_line_length:
+                return line
+            else:
+                return "{\n" + ",\n".join([f"{indent}    {item}" for item in items]) + "\n" + indent + "}"
+        elif isinstance(obj, list):
+            items = [_format_json(item, indent_level + 1) for item in obj]
+            line = "[" + ", ".join(items) + "]"
+            if len(line) <= max_line_length:
+                return line
+            else:
+                return "[\n" + ",\n".join([f"{indent}    {item}" for item in items]) + "\n" + indent + "]"
+        else:
+            return json.dumps(obj, default=default)
+
+    return _format_json(obj)

anemoi/utils/s3.py

@@ -18,7 +18,7 @@ to use a different S3 compatible service::
 
 """
 
-import concurrent
+import concurrent.futures
 import logging
 import os
 import threading
@@ -26,7 +26,6 @@ from copy import deepcopy
 
 import tqdm
 
-from .config import check_config_mode
 from .config import load_config
 from .humanize import bytes
 
@@ -41,9 +40,7 @@ thread_local = threading.local()
 def s3_client(bucket):
     import boto3
 
-    config = load_config()
-    if "object-storage" in config:
-        check_config_mode()
+    config = load_config(secrets=["aws_access_key_id", "aws_secret_access_key"])
 
     if not hasattr(thread_local, "s3_clients"):
         thread_local.s3_clients = {}
@@ -51,8 +48,14 @@ def s3_client(bucket):
     if bucket not in thread_local.s3_clients:
 
         options = {}
-        options.update(config.get("object-storage", {}))
-        options.update(config.get("object-storage", {}).get(bucket, {}))
+        cfg = config.get("object-storage", {})
+        for k, v in cfg.items():
+            if isinstance(v, (str, int, float, bool)):
+                options[k] = v
+
+        for k, v in cfg.get(bucket, {}).items():
+            if isinstance(v, (str, int, float, bool)):
+                options[k] = v
 
         type = options.pop("type", "s3")
         if type != "s3":
@@ -420,11 +423,13 @@ def list_folder(folder):
         A list of the subfolders names in the folder.
     """
 
+    print(folder)
     assert folder.startswith("s3://")
     if not folder.endswith("/"):
         folder += "/"
 
     _, _, bucket, prefix = folder.split("/", 3)
+    print(bucket, prefix)
 
     s3 = s3_client(bucket)
     paginator = s3.get_paginator("list_objects_v2")

{anemoi_utils-0.3.7.dist-info → anemoi_utils-0.3.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.7
+Version: 0.3.9
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License: Apache License

anemoi_utils-0.3.9.dist-info/RECORD

@@ -0,0 +1,25 @@
+anemoi/utils/__init__.py,sha256=zZZpbKIoGWwdCOuo6YSruLR7C0GzvzI1Wzhyqaa0K7M,456
+anemoi/utils/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
+anemoi/utils/_version.py,sha256=nV2HEiFwTdaOZoFEyVxxG_D8Oq_nlSmX2vHL4jK4h6w,411
+anemoi/utils/caching.py,sha256=HrC9aFHlcCTaM2Z5u0ivGIXz7eFu35UQQhUuwwuG2pk,1743
+anemoi/utils/checkpoints.py,sha256=1_3mg4B-ykTVfIvIUEv7IxGyREx_ZcilVbB3U-V6O6I,5165
+anemoi/utils/cli.py,sha256=4uMy6LyL9VPg9ZVNKsimahT5BBVPQgWWk7LUel5dQZE,3545
+anemoi/utils/config.py,sha256=_Kzvu69_XlXRDh1Ane_VJ52z-3oyJ4DVXczBycvOGuY,8798
+anemoi/utils/dates.py,sha256=P-AW-dViogxn4t8Q10sO1HldJ2-lez6ILrx6z59tA10,7454
+anemoi/utils/grib.py,sha256=gVfo4KYQv31iRyoqRDwk5tiqZDUgOIvhag_kO0qjYD0,3067
+anemoi/utils/hindcasts.py,sha256=X8k-81ltmkTDHdviY0SJgvMg7XDu07xoc5ALlUxyPoo,1453
+anemoi/utils/humanize.py,sha256=v5fRfrEJduVDhre3TPRLYyYAcYmuUvVMoINtElu0YYU,12112
+anemoi/utils/provenance.py,sha256=v54L9jF1JgYcclOhg3iojRl1v3ajbiWz_oc289xTgO4,9574
+anemoi/utils/s3.py,sha256=lSJc0Oepl3qqwyiDW9ACf320VfPidncwTkIYAmDU754,16087
+anemoi/utils/text.py,sha256=4Zlc4r9dzRjkKL9xqp2vuQsoJY15bJ3y_Xv3YW_XsmU,8510
+anemoi/utils/timer.py,sha256=JKOgFkpJxmVRn57DEBolmTGwr25P-ePTWASBd8CLeqM,972
+anemoi/utils/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
+anemoi/utils/commands/config.py,sha256=lXbbP4HrNez0wh0Fa2eLmTpA3le0EGNLPO4g3JkcGxQ,826
+anemoi/utils/mars/__init__.py,sha256=RAeY8gJ7ZvsPlcIvrQ4fy9xVHs3SphTAPw_XJDtNIKo,1750
+anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
+anemoi_utils-0.3.9.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_utils-0.3.9.dist-info/METADATA,sha256=mDJUF1ukIQDF47RGDT-epp3pKMeN-Qf6O0-wPyGHS54,15513
+anemoi_utils-0.3.9.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
+anemoi_utils-0.3.9.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.3.9.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.3.9.dist-info/RECORD,,

{anemoi_utils-0.3.7.dist-info → anemoi_utils-0.3.9.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.1)
+Generator: setuptools (70.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

anemoi_utils-0.3.7.dist-info/RECORD

@@ -1,24 +0,0 @@
-anemoi/utils/__init__.py,sha256=zZZpbKIoGWwdCOuo6YSruLR7C0GzvzI1Wzhyqaa0K7M,456
-anemoi/utils/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
-anemoi/utils/_version.py,sha256=vVN20516E2VTC9JNgtvqrQNlj5XptaB_a5z2XL8NFxg,411
-anemoi/utils/caching.py,sha256=HrC9aFHlcCTaM2Z5u0ivGIXz7eFu35UQQhUuwwuG2pk,1743
-anemoi/utils/checkpoints.py,sha256=1_3mg4B-ykTVfIvIUEv7IxGyREx_ZcilVbB3U-V6O6I,5165
-anemoi/utils/cli.py,sha256=w6YVYfJV-50Zm9FrO0KNrrIWDdgj5hPjxJvgAh391NY,3308
-anemoi/utils/config.py,sha256=BKH-ZOIVZWhuvKxmhK33JVpIr3q7Tdx-tI8kwd4SvgA,6175
-anemoi/utils/dates.py,sha256=Ot9OTY1uFvHxW1EU4DPv3oUqmzvkXTwKuwhlfVlY788,8426
-anemoi/utils/grib.py,sha256=gVfo4KYQv31iRyoqRDwk5tiqZDUgOIvhag_kO0qjYD0,3067
-anemoi/utils/humanize.py,sha256=LD6dGnqChxA5j3tMhSybsAGRQzi33d_qS9pUoUHubkc,10330
-anemoi/utils/provenance.py,sha256=v54L9jF1JgYcclOhg3iojRl1v3ajbiWz_oc289xTgO4,9574
-anemoi/utils/s3.py,sha256=icYeOlCnb0vfKDMIR_Rq6Machl-3KB67sAFSi8OmftI,15910
-anemoi/utils/text.py,sha256=4Zlc4r9dzRjkKL9xqp2vuQsoJY15bJ3y_Xv3YW_XsmU,8510
-anemoi/utils/timer.py,sha256=JKOgFkpJxmVRn57DEBolmTGwr25P-ePTWASBd8CLeqM,972
-anemoi/utils/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
-anemoi/utils/commands/config.py,sha256=KEffXZh0ZQfn8t6LXresfd94kDY0gEyulx9Wto5ttW0,824
-anemoi/utils/mars/__init__.py,sha256=RAeY8gJ7ZvsPlcIvrQ4fy9xVHs3SphTAPw_XJDtNIKo,1750
-anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
-anemoi_utils-0.3.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_utils-0.3.7.dist-info/METADATA,sha256=TR-5CWm6hwvgDbTyu-P4cIoobZOBJmWWdrrdfQSu3eY,15513
-anemoi_utils-0.3.7.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
-anemoi_utils-0.3.7.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.3.7.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.3.7.dist-info/RECORD,,