anemoi-utils 0.3.8__tar.gz → 0.3.10__tar.gz

{anemoi_utils-0.3.8 → anemoi_utils-0.3.10}/.github/workflows/python-publish.yml

@@ -6,7 +6,7 @@ name: Upload Python Package
 on:
 
   push: {}
-
+  pull_request:
   release:
     types: [created]
 

{anemoi_utils-0.3.8/src/anemoi_utils.egg-info → anemoi_utils-0.3.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.8
+Version: 0.3.10
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.3.8 → anemoi_utils-0.3.10}/src/anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.8'
-__version_tuple__ = version_tuple = (0, 3, 8)
+__version__ = version = '0.3.10'
+__version_tuple__ = version_tuple = (0, 3, 10)

{anemoi_utils-0.3.8 → anemoi_utils-0.3.10}/src/anemoi/utils/cli.py

@@ -16,6 +16,8 @@ LOG = logging.getLogger(__name__)
 
 
 class Command:
+    accept_unknown_args = False
+
     def run(self, args):
         raise NotImplementedError(f"Command not implemented: {args.command}")
 

{anemoi_utils-0.3.8 → anemoi_utils-0.3.10}/src/anemoi/utils/config.py

@@ -104,12 +104,54 @@ class DotDict(dict):
 
 CONFIG = {}
 CHECKED = {}
-CONFIG_LOCK = threading.Lock()
+CONFIG_LOCK = threading.RLock()
 QUIET = False
 
 
+def _find(config, what, result=None):
+    if result is None:
+        result = []
+
+    if isinstance(config, list):
+        for i in config:
+            _find(i, what, result)
+        return result
+
+    if isinstance(config, dict):
+        if what in config:
+            result.append(config[what])
+
+        for k, v in config.items():
+            _find(v, what, result)
+
+    return result
+
+
+def _merge_dicts(a, b):
+    for k, v in b.items():
+        if k in a and isinstance(a[k], dict) and isinstance(v, dict):
+            _merge_dicts(a[k], v)
+        else:
+            a[k] = v
+
+
+def _set_defaults(a, b):
+    for k, v in b.items():
+        if k in a and isinstance(a[k], dict) and isinstance(v, dict):
+            _set_defaults(a[k], v)
+        else:
+            a.setdefault(k, v)
+
+
 def config_path(name="settings.toml"):
     global QUIET
+
+    if name.startswith("/") or name.startswith("."):
+        return name
+
+    if name.startswith("~"):
+        return os.path.expanduser(name)
+
     full = os.path.join(os.path.expanduser("~"), ".config", "anemoi", name)
     os.makedirs(os.path.dirname(full), exist_ok=True)
 
@@ -133,7 +175,15 @@ def config_path(name="settings.toml"):
     return full
 
 
-def _load(path):
+def load_any_dict_format(path):
+    """Load a configuration file in any supported format: JSON, YAML and TOML.
+
+    Returns
+    -------
+    dict
+        The decoded configuration file.
+    """
+
     try:
         if path.endswith(".json"):
             with open(path, "rb") as f:
@@ -153,23 +203,42 @@ def _load(path):
     return open(path).read()
 
 
-def _load_config(name="settings.toml"):
+def _load_config(name="settings.toml", secrets=None, defaults=None):
 
     if name in CONFIG:
         return CONFIG[name]
 
-    conf = config_path(name)
-
-    if os.path.exists(conf):
-        config = _load(conf)
+    path = config_path(name)
+    if os.path.exists(path):
+        config = load_any_dict_format(path)
     else:
         config = {}
 
-    if isinstance(config, dict):
-        CONFIG[name] = DotDict(config)
-    else:
-        CONFIG[name] = config
+    if defaults is not None:
+        if isinstance(defaults, str):
+            defaults = load_raw_config(defaults)
+        _set_defaults(config, defaults)
+
+    if secrets is not None:
+        if isinstance(secrets, str):
+            secrets = [secrets]
 
+        base, ext = os.path.splitext(path)
+        secret_name = base + ".secrets" + ext
+
+        found = set()
+        for secret in secrets:
+            if _find(config, secret):
+                found.add(secret)
+
+        if found:
+            check_config_mode(name, secret_name, found)
+
+        check_config_mode(secret_name, None)
+        secret_config = _load_config(secret_name)
+        _merge_dicts(config, secret_config)
+
+    CONFIG[name] = DotDict(config)
     return CONFIG[name]
 
 
@@ -211,7 +280,7 @@ def save_config(name, data):
         _save_config(name, data)
 
 
-def load_config(name="settings.toml"):
+def load_config(name="settings.toml", secrets=None, defaults=None):
     """Read a configuration file.
 
     Parameters
@@ -224,20 +293,21 @@ def load_config(name="settings.toml"):
     DotDict or str
         Return DotDict if it is a dictionary, otherwise the raw data
     """
+
     with CONFIG_LOCK:
-        return _load_config(name)
+        return _load_config(name, secrets, defaults)
 
 
 def load_raw_config(name, default=None):
 
     path = config_path(name)
     if os.path.exists(path):
-        return _load(path)
+        return load_any_dict_format(path)
 
     return default
 
 
-def check_config_mode(name="settings.toml"):
+def check_config_mode(name="settings.toml", secrets_name=None, secrets=None):
     """Check that a configuration file is secure.
 
     Parameters
@@ -253,8 +323,18 @@ def check_config_mode(name="settings.toml"):
     with CONFIG_LOCK:
         if name in CHECKED:
             return
+
         conf = config_path(name)
+        if not os.path.exists(conf):
+            return
         mode = os.stat(conf).st_mode
         if mode & 0o777 != 0o600:
-            raise SystemError(f"Configuration file {conf} is not secure. " "Please run `chmod 600 {conf}`.")
+            if secrets_name:
+                secret_path = config_path(secrets_name)
+                raise SystemError(
+                    f"Configuration file {conf} should not hold entries {secrets}.\n"
+                    f"Please move them to {secret_path}."
+                )
+            raise SystemError(f"Configuration file {conf} is not secure.\n" f"Please run `chmod 600 {conf}`.")
+
         CHECKED[name] = True

{anemoi_utils-0.3.8 → anemoi_utils-0.3.10}/src/anemoi/utils/humanize.py

@@ -190,7 +190,7 @@ def __(n):
     return "th"
 
 
-def when(then, now=None, short=True):
+def when(then, now=None, short=True, use_utc=False):
     """Generate a human readable string for a date, relative to now
 
     >>> when(datetime.datetime.now() - datetime.timedelta(hours=2))
@@ -226,7 +226,10 @@ def when(then, now=None, short=True):
     last = "last"
 
     if now is None:
-        now = datetime.datetime.now()
+        if use_utc:
+            now = datetime.datetime.utcnow()
+        else:
+            now = datetime.datetime.now()
 
     diff = (now - then).total_seconds()
 

{anemoi_utils-0.3.8 → anemoi_utils-0.3.10}/src/anemoi/utils/s3.py

@@ -18,7 +18,7 @@ to use a different S3 compatible service::
 
 """
 
-import concurrent
+import concurrent.futures
 import logging
 import os
 import threading
@@ -26,7 +26,6 @@ from copy import deepcopy
 
 import tqdm
 
-from .config import check_config_mode
 from .config import load_config
 from .humanize import bytes
 
@@ -41,9 +40,7 @@ thread_local = threading.local()
 def s3_client(bucket):
     import boto3
 
-    config = load_config()
-    if "object-storage" in config:
-        check_config_mode()
+    config = load_config(secrets=["aws_access_key_id", "aws_secret_access_key"])
 
     if not hasattr(thread_local, "s3_clients"):
         thread_local.s3_clients = {}
@@ -51,8 +48,14 @@ def s3_client(bucket):
     if bucket not in thread_local.s3_clients:
 
         options = {}
-        options.update(config.get("object-storage", {}))
-        options.update(config.get("object-storage", {}).get(bucket, {}))
+        cfg = config.get("object-storage", {})
+        for k, v in cfg.items():
+            if isinstance(v, (str, int, float, bool)):
+                options[k] = v
+
+        for k, v in cfg.get(bucket, {}).items():
+            if isinstance(v, (str, int, float, bool)):
+                options[k] = v
 
         type = options.pop("type", "s3")
         if type != "s3":
@@ -420,11 +423,13 @@ def list_folder(folder):
         A list of the subfolders names in the folder.
     """
 
+    print(folder)
     assert folder.startswith("s3://")
     if not folder.endswith("/"):
         folder += "/"
 
     _, _, bucket, prefix = folder.split("/", 3)
+    print(bucket, prefix)
 
     s3 = s3_client(bucket)
     paginator = s3.get_paginator("list_objects_v2")

{anemoi_utils-0.3.8 → anemoi_utils-0.3.10/src/anemoi_utils.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.8
+Version: 0.3.10
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.3.8 → anemoi_utils-0.3.10}/tests/test_utils.py

@@ -7,6 +7,8 @@
 
 
 from anemoi.utils.config import DotDict
+from anemoi.utils.config import _merge_dicts
+from anemoi.utils.config import _set_defaults
 from anemoi.utils.grib import paramid_to_shortname
 from anemoi.utils.grib import shortname_to_paramid
 
@@ -30,10 +32,24 @@ def test_dotdict():
     assert d.e[1].a == 3
 
 
+def test_merge_dicts():
+    a = dict(a=1, b=2, c=dict(d=3, e=4))
+    b = dict(a=10, c=dict(a=30, e=40), d=9)
+    _merge_dicts(a, b)
+    assert a == {"a": 10, "b": 2, "c": {"d": 3, "e": 40, "a": 30}, "d": 9}
+
+
+def test_set_defaults():
+    a = dict(a=1, b=2, c=dict(d=3, e=4))
+    b = dict(a=10, c=dict(a=30, e=40), d=9)
+    _set_defaults(a, b)
+    assert a == {"a": 1, "b": 2, "c": {"d": 3, "e": 4, "a": 30}, "d": 9}
+
+
 def test_grib():
     assert shortname_to_paramid("2t") == 167
     assert paramid_to_shortname(167) == "2t"
 
 
 if __name__ == "__main__":
-    test_grib()
+    test_set_defaults()