anemoi-utils 0.4.8__py3-none-any.whl → 0.4.10__py3-none-any.whl

anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.8'
-__version_tuple__ = version_tuple = (0, 4, 8)
+__version__ = version = '0.4.10'
+__version_tuple__ = version_tuple = (0, 4, 10)

anemoi/utils/commands/requests.py

@@ -0,0 +1,44 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import json
+
+from anemoi.utils.mars.requests import print_request
+
+from . import Command
+
+
+class Requests(Command):
+    """Convert a JSON requests file to MARS format."""
+
+    def add_arguments(self, command_parser):
+        command_parser.add_argument("input")
+        command_parser.add_argument("output")
+        command_parser.add_argument("--verb", default="retrieve")
+        command_parser.add_argument("--only-one-field", action="store_true")
+
+    def run(self, args):
+        with open(args.input) as f:
+            requests = json.load(f)
+
+        if args.only_one_field:
+            for r in requests:
+                for key in (
+                    "grid",
+                    "area",
+                ):
+                    r.pop(key, None)
+                for k, v in list(r.items()):
+                    if isinstance(v, list):
+                        r[k] = v[-1]
+
+        with open(args.output, "w") as f:
+            for r in requests:
+                print_request(args.verb, r, file=f)
+
+
+command = Requests

anemoi/utils/config.py

@@ -50,14 +50,14 @@ class DotDict(dict):
         super().__init__(*args, **kwargs)
 
         for k, v in self.items():
-            if isinstance(v, dict):
+            if isinstance(v, dict) or is_omegaconf_dict(v):
                 self[k] = DotDict(v)
 
-            if isinstance(v, list):
-                self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
+            if isinstance(v, list) or is_omegaconf_list(v):
+                self[k] = [DotDict(i) if isinstance(i, dict) or is_omegaconf_dict(i) else i for i in v]
 
             if isinstance(v, tuple):
-                self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
+                self[k] = [DotDict(i) if isinstance(i, dict) or is_omegaconf_dict(i) else i for i in v]
 
     @classmethod
     def from_file(cls, path: str):
@@ -106,6 +106,24 @@ class DotDict(dict):
         return f"DotDict({super().__repr__()})"
 
 
+def is_omegaconf_dict(value) -> bool:
+    try:
+        from omegaconf import DictConfig
+
+        return isinstance(value, DictConfig)
+    except ImportError:
+        return False
+
+
+def is_omegaconf_list(value) -> bool:
+    try:
+        from omegaconf import ListConfig
+
+        return isinstance(value, ListConfig)
+    except ImportError:
+        return False
+
+
 CONFIG = {}
 CHECKED = {}
 CONFIG_LOCK = threading.RLock()
@@ -376,3 +394,11 @@ def find(metadata, what, result=None, *, select: callable = None):
             find(v, what, result)
 
     return result
+
+
+def merge_configs(*configs):
+    result = {}
+    for config in configs:
+        _merge_dicts(result, config)
+
+    return result

anemoi/utils/logs.py

@@ -0,0 +1,40 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+"""Logging utilities."""
+
+import logging
+import threading
+
+thread_local = threading.local()
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+def set_logging_name(name):
+    thread_local.logging_name = name
+
+
+class ThreadCustomFormatter(logging.Formatter):
+    def format(self, record):
+        record.logging_name = thread_local.logging_name
+        return super().format(record)
+
+
+def enable_logging_name(name="main"):
+    thread_local.logging_name = name
+
+    formatter = ThreadCustomFormatter("%(asctime)s - %(logging_name)s - %(levelname)s - %(message)s")
+
+    logger = logging.getLogger()
+
+    for handler in logger.handlers:
+        handler.setFormatter(formatter)

anemoi/utils/mars/requests.py

@@ -0,0 +1,22 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import sys
+
+
+def print_request(verb, request, file=sys.stdout):
+    r = [verb]
+    for k, v in request.items():
+        if not isinstance(v, (list, tuple, set)):
+            v = [v]
+        v = [str(_) for _ in v]
+        v = "/".join(v)
+        r.append(f"{k}={v}")
+
+    r = ",\n   ".join(r)
+    print(r, file=file)
+    print(file=file)

anemoi/utils/remote/__init__.py

@@ -199,6 +199,8 @@ class Transfer:
     ):
         if target == ".":
             target = os.path.basename(source)
+            if not target:
+                target = os.path.basename(os.path.dirname(source))
 
         temporary_target = {
             False: None,

{anemoi_utils-0.4.8.dist-info → anemoi_utils-0.4.10.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.4.8
+Version: 0.4.10
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
-License: Apache License
+License:                                  Apache License
                                    Version 2.0, January 2004
                                 http://www.apache.org/licenses/
         
@@ -225,31 +225,30 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: aniso8601
+Requires-Dist: importlib-metadata; python_version < "3.10"
 Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
+Requires-Dist: tomli; python_version < "3.11"
 Requires-Dist: tqdm
-Requires-Dist: importlib-metadata ; python_version < "3.10"
-Requires-Dist: tomli ; python_version < "3.11"
 Provides-Extra: all
-Requires-Dist: anemoi-utils[grib,provenance,text] ; extra == 'all'
+Requires-Dist: anemoi-utils[grib,provenance,text]; extra == "all"
 Provides-Extra: dev
-Requires-Dist: anemoi-utils[all,docs,tests] ; extra == 'dev'
+Requires-Dist: anemoi-utils[all,docs,tests]; extra == "dev"
 Provides-Extra: docs
-Requires-Dist: nbsphinx ; extra == 'docs'
-Requires-Dist: pandoc ; extra == 'docs'
-Requires-Dist: requests ; extra == 'docs'
-Requires-Dist: sphinx ; extra == 'docs'
-Requires-Dist: sphinx-argparse <0.5 ; extra == 'docs'
-Requires-Dist: sphinx-rtd-theme ; extra == 'docs'
-Requires-Dist: termcolor ; extra == 'docs'
+Requires-Dist: nbsphinx; extra == "docs"
+Requires-Dist: pandoc; extra == "docs"
+Requires-Dist: requests; extra == "docs"
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-argparse<0.5; extra == "docs"
+Requires-Dist: sphinx-rtd-theme; extra == "docs"
+Requires-Dist: termcolor; extra == "docs"
 Provides-Extra: grib
-Requires-Dist: requests ; extra == 'grib'
+Requires-Dist: requests; extra == "grib"
 Provides-Extra: provenance
-Requires-Dist: gitpython ; extra == 'provenance'
-Requires-Dist: nvsmi ; extra == 'provenance'
+Requires-Dist: gitpython; extra == "provenance"
+Requires-Dist: nvsmi; extra == "provenance"
 Provides-Extra: tests
-Requires-Dist: pytest ; extra == 'tests'
+Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text
-Requires-Dist: termcolor ; extra == 'text'
-Requires-Dist: wcwidth ; extra == 'text'
-
+Requires-Dist: termcolor; extra == "text"
+Requires-Dist: wcwidth; extra == "text"

{anemoi_utils-0.4.8.dist-info → anemoi_utils-0.4.10.dist-info}/RECORD

@@ -1,15 +1,16 @@
 anemoi/utils/__init__.py,sha256=0u0eIdu5-H1frf6V4KHpNmlh_SS-bJnxjzIejlsLqdw,702
 anemoi/utils/__main__.py,sha256=5NW2A3OgTimB4ptwYThivIRSeCrvabMuvnr8mmnVx0E,715
-anemoi/utils/_version.py,sha256=WKH2DAU2byiQfZy9XW8vTna5aTSXgkj8aA__Cy4DLwQ,411
+anemoi/utils/_version.py,sha256=OK_y1wkdggDGF27DpKS31C6p5FSDPep8dDgBhyKlfYE,413
 anemoi/utils/caching.py,sha256=0cznpvaaox14NSVi-Q3PqumfuGtXo0YNcEFwDPxvMZw,1948
 anemoi/utils/checkpoints.py,sha256=q8QqKlZ6qChjzEfq7KM1gVXuyqgsVRGIb4dJFtkGk58,7774
 anemoi/utils/cli.py,sha256=rmMP60VY3em99rQP6TCrKibMngWwVe5h_0GDcf16c5U,4117
 anemoi/utils/compatibility.py,sha256=0_nIcbdQbNMrS6AkqrBgJGJlSJXW8R23ncaZaDwdJ4c,2190
-anemoi/utils/config.py,sha256=At0k_NtbJ74S72sEZIYgdEaLf5KCEvxIJi9FGLcUwrE,9872
+anemoi/utils/config.py,sha256=c7dh9jRKHQ4DmwNTuniOZcrqZQgxU5VLQqH6rHSE7bI,10460
 anemoi/utils/dates.py,sha256=wwYD5_QI7EWY_jhpENNYtL5O7fjwYkzmqHkNoayvmrY,12452
 anemoi/utils/grib.py,sha256=zBICyOsYtR_9px1C5UDT6wL_D6kiIhUi_00kjFmas5c,3047
 anemoi/utils/hindcasts.py,sha256=TEYDmrZUajuhp_dfWeg6z5c6XfntE-mwugUQJyAgUco,1419
 anemoi/utils/humanize.py,sha256=tSQkiUHiDj3VYk-DeruHp9P79sJO1b0whsPBphqy9qA,16627
+anemoi/utils/logs.py,sha256=o0xXiO2BdG_bZkljxxI2TKlCiA5QbWHgAUlYM53lirE,1058
 anemoi/utils/provenance.py,sha256=SqOiNoY1y36Zec83Pjt7OhihbwxMyknscfmogHCuriA,10894
 anemoi/utils/registry.py,sha256=Iit_CfTGuoVffXkZA2A5mUXb4AdGIUX9TpnUqWT4HJ0,4291
 anemoi/utils/s3.py,sha256=UOEETko08hnIXeFy8u10eQbqpcape9d-L6IgsjFMe18,2473
@@ -19,14 +20,16 @@ anemoi/utils/text.py,sha256=Xfr_3wvsjg7m-BwvdJVz1bV6f5KNMnGIIFRtXaiMfbs,10496
 anemoi/utils/timer.py,sha256=Twnr3GZu-n0WzgboELRKJWs87qyDYqy6Dwr9cQ_JG18,1803
 anemoi/utils/commands/__init__.py,sha256=O5W3yHZywRoAqmRUioAr3zMCh0hGVV18wZYGvc00ioM,698
 anemoi/utils/commands/config.py,sha256=zt4PFATYJ-zs0C5mpUlrQ4Fj5m1kM3CcsszUP1VBbzA,816
+anemoi/utils/commands/requests.py,sha256=7joRYnJUzJh5O8Pqkqa-s9M9woHy-Z86czp00uCZXGc,1448
 anemoi/utils/mars/__init__.py,sha256=kvbu-gSaYI9jSNEzfQltrtHPVIameYGoLjOJKwI7x_U,1723
 anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
-anemoi/utils/remote/__init__.py,sha256=f6pfM0PiCabyRmqDFOsXegnXbOvkaUiHGeC9ThU9Ero,11545
+anemoi/utils/mars/requests.py,sha256=0khe_mbq4GNueR_B8fGPTBoWHtCfjQvtoKXOSVm6La4,759
+anemoi/utils/remote/__init__.py,sha256=-_AA1xm9GpagW5zP0PGpz-3SRKEUjw_AGSNd_bhuh7g,11639
 anemoi/utils/remote/s3.py,sha256=hykbVlh1_aFI00FWjgm_FWIMfVCTFiQf_cq8_gAo31s,11976
 anemoi/utils/remote/ssh.py,sha256=3lqFpY9CEor_DvIK9ZxSmj3rND-366Sm9R3Vw61sWSs,4695
-anemoi_utils-0.4.8.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_utils-0.4.8.dist-info/METADATA,sha256=mfdZdTOTRN6Ata0zc4V7UkIKgKImSXI3IO1JumHWwss,15222
-anemoi_utils-0.4.8.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-anemoi_utils-0.4.8.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.4.8.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.4.8.dist-info/RECORD,,
+anemoi_utils-0.4.10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_utils-0.4.10.dist-info/METADATA,sha256=CHhddg_iojS8zfV67wBSG0DIlGeZq7ShPf7UiSAdu3Y,15237
+anemoi_utils-0.4.10.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+anemoi_utils-0.4.10.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.4.10.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.4.10.dist-info/RECORD,,

{anemoi_utils-0.4.8.dist-info → anemoi_utils-0.4.10.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.5.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any