anemoi-utils 0.4.37__py3-none-any.whl → 0.4.38__py3-none-any.whl

anemoi/utils/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.4.37'
-__version_tuple__ = version_tuple = (0, 4, 37)
+__version__ = version = '0.4.38'
+__version_tuple__ = version_tuple = (0, 4, 38)
 
 __commit_id__ = commit_id = None

anemoi/utils/checkpoints.py

@@ -20,6 +20,7 @@ import zipfile
 from collections.abc import Callable
 from tempfile import TemporaryDirectory
 
+import numpy as np
 import tqdm
 
 LOG = logging.getLogger(__name__)
@@ -147,6 +148,41 @@ def load_supporting_arrays(zipf: zipfile.ZipFile, entries: dict) -> dict:
     return supporting_arrays
 
 
+def _get_supporting_arrays_paths(directory: str, folder: str, supporting_arrays: dict | np.ndarray) -> dict:
+    """Get the paths of supporting arrays."""
+    if supporting_arrays is None:
+        return {}
+
+    if isinstance(supporting_arrays, dict):
+        return {
+            new_key: _get_supporting_arrays_paths(f"{directory}/{folder}", new_key, new_value)
+            for new_key, new_value in supporting_arrays.items()
+        }
+
+    return dict(
+        path=f"{directory}/{folder}.numpy",
+        shape=supporting_arrays.shape,
+        dtype=str(supporting_arrays.dtype),
+    )
+
+
+def _write_array_to_bytes(array: dict | np.ndarray, name: str, entry: dict, zipf: zipfile.ZipFile) -> None:
+    """Write a supporting array to bytes in a zip file."""
+    if isinstance(array, dict):
+        for sub_name, sub_array in array.items():
+            _write_array_to_bytes(sub_array, sub_name, entry[sub_name], zipf)
+        return None
+
+    LOG.info(
+        "Saving supporting array `%s` to %s (shape=%s, dtype=%s)",
+        name,
+        entry["path"],
+        entry["shape"],
+        entry["dtype"],
+    )
+    zipf.writestr(entry["path"], array.tobytes())
+
+
 def save_metadata(
     path: str, metadata: dict, *, supporting_arrays: dict = None, name: str = DEFAULT_NAME, folder: str = DEFAULT_FOLDER
 ) -> None:
@@ -189,29 +225,14 @@ def save_metadata(
         LOG.info("Saving metadata to %s/%s/%s", directory, folder, name)
 
         metadata = metadata.copy()
-        if supporting_arrays is not None:
-            metadata["supporting_arrays_paths"] = {
-                key: dict(path=f"{directory}/{folder}/{key}.numpy", shape=value.shape, dtype=str(value.dtype))
-                for key, value in supporting_arrays.items()
-            }
-        else:
-            metadata["supporting_arrays_paths"] = {}
+        metadata["supporting_arrays_paths"] = _get_supporting_arrays_paths(directory, folder, supporting_arrays)
 
         zipf.writestr(
             f"{directory}/{folder}/{name}",
             json.dumps(metadata),
         )
 
-        for name, entry in metadata["supporting_arrays_paths"].items():
-            value = supporting_arrays[name]
-            LOG.info(
-                "Saving supporting array `%s` to %s (shape=%s, dtype=%s)",
-                name,
-                entry["path"],
-                entry["shape"],
-                entry["dtype"],
-            )
-            zipf.writestr(entry["path"], value.tobytes())
+        _write_array_to_bytes(supporting_arrays, "", metadata["supporting_arrays_paths"], zipf)
 
 
 def _edit_metadata(path: str, name: str, callback: Callable, supporting_arrays: dict | None = None) -> None:

anemoi/utils/dates.py

@@ -260,39 +260,21 @@ def frequency_to_string(frequency: datetime.timedelta) -> str:
         A string representation of the frequency.
     """
 
-    frequency = frequency_to_timedelta(frequency)
+    total_seconds = int(frequency.total_seconds())
 
-    total_seconds = frequency.total_seconds()
     if total_seconds < 0:
         return f"-{frequency_to_string(-frequency)}"
-    assert int(total_seconds) == total_seconds, total_seconds
-    total_seconds = int(total_seconds)
 
-    seconds = total_seconds
+    if total_seconds % (24 * 3600) == 0:
+        return f"{total_seconds // (24 * 3600)}d"
 
-    days = seconds // (24 * 3600)
-    seconds %= 24 * 3600
-    hours = seconds // 3600
-    seconds %= 3600
-    minutes = seconds // 60
-    seconds %= 60
+    if total_seconds % 3600 == 0:
+        return f"{total_seconds // 3600}h"
 
-    if days > 0 and hours == 0 and minutes == 0 and seconds == 0:
-        return f"{days}d"
+    if total_seconds % 60 == 0:
+        return f"{total_seconds // 60}m"
 
-    if days == 0 and hours > 0 and minutes == 0 and seconds == 0:
-        return f"{hours}h"
-
-    if days == 0 and hours == 0 and minutes > 0 and seconds == 0:
-        return f"{minutes}m"
-
-    if days == 0 and hours == 0 and minutes == 0 and seconds > 0:
-        return f"{seconds}s"
-
-    if days > 0:
-        return f"{total_seconds}s"
-
-    return str(frequency)
+    return f"{total_seconds}s"
 
 
 def frequency_to_seconds(frequency: int | str | datetime.timedelta) -> int:

anemoi/utils/testing.py

@@ -114,7 +114,7 @@ class GetTestData:
 
         LOG.info(f"Downloading test data from {url} to {target}")
 
-        download(url, target)
+        download(url, target, maximum_retries=5, retry_after=60, timeout=60)
 
         if gzipped:
             import gzip

{anemoi_utils-0.4.37.dist-info → anemoi_utils-0.4.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.37
+Version: 0.4.38
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_utils-0.4.37.dist-info → anemoi_utils-0.4.38.dist-info}/RECORD

@@ -1,13 +1,13 @@
 anemoi/utils/__init__.py,sha256=G1BlBl5IGz1AKHYkEUNT5x69WOTV0ALEe8appAdYr38,806
 anemoi/utils/__main__.py,sha256=6LlE4MYrPvqqrykxXh7XMi50UZteUY59NeM8P9Zs2dU,910
 anemoi/utils/_environment.py,sha256=jN4y-fNEkIQaz87dJJNBQU-UttFsEcL_edjlnaA6lVc,1488
-anemoi/utils/_version.py,sha256=rrk9b9t6FcjvVt4Tp41VGDj03xKbwyFWENJYjCLYQ_U,706
+anemoi/utils/_version.py,sha256=0idbjr4U1OcAqhKQqoELJq61WRFFz5A1vJU4qmnJRMk,706
 anemoi/utils/caching.py,sha256=wKkpsACniNCFK1nc5Tw18EAv9FJGYdhs5p8qsr8i_FI,6153
-anemoi/utils/checkpoints.py,sha256=DME5s-OG5bwJJscjlO17RWrQe_gbyxlGETeBBzGLO9E,10407
+anemoi/utils/checkpoints.py,sha256=4tfwG7KwCp2Kr4dnMyKiFtZ2bl0Vo8czVq7V69NBIBk,11050
 anemoi/utils/cli.py,sha256=w_edLlUa_S-RoPU3HKSmjjh949QWTl4FXPfR_HKLYDI,7522
 anemoi/utils/compatibility.py,sha256=Yx4Yj3xL5q0PxYccM3N5qGeQxTtEPfaCz4EqjrS7CUc,2175
 anemoi/utils/config.py,sha256=gyWr_4ZDEf5wIFoV9gVf8pHeBFTL9WS81s4et8K-RHE,17934
-anemoi/utils/dates.py,sha256=35KyiyFW8SeWLIm7VeMKBJvNvl2nVu-edRWLX-04uT4,17307
+anemoi/utils/dates.py,sha256=kD72mxNmKyDAkkyiLTZzlueWjAjSA48yK5fN4Wup4dE,16795
 anemoi/utils/devtools.py,sha256=W3OBu96MkXRIl7Qh1SE5Zd6aB1R0QlnmlrlpBYM0fVY,3527
 anemoi/utils/grib.py,sha256=eHz3BVoSB4O-3RARxXFcKRSWtYVbt6lqoLBcssHOp70,3543
 anemoi/utils/grids.py,sha256=l09Hkvq8pZoh5XE8Cy29584-b1GouQSh_WqZks8uuEs,5963
@@ -20,7 +20,7 @@ anemoi/utils/rules.py,sha256=YtJcwkyBJwPjHlM4ZM6icGJXLCRZmcuHCXc-SSR8wgw,6856
 anemoi/utils/s3.py,sha256=bBVlfg3R4sJYOfgrSkhX7MxSVUQEur4LjE86ug8vkNw,4052
 anemoi/utils/sanitise.py,sha256=XkQzibDbu-VFJkJC4WcB9ovkcTkVAynXtkn1Tlc2CC4,4019
 anemoi/utils/sanitize.py,sha256=43ZKDcfVpeXSsJ9TFEc9aZnD6oe2cUh151XnDspM98M,462
-anemoi/utils/testing.py,sha256=u6VyarS9li5X_kqZr6hrqvwEAVjtgY9ZYdIyIsv7kyQ,10789
+anemoi/utils/testing.py,sha256=Zy-HVhqQ_rQ3kXzj9ff5QOSUe_M3OibS5bP9hBGYfRE,10836
 anemoi/utils/text.py,sha256=9M7VepzZLL-jGfFGXDqNxjwpNl7tiU8arkkOqhT9iSE,14361
 anemoi/utils/timer.py,sha256=_leKMYza2faM7JKlGE7LCNy13rbdPnwaCF7PSrI_NmI,3895
 anemoi/utils/commands/__init__.py,sha256=5u_6EwdqYczIAgJfCwRSyQAYFEqh2ZuHHT57g9g7sdI,808
@@ -40,9 +40,9 @@ anemoi/utils/remote/s3.py,sha256=4UMqC4qFRNpZ2YJWhN6L71hqn0E-3tNAhUtkLZ_R1Tw,178
 anemoi/utils/remote/ssh.py,sha256=xNtsawh8okytCKRehkRCVExbHZj-CRUQNormEHglfuw,8088
 anemoi/utils/schemas/__init__.py,sha256=nkinKlsPLPXEjfTYQT1mpKC4cvs-14w_zBkDRxakwxw,698
 anemoi/utils/schemas/errors.py,sha256=lgOXzVTYzAE0qWQf3OZ42vCWixv8lilSqLLhzARBmvI,1831
-anemoi_utils-0.4.37.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
-anemoi_utils-0.4.37.dist-info/METADATA,sha256=kgnjqlC2q10qDTEI5BAkYI3SAfVVnQgrd7Zd1CFBTtg,15692
-anemoi_utils-0.4.37.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-anemoi_utils-0.4.37.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.4.37.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.4.37.dist-info/RECORD,,
+anemoi_utils-0.4.38.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
+anemoi_utils-0.4.38.dist-info/METADATA,sha256=Ct1xBFzI5S9jwsLg-fk7btPCo-iwzuwmssJvwbiMJE0,15692
+anemoi_utils-0.4.38.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+anemoi_utils-0.4.38.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.4.38.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.4.38.dist-info/RECORD,,