anemoi-utils 0.4.28__py3-none-any.whl → 0.4.29__py3-none-any.whl

anemoi/utils/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.28'
-__version_tuple__ = version_tuple = (0, 4, 28)
+__version__ = version = '0.4.29'
+__version_tuple__ = version_tuple = (0, 4, 29)

anemoi/utils/dates.py

@@ -199,6 +199,15 @@ def as_timedelta(frequency: Union[int, str, datetime.timedelta]) -> datetime.tim
     except ValueError:
         pass
 
+    if frequency.startswith(" ") or frequency.startswith(" "):
+        frequency = frequency.strip()
+
+    if frequency.startswith("-"):
+        return -as_timedelta(frequency[1:])
+
+    if frequency.startswith("+"):
+        return as_timedelta(frequency[1:])
+
     if re.match(r"^\d+[hdms]$", frequency, re.IGNORECASE):
         unit = frequency[-1].lower()
         v = int(frequency[:-1])
@@ -261,6 +270,8 @@ def frequency_to_string(frequency: datetime.timedelta) -> str:
     frequency = frequency_to_timedelta(frequency)
 
     total_seconds = frequency.total_seconds()
+    if total_seconds < 0:
+        return f"-{frequency_to_string(-frequency)}"
     assert int(total_seconds) == total_seconds, total_seconds
     total_seconds = int(total_seconds)
 

anemoi/utils/mlflow/auth.py

@@ -22,6 +22,7 @@ from typing import TYPE_CHECKING
 import requests
 from requests.exceptions import HTTPError
 
+from ..config import config_path
 from ..config import load_config
 from ..config import save_config
 from ..remote import robust
@@ -87,6 +88,14 @@ class TokenAuth:
 
     @staticmethod
     def load_config() -> dict:
+        path = config_path(TokenAuth.config_file)
+
+        if not os.path.exists(path):
+            save_config(TokenAuth.config_file, {})
+
+        if os.path.exists(path) and os.stat(path).st_mode & 0o777 != 0o600:
+            os.chmod(path, 0o600)
+
         return load_config(TokenAuth.config_file)
 
     def enabled(fn: Callable) -> Callable:  # noqa: N805

anemoi/utils/mlflow/utils.py

@@ -8,9 +8,7 @@
 # nor does it submit to any jurisdiction.
 from __future__ import annotations
 
-import functools
 import os
-from typing import Any
 
 import requests
 
@@ -44,116 +42,3 @@ def health_check(tracking_uri: str) -> None:
     if not token:
         error_msg += "The server may require authentication, did you forget to turn it on?"
     raise ConnectionError(error_msg)
-
-
-def expand_iterables(
-    params: dict[str, Any],
-    *,
-    size_threshold: int | None = None,
-    recursive: bool = True,
-    delimiter: str = ".",
-) -> dict[str, Any]:
-    """Expand any iterable values to the form {key.i: value_i}.
-
-    If expanded will also add {key.all: [value_0, value_1, ...], key.length: len([value_0, value_1, ...])}.
-
-    If `size_threshold` is not None, expand the iterable only if the length of str(value) is
-    greater than `size_threshold`.
-
-    Parameters
-    ----------
-    params : dict[str, Any]
-        Parameters to be expanded.
-    size_threshold : int | None, optional
-        Threshold of str(value) to expand iterable at.
-        Default is None.
-    recursive : bool, optional
-        Expand nested dictionaries.
-        Default is True.
-    delimiter: str, optional
-        Delimiter to use for keys.
-        Default is ".".
-
-    Returns
-    -------
-    dict[str, Any]
-        Dictionary with all iterable values expanded.
-
-    Examples
-    --------
-        >>> expand_iterables({'a': ['a', 'b', 'c']})
-        {'a.0': 'a', 'a.1': 'b', 'a.2': 'c', 'a.all': ['a', 'b', 'c'], 'a.length': 3}
-        >>> expand_iterables({'a': {'b': ['a', 'b', 'c']}})
-        {'a': {'b.0': 'a', 'b.1': 'b', 'b.2': 'c', 'b.all': ['a', 'b', 'c'], 'b.length': 3}}
-        >>> expand_iterables({'a': ['a', 'b', 'c']}, size_threshold=100)
-        {'a': ['a', 'b', 'c']}
-        >>> expand_iterables({'a': [[0,1,2], 'b', 'c']})
-        {'a.0': {0: 0, 1: 1, 2: 2}, 'a.1': 'b', 'a.2': 'c', 'a.all': [[0, 1, 2], 'b', 'c'], 'a.length': 3}
-    """
-
-    def should_be_expanded(x: Any) -> bool:
-        return size_threshold is None or len(str(x)) > size_threshold
-
-    nested_func = functools.partial(expand_iterables, size_threshold=size_threshold, recursive=recursive)
-
-    def expand(val: dict | list) -> dict[str, Any]:
-        if not recursive:
-            return val
-        if isinstance(val, dict):
-            return nested_func(val)
-        if isinstance(val, list):
-            return nested_func(dict(enumerate(val)))
-        return val
-
-    expanded_params = {}
-
-    for key, value in params.items():
-        if isinstance(value, (list, tuple)):
-            if should_be_expanded(value):
-                for i, v in enumerate(value):
-                    expanded_params[f"{key}{delimiter}{i}"] = expand(v)
-
-                expanded_params[f"{key}{delimiter}all"] = value
-                expanded_params[f"{key}{delimiter}length"] = len(value)
-            else:
-                expanded_params[key] = value
-        else:
-            expanded_params[key] = expand(value)
-    return expanded_params
-
-
-def clean_config_params(params: dict[str, Any]) -> dict[str, Any]:
-    """Clean up params to avoid issues with mlflow.
-
-    Too many logged params will make the server take longer to render the
-    experiment.
-
-    Parameters
-    ----------
-    params : dict[str, Any]
-        Parameters to clean up.
-
-    Returns
-    -------
-    dict[str, Any]
-        Cleaned up params ready for MlFlow.
-    """
-    prefixes_to_remove = [
-        "hardware",
-        "data",
-        "dataloader",
-        "model",
-        "training",
-        "diagnostics",
-        "graph",
-        "metadata.config",
-        "config.dataset.sourcesmetadata.dataset.variables_metadata",
-        "metadata.dataset.sources",
-        "metadata.dataset.specific",
-        "metadata.dataset.variables_metadata",
-    ]
-
-    keys_to_remove = [key for key in params if any(key.startswith(prefix) for prefix in prefixes_to_remove)]
-    for key in keys_to_remove:
-        del params[key]
-    return params

anemoi/utils/sanitise.py

@@ -17,19 +17,24 @@ from urllib.parse import urlencode
 from urllib.parse import urlparse
 from urllib.parse import urlunparse
 
-# Patterns used but earthkit-data for url-patterns and path-patterns
+# Patterns used by earthkit-data for url-patterns and path-patterns
 
-RE1 = re.compile(r"{([^}]*)}")
-RE2 = re.compile(r"\(([^}]*)\)")
+RE1 = re.compile(r"{([^}]*)}")  # {*}
+RE2 = re.compile(r"\(([^}]*)\)")  # (*)
 
 
-def sanitise(obj: Any) -> Any:
-    """Sanitise an object by replacing all full paths with shortened versions and URL passwords with '***'.
+def sanitise(obj: Any, level=1) -> Any:
+    """Sanitise an object by replacing all full paths with shortened versions and URL credentials with '***'.
 
     Parameters
     ----------
     obj : Any
         The object to sanitise.
+    level : int, optional
+        The level of sanitation. The higher levels will also apply the levels below it.
+        - 1: Shorten file paths to file name and hide credentials in URLs (default).
+        - 2: Hide hostnames in URLs.
+        - 3: Hide full file paths and URLs.
 
     Returns
     -------
@@ -37,6 +42,8 @@ def sanitise(obj: Any) -> Any:
         The sanitised object.
     """
 
+    assert level in (1, 2, 3), "level must be 1, 2 or 3"
+
     if isinstance(obj, dict):
         return {sanitise(k): sanitise(v) for k, v in obj.items()}
 
@@ -47,29 +54,21 @@ def sanitise(obj: Any) -> Any:
         return tuple(sanitise(v) for v in obj)
 
     if isinstance(obj, str):
-        return _sanitise_string(obj)
+        return _sanitise_string(obj, level)
 
     return obj
 
 
-def _sanitise_string(obj: str) -> str:
-    """Sanitise a string by replacing full paths and URL passwords.
-
-    Parameters
-    ----------
-    obj : str
-        The string to sanitise.
-
-    Returns
-    -------
-    str
-        The sanitised string.
-    """
+def _sanitise_string(obj: str, level=1) -> str:
+    """Sanitise a string by replacing full paths and URL passwords."""
 
     parsed = urlparse(obj, allow_fragments=True)
 
     if parsed.scheme and parsed.scheme[0].isalpha():
-        return _sanitise_url(parsed)
+        return _sanitise_url(parsed, level)
+
+    if level > 2:
+        return "hidden"
 
     if obj.startswith("/") or obj.startswith("~"):
         return _sanitise_path(obj)
@@ -77,19 +76,8 @@ def _sanitise_string(obj: str) -> str:
     return obj
 
 
-def _sanitise_url(parsed: Any) -> str:
-    """Sanitise a URL by replacing passwords with '***'.
-
-    Parameters
-    ----------
-    parsed : Any
-        The parsed URL.
-
-    Returns
-    -------
-    str
-        The sanitised URL.
-    """
+def _sanitise_url(parsed: Any, level=1) -> str:
+    """Sanitise a URL by replacing passwords with '***'."""
 
     LIST = [
         "pass",
@@ -107,6 +95,9 @@ def _sanitise_url(parsed: Any) -> str:
         "_api_key",
         "username",
         "login",
+        "auth",
+        "auth_token",
+        "auth_key",
     ]
 
     scheme, netloc, path, params, query, fragment = parsed
@@ -130,26 +121,25 @@ def _sanitise_url(parsed: Any) -> str:
                 qs[k] = "hidden"
         params = urlencode(qs, doseq=True)
 
-    return urlunparse([scheme, netloc, path, params, query, fragment])
+    if level > 1:
+        if (bits := netloc.split("@")) and len(bits) > 1:
+            netloc = f"{bits[0]}@hidden"
+        else:
+            netloc = "hidden"
 
+    if level > 2:
+        return urlunparse([scheme, netloc, "", "", "", ""])
 
-def _sanitise_path(path: str) -> str:
-    """Sanitise a file path by shortening it.
+    return urlunparse([scheme, netloc, path, params, query, fragment])
 
-    Parameters
-    ----------
-    path : str
-        The file path to sanitise.
 
-    Returns
-    -------
-    str
-        The sanitised file path.
-    """
+def _sanitise_path(path: str) -> str:
+    """Sanitise a file path by shortening it."""
     bits = list(reversed(Path(path).parts))
     result = [bits.pop(0)]
     for bit in bits:
         if RE1.match(bit) or RE2.match(bit):
+            # keep earthkit-data folder patterns
             result.append(bit)
             continue
         if result[-1] == "...":

{anemoi_utils-0.4.28.dist-info → anemoi_utils-0.4.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-utils
-Version: 0.4.28
+Version: 0.4.29
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -251,7 +251,7 @@ Requires-Dist: termcolor; extra == "docs"
 Provides-Extra: grib
 Requires-Dist: requests; extra == "grib"
 Provides-Extra: mlflow
-Requires-Dist: mlflow>=2.11.1; extra == "mlflow"
+Requires-Dist: mlflow-skinny>=2.11.1; extra == "mlflow"
 Requires-Dist: requests; extra == "mlflow"
 Provides-Extra: provenance
 Requires-Dist: gitpython; extra == "provenance"
@@ -259,6 +259,7 @@ Requires-Dist: nvsmi; extra == "provenance"
 Provides-Extra: s3
 Requires-Dist: boto3>1.36; extra == "s3"
 Provides-Extra: tests
+Requires-Dist: anemoi-utils[mlflow]; extra == "tests"
 Requires-Dist: pytest; extra == "tests"
 Requires-Dist: pytest-mock>=3; extra == "tests"
 Provides-Extra: text

{anemoi_utils-0.4.28.dist-info → anemoi_utils-0.4.29.dist-info}/RECORD

@@ -1,12 +1,12 @@
 anemoi/utils/__init__.py,sha256=uVhpF-VjIl_4mMywOVtgTutgsdIsqz-xdkwxeMhzuag,730
 anemoi/utils/__main__.py,sha256=6LlE4MYrPvqqrykxXh7XMi50UZteUY59NeM8P9Zs2dU,910
-anemoi/utils/_version.py,sha256=B7w9p3e_jRrhCAcRVz7MulIQ9D0PoLvF57YvgUIilfM,513
+anemoi/utils/_version.py,sha256=M33mpOYpqw35qLoEcg5GxMG4YQ25eohx8wEQkM6FTT0,513
 anemoi/utils/caching.py,sha256=rXbeAmpBcMbbfN4EVblaHWKicsrtx1otER84FEBtz98,6183
 anemoi/utils/checkpoints.py,sha256=PydBqA-wI8_05zU-3yT-ZP53GnvuqB7vPXl-w6-9XX8,9541
 anemoi/utils/cli.py,sha256=Cd3ESQkNWecbGnJjkR_SKHdFPETJWFrHqg5ovtANDKs,6522
 anemoi/utils/compatibility.py,sha256=wRBRMmxQP88rNcWiP5gqXliwYQbBv1iCAsDjcCRi5UY,2234
 anemoi/utils/config.py,sha256=EEfcSxW2CD6fFOzDtqz_uYlMKuYq4X5QinJW_8GBYj4,17325
-anemoi/utils/dates.py,sha256=CnY6JOdpk0T-waPEowMRTkcMzxcN0GcjPVtLkwH_byw,17196
+anemoi/utils/dates.py,sha256=Ew11G-Upl0VM9R9Sgg3Be6KIkXfpMCD69kgiao7Xv48,17534
 anemoi/utils/devtools.py,sha256=W3OBu96MkXRIl7Qh1SE5Zd6aB1R0QlnmlrlpBYM0fVY,3527
 anemoi/utils/grib.py,sha256=201WcxjjAl92Y2HX2kZ2S8Qr5dN-oG7nV-vQLaybzP4,3610
 anemoi/utils/grids.py,sha256=VlhuN8MZDBu-G50QFI1XHV9IXLway2LpvI9X8sDjb5s,6047
@@ -17,7 +17,7 @@ anemoi/utils/provenance.py,sha256=iTsn4r-VPq2D8tSHPSuAIqG077_opkqMT42G03DRWJg,14
 anemoi/utils/registry.py,sha256=e3nOIRyMYQ-mpEvaHAv5tuvMYNbkJ5yz94ns7BnvkjM,9717
 anemoi/utils/rules.py,sha256=VspUoPmw7tijrs6l_wl4vDjr_zVQsFjx9ITiBSvxgc8,6972
 anemoi/utils/s3.py,sha256=xMT48kbcelcjjqsaU567WI3oZ5eqo88Rlgyx5ECszAU,4074
-anemoi/utils/sanitise.py,sha256=ZYGdSX6qihQANr3pHZjbKnoapnzP1KcrWdW1Ul1mOGk,3668
+anemoi/utils/sanitise.py,sha256=XkQzibDbu-VFJkJC4WcB9ovkcTkVAynXtkn1Tlc2CC4,4019
 anemoi/utils/sanitize.py,sha256=43ZKDcfVpeXSsJ9TFEc9aZnD6oe2cUh151XnDspM98M,462
 anemoi/utils/testing.py,sha256=DeTAkmg-RCMPXBBRUy_Gti5UJriUPRKT6ruE3JL2GVc,10372
 anemoi/utils/text.py,sha256=HkzIvi24obDceFLpJEwBJ9PmPrJUkQN2TrElJ-A87gU,14441
@@ -31,17 +31,17 @@ anemoi/utils/mars/__init__.py,sha256=b-Lc3L1TAQd9ODs0Z1YSJzgZCO1K_M3DSgx_yd2qXvM
 anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
 anemoi/utils/mars/requests.py,sha256=VFMHBVAAl0_2lOcMBa1lvaKHctN0lDJsI6_U4BucGew,1142
 anemoi/utils/mlflow/__init__.py,sha256=hCW0QcLHJmE-C1r38P27_ZOvCLNewex5iQEtZqx2ckI,393
-anemoi/utils/mlflow/auth.py,sha256=hMuS3vrX4VDfEFiIlRM_7U6XHvYRGU149doi4jlgsW8,8092
+anemoi/utils/mlflow/auth.py,sha256=Zgn2Ru7GE99_ZTBeskl5M1WnrlEGNU9XxsRVgVoUwmM,8376
 anemoi/utils/mlflow/client.py,sha256=Y34ceLcp1-H0XTt8h8-IhHKX9bApc_QJcgVrzZKtabY,2752
-anemoi/utils/mlflow/utils.py,sha256=2zwNqkS69TXKz-vUy8cdGckJA7jjtjbUZ6P7cj9U8OM,4940
+anemoi/utils/mlflow/utils.py,sha256=0d-dp7YmDLJ59ikJWUcZooqBIdrNDxPShDQubcX-a3k,1310
 anemoi/utils/remote/__init__.py,sha256=7nHu-LRspYW2Fx9GNLjsxpytAUIvhIbOjb0Xmxb-33s,20988
 anemoi/utils/remote/s3.py,sha256=DxO_TjmetX_r3ZvGaHjpz40oqvcQYP3Vd_A4ojMGlSA,21379
 anemoi/utils/remote/ssh.py,sha256=xNtsawh8okytCKRehkRCVExbHZj-CRUQNormEHglfuw,8088
 anemoi/utils/schemas/__init__.py,sha256=nkinKlsPLPXEjfTYQT1mpKC4cvs-14w_zBkDRxakwxw,698
 anemoi/utils/schemas/errors.py,sha256=lgOXzVTYzAE0qWQf3OZ42vCWixv8lilSqLLhzARBmvI,1831
-anemoi_utils-0.4.28.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
-anemoi_utils-0.4.28.dist-info/METADATA,sha256=CxRjsdH4Audnqnun7v-FSgCNxxSVFtwIuSQqgnoIHUk,15656
-anemoi_utils-0.4.28.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-anemoi_utils-0.4.28.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.4.28.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.4.28.dist-info/RECORD,,
+anemoi_utils-0.4.29.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
+anemoi_utils-0.4.29.dist-info/METADATA,sha256=v5sHgt4rZA9mZ2YW1pMO0S76mlvZ09hoB3Q-G7xp6-Y,15717
+anemoi_utils-0.4.29.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+anemoi_utils-0.4.29.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.4.29.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.4.29.dist-info/RECORD,,