anemoi-utils 0.3.14__py3-none-any.whl → 0.3.17__py3-none-any.whl

anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.14'
-__version_tuple__ = version_tuple = (0, 3, 14)
+__version__ = version = '0.3.17'
+__version_tuple__ = version_tuple = (0, 3, 17)

anemoi/utils/caching.py

@@ -49,6 +49,7 @@ def cache(key, proc, collection="default", expires=None):
 
 
 class cached:
+    """Decorator to cache the result of a function."""
 
     def __init__(self, collection="default", expires=None):
         self.collection = collection

anemoi/utils/checkpoints.py

@@ -47,7 +47,7 @@ def has_metadata(path: str, name: str = DEFAULT_NAME) -> bool:
     return False
 
 
-def load_metadata(path: str, name: str = DEFAULT_NAME):
+def load_metadata(path: str, name: str = DEFAULT_NAME) -> dict:
     """Load metadata from a checkpoint file
 
     Parameters
@@ -59,8 +59,8 @@ def load_metadata(path: str, name: str = DEFAULT_NAME):
 
     Returns
     -------
-    JSON
-        The content of the metadata file
+    dict
+        The content of the metadata file from JSON
 
     Raises
     ------
@@ -82,7 +82,7 @@ def load_metadata(path: str, name: str = DEFAULT_NAME):
         raise ValueError(f"Could not find '{name}' in {path}.")
 
 
-def save_metadata(path, metadata, name=DEFAULT_NAME, folder=DEFAULT_FOLDER):
+def save_metadata(path, metadata, name=DEFAULT_NAME, folder=DEFAULT_FOLDER) -> None:
     """Save metadata to a checkpoint file
 
     Parameters
@@ -93,6 +93,8 @@ def save_metadata(path, metadata, name=DEFAULT_NAME, folder=DEFAULT_FOLDER):
         A JSON serializable object
     name : str, optional
         The name of the metadata file in the zip archive
+    folder : str, optional
+        The folder where the metadata file will be saved
     """
     with zipfile.ZipFile(path, "a") as zipf:
 

anemoi/utils/cli.py

@@ -55,6 +55,8 @@ def make_parser(description, commands):
 
 
 class Failed(Command):
+    """Command not available."""
+
     def __init__(self, name, error):
         self.name = name
         self.error = error

anemoi/utils/config.py

@@ -5,6 +5,7 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+from __future__ import annotations
 
 import json
 import logging
@@ -175,9 +176,14 @@ def config_path(name="settings.toml"):
     return full
 
 
-def load_any_dict_format(path):
+def load_any_dict_format(path) -> dict:
     """Load a configuration file in any supported format: JSON, YAML and TOML.
 
+    Parameters
+    ----------
+    path : str
+        The path to the configuration file.
+
     Returns
     -------
     dict
@@ -243,7 +249,7 @@ def _load_config(name="settings.toml", secrets=None, defaults=None):
     return CONFIG[key]
 
 
-def _save_config(name, data):
+def _save_config(name, data) -> None:
     CONFIG.pop(name, None)
 
     conf = config_path(name)
@@ -265,7 +271,7 @@ def _save_config(name, data):
         f.write(data)
 
 
-def save_config(name, data):
+def save_config(name, data) -> None:
     """Save a configuration file.
 
     Parameters
@@ -281,13 +287,17 @@ def save_config(name, data):
         _save_config(name, data)
 
 
-def load_config(name="settings.toml", secrets=None, defaults=None):
+def load_config(name="settings.toml", secrets=None, defaults=None) -> DotDict | str:
     """Read a configuration file.
 
     Parameters
     ----------
     name : str, optional
         The name of the config file to read, by default "settings.toml"
+    secrets : str or list, optional
+        The name of the secrets file, by default None
+    defaults : str or dict, optional
+        The name of the defaults file, by default None
 
     Returns
     -------
@@ -299,7 +309,7 @@ def load_config(name="settings.toml", secrets=None, defaults=None):
         return _load_config(name, secrets, defaults)
 
 
-def load_raw_config(name, default=None):
+def load_raw_config(name, default=None) -> DotDict | str:
 
     path = config_path(name)
     if os.path.exists(path):
@@ -308,13 +318,17 @@ def load_raw_config(name, default=None):
     return default
 
 
-def check_config_mode(name="settings.toml", secrets_name=None, secrets=None):
+def check_config_mode(name="settings.toml", secrets_name=None, secrets=None) -> None:
     """Check that a configuration file is secure.
 
     Parameters
     ----------
     name : str, optional
         The name of the configuration file, by default "settings.toml"
+    secrets_name : str, optional
+        The name of the secrets file, by default None
+    secrets : list, optional
+        The list of secrets to check, by default None
 
     Raises
     ------

anemoi/utils/dates.py

@@ -10,9 +10,7 @@ import calendar
 import datetime
 import re
 
-import isodate
-
-from .hindcasts import HindcastDatesTimes
+import aniso8601
 
 
 def normalise_frequency(frequency):
@@ -25,7 +23,7 @@ def normalise_frequency(frequency):
     return {"h": v, "d": v * 24}[unit]
 
 
-def no_time_zone(date):
+def _no_time_zone(date) -> datetime.datetime:
     """Remove time zone information from a date.
 
     Parameters
@@ -43,13 +41,15 @@ def no_time_zone(date):
 
 
 # this function is use in anemoi-datasets
-def as_datetime(date):
+def as_datetime(date, keep_time_zone=False) -> datetime.datetime:
     """Convert a date to a datetime object, removing any time zone information.
 
     Parameters
     ----------
     date : datetime.date or datetime.datetime or str
         The date to convert.
+    keep_time_zone : bool, optional
+        If True, the time zone information is kept, by default False.
 
     Returns
     -------
@@ -57,92 +57,82 @@ def as_datetime(date):
         The datetime object.
     """
 
+    tidy = _no_time_zone if not keep_time_zone else lambda x: x
+
     if isinstance(date, datetime.datetime):
-        return no_time_zone(date)
+        return tidy(date)
 
     if isinstance(date, datetime.date):
-        return no_time_zone(datetime.datetime(date.year, date.month, date.day))
+        return tidy(datetime.datetime(date.year, date.month, date.day))
 
     if isinstance(date, str):
-        return no_time_zone(datetime.datetime.fromisoformat(date))
+        return tidy(datetime.datetime.fromisoformat(date))
 
     raise ValueError(f"Invalid date type: {type(date)}")
 
 
-def _compress_dates(dates):
-    dates = sorted(dates)
-    if len(dates) < 3:
-        yield dates
-        return
-
-    prev = first = dates.pop(0)
-    curr = dates.pop(0)
-    delta = curr - prev
-    while curr - prev == delta:
-        prev = curr
-        if not dates:
-            break
-        curr = dates.pop(0)
-
-    yield (first, prev, delta)
-    if dates:
-        yield from _compress_dates([curr] + dates)
-
-
-def compress_dates(dates):
-    dates = [as_datetime(_) for _ in dates]
-    result = []
+def _as_datetime_list(date, default_increment):
+    if isinstance(date, (list, tuple)):
+        for d in date:
+            yield from _as_datetime_list(d, default_increment)
 
-    for n in _compress_dates(dates):
-        if isinstance(n, list):
-            result.extend([str(_) for _ in n])
-        else:
-            result.append(" ".join([str(n[0]), "to", str(n[1]), "by", str(n[2])]))
-
-    return result
+    if isinstance(date, str):
+        # Check for ISO format
+        try:
+            start, end = aniso8601.parse_interval(date)
+            while start <= end:
+                yield as_datetime(start)
+                start += default_increment
 
+            return
 
-def print_dates(dates):
-    print(compress_dates(dates))
+        except aniso8601.exceptions.ISOFormatError:
+            pass
 
+        try:
+            intervals = aniso8601.parse_repeating_interval(date)
+            for date in intervals:
+                yield as_datetime(date)
+            return
+        except aniso8601.exceptions.ISOFormatError:
+            pass
 
-def frequency_to_string(frequency):
-    # TODO: use iso8601
-    frequency = frequency_to_timedelta(frequency)
+    yield as_datetime(date)
 
-    total_seconds = frequency.total_seconds()
-    assert int(total_seconds) == total_seconds, total_seconds
-    total_seconds = int(total_seconds)
 
-    seconds = total_seconds
-
-    days = seconds // (24 * 3600)
-    seconds %= 24 * 3600
-    hours = seconds // 3600
-    seconds %= 3600
-    minutes = seconds // 60
-    seconds %= 60
-
-    if days > 0 and hours == 0 and minutes == 0 and seconds == 0:
-        return f"{days}d"
+def as_datetime_list(date, default_increment=1):
+    default_increment = frequency_to_timedelta(default_increment)
+    return list(_as_datetime_list(date, default_increment))
 
-    if days == 0 and hours > 0 and minutes == 0 and seconds == 0:
-        return f"{hours}h"
 
-    if days == 0 and hours == 0 and minutes > 0 and seconds == 0:
-        return f"{minutes}m"
+def frequency_to_timedelta(frequency) -> datetime.timedelta:
+    """Convert a frequency to a timedelta object.
 
-    if days == 0 and hours == 0 and minutes == 0 and seconds > 0:
-        return f"{seconds}s"
+    Parameters
+    ----------
+    frequency : int or str or datetime.timedelta
+        The frequency to convert. If an integer, it is assumed to be in hours. If a string, it can be in the format:
 
-    if days > 0:
-        return f"{total_seconds}s"
+        - "1h" for 1 hour
+        - "1d" for 1 day
+        - "1m" for 1 minute
+        - "1s" for 1 second
+        - "1:30" for 1 hour and 30 minutes
+        - "1:30:10" for 1 hour, 30 minutes and 10 seconds
+        - "PT10M" for 10 minutes (ISO8601)
 
-    return str(frequency)
+        If a timedelta object is provided, it is returned as is.
 
+    Returns
+    -------
+    datetime.timedelta
+        The timedelta object.
 
-def frequency_to_timedelta(frequency):
-    # TODO: use iso8601 or check pytimeparse
+    Raises
+    ------
+    ValueError
+        Exception raised if the frequency cannot be converted to a timedelta.
+    """
 
     if isinstance(frequency, datetime.timedelta):
         return frequency
@@ -172,17 +162,77 @@ def frequency_to_timedelta(frequency):
 
     # ISO8601
     try:
-        return isodate.parse_duration(frequency)
-    except isodate.isoerror.ISO8601Error:
+        return aniso8601.parse_duration(frequency)
+    except aniso8601.exceptions.ISOFormatError:
         pass
 
     raise ValueError(f"Cannot convert frequency {frequency} to timedelta")
 
 
-def normalize_date(x):
-    if isinstance(x, str):
-        return no_time_zone(datetime.datetime.fromisoformat(x))
-    return x
+def frequency_to_string(frequency) -> str:
+    """Convert a frequency (i.e. a datetime.timedelta) to a string.
+
+    Parameters
+    ----------
+    frequency : datetime.timedelta
+        The frequency to convert.
+
+    Returns
+    -------
+    str
+        A string representation of the frequency.
+    """
+
+    frequency = frequency_to_timedelta(frequency)
+
+    total_seconds = frequency.total_seconds()
+    assert int(total_seconds) == total_seconds, total_seconds
+    total_seconds = int(total_seconds)
+
+    seconds = total_seconds
+
+    days = seconds // (24 * 3600)
+    seconds %= 24 * 3600
+    hours = seconds // 3600
+    seconds %= 3600
+    minutes = seconds // 60
+    seconds %= 60
+
+    if days > 0 and hours == 0 and minutes == 0 and seconds == 0:
+        return f"{days}d"
+
+    if days == 0 and hours > 0 and minutes == 0 and seconds == 0:
+        return f"{hours}h"
+
+    if days == 0 and hours == 0 and minutes > 0 and seconds == 0:
+        return f"{minutes}m"
+
+    if days == 0 and hours == 0 and minutes == 0 and seconds > 0:
+        return f"{seconds}s"
+
+    if days > 0:
+        return f"{total_seconds}s"
+
+    return str(frequency)
+
+
+def frequency_to_seconds(frequency) -> int:
+    """Convert a frequency to seconds.
+
+    Parameters
+    ----------
+    frequency : _type_
+        _description_
+
+    Returns
+    -------
+    int
+        Number of seconds.
+    """
+
+    result = frequency_to_timedelta(frequency).total_seconds()
+    assert int(result) == result, result
+    return int(result)
 
 
 DOW = {
@@ -261,7 +311,7 @@ class DateTimes:
         """
         self.start = as_datetime(start)
         self.end = as_datetime(end)
-        self.increment = datetime.timedelta(hours=increment)
+        self.increment = frequency_to_timedelta(increment)
         self.day_of_month = _make_day(day_of_month)
         self.day_of_week = _make_week(day_of_week)
         self.calendar_months = _make_months(calendar_months)
@@ -354,6 +404,8 @@ class Autumn(DateTimes):
 
 
 class ConcatDateTimes:
+    """ConcatDateTimes is an iterator that generates datetime objects from a list of dates."""
+
     def __init__(self, *dates):
         if len(dates) == 1 and isinstance(dates[0], list):
             dates = dates[0]
@@ -366,6 +418,8 @@ class ConcatDateTimes:
 
 
 class EnumDateTimes:
+    """EnumDateTimes is an iterator that generates datetime objects from a list of dates."""
+
     def __init__(self, dates):
         self.dates = dates
 
@@ -385,6 +439,8 @@ def datetimes_factory(*args, **kwargs):
         name = kwargs.get("name")
 
         if name == "hindcast":
+            from .hindcasts import HindcastDatesTimes
+
             reference_dates = kwargs["reference_dates"]
             reference_dates = datetimes_factory(reference_dates)
             years = kwargs["years"]
@@ -392,8 +448,7 @@ def datetimes_factory(*args, **kwargs):
 
         kwargs = kwargs.copy()
         if "frequency" in kwargs:
-            freq = kwargs.pop("frequency")
-            kwargs["increment"] = normalise_frequency(freq)
+            kwargs["increment"] = kwargs.pop("frequency")
         return DateTimes(**kwargs)
 
     if not any((isinstance(x, dict) or isinstance(x, list)) for x in args):
@@ -409,3 +464,8 @@ def datetimes_factory(*args, **kwargs):
             return datetimes_factory(*a)
 
     return ConcatDateTimes(*[datetimes_factory(a) for a in args])
+
+
+if __name__ == "__main__":
+    print(as_datetime_list("R10/2023-01-01T00:00:00Z/P1D"))
+    print(as_datetime_list("2007-03-01T13:00:00/2008-05-11T15:30:00", "200h"))

anemoi/utils/grib.py

@@ -95,8 +95,8 @@ def units(param) -> str:
 
     Parameters
     ----------
-    paramid : int or str
-        Parameter id ir name.
+    param : int or str
+        Parameter id or name.
 
     Returns
     -------
@@ -112,7 +112,7 @@ def units(param) -> str:
     return _units()[unit_id]
 
 
-def must_be_positive(param):
+def must_be_positive(param) -> bool:
     """Check if a parameter must be positive.
 
     Parameters

anemoi/utils/humanize.py

@@ -15,6 +15,8 @@ import re
 import warnings
 from collections import defaultdict
 
+from anemoi.utils.dates import as_datetime
+
 
 def bytes_to_human(n: float) -> str:
     """Convert a number of bytes to a human readable string
@@ -215,7 +217,7 @@ def __(n):
     return "th"
 
 
-def when(then, now=None, short=True, use_utc=False):
+def when(then, now=None, short=True, use_utc=False) -> str:
     """Generate a human readable string for a date, relative to now
 
     >>> when(datetime.datetime.now() - datetime.timedelta(hours=2))
@@ -241,6 +243,8 @@ def when(then, now=None, short=True, use_utc=False):
         The reference date, by default NOW
     short : bool, optional
         Genererate shorter strings, by default True
+    use_utc : bool, optional
+        Use UTC time, by default False
 
     Returns
     -------
@@ -364,7 +368,7 @@ def string_distance(s, t):
     return d[m, n]
 
 
-def did_you_mean(word, vocabulary):
+def did_you_mean(word, vocabulary) -> str:
     """Pick the closest word in a vocabulary
 
     >>> did_you_mean("aple", ["banana", "lemon", "apple", "orange"])
@@ -393,7 +397,7 @@ def dict_to_human(query):
     return list_to_human(lst)
 
 
-def list_to_human(lst, conjunction="and"):
+def list_to_human(lst, conjunction="and") -> str:
     """Convert a list of strings to a human readable string
 
     >>> list_to_human(["banana", "lemon", "apple", "orange"])
@@ -408,8 +412,8 @@ def list_to_human(lst, conjunction="and"):
 
     Returns
     -------
-    _type_
-        _description_
+    str
+        Human readable string of list
     """
     if not lst:
         return "??"
@@ -548,19 +552,21 @@ def rounded_datetime(d):
     return d
 
 
-def json_pretty_dump(obj, max_line_length=120, default=str):
+def json_pretty_dump(obj, max_line_length=120, default=str) -> str:
     """Custom JSON dump function that keeps dicts and lists on one line if they are short enough.
 
     Parameters
     ----------
-    obj
+    obj : Any
         The object to be dumped as JSON.
-    max_line_length
-        Maximum allowed line length for pretty-printing.
+    max_line_length : int, optional
+        Maximum allowed line length for pretty-printing. Default is 120.
+    default : function, optional
+        Default function to convert non-serializable objects. Default is str.
 
     Returns
     -------
-    unknown
+    str
         JSON string.
     """
 
@@ -569,14 +575,14 @@ def json_pretty_dump(obj, max_line_length=120, default=str):
 
         Parameters
         ----------
-        obj
+        obj : Any
             The object to format.
-        indent_level
-            Current indentation level.
+        indent_level : int, optional
+            Current indentation level. Default is 0.
 
         Returns
         -------
-        unknown
+        str
             Formatted JSON string.
         """
         indent = " " * 4 * indent_level
@@ -602,15 +608,15 @@ def json_pretty_dump(obj, max_line_length=120, default=str):
     return _format_json(obj)
 
 
-def shorten_list(lst, max_length=5):
+def shorten_list(lst, max_length=5) -> list:
     """Shorten a list to a maximum length.
 
     Parameters
     ----------
-    lst
+    lst : list
         The list to be shortened.
-    max_length
-        Maximum length of the shortened list.
+    max_length : int, optional
+        Maximum length of the shortened list. Default is 5.
 
     Returns
     -------
@@ -625,3 +631,60 @@ def shorten_list(lst, max_length=5):
         if isinstance(lst, tuple):
             return tuple(result)
         return result
+
+
+def _compress_dates(dates):
+    dates = sorted(dates)
+    if len(dates) < 3:
+        yield dates
+        return
+
+    prev = first = dates.pop(0)
+    curr = dates.pop(0)
+    delta = curr - prev
+    while curr - prev == delta:
+        prev = curr
+        if not dates:
+            break
+        curr = dates.pop(0)
+
+    yield (first, prev, delta)
+    if dates:
+        yield from _compress_dates([curr] + dates)
+
+
+def compress_dates(dates) -> str:
+    """Compress a list of dates into a human-readable format.
+
+    Parameters
+    ----------
+    dates : list
+        A list of dates, as datetime objects or strings.
+
+    Returns
+    -------
+    str
+        A human-readable string representing the compressed dates.
+    """
+
+    dates = [as_datetime(_) for _ in dates]
+    result = []
+
+    for n in _compress_dates(dates):
+        if isinstance(n, list):
+            result.extend([str(_) for _ in n])
+        else:
+            result.append(" ".join([str(n[0]), "to", str(n[1]), "by", str(n[2])]))
+
+    return result
+
+
+def print_dates(dates) -> None:
+    """Print a list of dates in a human-readable format.
+
+    Parameters
+    ----------
+    dates : list
+        A list of dates, as datetime objects or strings.
+    """
+    print(compress_dates(dates))

anemoi/utils/provenance.py

@@ -199,7 +199,7 @@ def _paths(path_or_object):
     return paths
 
 
-def git_check(*args):
+def git_check(*args) -> dict:
     """Return the git information for the given arguments.
 
     Arguments can be:
@@ -209,6 +209,11 @@ def git_check(*args):
         - an object or a class
         - a path to a directory
 
+    Parameters
+    ----------
+    args : list
+        The list of arguments to check
+
     Returns
     -------
     dict

anemoi/utils/s3.py

@@ -321,7 +321,7 @@ class Download(Transfer):
         return size
 
 
-def upload(source, target, *, overwrite=False, resume=False, verbosity=1, progress=None, threads=1):
+def upload(source, target, *, overwrite=False, resume=False, verbosity=1, progress=None, threads=1) -> None:
     """Upload a file or a folder to S3.
 
     Parameters
@@ -335,6 +335,8 @@ def upload(source, target, *, overwrite=False, resume=False, verbosity=1, progre
     resume : bool, optional
         If the data is alreay on S3 it will not be uploaded, unless the remote file
         has a different size, by default False
+    verbosity : int, optional
+        The level of verbosity, by default 1
     progress: callable, optional
         A callable that will be called with the number of files, the total size of the files, the total size
         transferred and a boolean indicating if the transfer has started. By default None
@@ -365,7 +367,7 @@ def upload(source, target, *, overwrite=False, resume=False, verbosity=1, progre
         )
 
 
-def download(source, target, *, overwrite=False, resume=False, verbosity=1, progress=None, threads=1):
+def download(source, target, *, overwrite=False, resume=False, verbosity=1, progress=None, threads=1) -> None:
     """Download a file or a folder from S3.
 
     Parameters
@@ -381,6 +383,8 @@ def download(source, target, *, overwrite=False, resume=False, verbosity=1, prog
     resume : bool, optional
         If the data is alreay on local it will not be downloaded, unless the remote file
         has a different size, by default False
+    verbosity : int, optional
+        The level of verbosity, by default 1
     progress: callable, optional
         A callable that will be called with the number of files, the total size of the files, the total size
         transferred and a boolean indicating if the transfer has started. By default None
@@ -427,7 +431,7 @@ def _list_objects(target, batch=False):
                 yield from objects
 
 
-def _delete_folder(target):
+def _delete_folder(target) -> None:
     _, _, bucket, _ = target.split("/", 3)
     s3 = s3_client(bucket)
 
@@ -439,7 +443,7 @@ def _delete_folder(target):
         LOGGER.info(f"Deleted {len(batch):,} objects (total={total:,})")
 
 
-def _delete_file(target):
+def _delete_file(target) -> None:
     from botocore.exceptions import ClientError
 
     _, _, bucket, key = target.split("/", 3)
@@ -462,7 +466,7 @@ def _delete_file(target):
     LOGGER.info(f"{target} is deleted")
 
 
-def delete(target):
+def delete(target) -> None:
     """Delete a file or a folder from S3.
 
     Parameters
@@ -480,7 +484,7 @@ def delete(target):
         _delete_file(target)
 
 
-def list_folder(folder):
+def list_folder(folder) -> list:
     """List the sub folders in a folder on S3.
 
     Parameters
@@ -508,7 +512,7 @@ def list_folder(folder):
             yield from [folder + _["Prefix"] for _ in page.get("CommonPrefixes")]
 
 
-def object_info(target):
+def object_info(target) -> dict:
     """Get information about an object on S3.
 
     Parameters
@@ -533,7 +537,7 @@ def object_info(target):
         raise
 
 
-def object_acl(target):
+def object_acl(target) -> dict:
     """Get information about an object's ACL on S3.
 
     Parameters

anemoi/utils/sanetize.py

@@ -0,0 +1,10 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+from .sanetise import sanetise as sanetize
+
+__all__ = ["sanetize"]

anemoi/utils/sanitise.py

@@ -0,0 +1,115 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import os
+import re
+from pathlib import Path
+from urllib.parse import parse_qs
+from urllib.parse import urlencode
+from urllib.parse import urlparse
+from urllib.parse import urlunparse
+
+# Patterns used but earthkit-data for url-patterns and path-patterns
+
+RE1 = re.compile(r"{([^}]*)}")
+RE2 = re.compile(r"\(([^}]*)\)")
+
+
+def sanitise(obj):
+    """sanitise an object:
+    - by replacing all full paths with shortened versions.
+    - by replacing URL passwords with '***'.
+    """
+
+    if isinstance(obj, dict):
+        return {sanitise(k): sanitise(v) for k, v in obj.items()}
+
+    if isinstance(obj, list):
+        return [sanitise(v) for v in obj]
+
+    if isinstance(obj, tuple):
+        return tuple(sanitise(v) for v in obj)
+
+    if isinstance(obj, str):
+        return _sanitise_string(obj)
+
+    return obj
+
+
+def _sanitise_string(obj):
+
+    parsed = urlparse(obj, allow_fragments=True)
+
+    if parsed.scheme:
+        return _sanitise_url(parsed)
+
+    if obj.startswith("/") or obj.startswith("~"):
+        return _sanitise_path(obj)
+
+    return obj
+
+
+def _sanitise_url(parsed):
+
+    LIST = [
+        "pass",
+        "password",
+        "token",
+        "user",
+        "key",
+        "pwd",
+        "_key",
+        "_token",
+        "apikey",
+        "api_key",
+        "api_token",
+        "_api_token",
+        "_api_key",
+        "username",
+        "login",
+    ]
+
+    scheme, netloc, path, params, query, fragment = parsed
+
+    if parsed.password or parsed.username:
+        _, host = netloc.split("@")
+        user = "user:***" if parsed.password else "user"
+        netloc = f"{user}@{host}"
+
+    if query:
+        qs = parse_qs(query)
+        for k in LIST:
+            if k in qs:
+                qs[k] = "hidden"
+        query = urlencode(qs, doseq=True)
+
+    if params:
+        qs = parse_qs(params)
+        for k in LIST:
+            if k in qs:
+                qs[k] = "hidden"
+        params = urlencode(qs, doseq=True)
+
+    return urlunparse([scheme, netloc, path, params, query, fragment])
+
+
+def _sanitise_path(path):
+    bits = list(reversed(Path(path).parts))
+    result = [bits.pop(0)]
+    for bit in bits:
+        if RE1.match(bit) or RE2.match(bit):
+            result.append(bit)
+            continue
+        if result[-1] == "...":
+            continue
+        result.append("...")
+    result = os.path.join(*reversed(result))
+    if bits[-1] == "/":
+        result = os.path.join("/", result)
+
+    return result

anemoi/utils/text.py

@@ -174,6 +174,8 @@ def green(text):
 
 
 class Tree:
+    """Tree data structure."""
+
     def __init__(self, actor, parent=None):
         self._actor = actor
         self._kids = []
@@ -308,7 +310,7 @@ class Tree:
         }
 
 
-def table(rows, header, align, margin=0):
+def table(rows, header, align, margin=0) -> str:
     """Format a table
 
     >>> table([['Aa', 12, 5],

{anemoi_utils-0.3.14.dist-info → anemoi_utils-0.3.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.14
+Version: 0.3.17
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License: Apache License
@@ -223,26 +223,14 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: isodate
+Requires-Dist: aniso8601
 Requires-Dist: pyyaml
 Requires-Dist: tomli
 Requires-Dist: tqdm
 Provides-Extra: all
-Requires-Dist: gitpython ; extra == 'all'
-Requires-Dist: nvsmi ; extra == 'all'
-Requires-Dist: requests ; extra == 'all'
-Requires-Dist: termcolor ; extra == 'all'
+Requires-Dist: anemoi-utils[grib,provenance,text] ; extra == 'all'
 Provides-Extra: dev
-Requires-Dist: gitpython ; extra == 'dev'
-Requires-Dist: nbsphinx ; extra == 'dev'
-Requires-Dist: nvsmi ; extra == 'dev'
-Requires-Dist: pandoc ; extra == 'dev'
-Requires-Dist: pytest ; extra == 'dev'
-Requires-Dist: requests ; extra == 'dev'
-Requires-Dist: sphinx ; extra == 'dev'
-Requires-Dist: sphinx-argparse <0.5 ; extra == 'dev'
-Requires-Dist: sphinx-rtd-theme ; extra == 'dev'
-Requires-Dist: termcolor ; extra == 'dev'
+Requires-Dist: anemoi-utils[all,docs,tests] ; extra == 'dev'
 Provides-Extra: docs
 Requires-Dist: nbsphinx ; extra == 'docs'
 Requires-Dist: pandoc ; extra == 'docs'
@@ -260,4 +248,5 @@ Provides-Extra: tests
 Requires-Dist: pytest ; extra == 'tests'
 Provides-Extra: text
 Requires-Dist: termcolor ; extra == 'text'
+Requires-Dist: wcwidth ; extra == 'text'
 

anemoi_utils-0.3.17.dist-info/RECORD

@@ -0,0 +1,27 @@
+anemoi/utils/__init__.py,sha256=zZZpbKIoGWwdCOuo6YSruLR7C0GzvzI1Wzhyqaa0K7M,456
+anemoi/utils/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
+anemoi/utils/_version.py,sha256=EanR9QKHDmsyYNdsvPdG4re1cWDyd1A_td5gwFF-ouQ,413
+anemoi/utils/caching.py,sha256=bCOjP1jcDbwi7BID3XaR9BbkD1k3UipkP74NbgJuEFA,1974
+anemoi/utils/checkpoints.py,sha256=qWtqkQvMDlPIcfqz_GmOyUf0mmHZ6QEInpvgSJYIQjY,5275
+anemoi/utils/cli.py,sha256=9TrBXkDjBsos2d0z6wdFDRZIjft3HWGutSgAFi0zCK4,3712
+anemoi/utils/config.py,sha256=fHM6kENZhdD350WaEDekSdH_Fs0diovj-nPuv_a7bko,9408
+anemoi/utils/dates.py,sha256=dgGbTqpGOpYDGgWfXL_69HutXTCHDFI2DhvG-9I9WQI,12341
+anemoi/utils/grib.py,sha256=mrk1drJm2jaPYERQX45QfX2MP4eUqRv0J-Y8IRSRTRE,3073
+anemoi/utils/hindcasts.py,sha256=X8k-81ltmkTDHdviY0SJgvMg7XDu07xoc5ALlUxyPoo,1453
+anemoi/utils/humanize.py,sha256=-xQraQWMLwNaLQAWfPi4K05qieQLgkiyYmV6bfhr10U,16611
+anemoi/utils/provenance.py,sha256=EtlQrubTpEGkTG2UpoRlNGGjKMDcJ-9H3j96vedGyaI,9673
+anemoi/utils/s3.py,sha256=MuY-PrHpt6iKM2RK7v74YoCdqvVJ8UjBDJh0wxUR9Co,18720
+anemoi/utils/sanetize.py,sha256=cQ4r52bQuegMqsEzCN2flXYk_0y_amwpEhMx2-G78j4,488
+anemoi/utils/sanitise.py,sha256=VKIUiwm0EHPdkFUR6FkAxe94933yQx2obQtN6YROH5M,2862
+anemoi/utils/text.py,sha256=WvxkRlPpmK7HLOpWZmyqPQG29GMF8IFaCJp7frfIWNI,10436
+anemoi/utils/timer.py,sha256=EQcucuwUaGeSpt2S1APJlwSOu6kC47MK9f4h-r8c_AY,990
+anemoi/utils/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
+anemoi/utils/commands/config.py,sha256=KEffXZh0ZQfn8t6LXresfd94kDY0gEyulx9Wto5ttW0,824
+anemoi/utils/mars/__init__.py,sha256=RAeY8gJ7ZvsPlcIvrQ4fy9xVHs3SphTAPw_XJDtNIKo,1750
+anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
+anemoi_utils-0.3.17.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_utils-0.3.17.dist-info/METADATA,sha256=saT-_cFT11QQw7MU-ogdaBT1U_VKl_00zHqo5QRPVF4,15055
+anemoi_utils-0.3.17.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+anemoi_utils-0.3.17.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
+anemoi_utils-0.3.17.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_utils-0.3.17.dist-info/RECORD,,

{anemoi_utils-0.3.14.dist-info → anemoi_utils-0.3.17.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (73.0.1)
+Generator: setuptools (75.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

anemoi_utils-0.3.14.dist-info/RECORD

@@ -1,25 +0,0 @@
-anemoi/utils/__init__.py,sha256=zZZpbKIoGWwdCOuo6YSruLR7C0GzvzI1Wzhyqaa0K7M,456
-anemoi/utils/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
-anemoi/utils/_version.py,sha256=uXvaQV0x-j_stHaI7VjSOLUNVrd36A2OO0ftk9b0Dmw,413
-anemoi/utils/caching.py,sha256=EZ4bRG72aTvTxzrbYCgjFpdIn8OtA1rzoRmGg8caWsI,1919
-anemoi/utils/checkpoints.py,sha256=1_3mg4B-ykTVfIvIUEv7IxGyREx_ZcilVbB3U-V6O6I,5165
-anemoi/utils/cli.py,sha256=SWb5_itARlDCq6yEf-VvagTioSW2phKXXFMW2ihXu18,3678
-anemoi/utils/config.py,sha256=s8eqlHsuak058_NJXGMOoT2HenwiZJKcZ9plUWvO7tw,8865
-anemoi/utils/dates.py,sha256=yDIV8hYakNaC9iyYD9WRTNClFO6ZKyM8JKCRUiFJEv8,10413
-anemoi/utils/grib.py,sha256=gVfo4KYQv31iRyoqRDwk5tiqZDUgOIvhag_kO0qjYD0,3067
-anemoi/utils/hindcasts.py,sha256=X8k-81ltmkTDHdviY0SJgvMg7XDu07xoc5ALlUxyPoo,1453
-anemoi/utils/humanize.py,sha256=-MRGgGG_STTWzIagL4RQnOAooqAIigCcuN-RZ6CK75M,14982
-anemoi/utils/provenance.py,sha256=NL36lM_aCw3fG6VIAouZCRBAJv8a6M3x9cScrFxCMcA,9579
-anemoi/utils/s3.py,sha256=kDzbs4nVD2lQuppSe88NVSNpy0wSZpuzkmcAgN2irkU,18506
-anemoi/utils/text.py,sha256=5HBqNwhifus4d3OUnod5q1VgCBdEpzE7o0IR0S85knw,10397
-anemoi/utils/timer.py,sha256=EQcucuwUaGeSpt2S1APJlwSOu6kC47MK9f4h-r8c_AY,990
-anemoi/utils/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
-anemoi/utils/commands/config.py,sha256=KEffXZh0ZQfn8t6LXresfd94kDY0gEyulx9Wto5ttW0,824
-anemoi/utils/mars/__init__.py,sha256=RAeY8gJ7ZvsPlcIvrQ4fy9xVHs3SphTAPw_XJDtNIKo,1750
-anemoi/utils/mars/mars.yaml,sha256=R0dujp75lLA4wCWhPeOQnzJ45WZAYLT8gpx509cBFlc,66
-anemoi_utils-0.3.14.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_utils-0.3.14.dist-info/METADATA,sha256=CRVUi4_pMo2st6kwOUgQ0jyEFwl67ZcIHpDo7a4y2NE,15470
-anemoi_utils-0.3.14.dist-info/WHEEL,sha256=Mdi9PDNwEZptOjTlUcAth7XJDFtKrHYaQMPulZeBCiQ,91
-anemoi_utils-0.3.14.dist-info/entry_points.txt,sha256=LENOkn88xzFQo-V59AKoA_F_cfYQTJYtrNTtf37YgHY,60
-anemoi_utils-0.3.14.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_utils-0.3.14.dist-info/RECORD,,