anemoi-utils 0.3.13__tar.gz → 0.3.15__tar.gz

anemoi_utils-0.3.15/.github/workflows/changelog-pr-update.yml

@@ -0,0 +1,15 @@
+name: Check Changelog Update on PR
+on:
+  pull_request:
+    types: [assigned, opened, synchronize, reopened, labeled, unlabeled]
+    branches:
+      - main
+      - develop
+jobs:
+  Check-Changelog:
+    name: Check Changelog Action
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: tarides/changelog-check-action@v2
+        with:
+          changelog: CHANGELOG.md

anemoi_utils-0.3.15/.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: ci
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - 'main'
+      - 'develop'
+    tags-ignore:
+      - '**'
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manually installs
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      anemoi-utils: ecmwf/anemoi-utils@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+    secrets: inherit
+
+   # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      anemoi-utils: ecmwf/anemoi-utils@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit

anemoi_utils-0.3.15/.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/.github/workflows/python-publish.yml

@@ -35,7 +35,7 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -49,26 +49,6 @@ jobs:
 
   deploy:
 
-    if: ${{ github.event_name == 'release' }}
-    runs-on: ubuntu-latest
     needs: [checks, quality]
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.x
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install build wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: __token__
-        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
-      run: |
-        python -m build
-        twine upload dist/*
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit

anemoi_utils-0.3.15/.github/workflows/readthedocs-pr-update.yml

@@ -0,0 +1,22 @@
+name: Read the Docs PR Preview
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    # Execute this action only on PRs that touch
+    # documentation files.
+    paths:
+      - "docs/**"
+
+permissions:
+  pull-requests: write
+
+jobs:
+  documentation-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/actions/preview@v1
+        with:
+          project-slug: "anemoi-utils"

anemoi_utils-0.3.15/CHANGELOG.md

@@ -0,0 +1,39 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Please add your functional changes to the appropriate section in the PR.
+Keep it human-readable, your future self will thank you!
+
+## [Unreleased]
+
+### Added
+
+### Changed
+
+### Removed
+
+## [0.3.0] - Initial Release, utility functions
+
+### Added
+- Command line interface utility
+
+## [0.2.0] - Initial Release, utility functions
+
+### Changed
+- updated documentation
+
+## [0.1.0] - Initial Release, utility functions
+
+### Added
+- Documentation
+- Initial implementation for a series of utility functions for used by the rest of the Anemoi packages
+
+<!-- Add Git Diffs for Links above -->
+[unreleased]: https://github.com/ecmwf/anemoi-utils/compare/0.3.13...HEAD
+https://github.com/ecmwf/anemoi-utils/compare/0.2.0...0.3.0
+https://github.com/ecmwf/anemoi-utils/compare/0.1.0...0.2.0
+[0.1.0]: https://github.com/ecmwf/anemoi-utils/releases/tag/0.1.0

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.13
+Version: 0.3.15
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -223,6 +223,7 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
+Requires-Dist: isodate
 Requires-Dist: pyyaml
 Requires-Dist: tomli
 Requires-Dist: tqdm

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/pyproject.toml

@@ -49,6 +49,8 @@ dynamic = [
   "version",
 ]
 dependencies = [
+  "isodate",
+
   "pyyaml",
   "tomli",  # Only needed before 3.11
   "tqdm",

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/src/anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.13'
-__version_tuple__ = version_tuple = (0, 3, 13)
+__version__ = version = '0.3.15'
+__version_tuple__ = version_tuple = (0, 3, 15)

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/src/anemoi/utils/cli.py

@@ -12,6 +12,11 @@ import os
 import sys
 import traceback
 
+try:
+    import argcomplete
+except ImportError:
+    argcomplete = None
+
 LOG = logging.getLogger(__name__)
 
 
@@ -100,6 +105,8 @@ def register_commands(here, package, select, fail=None):
 def cli_main(version, description, commands):
     parser = make_parser(description, commands)
     args, unknown = parser.parse_known_args()
+    if argcomplete:
+        argcomplete.autocomplete(parser)
 
     if args.version:
         print(version)

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/src/anemoi/utils/dates.py

@@ -8,21 +8,14 @@
 
 import calendar
 import datetime
+import re
 
-from .hindcasts import HindcastDatesTimes
-
-
-def normalise_frequency(frequency):
-    if isinstance(frequency, int):
-        return frequency
-    assert isinstance(frequency, str), (type(frequency), frequency)
+import isodate
 
-    unit = frequency[-1].lower()
-    v = int(frequency[:-1])
-    return {"h": v, "d": v * 24}[unit]
+from .hindcasts import HindcastDatesTimes
 
 
-def no_time_zone(date):
+def _no_time_zone(date):
     """Remove time zone information from a date.
 
     Parameters
@@ -40,13 +33,15 @@ def no_time_zone(date):
 
 
 # this function is use in anemoi-datasets
-def as_datetime(date):
+def as_datetime(date, keep_time_zone=False):
     """Convert a date to a datetime object, removing any time zone information.
 
     Parameters
     ----------
     date : datetime.date or datetime.datetime or str
         The date to convert.
+    keep_time_zone : bool, optional
+        If True, the time zone information is kept, by default False.
 
     Returns
     -------
@@ -54,18 +49,150 @@ def as_datetime(date):
         The datetime object.
     """
 
+    tidy = _no_time_zone if not keep_time_zone else lambda x: x
+
     if isinstance(date, datetime.datetime):
-        return no_time_zone(date)
+        return tidy(date)
 
     if isinstance(date, datetime.date):
-        return no_time_zone(datetime.datetime(date.year, date.month, date.day))
+        return tidy(datetime.datetime(date.year, date.month, date.day))
 
     if isinstance(date, str):
-        return no_time_zone(datetime.datetime.fromisoformat(date))
+        return tidy(datetime.datetime.fromisoformat(date))
 
     raise ValueError(f"Invalid date type: {type(date)}")
 
 
+def frequency_to_timedelta(frequency):
+    """Convert a frequency to a timedelta object.
+
+    Parameters
+    ----------
+    frequency : int or str or datetime.timedelta
+        The frequency to convert. If an integer, it is assumed to be in hours. If a string, it can be in the format:
+
+        - "1h" for 1 hour
+        - "1d" for 1 day
+        - "1m" for 1 minute
+        - "1s" for 1 second
+        - "1:30" for 1 hour and 30 minutes
+        - "1:30:10" for 1 hour, 30 minutes and 10 seconds
+        - "PT10M" for 10 minutes (ISO8601)
+
+        If a timedelta object is provided, it is returned as is.
+
+    Returns
+    -------
+    datetime.timedelta
+        The timedelta object.
+
+    Raises
+    ------
+    ValueError
+        Exception raised if the frequency cannot be converted to a timedelta.
+    """
+
+    if isinstance(frequency, datetime.timedelta):
+        return frequency
+
+    if isinstance(frequency, int):
+        return datetime.timedelta(hours=frequency)
+
+    assert isinstance(frequency, str), (type(frequency), frequency)
+
+    try:
+        return frequency_to_timedelta(int(frequency))
+    except ValueError:
+        pass
+
+    if re.match(r"^\d+[hdms]$", frequency, re.IGNORECASE):
+        unit = frequency[-1].lower()
+        v = int(frequency[:-1])
+        unit = {"h": "hours", "d": "days", "s": "seconds", "m": "minutes"}[unit]
+        return datetime.timedelta(**{unit: v})
+
+    m = frequency.split(":")
+    if len(m) == 2:
+        return datetime.timedelta(hours=int(m[0]), minutes=int(m[1]))
+
+    if len(m) == 3:
+        return datetime.timedelta(hours=int(m[0]), minutes=int(m[1]), seconds=int(m[2]))
+
+    # ISO8601
+    try:
+        return isodate.parse_duration(frequency)
+    except isodate.isoerror.ISO8601Error:
+        pass
+
+    raise ValueError(f"Cannot convert frequency {frequency} to timedelta")
+
+
+def frequency_to_string(frequency):
+    """Convert a frequency (i.e. a datetime.timedelta) to a string.
+
+    Parameters
+    ----------
+    frequency : datetime.timedelta
+        The frequency to convert.
+
+    Returns
+    -------
+    str
+        A string representation of the frequency.
+    """
+
+    frequency = frequency_to_timedelta(frequency)
+
+    total_seconds = frequency.total_seconds()
+    assert int(total_seconds) == total_seconds, total_seconds
+    total_seconds = int(total_seconds)
+
+    seconds = total_seconds
+
+    days = seconds // (24 * 3600)
+    seconds %= 24 * 3600
+    hours = seconds // 3600
+    seconds %= 3600
+    minutes = seconds // 60
+    seconds %= 60
+
+    if days > 0 and hours == 0 and minutes == 0 and seconds == 0:
+        return f"{days}d"
+
+    if days == 0 and hours > 0 and minutes == 0 and seconds == 0:
+        return f"{hours}h"
+
+    if days == 0 and hours == 0 and minutes > 0 and seconds == 0:
+        return f"{minutes}m"
+
+    if days == 0 and hours == 0 and minutes == 0 and seconds > 0:
+        return f"{seconds}s"
+
+    if days > 0:
+        return f"{total_seconds}s"
+
+    return str(frequency)
+
+
+def frequency_to_seconds(frequency):
+    """Convert a frequency to seconds.
+
+    Parameters
+    ----------
+    frequency : _type_
+        _description_
+
+    Returns
+    -------
+    int
+        Number of seconds.
+    """
+
+    result = frequency_to_timedelta(frequency).total_seconds()
+    assert int(result) == result, result
+    return int(result)
+
+
 DOW = {
     "monday": 0,
     "tuesday": 1,
@@ -142,7 +269,7 @@ class DateTimes:
         """
         self.start = as_datetime(start)
         self.end = as_datetime(end)
-        self.increment = datetime.timedelta(hours=increment)
+        self.increment = frequency_to_timedelta(increment)
         self.day_of_month = _make_day(day_of_month)
         self.day_of_week = _make_week(day_of_week)
         self.calendar_months = _make_months(calendar_months)
@@ -273,8 +400,7 @@ def datetimes_factory(*args, **kwargs):
 
         kwargs = kwargs.copy()
         if "frequency" in kwargs:
-            freq = kwargs.pop("frequency")
-            kwargs["increment"] = normalise_frequency(freq)
+            kwargs["increment"] = kwargs.pop("frequency")
         return DateTimes(**kwargs)
 
     if not any((isinstance(x, dict) or isinstance(x, list)) for x in args):

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/src/anemoi/utils/humanize.py

@@ -15,6 +15,8 @@ import re
 import warnings
 from collections import defaultdict
 
+from anemoi.utils.dates import as_datetime
+
 
 def bytes_to_human(n: float) -> str:
     """Convert a number of bytes to a human readable string
@@ -625,3 +627,60 @@ def shorten_list(lst, max_length=5):
         if isinstance(lst, tuple):
             return tuple(result)
         return result
+
+
+def _compress_dates(dates):
+    dates = sorted(dates)
+    if len(dates) < 3:
+        yield dates
+        return
+
+    prev = first = dates.pop(0)
+    curr = dates.pop(0)
+    delta = curr - prev
+    while curr - prev == delta:
+        prev = curr
+        if not dates:
+            break
+        curr = dates.pop(0)
+
+    yield (first, prev, delta)
+    if dates:
+        yield from _compress_dates([curr] + dates)
+
+
+def compress_dates(dates):
+    """Compress a list of dates into a human-readable format.
+
+    Parameters
+    ----------
+    dates : list
+        A list of dates, as datetime objects or strings.
+
+    Returns
+    -------
+    str
+        A human-readable string representing the compressed dates.
+    """
+
+    dates = [as_datetime(_) for _ in dates]
+    result = []
+
+    for n in _compress_dates(dates):
+        if isinstance(n, list):
+            result.extend([str(_) for _ in n])
+        else:
+            result.append(" ".join([str(n[0]), "to", str(n[1]), "by", str(n[2])]))
+
+    return result
+
+
+def print_dates(dates):
+    """Print a list of dates in a human-readable format.
+
+    Parameters
+    ----------
+    dates : list
+        A list of dates, as datetime objects or strings.
+    """
+    print(compress_dates(dates))

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/src/anemoi/utils/s3.py

@@ -343,6 +343,7 @@ def upload(source, target, *, overwrite=False, resume=False, verbosity=1, progre
     """
 
     uploader = Upload()
+
     if os.path.isdir(source):
         uploader.transfer_folder(
             source=source,

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/src/anemoi/utils/text.py

@@ -7,6 +7,7 @@
 
 """Text utilities"""
 
+import re
 from collections import defaultdict
 
 # https://en.wikipedia.org/wiki/Box-drawing_character
@@ -32,6 +33,62 @@ def dotted_line(width=84) -> str:
     return "┈" * width
 
 
+# Regular expression to match ANSI escape codes
+_ansi_escape = re.compile(r"\x1b\[([0-9;]*[mGKH])")
+
+
+def _has_ansi_escape(s):
+    return _ansi_escape.search(s) is not None
+
+
+def _split_tokens(s):
+    """Split a string into a list of visual characters with their lenghts."""
+    from wcwidth import wcswidth
+
+    initial = s
+    out = []
+
+    # Function to probe the number of bytes needed to encode the first character
+    def probe_utf8(s):
+        for i in range(1, 5):
+            try:
+                s[:i].encode("utf-8")
+            except UnicodeEncodeError:
+                return i - 1
+        return 1
+
+    while s:
+        match = _ansi_escape.match(s)
+        if match:
+            token = match.group(0)
+            s = s[len(token) :]
+            out.append((token, 0))
+        else:
+            i = probe_utf8(s)
+            token = s[:i]
+            s = s[i:]
+            out.append((token, wcswidth(token)))
+
+    assert "".join(token for (token, _) in out) == initial, (out, initial)
+    return out
+
+
+def visual_len(s):
+    """Compute the length of a string as it appears on the terminal."""
+    if isinstance(s, str):
+        s = _split_tokens(s)
+    assert isinstance(s, (tuple, list)), (type(s), s)
+    if len(s) == 0:
+        return 0
+    for _ in s:
+        assert isinstance(_, tuple), s
+        assert len(_) == 2, s
+    n = 0
+    for _, width in s:
+        n += width
+    return n
+
+
 def boxed(text, min_width=80, max_width=80) -> str:
     """Put a box around a text
 
@@ -57,27 +114,44 @@ def boxed(text, min_width=80, max_width=80) -> str:
 
     """
 
-    lines = text.split("\n")
-    width = max(len(_) for _ in lines)
+    lines = []
+    for line in text.split("\n"):
+        line = line.strip()
+        line = _split_tokens(line)
+        lines.append(line)
+
+    width = max(visual_len(_) for _ in lines)
 
     if min_width is not None:
         width = max(width, min_width)
 
     if max_width is not None:
+
+        def shorten_line(line, max_width):
+            if visual_len(line) > max_width:
+                while visual_len(line) >= max_width:
+                    line = line[:-1]
+                line.append(("…", 1))
+            return line
+
         width = min(width, max_width)
-        lines = []
-        for line in text.split("\n"):
-            if len(line) > max_width:
-                line = line[: max_width - 1] + "…"
-            lines.append(line)
-        text = "\n".join(lines)
+        lines = [shorten_line(line, max_width) for line in lines]
+
+    def pad_line(line, width):
+        line = line + [" "] * (width - visual_len(line))
+        return line
+
+    lines = [pad_line(line, width) for line in lines]
 
     box = []
     box.append("┌" + "─" * (width + 2) + "┐")
     for line in lines:
-        box.append(f"│ {line:{width}} │")
-
+        s = "".join(_[0] for _ in line)
+        if _has_ansi_escape(s):
+            s += "\x1b[0m"
+        box.append(f"│ {s} │")
     box.append("└" + "─" * (width + 2) + "┘")
+
     return "\n".join(box)
 
 
@@ -241,7 +315,7 @@ def table(rows, header, align, margin=0):
                ['B', 120, 1],
                ['C', 9, 123]],
                ['C1', 'C2', 'C3'],
-               ['<', '>', '>']))
+               ['<', '>', '>'])
         C1 │  C2 │  C3
         ───┼─────┼────
         Aa │  12 │   5

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/src/anemoi_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.13
+Version: 0.3.15
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -223,6 +223,7 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
+Requires-Dist: isodate
 Requires-Dist: pyyaml
 Requires-Dist: tomli
 Requires-Dist: tqdm

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/src/anemoi_utils.egg-info/SOURCES.txt

@@ -1,10 +1,15 @@
 .gitignore
 .pre-commit-config.yaml
 .readthedocs.yaml
+CHANGELOG.md
 LICENSE
 README.md
 pyproject.toml
+.github/workflows/changelog-pr-update.yml
+.github/workflows/ci.yml
+.github/workflows/label-public-pr.yml
 .github/workflows/python-publish.yml
+.github/workflows/readthedocs-pr-update.yml
 docs/Makefile
 docs/conf.py
 docs/index.rst
@@ -46,4 +51,5 @@ src/anemoi_utils.egg-info/entry_points.txt
 src/anemoi_utils.egg-info/requires.txt
 src/anemoi_utils.egg-info/top_level.txt
 tests/test_dates.py
+tests/test_frequency.py
 tests/test_utils.py

{anemoi_utils-0.3.13 → anemoi_utils-0.3.15}/src/anemoi_utils.egg-info/requires.txt

@@ -1,3 +1,4 @@
+isodate
 pyyaml
 tomli
 tqdm

anemoi_utils-0.3.15/tests/test_frequency.py

@@ -0,0 +1,37 @@
+# (C) Copyright 2023 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import datetime
+
+from anemoi.utils.dates import frequency_to_string
+from anemoi.utils.dates import frequency_to_timedelta
+
+
+def test_frequency_to_string():
+    assert frequency_to_string(datetime.timedelta(hours=1)) == "1h"
+    assert frequency_to_string(datetime.timedelta(hours=1, minutes=30)) == "1:30:00"
+    assert frequency_to_string(datetime.timedelta(days=10)) == "10d"
+    assert frequency_to_string(datetime.timedelta(minutes=10)) == "10m"
+    assert frequency_to_string(datetime.timedelta(minutes=90)) == "1:30:00"
+
+
+def test_frequency_to_timedelta():
+    assert frequency_to_timedelta("1s") == datetime.timedelta(seconds=1)
+    assert frequency_to_timedelta("3m") == datetime.timedelta(minutes=3)
+    assert frequency_to_timedelta("1h") == datetime.timedelta(hours=1)
+    assert frequency_to_timedelta("3d") == datetime.timedelta(days=3)
+    assert frequency_to_timedelta("90m") == datetime.timedelta(hours=1, minutes=30)
+    assert frequency_to_timedelta("0:30") == datetime.timedelta(minutes=30)
+    assert frequency_to_timedelta("0:30:10") == datetime.timedelta(minutes=30, seconds=10)
+    assert frequency_to_timedelta("1:30:10") == datetime.timedelta(hours=1, minutes=30, seconds=10)
+
+    assert frequency_to_timedelta("PT10M") == datetime.timedelta(minutes=10)
+
+
+if __name__ == "__main__":
+    test_frequency_to_string()
+    test_frequency_to_timedelta()