anemoi-utils 0.3.15__tar.gz → 0.3.17__tar.gz

anemoi_utils-0.3.17/.gitattributes

@@ -0,0 +1 @@
+CHANGELOG.md merge=union

anemoi_utils-0.3.17/.github/CODEOWNERS

@@ -0,0 +1,6 @@
+# CODEOWNERS file
+
+# Protect workflow files
+/.github/ @theissenhelen @jesperdramsch @gmertes
+/.pre-commit-config.yaml @theissenhelen @jesperdramsch @gmertes
+/pyproject.toml @theissenhelen @jesperdramsch @gmertes

anemoi_utils-0.3.17/.github/ci-hpc-config.yml

@@ -0,0 +1,8 @@
+build:
+  python: '3.10'
+  modules:
+    - ninja
+  parallel: 64
+
+  pytest_cmd: |
+    python -m pytest -vv -m 'not notebook and not no_cache_init' --cov=. --cov-report=xml

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/.github/workflows/changelog-pr-update.yml

@@ -5,6 +5,9 @@ on:
     branches:
       - main
       - develop
+    paths-ignore:
+      - .pre-commit-config.yaml
+      - .readthedocs.yaml
 jobs:
   Check-Changelog:
     name: Check Changelog Action

anemoi_utils-0.3.17/.github/workflows/changelog-release-update.yml

@@ -0,0 +1,35 @@
+# .github/workflows/update-changelog.yaml
+name: "Update Changelog"
+
+on:
+  release:
+    types: [released]
+  workflow_dispatch: ~
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.release.target_commitish }}
+
+    - name: Update Changelog
+      uses: stefanzweifel/changelog-updater-action@v1
+      with:
+        latest-version: ${{ github.event.release.tag_name }}
+        heading-text: ${{ github.event.release.name }}
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v6
+      with:
+        branch: docs/changelog-update-${{ github.event.release.tag_name }}
+        title: '[Changelog] Update to ${{ github.event.release.tag_name }}'
+        add-paths: |
+          CHANGELOG.md

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/.github/workflows/ci.yml

@@ -8,6 +8,9 @@ on:
       - 'develop'
     tags-ignore:
       - '**'
+    paths:
+      - "src/**"
+      - "tests/**"
 
   # Trigger the workflow on pull request
   pull_request: ~
@@ -34,7 +37,7 @@ jobs:
   downstream-ci-hpc:
     name: downstream-ci-hpc
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
     with:
       anemoi-utils: ecmwf/anemoi-utils@${{ github.event.pull_request.head.sha || github.sha }}
     secrets: inherit

anemoi_utils-0.3.17/.github/workflows/python-publish.yml

@@ -0,0 +1,27 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
+
+  checks:
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+    with:
+      python-version: ${{ matrix.python-version }}
+
+  deploy:
+    needs: [checks, quality]
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit

anemoi_utils-0.3.17/.github/workflows/python-pull-request.yml

@@ -0,0 +1,23 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Code Quality checks for PRs
+
+on:
+  push:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
+
+  checks:
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+    with:
+      python-version: ${{ matrix.python-version }}

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/.gitignore

@@ -121,6 +121,7 @@ celerybeat.pid
 
 # Environments
 .env
+.envrc
 .venv
 env/
 venv/

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/.pre-commit-config.yaml

@@ -20,8 +20,14 @@ repos:
   - id: no-commit-to-branch # Prevent committing to main / master
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0  # Use the ref you want to point at
+    hooks:
+    -   id: python-use-type-annotations # Check for missing type annotations
+    -   id: python-check-blanket-noqa # Check for # noqa: all
+    -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.4.2
+  rev: 24.8.0
   hooks:
   - id: black
     args: [--line-length=120]
@@ -34,7 +40,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.6
+  rev: v0.6.4
   hooks:
   - id: ruff
     # Next line if for documenation cod snippets
@@ -45,7 +51,7 @@ repos:
     - --exit-non-zero-on-fix
     - --preview
 - repo: https://github.com/sphinx-contrib/sphinx-lint
-  rev: v0.9.1
+  rev: v1.0.0
   hooks:
   - id: sphinx-lint
 # For now, we use it. But it does not support a lot of sphinx features
@@ -59,12 +65,23 @@ repos:
   hooks:
   - id: docconvert
     args: ["numpy"]
-- repo: https://github.com/b8raoult/optional-dependencies-all
-  rev: "0.0.6"
-  hooks:
-  - id: optional-dependencies-all
-    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "2.1.3"
+  rev: "2.2.3"
   hooks:
   - id: pyproject-fmt
+-   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
+    rev: v0.60.1
+    hooks:
+    -   id: docsig
+        args:
+        - --ignore-no-params # Allow docstrings without parameters
+        - --check-dunders    # Check dunder methods
+        - --check-overridden # Check overridden methods
+        - --check-protected  # Check protected methods
+        - --check-class      # Check class docstrings
+        - --disable=E113     # Disable empty docstrings
+        - --summary          # Print a summary
+ci:
+  autoupdate_schedule: monthly
+ci:
+  autoupdate_schedule: monthly

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/CHANGELOG.md

@@ -11,8 +11,17 @@ Keep it human-readable, your future self will thank you!
 ## [Unreleased]
 
 ### Added
+- Codeowners file
+- Pygrep precommit hooks
+- Docsig precommit hooks
+- Changelog merge strategy- Codeowners file
+- Create dependency on wcwidth. MIT licence.
+- Add anonimize() function.
 
 ### Changed
+- downstream-ci should only runs for changes in src and tests
+- bugfixes for CI
+- python3.9 support
 
 ### Removed
 

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-utils
-Version: 0.3.15
+Version: 0.3.17
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -223,26 +223,14 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: isodate
+Requires-Dist: aniso8601
 Requires-Dist: pyyaml
 Requires-Dist: tomli
 Requires-Dist: tqdm
 Provides-Extra: all
-Requires-Dist: gitpython; extra == "all"
-Requires-Dist: nvsmi; extra == "all"
-Requires-Dist: requests; extra == "all"
-Requires-Dist: termcolor; extra == "all"
+Requires-Dist: anemoi-utils[grib,provenance,text]; extra == "all"
 Provides-Extra: dev
-Requires-Dist: gitpython; extra == "dev"
-Requires-Dist: nbsphinx; extra == "dev"
-Requires-Dist: nvsmi; extra == "dev"
-Requires-Dist: pandoc; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: requests; extra == "dev"
-Requires-Dist: sphinx; extra == "dev"
-Requires-Dist: sphinx-argparse<0.5; extra == "dev"
-Requires-Dist: sphinx-rtd-theme; extra == "dev"
-Requires-Dist: termcolor; extra == "dev"
+Requires-Dist: anemoi-utils[all,docs,tests]; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: nbsphinx; extra == "docs"
 Requires-Dist: pandoc; extra == "docs"
@@ -260,3 +248,4 @@ Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
 Provides-Extra: text
 Requires-Dist: termcolor; extra == "text"
+Requires-Dist: wcwidth; extra == "text"

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/pyproject.toml

@@ -10,19 +10,13 @@
 # https://packaging.python.org/en/latest/guides/writing-pyproject-toml/
 
 [build-system]
-requires = [
-  "setuptools>=60",
-  "setuptools-scm>=8",
-]
+requires = [ "setuptools>=60", "setuptools-scm>=8" ]
 
 [project]
 name = "anemoi-utils"
 
 description = "A package to hold various functions to support training of ML models on ECMWF data."
-keywords = [
-  "ai",
-  "tools",
-]
+keywords = [ "ai", "tools" ]
 
 license = { file = "LICENSE" }
 authors = [
@@ -45,35 +39,16 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
 
-dynamic = [
-  "version",
-]
+dynamic = [ "version" ]
 dependencies = [
-  "isodate",
-
+  "aniso8601",
   "pyyaml",
-  "tomli",  # Only needed before 3.11
+  "tomli",     # Only needed before 3.11
   "tqdm",
 ]
 
-optional-dependencies.all = [
-  "gitpython",
-  "nvsmi",
-  "requests",
-  "termcolor",
-]
-optional-dependencies.dev = [
-  "gitpython",
-  "nbsphinx",
-  "nvsmi",
-  "pandoc",
-  "pytest",
-  "requests",
-  "sphinx",
-  "sphinx-argparse<0.5",
-  "sphinx-rtd-theme",
-  "termcolor",
-]
+optional-dependencies.all = [ "anemoi-utils[grib,provenance,text]" ]
+optional-dependencies.dev = [ "anemoi-utils[all,docs,tests]" ]
 
 optional-dependencies.docs = [
   "nbsphinx",
@@ -85,21 +60,14 @@ optional-dependencies.docs = [
   "termcolor",
 ]
 
-optional-dependencies.grib = [
-  "requests",
-]
+optional-dependencies.grib = [ "requests" ]
 
-optional-dependencies.provenance = [
-  "gitpython",
-  "nvsmi",
-]
-optional-dependencies.tests = [
-  "pytest",
-]
+optional-dependencies.provenance = [ "gitpython", "nvsmi" ]
+
+optional-dependencies.tests = [ "pytest" ]
+
+optional-dependencies.text = [ "termcolor", "wcwidth" ]
 
-optional-dependencies.text = [
-  "termcolor",
-]
 urls.Documentation = "https://anemoi-utils.readthedocs.io/"
 urls.Homepage = "https://github.com/ecmwf/anemoi-utils/"
 urls.Issues = "https://github.com/ecmwf/anemoi-utils/issues"
@@ -108,9 +76,7 @@ urls.Repository = "https://github.com/ecmwf/anemoi-utils/"
 scripts.anemoi-utils = "anemoi.utils.__main__:main"
 
 [tool.setuptools.package-data]
-"anemoi.utils.mars" = [
-  "*.yaml",
-]
+"anemoi.utils.mars" = [ "*.yaml" ]
 
 [tool.setuptools_scm]
 version_file = "src/anemoi/utils/_version.py"

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/src/anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.15'
-__version_tuple__ = version_tuple = (0, 3, 15)
+__version__ = version = '0.3.17'
+__version_tuple__ = version_tuple = (0, 3, 17)

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/src/anemoi/utils/caching.py

@@ -49,6 +49,7 @@ def cache(key, proc, collection="default", expires=None):
 
 
 class cached:
+    """Decorator to cache the result of a function."""
 
     def __init__(self, collection="default", expires=None):
         self.collection = collection

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/src/anemoi/utils/checkpoints.py

@@ -47,7 +47,7 @@ def has_metadata(path: str, name: str = DEFAULT_NAME) -> bool:
     return False
 
 
-def load_metadata(path: str, name: str = DEFAULT_NAME):
+def load_metadata(path: str, name: str = DEFAULT_NAME) -> dict:
     """Load metadata from a checkpoint file
 
     Parameters
@@ -59,8 +59,8 @@ def load_metadata(path: str, name: str = DEFAULT_NAME):
 
     Returns
     -------
-    JSON
-        The content of the metadata file
+    dict
+        The content of the metadata file from JSON
 
     Raises
     ------
@@ -82,7 +82,7 @@ def load_metadata(path: str, name: str = DEFAULT_NAME):
         raise ValueError(f"Could not find '{name}' in {path}.")
 
 
-def save_metadata(path, metadata, name=DEFAULT_NAME, folder=DEFAULT_FOLDER):
+def save_metadata(path, metadata, name=DEFAULT_NAME, folder=DEFAULT_FOLDER) -> None:
     """Save metadata to a checkpoint file
 
     Parameters
@@ -93,6 +93,8 @@ def save_metadata(path, metadata, name=DEFAULT_NAME, folder=DEFAULT_FOLDER):
         A JSON serializable object
     name : str, optional
         The name of the metadata file in the zip archive
+    folder : str, optional
+        The folder where the metadata file will be saved
     """
     with zipfile.ZipFile(path, "a") as zipf:
 

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/src/anemoi/utils/cli.py

@@ -55,6 +55,8 @@ def make_parser(description, commands):
 
 
 class Failed(Command):
+    """Command not available."""
+
     def __init__(self, name, error):
         self.name = name
         self.error = error

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/src/anemoi/utils/config.py

@@ -5,6 +5,7 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+from __future__ import annotations
 
 import json
 import logging
@@ -175,9 +176,14 @@ def config_path(name="settings.toml"):
     return full
 
 
-def load_any_dict_format(path):
+def load_any_dict_format(path) -> dict:
     """Load a configuration file in any supported format: JSON, YAML and TOML.
 
+    Parameters
+    ----------
+    path : str
+        The path to the configuration file.
+
     Returns
     -------
     dict
@@ -243,7 +249,7 @@ def _load_config(name="settings.toml", secrets=None, defaults=None):
     return CONFIG[key]
 
 
-def _save_config(name, data):
+def _save_config(name, data) -> None:
     CONFIG.pop(name, None)
 
     conf = config_path(name)
@@ -265,7 +271,7 @@ def _save_config(name, data):
         f.write(data)
 
 
-def save_config(name, data):
+def save_config(name, data) -> None:
     """Save a configuration file.
 
     Parameters
@@ -281,13 +287,17 @@ def save_config(name, data):
         _save_config(name, data)
 
 
-def load_config(name="settings.toml", secrets=None, defaults=None):
+def load_config(name="settings.toml", secrets=None, defaults=None) -> DotDict | str:
     """Read a configuration file.
 
     Parameters
     ----------
     name : str, optional
         The name of the config file to read, by default "settings.toml"
+    secrets : str or list, optional
+        The name of the secrets file, by default None
+    defaults : str or dict, optional
+        The name of the defaults file, by default None
 
     Returns
     -------
@@ -299,7 +309,7 @@ def load_config(name="settings.toml", secrets=None, defaults=None):
         return _load_config(name, secrets, defaults)
 
 
-def load_raw_config(name, default=None):
+def load_raw_config(name, default=None) -> DotDict | str:
 
     path = config_path(name)
     if os.path.exists(path):
@@ -308,13 +318,17 @@ def load_raw_config(name, default=None):
     return default
 
 
-def check_config_mode(name="settings.toml", secrets_name=None, secrets=None):
+def check_config_mode(name="settings.toml", secrets_name=None, secrets=None) -> None:
     """Check that a configuration file is secure.
 
     Parameters
     ----------
     name : str, optional
         The name of the configuration file, by default "settings.toml"
+    secrets_name : str, optional
+        The name of the secrets file, by default None
+    secrets : list, optional
+        The list of secrets to check, by default None
 
     Raises
     ------

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/src/anemoi/utils/dates.py

@@ -10,12 +10,20 @@ import calendar
 import datetime
 import re
 
-import isodate
+import aniso8601
 
-from .hindcasts import HindcastDatesTimes
+
+def normalise_frequency(frequency):
+    if isinstance(frequency, int):
+        return frequency
+    assert isinstance(frequency, str), (type(frequency), frequency)
+
+    unit = frequency[-1].lower()
+    v = int(frequency[:-1])
+    return {"h": v, "d": v * 24}[unit]
 
 
-def _no_time_zone(date):
+def _no_time_zone(date) -> datetime.datetime:
     """Remove time zone information from a date.
 
     Parameters
@@ -33,7 +41,7 @@ def _no_time_zone(date):
 
 
 # this function is use in anemoi-datasets
-def as_datetime(date, keep_time_zone=False):
+def as_datetime(date, keep_time_zone=False) -> datetime.datetime:
     """Convert a date to a datetime object, removing any time zone information.
 
     Parameters
@@ -63,7 +71,41 @@ def as_datetime(date, keep_time_zone=False):
     raise ValueError(f"Invalid date type: {type(date)}")
 
 
-def frequency_to_timedelta(frequency):
+def _as_datetime_list(date, default_increment):
+    if isinstance(date, (list, tuple)):
+        for d in date:
+            yield from _as_datetime_list(d, default_increment)
+
+    if isinstance(date, str):
+        # Check for ISO format
+        try:
+            start, end = aniso8601.parse_interval(date)
+            while start <= end:
+                yield as_datetime(start)
+                start += default_increment
+
+            return
+
+        except aniso8601.exceptions.ISOFormatError:
+            pass
+
+        try:
+            intervals = aniso8601.parse_repeating_interval(date)
+            for date in intervals:
+                yield as_datetime(date)
+            return
+        except aniso8601.exceptions.ISOFormatError:
+            pass
+
+    yield as_datetime(date)
+
+
+def as_datetime_list(date, default_increment=1):
+    default_increment = frequency_to_timedelta(default_increment)
+    return list(_as_datetime_list(date, default_increment))
+
+
+def frequency_to_timedelta(frequency) -> datetime.timedelta:
     """Convert a frequency to a timedelta object.
 
     Parameters
@@ -120,14 +162,14 @@ def frequency_to_timedelta(frequency):
 
     # ISO8601
     try:
-        return isodate.parse_duration(frequency)
-    except isodate.isoerror.ISO8601Error:
+        return aniso8601.parse_duration(frequency)
+    except aniso8601.exceptions.ISOFormatError:
         pass
 
     raise ValueError(f"Cannot convert frequency {frequency} to timedelta")
 
 
-def frequency_to_string(frequency):
+def frequency_to_string(frequency) -> str:
     """Convert a frequency (i.e. a datetime.timedelta) to a string.
 
     Parameters
@@ -174,7 +216,7 @@ def frequency_to_string(frequency):
     return str(frequency)
 
 
-def frequency_to_seconds(frequency):
+def frequency_to_seconds(frequency) -> int:
     """Convert a frequency to seconds.
 
     Parameters
@@ -362,6 +404,8 @@ class Autumn(DateTimes):
 
 
 class ConcatDateTimes:
+    """ConcatDateTimes is an iterator that generates datetime objects from a list of dates."""
+
     def __init__(self, *dates):
         if len(dates) == 1 and isinstance(dates[0], list):
             dates = dates[0]
@@ -374,6 +418,8 @@ class ConcatDateTimes:
 
 
 class EnumDateTimes:
+    """EnumDateTimes is an iterator that generates datetime objects from a list of dates."""
+
     def __init__(self, dates):
         self.dates = dates
 
@@ -393,6 +439,8 @@ def datetimes_factory(*args, **kwargs):
         name = kwargs.get("name")
 
         if name == "hindcast":
+            from .hindcasts import HindcastDatesTimes
+
             reference_dates = kwargs["reference_dates"]
             reference_dates = datetimes_factory(reference_dates)
             years = kwargs["years"]
@@ -416,3 +464,8 @@ def datetimes_factory(*args, **kwargs):
             return datetimes_factory(*a)
 
     return ConcatDateTimes(*[datetimes_factory(a) for a in args])
+
+
+if __name__ == "__main__":
+    print(as_datetime_list("R10/2023-01-01T00:00:00Z/P1D"))
+    print(as_datetime_list("2007-03-01T13:00:00/2008-05-11T15:30:00", "200h"))

{anemoi_utils-0.3.15 → anemoi_utils-0.3.17}/src/anemoi/utils/grib.py

@@ -95,8 +95,8 @@ def units(param) -> str:
 
     Parameters
     ----------
-    paramid : int or str
-        Parameter id ir name.
+    param : int or str
+        Parameter id or name.
 
     Returns
     -------
@@ -112,7 +112,7 @@ def units(param) -> str:
     return _units()[unit_id]
 
 
-def must_be_positive(param):
+def must_be_positive(param) -> bool:
     """Check if a parameter must be positive.
 
     Parameters