anemoi-utils 0.4.9__tar.gz → 0.4.11__tar.gz

anemoi_utils-0.4.11/.github/release.yml

@@ -0,0 +1,20 @@
+# .github/release.yml
+# https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes
+
+changelog:
+  exclude:
+    labels:
+      - ignore-for-release
+      - no-changelog
+    authors:
+      - pre-commit-ci
+  categories:
+    - title: Breaking Changes 🛠
+      labels:
+        - breaking-change
+    - title: Exciting New Features 🎉
+      labels:
+        - enhancement
+    - title: Other Changes
+      labels:
+        - "*"

anemoi_utils-0.4.11/.github/workflows/merge-main-into-develop.yml

@@ -0,0 +1,31 @@
+name: Merge main into develop
+on:
+  push:
+    branches:
+      - main
+
+
+permissions:
+  # Needed to read branches
+  contents: read
+  # Needed to create PR's
+  pull-requests: write
+
+jobs:
+  sync-branches:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Opening pull request
+        id: pull
+        uses: jdtx0/branch-sync@1.5.1
+        with:
+          GITHUB_TOKEN: ${{ secrets.REPO_SYNC_ACTION_PAT }}
+          FROM_BRANCH: "main"
+          TO_BRANCH: "develop"
+          PULL_REQUEST_AUTO_MERGE_METHOD: "merge"

anemoi_utils-0.4.11/.github/workflows/pr-conventional-commit.yml

@@ -0,0 +1,21 @@
+name: "Ensure Conventional Commit in PR title"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+
+permissions:
+  pull-requests: read
+
+jobs:
+  main:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

{anemoi_utils-0.4.9 → anemoi_utils-0.4.11}/.github/workflows/python-publish.yml

@@ -8,20 +8,6 @@ on:
     types: [created]
 
 jobs:
-  quality:
-    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
-    with:
-      skip-hooks: "no-commit-to-branch"
-
-  checks:
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
-    with:
-      python-version: ${{ matrix.python-version }}
-
   deploy:
-    needs: [checks, quality]
     uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
     secrets: inherit

anemoi_utils-0.4.11/.github/workflows/release-please.yml

@@ -0,0 +1,24 @@
+name: Run Release Please
+on:
+  push:
+    branches:
+      - develop
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  release-please:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: googleapis/release-please-action@v4
+        with:
+          # this assumes that you have created a personal access token
+          # (PAT) and configured it as a GitHub action secret named
+          # `MY_RELEASE_PLEASE_TOKEN` (this secret name is not important).
+          token: ${{ secrets.RELEASE_PLEASE_TOKEN }}
+          # optional. customize path to .release-please-config.json
+          config-file: .release-please-config.json
+          # Currently releases are done "from main" to have a stable branch
+          # target-branch: main

{anemoi_utils-0.4.9 → anemoi_utils-0.4.11}/.gitignore

@@ -173,7 +173,6 @@ bar
 *.grib
 *.nc
 *.npz
-*.json
 *.zarr/
 ~$images.pptx
 test.py

{anemoi_utils-0.4.9 → anemoi_utils-0.4.11}/.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.7.2
+  rev: v0.8.6
   hooks:
   - id: ruff
     args:
@@ -64,7 +64,7 @@ repos:
   hooks:
   - id: pyproject-fmt
 -   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
-    rev: v0.64.0
+    rev: v0.66.1
     hooks:
     -   id: docsig
         args:
@@ -74,7 +74,6 @@ repos:
         - --check-protected  # Check protected methods
         - --check-class      # Check class docstrings
         - --disable=E113     # Disable empty docstrings
-        - --summary          # Print a summary
 ci:
   autoupdate_schedule: monthly
 ci:

anemoi_utils-0.4.11/.release-please-config.json

@@ -0,0 +1,19 @@
+{
+  "release-type": "python",
+  "bump-minor-pre-major": true,
+  "bump-patch-for-minor-pre-major": true,
+  "separate-pull-requests": true,
+  "always-update": true,
+  "changelog-type": "github",
+  "include-component-in-tag": false,
+  "include-v-in-tag": false,
+  "pull-request-title-pattern": "chore${scope}: Preparing Next Release for ${component} ${version}",
+  "pull-request-header": ":robot:This is an automated PR using `release-please`.\n\nThe following changes will be included in the release, once the next version is triggered.\n\nContent of the next Release",
+  "pull-request-footer": "> [!IMPORTANT]\n> Merging this PR creates a release. Please refer to the [documentation](https://github.com/googleapis/release-please) if you're unsure.",
+  "packages": {
+    ".": {
+        "package-name": "anemoi-utils"
+    }
+  },
+  "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json"
+}

anemoi_utils-0.4.11/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.4.11"
+}

{anemoi_utils-0.4.9 → anemoi_utils-0.4.11}/CHANGELOG.md

@@ -8,6 +8,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## 0.4.11 (2025-01-17)
+
+<!-- Release notes generated using configuration in .github/release.yml at develop -->
+
+## What's Changed
+### Other Changes
+* Feature request: Add option to read configuration from stdin by @mpartio in https://github.com/ecmwf/anemoi-utils/pull/59
+* feat(plots): Add quick map plot for debugging by @b8raoult in https://github.com/ecmwf/anemoi-utils/pull/69
+* feat: added-anemoi-utils-grids-and-tests by @floriankrb in https://github.com/ecmwf/anemoi-utils/pull/74
+* feat(plot): added plotting options by @NRaoult in https://github.com/ecmwf/anemoi-utils/pull/72
+* ci(release): Simplify Release Workflow to Minimum by @JesperDramsch in https://github.com/ecmwf/anemoi-utils/pull/78
+* feat: adding-tools-for-grids by @floriankrb in https://github.com/ecmwf/anemoi-utils/pull/76
+
+## New Contributors
+* @mpartio made their first contribution in https://github.com/ecmwf/anemoi-utils/pull/59
+* @NRaoult made their first contribution in https://github.com/ecmwf/anemoi-utils/pull/72
+
+**Full Changelog**: https://github.com/ecmwf/anemoi-utils/compare/0.4.10...0.4.11
+
 ## [0.4.5](https://github.com/ecmwf/anemoi-utils/compare/0.4.4...0.4.5) - 2024-11-06
 
 ### What's Changed

{anemoi_utils-0.4.9 → anemoi_utils-0.4.11}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: anemoi-utils
-Version: 0.4.9
+Version: 0.4.11
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -226,6 +226,7 @@ Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: aniso8601
 Requires-Dist: importlib-metadata; python_version < "3.10"
+Requires-Dist: numpy
 Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
 Requires-Dist: tomli; python_version < "3.11"

{anemoi_utils-0.4.9 → anemoi_utils-0.4.11}/pyproject.toml

@@ -42,6 +42,7 @@ dynamic = [ "version" ]
 dependencies = [
   "aniso8601",
   "importlib-metadata; python_version<'3.10'",
+  "numpy",
   "python-dateutil",
   "pyyaml",
   "tomli; python_version<'3.11'",

{anemoi_utils-0.4.9 → anemoi_utils-0.4.11}/src/anemoi/utils/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.4.9'
-__version_tuple__ = version_tuple = (0, 4, 9)
+__version__ = version = '0.4.11'
+__version_tuple__ = version_tuple = (0, 4, 11)

anemoi_utils-0.4.11/src/anemoi/utils/caching.py

@@ -0,0 +1,124 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import hashlib
+import json
+import os
+import time
+from threading import Lock
+
+import numpy as np
+
+LOCK = Lock()
+CACHE = {}
+
+
+def _get_cache_path(collection):
+    return os.path.join(os.path.expanduser("~"), ".cache", "anemoi", collection)
+
+
+def clean_cache(collection="default"):
+    global CACHE
+    CACHE = {}
+    path = _get_cache_path(collection)
+    if not os.path.exists(path):
+        return
+    for filename in os.listdir(path):
+        os.remove(os.path.join(path, filename))
+
+
+class Cacher:
+    """This class implements a simple caching mechanism.
+    Private class, do not use directly"""
+
+    def __init__(self, collection, expires):
+        self.collection = collection
+        self.expires = expires
+
+    def __call__(self, func):
+
+        full = f"{func.__module__}.{func.__name__}"
+
+        def wrapped(*args, **kwargs):
+            with LOCK:
+                return self.cache(
+                    (full, args, kwargs),
+                    lambda: func(*args, **kwargs),
+                )
+
+        return wrapped
+
+    def cache(self, key, proc):
+
+        key = json.dumps(key, sort_keys=True)
+        m = hashlib.md5()
+        m.update(key.encode("utf-8"))
+        m = m.hexdigest()
+
+        if m in CACHE:
+            return CACHE[m]
+
+        path = _get_cache_path(self.collection)
+
+        filename = os.path.join(path, m) + self.ext
+        if os.path.exists(filename):
+            data = self.load(filename)
+            if self.expires is None or data["expires"] > time.time():
+                if data["key"] == key:
+                    return data["value"]
+
+        value = proc()
+        data = {"key": key, "value": value}
+        if self.expires is not None:
+            data["expires"] = time.time() + self.expires
+
+        os.makedirs(path, exist_ok=True)
+        temp_filename = self.save(filename, data)
+        os.rename(temp_filename, filename)
+
+        CACHE[m] = value
+        return value
+
+
+class JsonCacher(Cacher):
+    ext = ""
+
+    def save(self, path, data):
+        temp_path = path + ".tmp"
+        with open(temp_path, "w") as f:
+            json.dump(data, f)
+        return temp_path
+
+    def load(self, path):
+        with open(path, "r") as f:
+            return json.load(f)
+
+
+class NpzCacher(Cacher):
+    ext = ".npz"
+
+    def save(self, path, data):
+        temp_path = path + ".tmp.npz"
+        np.savez(temp_path, **data)
+        return temp_path
+
+    def load(self, path):
+        return np.load(path, allow_pickle=True)
+
+
+# PUBLIC API
+def cached(collection="default", expires=None, encoding="json"):
+    """Decorator to cache the result of a function.
+
+    Default is to use a json file to store the cache, but you can also use npz files
+    to cache dict of numpy arrays.
+
+    """
+    return dict(json=JsonCacher, npz=NpzCacher)[encoding](collection, expires)

anemoi_utils-0.4.11/src/anemoi/utils/commands/requests.py

@@ -0,0 +1,44 @@
+# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import json
+
+from anemoi.utils.mars.requests import print_request
+
+from . import Command
+
+
+class Requests(Command):
+    """Convert a JSON requests file to MARS format."""
+
+    def add_arguments(self, command_parser):
+        command_parser.add_argument("input")
+        command_parser.add_argument("output")
+        command_parser.add_argument("--verb", default="retrieve")
+        command_parser.add_argument("--only-one-field", action="store_true")
+
+    def run(self, args):
+        with open(args.input) as f:
+            requests = json.load(f)
+
+        if args.only_one_field:
+            for r in requests:
+                for key in (
+                    "grid",
+                    "area",
+                ):
+                    r.pop(key, None)
+                for k, v in list(r.items()):
+                    if isinstance(v, list):
+                        r[k] = v[-1]
+
+        with open(args.output, "w") as f:
+            for r in requests:
+                print_request(args.verb, r, file=f)
+
+
+command = Requests

{anemoi_utils-0.4.9 → anemoi_utils-0.4.11}/src/anemoi/utils/config.py

@@ -50,14 +50,14 @@ class DotDict(dict):
         super().__init__(*args, **kwargs)
 
         for k, v in self.items():
-            if isinstance(v, dict):
+            if isinstance(v, dict) or is_omegaconf_dict(v):
                 self[k] = DotDict(v)
 
-            if isinstance(v, list):
-                self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
+            if isinstance(v, list) or is_omegaconf_list(v):
+                self[k] = [DotDict(i) if isinstance(i, dict) or is_omegaconf_dict(i) else i for i in v]
 
             if isinstance(v, tuple):
-                self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v]
+                self[k] = [DotDict(i) if isinstance(i, dict) or is_omegaconf_dict(i) else i for i in v]
 
     @classmethod
     def from_file(cls, path: str):
@@ -106,6 +106,24 @@ class DotDict(dict):
         return f"DotDict({super().__repr__()})"
 
 
+def is_omegaconf_dict(value) -> bool:
+    try:
+        from omegaconf import DictConfig
+
+        return isinstance(value, DictConfig)
+    except ImportError:
+        return False
+
+
+def is_omegaconf_list(value) -> bool:
+    try:
+        from omegaconf import ListConfig
+
+        return isinstance(value, ListConfig)
+    except ImportError:
+        return False
+
+
 CONFIG = {}
 CHECKED = {}
 CONFIG_LOCK = threading.RLock()
@@ -205,6 +223,23 @@ def load_any_dict_format(path) -> dict:
         if path.endswith(".toml"):
             with open(path, "rb") as f:
                 return tomllib.load(f)
+
+        if path == "-":
+            import sys
+
+            config = sys.stdin.read()
+
+            parsers = [(yaml.safe_load, "yaml"), (json.loads, "json"), (tomllib.loads, "toml")]
+
+            for parser, parser_type in parsers:
+                try:
+                    LOG.debug(f"Trying {parser_type} parser for stdin")
+                    return parser(config)
+                except Exception:
+                    pass
+
+            raise ValueError("Failed to parse configuration from stdin")
+
     except (json.JSONDecodeError, yaml.YAMLError, tomllib.TOMLDecodeError) as e:
         LOG.warning(f"Failed to parse config file {path}", exc_info=e)
         raise ValueError(f"Failed to parse config file {path} [{e}]")
@@ -376,3 +411,11 @@ def find(metadata, what, result=None, *, select: callable = None):
             find(v, what, result)
 
     return result
+
+
+def merge_configs(*configs):
+    result = {}
+    for config in configs:
+        _merge_dicts(result, config)
+
+    return result

anemoi_utils-0.4.11/src/anemoi/utils/devtools.py

@@ -0,0 +1,83 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import cartopy.crs as ccrs
+import cartopy.feature as cfeature
+import matplotlib.pyplot as plt
+import matplotlib.tri as tri
+import numpy as np
+
+"""FOR DEVELOPMENT PURPOSES ONLY
+
+This module contains
+
+"""
+
+# TODO: use earthkit-plots
+
+
+def fix(lons):
+    return np.where(lons > 180, lons - 360, lons)
+
+
+def plot_values(
+    values, latitudes, longitudes, title=None, missing_value=None, min_value=None, max_value=None, **kwargs
+):
+
+    _, ax = plt.subplots(subplot_kw={"projection": ccrs.PlateCarree()})
+    ax.coastlines()
+    ax.add_feature(cfeature.BORDERS, linestyle=":")
+
+    missing_values = np.isnan(values)
+
+    if missing_value is None:
+        values = values[~missing_values]
+        longitudes = longitudes[~missing_values]
+        latitudes = latitudes[~missing_values]
+    else:
+        values = np.where(missing_values, missing_value, values)
+
+    if max_value is not None:
+        values = np.where(values > max_value, max_value, values)
+
+    if min_value is not None:
+        values = np.where(values < min_value, min_value, values)
+
+    triangulation = tri.Triangulation(fix(longitudes), latitudes)
+
+    levels = kwargs.pop("levels", 10)
+
+    _ = ax.tricontourf(triangulation, values, levels=levels, transform=ccrs.PlateCarree())
+
+    options = dict(
+        levels=levels,
+        colors="black",
+        linewidths=0.5,
+        transform=ccrs.PlateCarree(),
+    )
+
+    options.update(kwargs)
+
+    ax.tricontour(
+        triangulation,
+        values,
+        **options,
+    )
+
+    if title is not None:
+        ax.set_title(title)
+
+    return ax
+
+
+def plot_field(field, title=None, **kwargs):
+    values = field.to_numpy(flatten=True)
+    latitudes, longitudes = field.grid_points()
+    return plot_values(values, latitudes, longitudes, title=title, **kwargs)

anemoi_utils-0.4.11/src/anemoi/utils/grids.py

@@ -0,0 +1,97 @@
+# (C) Copyright 2025 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+"""Utilities for working with grids.
+
+"""
+
+import logging
+import os
+from io import BytesIO
+
+import numpy as np
+import requests
+
+from .caching import cached
+
+LOG = logging.getLogger(__name__)
+
+
+GRIDS_URL_PATTERN = "https://get.ecmwf.int/repository/anemoi/grids/grid-{name}.npz"
+
+
+def xyz_to_latlon(x, y, z):
+    return (
+        np.rad2deg(np.arcsin(np.minimum(1.0, np.maximum(-1.0, z)))),
+        np.rad2deg(np.arctan2(y, x)),
+    )
+
+
+def latlon_to_xyz(lat, lon, radius=1.0):
+    # https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#From_geodetic_to_ECEF_coordinates
+    # We assume that the Earth is a sphere of radius 1 so N(phi) = 1
+    # We assume h = 0
+    #
+    phi = np.deg2rad(lat)
+    lda = np.deg2rad(lon)
+
+    cos_phi = np.cos(phi)
+    cos_lda = np.cos(lda)
+    sin_phi = np.sin(phi)
+    sin_lda = np.sin(lda)
+
+    x = cos_phi * cos_lda * radius
+    y = cos_phi * sin_lda * radius
+    z = sin_phi * radius
+
+    return x, y, z
+
+
+def nearest_grid_points(source_latitudes, source_longitudes, target_latitudes, target_longitudes):
+    from scipy.spatial import cKDTree
+
+    source_xyz = latlon_to_xyz(source_latitudes, source_longitudes)
+    source_points = np.array(source_xyz).transpose()
+
+    target_xyz = latlon_to_xyz(target_latitudes, target_longitudes)
+    target_points = np.array(target_xyz).transpose()
+
+    _, indices = cKDTree(source_points).query(target_points, k=1)
+    return indices
+
+
+@cached(collection="grids", encoding="npz")
+def _grids(name):
+    from anemoi.utils.config import load_config
+
+    user_path = load_config().get("utils", {}).get("grids_path")
+    if user_path:
+        path = os.path.expanduser(os.path.join(user_path, f"grid-{name}.npz"))
+        if os.path.exists(path):
+            LOG.warning("Loading grids from custom user path %s", path)
+            with open(path, "rb") as f:
+                return f.read()
+        else:
+            LOG.warning("Custom user path %s does not exist", path)
+
+    url = GRIDS_URL_PATTERN.format(name=name.lower())
+    LOG.warning("Downloading grids from %s", url)
+    response = requests.get(url)
+    response.raise_for_status()
+    return response.content
+
+
+def grids(name):
+    if name.endswith(".npz"):
+        return dict(np.load(name))
+
+    data = _grids(name)
+    npz = np.load(BytesIO(data))
+    return dict(npz)