anemoi-datasets 0.5.13__tar.gz → 0.5.14__tar.gz

anemoi_datasets-0.5.14/.github/workflows/python-publish.yml

@@ -0,0 +1,27 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  # quality:
+  #   if: ${{ !contains(github.repository, 'private') }}
+  #   uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+  #   with:
+  #     skip-hooks: "no-commit-to-branch"
+
+  # checks:
+  #   strategy:
+  #     matrix:
+  #       python-version: ["3.9", "3.10", "3.11", "3.12"]
+  #   uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+  #   with:
+  #     python-version: ${{ matrix.python-version }}
+
+  deploy:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.1
+  rev: v0.8.6
   hooks:
   - id: ruff
     args:

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/CHANGELOG.md

@@ -18,6 +18,8 @@ Keep it human-readable, your future self will thank you!
 - Use cKDTree instead of KDTree
 - Implement 'complement' feature
 - Add ability to patch xarrays (#160)
+- Add support of ECCC file formats (fstd)
+- Add `use_cdsapi_dataset` option to mars and accumulations
 
 ### Added
 

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: anemoi-datasets
-Version: 0.5.13
+Version: 0.5.14
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.5.13'
-__version_tuple__ = version_tuple = (0, 5, 13)
+__version__ = version = '0.5.14'
+__version_tuple__ = version_tuple = (0, 5, 14)

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi/datasets/commands/create.py

@@ -83,11 +83,12 @@ class Create(Command):
         task("load", options)
         task("finalise", options)
 
-        task("patch", options)
-
         task("init_additions", options)
         task("run_additions", options)
         task("finalise_additions", options)
+
+        task("patch", options)
+
         task("cleanup", options)
         task("verify", options)
 
@@ -153,6 +154,7 @@ class Create(Command):
 
         with ExecutorClass(max_workers=1) as executor:
             executor.submit(task, "finalise-additions", options).result()
+            executor.submit(task, "patch", options).result()
             executor.submit(task, "cleanup", options).result()
             executor.submit(task, "verify", options).result()
 

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi/datasets/create/__init__.py

@@ -79,7 +79,10 @@ def json_tidy(o):
         )
         return o.isoformat()
 
-    raise TypeError(repr(o) + " is not JSON serializable")
+    if isinstance(o, (np.float32, np.float64)):
+        return float(o)
+
+    raise TypeError(f"{repr(o)} is not JSON serializable {type(o)}")
 
 
 def build_statistics_dates(dates, start, end):
@@ -596,6 +599,8 @@ class Load(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixi
         # There is one cube to load for each result.
         dates = list(result.group_of_dates)
 
+        LOG.debug(f"Loading cube for {len(dates)} dates")
+
         cube = result.get_cube()
         shape = cube.extended_user_shape
         dates_in_data = cube.user_coords["valid_datetime"]
@@ -642,12 +647,14 @@ class Load(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixi
         indexes = dates_to_indexes(self.dates, dates_in_data)
 
         array = ViewCacheArray(self.data_array, shape=shape, indexes=indexes)
+        LOG.info(f"Loading array shape={shape}, indexes={len(indexes)}")
         self.load_cube(cube, array)
 
         stats = compute_statistics(array.cache, self.variables_names, allow_nans=self._get_allow_nans())
         self.tmp_statistics.write(indexes, stats, dates=dates_in_data)
-
+        LOG.info("Flush data array")
         array.flush()
+        LOG.info("Flushed data array")
 
     def _get_allow_nans(self):
         config = self.main_config
@@ -740,6 +747,11 @@ class AdditionsMixin:
         if not self.delta.total_seconds() % frequency.total_seconds() == 0:
             LOG.debug(f"Delta {self.delta} is not a multiple of frequency {frequency}. Skipping.")
             return True
+
+        if self.dataset.zarr_metadata.get("build", {}).get("additions", None) is False:
+            LOG.warning(f"Additions are disabled for {self.path} in the recipe.")
+            return True
+
         return False
 
     @cached_property

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi/datasets/create/functions/sources/accumulations.py

@@ -253,6 +253,7 @@ def _compute_accumulations(
     data_accumulation_period=None,
     patch=_identity,
     base_times=None,
+    use_cdsapi_dataset=None,
 ):
     adjust_step = isinstance(user_accumulation_period, int)
 
@@ -311,7 +312,9 @@ def _compute_accumulations(
 
                 requests.append(patch(r))
 
-    ds = mars(context, dates, *requests, request_already_using_valid_datetime=True)
+    ds = mars(
+        context, dates, *requests, request_already_using_valid_datetime=True, use_cdsapi_dataset=use_cdsapi_dataset
+    )
 
     accumulations = {}
     for a in [AccumulationClass(out, frequency=frequency, **r) for r in requests]:
@@ -366,7 +369,7 @@ def _scda(request):
     return request
 
 
-def accumulations(context, dates, **request):
+def accumulations(context, dates, use_cdsapi_dataset=None, **request):
     _to_list(request["param"])
     class_ = request.get("class", "od")
     stream = request.get("stream", "oper")
@@ -395,6 +398,7 @@ def accumulations(context, dates, **request):
         dates,
         request,
         user_accumulation_period=user_accumulation_period,
+        use_cdsapi_dataset=use_cdsapi_dataset,
         **kwargs,
     )
 

anemoi_datasets-0.5.14/src/anemoi/datasets/create/functions/sources/eccc_fstd.py

@@ -0,0 +1,16 @@
+# (C) Copyright 2025 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+from .xarray import load_many
+
+
+def execute(context, dates, path, *args, **kwargs):
+    options = {"engine": "fstd"}
+    return load_many("🍁", context, dates, path, *args, options=options, **kwargs)

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi/datasets/create/functions/sources/mars.py

@@ -246,6 +246,7 @@ def mars(
     *requests,
     request_already_using_valid_datetime=False,
     date_key="date",
+    use_cdsapi_dataset=None,
     **kwargs,
 ):
 
@@ -305,7 +306,10 @@ def mars(
                     f"⚠️ Unknown key {k}={v} in MARS request. Did you mean '{did_you_mean(k, MARS_KEYS)}' ?"
                 )
         try:
-            ds = ds + from_source("mars", **r)
+            if use_cdsapi_dataset:
+                ds = ds + from_source("cds", use_cdsapi_dataset, r)
+            else:
+                ds = ds + from_source("mars", **r)
         except Exception as e:
             if "File is empty:" not in str(e):
                 raise

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi/datasets/create/input/result.py

@@ -459,7 +459,7 @@ class Result:
         if self.group_of_dates is not None:
             dates = f" {len(self.group_of_dates)} dates"
             dates += " ("
-            dates += "/".join(d.strftime("%Y-%m-%d:%H") for d in self.group_of_dates)
+            dates += "/".join(d.strftime("%Y-%m-%dT%H:%M") for d in self.group_of_dates)
             if len(dates) > 100:
                 dates = dates[:100] + "..."
             dates += ")"

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi/datasets/create/statistics/__init__.py

@@ -18,6 +18,7 @@ import shutil
 import socket
 
 import numpy as np
+import tqdm
 from anemoi.utils.provenance import gather_provenance_info
 
 from ..check import check_data_values
@@ -134,7 +135,7 @@ def check_variance(x, variables_names, minimum, maximum, mean, count, sums, squa
 
 def compute_statistics(array, check_variables_names=None, allow_nans=False):
     """Compute statistics for a given array, provides minimum, maximum, sum, squares, count and has_nans as a dictionary."""
-
+    LOG.info(f"Computing statistics for {array.shape} array")
     nvars = array.shape[1]
 
     LOG.debug(f"Stats {nvars}, {array.shape}, {check_variables_names}")
@@ -149,7 +150,7 @@ def compute_statistics(array, check_variables_names=None, allow_nans=False):
     maximum = np.zeros(stats_shape, dtype=np.float64)
     has_nans = np.zeros(stats_shape, dtype=np.bool_)
 
-    for i, chunk in enumerate(array):
+    for i, chunk in tqdm.tqdm(enumerate(array), delay=1, total=array.shape[0], desc="Computing statistics"):
         values = chunk.reshape((nvars, -1))
 
         for j, name in enumerate(check_variables_names):
@@ -166,6 +167,8 @@ def compute_statistics(array, check_variables_names=None, allow_nans=False):
         count[i] = np.sum(~np.isnan(values), axis=1)
         has_nans[i] = np.isnan(values).any()
 
+    LOG.info(f"Statistics computed for {nvars} variables.")
+
     return {
         "minimum": minimum,
         "maximum": maximum,

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi/datasets/data/misc.py

@@ -103,6 +103,30 @@ def _as_date(d, dates, last):
 
     if isinstance(d, str):
 
+        def isfloat(s):
+            try:
+                float(s)
+                return True
+            except ValueError:
+                return False
+
+        if d.endswith("%") and isfloat(d[:-1]):
+            x = float(d[:-1])
+            if not 0 <= x <= 100:
+                raise ValueError(f"Invalid date: {d}")
+            i_float = x * len(dates) / 100
+
+            epsilon = 2 ** (-30)
+            if len(dates) > 1 / epsilon:
+                LOG.warning("Too many dates to use percentage, one date may be lost in rounding")
+
+            if last:
+                index = int(i_float + epsilon) - 1
+            else:
+                index = int(i_float - epsilon)
+            index = max(0, min(len(dates) - 1, index))
+            return dates[index]
+
         if "-" in d and ":" in d:
             date, time = d.replace(" ", "T").split("T")
             year, month, day = [int(_) for _ in date.split("-")]

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi/datasets/grids.py

@@ -62,6 +62,8 @@ def plot_mask(path, mask, lats, lons, global_lats, global_lons):
         plt.savefig(path + "-global-zoomed.png")
 
 
+# TODO: Use the one from anemoi.utils.grids instead
+# from anemoi.utils.grids import ...
 def xyz_to_latlon(x, y, z):
     return (
         np.rad2deg(np.arcsin(np.minimum(1.0, np.maximum(-1.0, z)))),
@@ -69,6 +71,8 @@ def xyz_to_latlon(x, y, z):
     )
 
 
+# TODO: Use the one from anemoi.utils.grids instead
+# from anemoi.utils.grids import ...
 def latlon_to_xyz(lat, lon, radius=1.0):
     # https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#From_geodetic_to_ECEF_coordinates
     # We assume that the Earth is a sphere of radius 1 so N(phi) = 1
@@ -380,6 +384,8 @@ def serialise_mask(mask):
 
 
 def nearest_grid_points(source_latitudes, source_longitudes, target_latitudes, target_longitudes):
+    # TODO: Use the one from anemoi.utils.grids instead
+    # from anemoi.utils.grids import ...
     from scipy.spatial import cKDTree
 
     source_xyz = latlon_to_xyz(source_latitudes, source_longitudes)

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi_datasets.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: anemoi-datasets
-Version: 0.5.13
+Version: 0.5.14
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/src/anemoi_datasets.egg-info/SOURCES.txt

@@ -260,6 +260,7 @@ src/anemoi/datasets/create/functions/filters/wz_to_w.py
 src/anemoi/datasets/create/functions/sources/__init__.py
 src/anemoi/datasets/create/functions/sources/accumulations.py
 src/anemoi/datasets/create/functions/sources/constants.py
+src/anemoi/datasets/create/functions/sources/eccc_fstd.py
 src/anemoi/datasets/create/functions/sources/empty.py
 src/anemoi/datasets/create/functions/sources/forcings.py
 src/anemoi/datasets/create/functions/sources/grib.py
@@ -347,6 +348,7 @@ tests/create/missing.yaml
 tests/create/nan.yaml
 tests/create/pipe.yaml
 tests/create/recentre.yaml
+tests/create/regrid.yaml
 tests/create/run.sh
 tests/create/test_create.py
 tests/xarray/test_kerchunk.py

anemoi_datasets-0.5.14/tests/create/regrid.yaml

@@ -0,0 +1,31 @@
+
+dates:
+  start: 2020-12-30 00:00:00
+  end: 2021-01-03 12:00:00
+  frequency: 12h
+
+input:
+  join:
+    - mars:
+        expver: "0001"
+        class: ea
+        grid: o48
+        param: [2t]
+        levtype: sfc
+        stream: oper
+        type: an
+    - pipe:
+      - mars:
+          expver: "0001"
+          class: ea
+          grid: o32
+          param: [q]
+          levtype: pl
+          level: [50]
+          stream: oper
+          type: an
+      - regrid:
+          # method: linear
+          method: nearest
+          in_grid: o32
+          out_grid: o48

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/tests/test_data.py

@@ -880,6 +880,51 @@ def test_dates_using_list():
     assert as_last_date("2021", dates) == np.datetime64("2021-12-31T06:00:00")
 
 
+@mockup_open_zarr
+def test_dates_using_list_2():
+    dates = [np.datetime64("2021-01-01T00:00:00") + i * np.timedelta64(24, "h") for i in range(0, 10)]
+    assert len(dates) == 10
+
+    assert dates[0] == as_first_date("0%", dates)
+    assert dates[0] == as_last_date("0%", dates)
+
+    assert dates[0] == as_first_date("0.01%", dates)
+    assert dates[0] == as_last_date("0.01%", dates)
+
+    assert dates[0] == as_first_date("9.99%", dates)
+    assert dates[0] == as_last_date("9.99%", dates)
+
+    assert dates[0] == as_first_date("10%", dates)
+    assert dates[0] == as_last_date("10%", dates)
+
+    assert dates[1] == as_first_date("10.01%", dates)
+    assert dates[0] == as_last_date("10.01%", dates)
+
+    assert dates[1] == as_first_date("19.99%", dates)
+    assert dates[0] == as_last_date("19.99%", dates)
+
+    assert dates[1] == as_first_date("20%", dates)
+    assert dates[1] == as_last_date("20%", dates)
+
+    assert dates[2] == as_first_date("20.01%", dates)
+    assert dates[1] == as_last_date("20.01%", dates)
+
+    assert dates[-2] == as_first_date("89.99%", dates)
+    assert dates[-3] == as_last_date("89.99%", dates)
+
+    assert dates[-2] == as_first_date("90%", dates)
+    assert dates[-2] == as_last_date("90%", dates)
+
+    assert dates[-1] == as_first_date("90.01%", dates)
+    assert dates[-2] == as_last_date("90.01%", dates)
+
+    assert dates[-1] == as_first_date("99.99%", dates)
+    assert dates[-2] == as_last_date("99.99%", dates)
+
+    assert dates[-1] == as_first_date("100%", dates)
+    assert dates[-1] == as_last_date("100%", dates)
+
+
 @mockup_open_zarr
 def test_slice_1():
     test = DatasetTester("test-2021-2021-6h-o96-abcd")

{anemoi_datasets-0.5.13 → anemoi_datasets-0.5.14}/tests/xarray/test_opendap.py

@@ -7,12 +7,18 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+import os
+
+import pytest
 import xarray as xr
 
 from anemoi.datasets.create.functions.sources.xarray import XarrayFieldList
 from anemoi.datasets.testing import assert_field_list
 
 
+# when the opendap server is under maintainance, this test will fail
+# it is skipped by default, and is only run when the SLOW_TESTS env var is set
+@pytest.mark.skipif(not os.environ.get("SLOW_TESTS"), reason="No SLOW_TESTS env var")
 def test_opendap():
 
     ds = xr.open_dataset(

anemoi_datasets-0.5.13/.github/workflows/python-publish.yml

@@ -1,28 +0,0 @@
-# This workflow will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
-
-name: Upload Python Package
-
-on:
-  release:
-    types: [created]
-
-jobs:
-  quality:
-    if: ${{ !contains(github.repository, 'private') }}
-    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
-    with:
-      skip-hooks: "no-commit-to-branch"
-
-  checks:
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
-    with:
-      python-version: ${{ matrix.python-version }}
-
-  deploy:
-    needs: [checks, quality]
-    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
-    secrets: inherit