anemoi-datasets 0.5.27__tar.gz → 0.5.28__tar.gz

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/.gitignore

@@ -141,3 +141,13 @@ _version.py
 *.to_upload
 tempCodeRunnerFile.python
 Untitled-*.py
+*.zip
+*.json
+*.db
+*.tgz
+_api/
+trace.txt
+?/
+*.prof
+prof/
+*.gz

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/.pre-commit-config.yaml

@@ -27,12 +27,12 @@ repos:
     -   id: python-check-blanket-noqa # Check for # noqa: all
     -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 25.1.0
+  rev: 25.9.0
   hooks:
   - id: black
     args: [--line-length=120]
 - repo: https://github.com/pycqa/isort
-  rev: 6.0.1
+  rev: 6.1.0
   hooks:
   - id: isort
     args:
@@ -41,7 +41,7 @@ repos:
     - --profile black
     - --project anemoi
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.12.11
+  rev: v0.13.3
   hooks:
   - id: ruff
     args:
@@ -65,7 +65,7 @@ repos:
   - id: docconvert
     args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "v2.6.0"
+  rev: "v2.7.0"
   hooks:
   - id: pyproject-fmt
 -   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
@@ -79,6 +79,7 @@ repos:
         - --check-protected  # Check protected methods
         - --check-class      # Check class docstrings
         - --disable=SIG101,SIG102     # Disable empty docstrings
+
 ci:
   autoupdate_schedule: monthly
   autoupdate_commit_msg: "chore(deps): pre-commit.ci autoupdate"

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/.readthedocs.yaml

@@ -4,9 +4,9 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.11"
-#  jobs:
-#    pre_build:
-#    - bash docs/scripts/api_build.sh
+  jobs:
+    pre_build:
+    - (cd docs/scripts && make rst)
 
 sphinx:
   configuration: docs/conf.py

anemoi_datasets-0.5.28/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.5.28"
+}

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/CHANGELOG.md

@@ -8,6 +8,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.5.28](https://github.com/ecmwf/anemoi-datasets/compare/0.5.27...0.5.28) (2025-10-30)
+
+
+### Features
+
+* Added rolling average on read ([#448](https://github.com/ecmwf/anemoi-datasets/issues/448)) ([5cef720](https://github.com/ecmwf/anemoi-datasets/commit/5cef7207b7900190272d970bf44f35a7011e3708))
+* Fix repeat dates ([#458](https://github.com/ecmwf/anemoi-datasets/issues/458)) ([16b5c83](https://github.com/ecmwf/anemoi-datasets/commit/16b5c836172a37c13b210b35baee1a5e97731e5d))
+* Refactor dataset create ([#379](https://github.com/ecmwf/anemoi-datasets/issues/379)) ([c5408da](https://github.com/ecmwf/anemoi-datasets/commit/c5408dafd34c80bcb4b0cd9623dbebfe00959151))
+
+
+### Bug Fixes
+
+* Allow grib source to be used for constant data ([#428](https://github.com/ecmwf/anemoi-datasets/issues/428)) ([c1d7ff1](https://github.com/ecmwf/anemoi-datasets/commit/c1d7ff1035848e26d3034903577c267973985c07))
+* Broken source imports ([#450](https://github.com/ecmwf/anemoi-datasets/issues/450)) ([#454](https://github.com/ecmwf/anemoi-datasets/issues/454)) ([d6ed174](https://github.com/ecmwf/anemoi-datasets/commit/d6ed174cf4f6da22464c050d3304b026fc0ae4a3))
+* Ensure missing value indicator in not a valid value in accumulat… ([#316](https://github.com/ecmwf/anemoi-datasets/issues/316)) ([3541a1f](https://github.com/ecmwf/anemoi-datasets/commit/3541a1f7068b54ceeb35dff93c3f94b4b138cb9d))
+* S3 access [#453](https://github.com/ecmwf/anemoi-datasets/issues/453) ([#459](https://github.com/ecmwf/anemoi-datasets/issues/459)) ([2d810e3](https://github.com/ecmwf/anemoi-datasets/commit/2d810e3775ace78ffe2c43cc9bac66d704c0754d))
+* Xarray-zarr filters ([#457](https://github.com/ecmwf/anemoi-datasets/issues/457)) ([5f2973f](https://github.com/ecmwf/anemoi-datasets/commit/5f2973fb7d128a3f6de5ed538e39cc7421eeb5dc))
+
+
+### Documentation
+
+* Documentation updates ([#419](https://github.com/ecmwf/anemoi-datasets/issues/419)) ([4588175](https://github.com/ecmwf/anemoi-datasets/commit/458817533e805652527945eef4de781c62d3abf9))
+
 ## [0.5.27](https://github.com/ecmwf/anemoi-datasets/compare/0.5.26...0.5.27) (2025-10-06)
 
 

{anemoi_datasets-0.5.27/src/anemoi_datasets.egg-info → anemoi_datasets-0.5.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-datasets
-Version: 0.5.27
+Version: 0.5.28
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -225,11 +225,14 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.10
 License-File: LICENSE
 Requires-Dist: anemoi-transform>=0.1.10
-Requires-Dist: anemoi-utils[provenance]>=0.4.32
+Requires-Dist: anemoi-utils[provenance]>=0.4.38
 Requires-Dist: cfunits
+Requires-Dist: glom
+Requires-Dist: jsonschema
 Requires-Dist: numcodecs<0.16
 Requires-Dist: numpy
 Requires-Dist: pyyaml
+Requires-Dist: ruamel-yaml
 Requires-Dist: semantic-version
 Requires-Dist: tqdm
 Requires-Dist: zarr<=2.18.4

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/docs/Makefile

@@ -19,5 +19,5 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	bash $(SOURCEDIR)/scripts/api_build.sh
+	@cd scripts && $(MAKE) $@
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

anemoi_datasets-0.5.28/docs/building/filters.rst

@@ -0,0 +1,10 @@
+.. _filters:
+
+#########
+ Filters
+#########
+
+Filters can be used to perform data transformations when creating a
+dataset. See the :ref:`anemoi-transform
+<anemoi-transform:list-of-filters>` documentation for a list of filters
+and example configurations.

{anemoi_datasets-0.5.27/docs/datasets → anemoi_datasets-0.5.28/docs}/building/introduction.rst

@@ -14,7 +14,7 @@ computed as the dataset is built, and stored in the metadata, with other
 information such as the locations of the grid points, the list of
 variables, etc.
 
-.. figure:: ../../_static/schemas/recipe.png
+.. figure:: ../_static/schemas/recipe.png
    :alt: Building datasets
    :align: center
 

{anemoi_datasets-0.5.27/docs/datasets → anemoi_datasets-0.5.28/docs}/building/sources/repeated-dates.rst

@@ -10,7 +10,7 @@ dates of the dataset.
 
 The general format of the `repeated-dates` source is:
 
-.. literalinclude:: yaml/repeated_dates1.yaml
+.. literalinclude:: yaml/repeated-dates1.yaml
    :language: yaml
 
 where ``source`` is any of the :ref:`operations <operations>` or

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/docs/cli/create.rst

@@ -4,7 +4,7 @@ Create Command
 ==============
 
 Use this command to create a dataset from a recipe file.
-The syntax of the recipe file is described in :doc:`building datasets <../datasets/building/introduction>`.
+The syntax of the recipe file is described in :ref:`building-introduction`.
 
 .. argparse::
     :module: anemoi.datasets.__main__

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/docs/cli/grib-index.rst

@@ -1,7 +1,7 @@
 .. _grib-index_command:
 
 Grib-index Command
-============
+==================
 
 The `grib-index` command is used to create an index file for GRIB files. The index file is then used
 by the `grib-index` :ref:`source <grib-index_source>`.

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/docs/cli/inspect.rst

@@ -15,7 +15,7 @@ This command will output the metadata of the dataset, including the variables, d
 
 which will output something like the following. The output should be self-explanatory.
 
-.. literalinclude:: ../datasets/yaml/building1.txt
+.. literalinclude:: ../yaml/building1.txt
    :language: console
 
 .. argparse::

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/docs/cli/introduction.rst

@@ -1,8 +1,8 @@
 .. _cli-introduction:
 
-##################
-Command line tool
-##################
+############
+Introduction
+############
 
 When you install the `anemoi-datasets` package, this will also install command line tool
 called ``anemoi-datasets`` which can be used to manage the zarr datasets.

anemoi_datasets-0.5.27/docs/usage/getting_started.rst → anemoi_datasets-0.5.28/docs/howtos/create/00-redo-sample-dataset.rst

@@ -1,8 +1,8 @@
-.. _usage-getting-started:
+.. _redo-sample-dataset:
 
-###########################
- Create Your First Dataset
-###########################
+##############################
+ Re-create the sample dataset
+##############################
 
 The following example presents a simple but comprehensive recipe to
 create an Anemoi dataset based on the atmospheric reanalysis ERA5. Once

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/docs/howtos/introduction.rst

@@ -35,4 +35,4 @@ good starting point.
    :hidden:
    :caption: Using anemoi-datasets
 
-   usage/*
+   using/*

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/docs/index.rst

@@ -4,10 +4,6 @@
  Welcome to `anemoi-datasets` documentation!
 #############################################
 
-.. warning::
-
-   This documentation is a work in progress.
-
 An *Anemoi dataset* is a thin wrapper around a Zarr_ store that is
 optimised for training data-driven weather forecasting models.
 anemoi-datasets are organised in such a way that I/O operations are
@@ -58,8 +54,7 @@ datasets for compatibility with machine learning models.
 In the rest of this documentation, you will learn how to configure and
 create anemoi datasets using YAML files, as well as how to load and read
 existing ones. A full example of a dataset preparation process can be
-found in the :ref:`Create Your First Dataset <usage-getting-started>`
-section.
+found in the :ref:`redo-sample-dataset` section.
 
 ************
  Installing
@@ -125,24 +120,15 @@ You may also have to install pandoc on macOS:
    :caption: Introduction
 
    overview
-   cli/introduction
    installing
 
-.. toctree::
-   :maxdepth: 1
-   :hidden:
-   :caption: Recipe examples
-
-   usage/getting_started
-
 .. toctree::
    :maxdepth: 1
    :hidden:
    :caption: User Guide
 
-   datasets/introduction
-   datasets/building/introduction
-   datasets/using/introduction
+   building/introduction
+   using/introduction
 
 .. toctree::
    :maxdepth: 3
@@ -156,6 +142,7 @@ You may also have to install pandoc on macOS:
    :hidden:
    :caption: CLI
 
+   cli/introduction
    cli/create
    cli/inspect
    cli/grib-index

anemoi_datasets-0.5.28/docs/installing.rst

@@ -0,0 +1,18 @@
+.. _installing:
+
+############
+ Installing
+############
+
+To install the package, you can use the following command:
+
+.. code:: bash
+
+   python -m pip install anemoi-datasets
+
+If you are interested in creating datasets, you can install the package
+with the following command:
+
+.. code:: bash
+
+   pip install anemoi-datasets[create]

anemoi_datasets-0.5.28/docs/modules/dataset.rst

@@ -0,0 +1,8 @@
+.. _dataset-autodoc:
+
+#########
+ Dataset
+#########
+
+.. autoclass:: anemoi.datasets.data.dataset.Dataset
+   :members: dates, latitudes, longitudes, dates_interval_to_indices, provenance, typed_variables, metadata, start_date, end_date, supporting_arrays, grids, variables, frequency, name_to_index, shape, field_shape, dtype, missing, statistics,statistics_tendencies

anemoi_datasets-0.5.28/docs/scripts/Makefile

@@ -0,0 +1,8 @@
+html: rst
+
+clean:
+	rm -rf ../api
+
+rst:
+	mkdir -p ../api
+# 	sphinx-apidoc --implicit-namespaces --separate  -o ../api ../../src/anemoi -t ../_templates/apidoc

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/docs/scripts/api_build.sh

@@ -9,5 +9,5 @@ source_dir="$script_dir/../../src/"
 
 trap 'rm -f $source_dir/anemoi/__init__.py' EXIT
 
-touch "$source_dir/anemoi/__init__.py"
+# touch "$source_dir/anemoi/__init__.py"
 sphinx-apidoc -M -f -o "$docs_dir/_api" "$source_dir/anemoi" -t "$docs_dir/_templates/apidoc"

{anemoi_datasets-0.5.27/docs/datasets → anemoi_datasets-0.5.28/docs}/using/combining.rst

@@ -67,7 +67,7 @@ concatenate datasets that have gaps between them, check the
 
    ds = open_dataset(concat=["dataset-1979-2000", "dataset-2001-2022"])
 
-.. image:: ../../_static/concat.png
+.. image:: ../_static/concat.png
    :align: center
    :alt: Concatenation
 
@@ -101,7 +101,7 @@ variables.
 
    ds = open_dataset(join=["dataset1-1979-2022", "dataset2-1979-2022"])
 
-.. image:: ../../_static/join.png
+.. image:: ../_static/join.png
    :align: center
    :alt: Join
 
@@ -109,7 +109,7 @@ If a variable is present in more than one file, the last occurrence of
 that variable will be used and will be at the position of the first
 occurrence of that name.
 
-.. image:: ../../_static//overlay.png
+.. image:: ../_static//overlay.png
    :align: center
    :alt: Overlay
 
@@ -164,14 +164,14 @@ the bounding box of the second dataset.
 
 The image below shows the global dataset:
 
-.. image:: ../../_static/cutout-1.png
+.. image:: ../_static/cutout-1.png
    :width: 75%
    :align: center
    :alt: Cutout
 
 The image below shows the LAM dataset:
 
-.. image:: ../../_static//cutout-2.png
+.. image:: ../_static//cutout-2.png
    :width: 75%
    :align: center
    :alt: Cutout
@@ -180,7 +180,7 @@ A 'cutout' is performed by removing the grid points from the global
 dataset that are contained in the LAM dataset. The result is shown
 below:
 
-.. image:: ../../_static//cutout-3.png
+.. image:: ../_static//cutout-3.png
    :width: 75%
    :align: center
    :alt: Cutout
@@ -188,7 +188,7 @@ below:
 The final dataset is the concatenation of the LAM dataset and the
 cutout:
 
-.. image:: ../../_static//cutout-4.png
+.. image:: ../_static//cutout-4.png
    :width: 75%
    :align: center
    :alt: Cutout
@@ -206,11 +206,11 @@ The plots below illustrate how the cutout differs if `min_distance_km`
 is not given (top) or if `min_distance_km` is set to `0` (bottom). The
 difference can be seen at the boundary between the two grids:
 
-.. image:: ../../_static//cutout-5.png
+.. image:: ../_static//cutout-5.png
    :align: center
    :alt: Cutout
 
-.. image:: ../../_static//cutout-6.png
+.. image:: ../_static//cutout-6.png
    :align: center
    :alt: Cutout
 

{anemoi_datasets-0.5.27/docs/datasets → anemoi_datasets-0.5.28/docs}/using/grids.rst

@@ -26,14 +26,14 @@ the thinning will apply to the only dimension.
 The following example shows the effect of thinning a dataset with a 1
 degree resolution:
 
-.. image:: ../../_static/thinning-before.png
+.. image:: ../_static/thinning-before.png
    :width: 75%
    :align: center
 
 Thinning the dataset with ``thinning=4`` will result in the following
 dataset:
 
-.. image:: ../../_static/thinning-after.png
+.. image:: ../_static/thinning-after.png
    :width: 75%
    :align: center
 
@@ -53,7 +53,7 @@ use:
 
 Which will result in the following dataset:
 
-.. image:: ../../_static/area-1.png
+.. image:: ../_static/area-1.png
    :width: 75%
    :align: center
 

{anemoi_datasets-0.5.27/docs/datasets → anemoi_datasets-0.5.28/docs}/using/missing.rst

@@ -71,7 +71,7 @@ the dates in the dataset, and the red cells are the missing dates. Given
 matching dates that are not missing (in blue). The interval between each
 date of a group is guaranteed to be constant across all groups.
 
-.. image:: ../../_static/skip-missing.png
+.. image:: ../_static/skip-missing.png
    :align: center
 
 .. code:: python

{anemoi_datasets-0.5.27/docs/datasets → anemoi_datasets-0.5.28/docs}/using/opening.rst

@@ -157,7 +157,7 @@ it with standard Python tools. For example:
    ax.gridlines(draw_labels=True)
    plt.colorbar(p, label="K", orientation="horizontal")
 
-.. figure:: ../../_static/2t_map_example.png
+.. figure:: ../_static/2t_map_example.png
    :alt: example map plot
    :align: center
 

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/pyproject.toml

@@ -49,18 +49,21 @@ dynamic = [
 ]
 dependencies = [
   "anemoi-transform>=0.1.10",
-  "anemoi-utils[provenance]>=0.4.32",
+  "anemoi-utils[provenance]>=0.4.38",
   "cfunits",
+  "glom",
+  "jsonschema",
   "numcodecs<0.16",                   # Until we move to zarr3
   "numpy",
   "pyyaml",
+  "ruamel-yaml",
   "semantic-version",
   "tqdm",
   "zarr<=2.18.4",
 ]
 
 optional-dependencies.all = [
-  "anemoi-datasets[create,remote,xarray,comparelam]",
+  "anemoi-datasets[comparelam,create,remote,xarray]",
 ]
 
 optional-dependencies.comparelam = [

{anemoi_datasets-0.5.27 → anemoi_datasets-0.5.28}/src/anemoi/datasets/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.5.27'
-__version_tuple__ = version_tuple = (0, 5, 27)
+__version__ = version = '0.5.28'
+__version_tuple__ = version_tuple = (0, 5, 28)
 
-__commit_id__ = commit_id = 'g2b7d13494'
+__commit_id__ = commit_id = 'g2c8e4cd05'

anemoi_datasets-0.5.28/src/anemoi/datasets/commands/recipe/__init__.py

@@ -0,0 +1,93 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import argparse
+import logging
+import sys
+from typing import Any
+
+import yaml
+
+from anemoi.datasets.create import validate_config
+
+from .. import Command
+from .format import format_recipe
+from .migrate import migrate_recipe
+
+LOG = logging.getLogger(__name__)
+
+
+class Recipe(Command):
+    def add_arguments(self, command_parser: Any) -> None:
+        """Add arguments to the command parser.
+
+        Parameters
+        ----------
+        command_parser : Any
+            Command parser object.
+        """
+
+        command_parser.add_argument("--validate", action="store_true", help="Validate recipe.")
+        command_parser.add_argument("--format", action="store_true", help="Format the recipe.")
+        command_parser.add_argument("--migrate", action="store_true", help="Migrate the recipe to the latest version.")
+
+        group = command_parser.add_mutually_exclusive_group()
+        group.add_argument("--inplace", action="store_true", help="Overwrite the recipe file in place.")
+        group.add_argument("--output", type=str, help="Output file path for the converted recipe.")
+
+        command_parser.add_argument(
+            "path",
+            help="Path to recipe.",
+        )
+
+    def run(self, args: Any) -> None:
+
+        if not args.validate and not args.format and not args.migrate:
+            args.validate = True
+
+        with open(args.path) as file:
+            config = yaml.safe_load(file)
+
+        assert isinstance(config, dict)
+
+        if args.validate:
+            if args.inplace and (not args.format and not args.migrate):
+                argparse.ArgumentError(None, "--inplace is not supported with --validate.")
+
+            if args.output and (not args.format and not args.migrate):
+                argparse.ArgumentError(None, "--output is not supported with --validate.")
+
+            validate_config(config)
+            LOG.info(f"{args.path}: Recipe is valid.")
+            return
+
+        if args.migrate:
+            config = migrate_recipe(args, config)
+            if config is None:
+                LOG.info(f"{args.path}: No changes needed.")
+                return
+
+            args.format = True
+
+        if args.format:
+            formatted = format_recipe(args, config)
+            assert "dates" in formatted
+            f = sys.stdout
+            if args.output:
+                f = open(args.output, "w")
+
+            if args.inplace:
+                f = open(args.path, "w")
+
+            print(formatted, file=f)
+            f.close()
+
+
+command = Recipe

anemoi_datasets-0.5.28/src/anemoi/datasets/commands/recipe/format.py

@@ -0,0 +1,55 @@
+# (C) Copyright 2025 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import datetime
+import logging
+
+from ...dumper import yaml_dump
+
+LOG = logging.getLogger(__name__)
+
+
+def make_dates(config):
+    if isinstance(config, dict):
+        return {k: make_dates(v) for k, v in config.items()}
+    if isinstance(config, list):
+        return [make_dates(v) for v in config]
+    if isinstance(config, str):
+        try:
+            return datetime.datetime.fromisoformat(config)
+        except ValueError:
+            return config
+    return config
+
+
+ORDER = (
+    "name",
+    "description",
+    "dataset_status",
+    "licence",
+    "attribution",
+    "env",
+    "dates",
+    "common",
+    "data_sources",
+    "input",
+    "output",
+    "statistics",
+    "build",
+    "platform",
+)
+
+
+def format_recipe(args, config: dict) -> str:
+
+    config = make_dates(config)
+    assert config
+
+    return yaml_dump(config, order=ORDER)