anemoi-datasets 0.5.26__tar.gz → 0.5.28__tar.gz

anemoi_datasets-0.5.28/.github/workflows/pr-label-ats.yml

@@ -0,0 +1,66 @@
+# This workflow checks that the appropriate ATS labels are applied to a pull request:
+# - "ATS Approval Not Needed": pass
+# - "ATS Approved": pass
+# - "ATS Approval Needed": fail
+# - Missing ATS label: fail
+
+name: "[PR] ATS labels"
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - labeled
+      - unlabeled
+      - synchronize
+
+permissions:
+  pull-requests: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR details
+        uses: actions/github-script@v7
+        id: check-pr
+        with:
+          script: |
+            const pr = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.payload.pull_request.number,
+            });
+            const labels = pr.data.labels.map(label => label.name);
+            core.setOutput('labels', JSON.stringify(labels));
+            core.setOutput('author', context.payload.pull_request.user.login);
+
+      - name: Evaluate ATS labels
+        run: |
+          AUTHOR='${{ steps.check-pr.outputs.author }}'
+          if [ "$AUTHOR" == "DeployDuck" ] || [ "$AUTHOR" == "pre-commit-ci[bot]" ]; then
+            echo "Bot PR, skipping."
+            exit 0
+          fi
+          LABELS=$(echo '${{ steps.check-pr.outputs.labels }}' | jq -r '.[]')
+          echo "Labels found:"
+          echo -e "$LABELS\n"
+          echo "Result:"
+          if echo "$LABELS" | grep -qi "ATS approval not needed"; then
+            echo "ATS approval not needed. Passing."
+            EXIT_CODE=0
+          elif echo "$LABELS" | grep -qi "ATS approved"; then
+            echo "ATS Approved. Passing."
+            EXIT_CODE=0
+          elif echo "$LABELS" | grep -qi "ATS approval needed"; then
+            echo "ATS Approval Needed. Failing."
+            EXIT_CODE=1
+          else
+            echo "No ATS approval labels found. Please assign the appropriate ATS label. Failing."
+            EXIT_CODE=1
+          fi
+          echo -e "\nFor more information on ATS labels, see:"
+          echo "https://anemoi.readthedocs.io/en/latest/contributing/guidelines.html#labelling-guidelines"
+          exit $EXIT_CODE

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/.github/workflows/python-pull-request.yml

@@ -19,7 +19,7 @@ jobs:
   checks:
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
     uses: ecmwf/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
     with:
       python-version: ${{ matrix.python-version }}

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/.gitignore

@@ -141,3 +141,13 @@ _version.py
 *.to_upload
 tempCodeRunnerFile.python
 Untitled-*.py
+*.zip
+*.json
+*.db
+*.tgz
+_api/
+trace.txt
+?/
+*.prof
+prof/
+*.gz

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
     entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
     additional_dependencies: [jupyter]
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
+  rev: v6.0.0
   hooks:
   - id: check-yaml # Check YAML files for syntax errors only
     args: [--unsafe, --allow-multiple-documents]
@@ -27,12 +27,12 @@ repos:
     -   id: python-check-blanket-noqa # Check for # noqa: all
     -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 25.1.0
+  rev: 25.9.0
   hooks:
   - id: black
     args: [--line-length=120]
 - repo: https://github.com/pycqa/isort
-  rev: 6.0.1
+  rev: 6.1.0
   hooks:
   - id: isort
     args:
@@ -41,7 +41,7 @@ repos:
     - --profile black
     - --project anemoi
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.11.12
+  rev: v0.13.3
   hooks:
   - id: ruff
     args:
@@ -65,11 +65,11 @@ repos:
   - id: docconvert
     args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "v2.6.0"
+  rev: "v2.7.0"
   hooks:
   - id: pyproject-fmt
 -   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
-    rev: v0.69.3
+    rev: v0.71.0
     hooks:
     -   id: docsig
         args:
@@ -79,6 +79,7 @@ repos:
         - --check-protected  # Check protected methods
         - --check-class      # Check class docstrings
         - --disable=SIG101,SIG102     # Disable empty docstrings
+
 ci:
   autoupdate_schedule: monthly
   autoupdate_commit_msg: "chore(deps): pre-commit.ci autoupdate"

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/.readthedocs.yaml

@@ -4,9 +4,9 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.11"
-#  jobs:
-#    pre_build:
-#    - bash docs/scripts/api_build.sh
+  jobs:
+    pre_build:
+    - (cd docs/scripts && make rst)
 
 sphinx:
   configuration: docs/conf.py

anemoi_datasets-0.5.28/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.5.28"
+}

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/CHANGELOG.md

@@ -8,6 +8,59 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.5.28](https://github.com/ecmwf/anemoi-datasets/compare/0.5.27...0.5.28) (2025-10-30)
+
+
+### Features
+
+* Added rolling average on read ([#448](https://github.com/ecmwf/anemoi-datasets/issues/448)) ([5cef720](https://github.com/ecmwf/anemoi-datasets/commit/5cef7207b7900190272d970bf44f35a7011e3708))
+* Fix repeat dates ([#458](https://github.com/ecmwf/anemoi-datasets/issues/458)) ([16b5c83](https://github.com/ecmwf/anemoi-datasets/commit/16b5c836172a37c13b210b35baee1a5e97731e5d))
+* Refactor dataset create ([#379](https://github.com/ecmwf/anemoi-datasets/issues/379)) ([c5408da](https://github.com/ecmwf/anemoi-datasets/commit/c5408dafd34c80bcb4b0cd9623dbebfe00959151))
+
+
+### Bug Fixes
+
+* Allow grib source to be used for constant data ([#428](https://github.com/ecmwf/anemoi-datasets/issues/428)) ([c1d7ff1](https://github.com/ecmwf/anemoi-datasets/commit/c1d7ff1035848e26d3034903577c267973985c07))
+* Broken source imports ([#450](https://github.com/ecmwf/anemoi-datasets/issues/450)) ([#454](https://github.com/ecmwf/anemoi-datasets/issues/454)) ([d6ed174](https://github.com/ecmwf/anemoi-datasets/commit/d6ed174cf4f6da22464c050d3304b026fc0ae4a3))
+* Ensure missing value indicator in not a valid value in accumulat… ([#316](https://github.com/ecmwf/anemoi-datasets/issues/316)) ([3541a1f](https://github.com/ecmwf/anemoi-datasets/commit/3541a1f7068b54ceeb35dff93c3f94b4b138cb9d))
+* S3 access [#453](https://github.com/ecmwf/anemoi-datasets/issues/453) ([#459](https://github.com/ecmwf/anemoi-datasets/issues/459)) ([2d810e3](https://github.com/ecmwf/anemoi-datasets/commit/2d810e3775ace78ffe2c43cc9bac66d704c0754d))
+* Xarray-zarr filters ([#457](https://github.com/ecmwf/anemoi-datasets/issues/457)) ([5f2973f](https://github.com/ecmwf/anemoi-datasets/commit/5f2973fb7d128a3f6de5ed538e39cc7421eeb5dc))
+
+
+### Documentation
+
+* Documentation updates ([#419](https://github.com/ecmwf/anemoi-datasets/issues/419)) ([4588175](https://github.com/ecmwf/anemoi-datasets/commit/458817533e805652527945eef4de781c62d3abf9))
+
+## [0.5.27](https://github.com/ecmwf/anemoi-datasets/compare/0.5.26...0.5.27) (2025-10-06)
+
+
+### Features
+
+* Add 'model_level_number' to coordinate guesser (ecmwf/anemoi-datasets[#402](https://github.com/ecmwf/anemoi-datasets/issues/402)) ([#403](https://github.com/ecmwf/anemoi-datasets/issues/403)) ([fa0b7e8](https://github.com/ecmwf/anemoi-datasets/commit/fa0b7e809740d918cddb98596a9178181fe900d2))
+* Extend xarray Variable.sel to handle scalar variables ([#381](https://github.com/ecmwf/anemoi-datasets/issues/381)) ([1306d62](https://github.com/ecmwf/anemoi-datasets/commit/1306d62d0c7e2be627460e35ba8a6780b0b26db7))
+* Improve conftest types ([#395](https://github.com/ecmwf/anemoi-datasets/issues/395)) ([bd8143d](https://github.com/ecmwf/anemoi-datasets/commit/bd8143d4baf94a5243c023632e62838226ae8fd4))
+* Remove python 3.9 from pyproject.toml. ([#392](https://github.com/ecmwf/anemoi-datasets/issues/392)) ([7f3a250](https://github.com/ecmwf/anemoi-datasets/commit/7f3a250cddf35bba7c83867933236587a1b48572))
+* **sources:** Add FDB source ([#405](https://github.com/ecmwf/anemoi-datasets/issues/405)) ([9222d59](https://github.com/ecmwf/anemoi-datasets/commit/9222d59280cafaf99f1471a028069fd6043522c9))
+* Validate dataset ([#330](https://github.com/ecmwf/anemoi-datasets/issues/330)) ([34d06df](https://github.com/ecmwf/anemoi-datasets/commit/34d06df5b8ea40568f1a413b30904a103d26acb9))
+
+
+### Bug Fixes
+
+* Allow multiple options to be provided to anemoi-datasets inspect ([#372](https://github.com/ecmwf/anemoi-datasets/issues/372)) ([dea680e](https://github.com/ecmwf/anemoi-datasets/commit/dea680e54a4daa43ce15400ff49adb2a29c39380))
+* Anemoi inspect wrong message ([#429](https://github.com/ecmwf/anemoi-datasets/issues/429)) ([984ccef](https://github.com/ecmwf/anemoi-datasets/commit/984ccef5561245a2a64b6e9b3c0543764b817664))
+* Better handling of xarray metadata ([#322](https://github.com/ecmwf/anemoi-datasets/issues/322)) ([4304cb4](https://github.com/ecmwf/anemoi-datasets/commit/4304cb40a6d6956492b5246a56f4af1a914d7cab))
+* Cleaning _build when building datasets ([#413](https://github.com/ecmwf/anemoi-datasets/issues/413)) ([046a7eb](https://github.com/ecmwf/anemoi-datasets/commit/046a7eb992653df34ba1c7550e38489e3df4d227))
+* Define the missing forwards_subclass_metadata_specific method in the ZipBase class ([#377](https://github.com/ecmwf/anemoi-datasets/issues/377)) ([2816314](https://github.com/ecmwf/anemoi-datasets/commit/28163143a76cec618e74a2b85d86a7d0d31168f5))
+* Remove config_path in Config ([#431](https://github.com/ecmwf/anemoi-datasets/issues/431)) ([2bf674f](https://github.com/ecmwf/anemoi-datasets/commit/2bf674fd966ce2a0c19cb72185173c999ae1c6af))
+* Remove-obsolete-code-and-rename ([#423](https://github.com/ecmwf/anemoi-datasets/issues/423)) ([ab3da59](https://github.com/ecmwf/anemoi-datasets/commit/ab3da591c9d7f809e21da8e354afff07b97cc490))
+* Save_dataset ([#383](https://github.com/ecmwf/anemoi-datasets/issues/383)) ([05b8f51](https://github.com/ecmwf/anemoi-datasets/commit/05b8f51f1b7c60cc3bc2695ccc274c149af2b9cf))
+* Support for sources with already opened datasets ([#417](https://github.com/ecmwf/anemoi-datasets/issues/417)) ([9bc2d17](https://github.com/ecmwf/anemoi-datasets/commit/9bc2d1736d457c09c042e8db45ac587c94e75218))
+
+
+### Documentation
+
+* Reference filters transform ([#389](https://github.com/ecmwf/anemoi-datasets/issues/389)) ([4e5d278](https://github.com/ecmwf/anemoi-datasets/commit/4e5d2785db5bb644d20a90e6cb1d3aa17e412718))
+
 ## [0.5.26](https://github.com/ecmwf/anemoi-datasets/compare/0.5.25...0.5.26) (2025-07-11)
 
 

{anemoi_datasets-0.5.26/src/anemoi_datasets.egg-info → anemoi_datasets-0.5.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-datasets
-Version: 0.5.26
+Version: 0.5.28
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -216,21 +216,23 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 License-File: LICENSE
 Requires-Dist: anemoi-transform>=0.1.10
-Requires-Dist: anemoi-utils[provenance]>=0.4.26
+Requires-Dist: anemoi-utils[provenance]>=0.4.38
 Requires-Dist: cfunits
+Requires-Dist: glom
+Requires-Dist: jsonschema
 Requires-Dist: numcodecs<0.16
 Requires-Dist: numpy
 Requires-Dist: pyyaml
+Requires-Dist: ruamel-yaml
 Requires-Dist: semantic-version
 Requires-Dist: tqdm
 Requires-Dist: zarr<=2.18.4
@@ -262,6 +264,7 @@ Requires-Dist: requests; extra == "remote"
 Provides-Extra: tests
 Requires-Dist: anemoi-datasets[xarray]; extra == "tests"
 Requires-Dist: pytest; extra == "tests"
+Requires-Dist: pytest-skip-slow; extra == "tests"
 Requires-Dist: pytest-xdist; extra == "tests"
 Provides-Extra: xarray
 Requires-Dist: adlfs; extra == "xarray"

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/docs/Makefile

@@ -19,5 +19,5 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	bash $(SOURCEDIR)/scripts/api_build.sh
+	@cd scripts && $(MAKE) $@
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

anemoi_datasets-0.5.28/docs/building/filters.rst

@@ -0,0 +1,10 @@
+.. _filters:
+
+#########
+ Filters
+#########
+
+Filters can be used to perform data transformations when creating a
+dataset. See the :ref:`anemoi-transform
+<anemoi-transform:list-of-filters>` documentation for a list of filters
+and example configurations.

{anemoi_datasets-0.5.26/docs/datasets → anemoi_datasets-0.5.28/docs}/building/introduction.rst

@@ -14,7 +14,7 @@ computed as the dataset is built, and stored in the metadata, with other
 information such as the locations of the grid points, the list of
 variables, etc.
 
-.. figure:: ../../_static/schemas/recipe.png
+.. figure:: ../_static/schemas/recipe.png
    :alt: Building datasets
    :align: center
 

{anemoi_datasets-0.5.26/docs/datasets → anemoi_datasets-0.5.28/docs}/building/sources/repeated-dates.rst

@@ -10,7 +10,7 @@ dates of the dataset.
 
 The general format of the `repeated-dates` source is:
 
-.. literalinclude:: yaml/repeated_dates1.yaml
+.. literalinclude:: yaml/repeated-dates1.yaml
    :language: yaml
 
 where ``source`` is any of the :ref:`operations <operations>` or

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/docs/cli/create.rst

@@ -4,7 +4,7 @@ Create Command
 ==============
 
 Use this command to create a dataset from a recipe file.
-The syntax of the recipe file is described in :doc:`building datasets <../datasets/building/introduction>`.
+The syntax of the recipe file is described in :ref:`building-introduction`.
 
 .. argparse::
     :module: anemoi.datasets.__main__

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/docs/cli/grib-index.rst

@@ -1,7 +1,7 @@
 .. _grib-index_command:
 
 Grib-index Command
-============
+==================
 
 The `grib-index` command is used to create an index file for GRIB files. The index file is then used
 by the `grib-index` :ref:`source <grib-index_source>`.

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/docs/cli/inspect.rst

@@ -15,7 +15,7 @@ This command will output the metadata of the dataset, including the variables, d
 
 which will output something like the following. The output should be self-explanatory.
 
-.. literalinclude:: ../datasets/yaml/building1.txt
+.. literalinclude:: ../yaml/building1.txt
    :language: console
 
 .. argparse::

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/docs/cli/introduction.rst

@@ -1,8 +1,8 @@
 .. _cli-introduction:
 
-##################
-Command line tool
-##################
+############
+Introduction
+############
 
 When you install the `anemoi-datasets` package, this will also install command line tool
 called ``anemoi-datasets`` which can be used to manage the zarr datasets.
@@ -20,4 +20,5 @@ The commands are:
 - :ref:`Inspect Command <Inspect_command>`
 - :ref:`Compare Command <compare_command>`
 - :ref:`Scan Command <scan_command>`
+- :ref:`Validate Command <validate_command>`
 - :ref:`Compare LAM Command <compare_lam_command>`

anemoi_datasets-0.5.28/docs/cli/validate.rst

@@ -0,0 +1,45 @@
+.. _validate_command:
+
+Validate Command
+============
+
+Use this command to validate a zarr dataset, or a class that implements the
+:class:`anemoi.datasets.Dataset` interface.
+
+
+This command has two modes of operation:
+
+1. Validate a local zarr dataset by specifying its path.
+2. Validate a dataset class by providing its fully qualified class name.
+
+Example usage
+-------------
+
+To validate a local zarr dataset:
+
+.. code-block:: bash
+
+    anemoi-datasets validate /path/to/dataset.zarr
+
+To validate a dataset class:
+
+.. code-block:: bash
+
+    anemoi-datasets validate --callable my_package.MyDatasetClass  some-args-relevant-to-the-class
+
+
+In the first case, the command will check the compatibility of the zarr dataset using anemoi-datasets own class, in the second case, it will use the class provided by the user.
+
+When running the command, emojis are used to categorise the different validation results:
+
+* ✅: validation succeeded
+* ⚠️: validation for the attribute/method is not implemented
+* 💣: validation failed because the test does not pass
+* 💥: validation failed because the validation tool is out of date
+
+
+.. argparse::
+    :module: anemoi.datasets.__main__
+    :func: create_parser
+    :prog: anemoi-datasets
+    :path: validate

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/docs/conf.py

@@ -40,9 +40,9 @@ year = datetime.datetime.now().year
 if year == 2024:
     years = "2024"
 else:
-    years = "2024-%s" % (year,)
+    years = f"2024-{year}"
 
-copyright = "%s, Anemoi contributors" % (years,)
+copyright = f"{years}, Anemoi contributors"
 
 
 try:

anemoi_datasets-0.5.26/docs/usage/getting_started.rst → anemoi_datasets-0.5.28/docs/howtos/create/00-redo-sample-dataset.rst

@@ -1,8 +1,8 @@
-.. _usage-getting-started:
+.. _redo-sample-dataset:
 
-###########################
- Create Your First Dataset
-###########################
+##############################
+ Re-create the sample dataset
+##############################
 
 The following example presents a simple but comprehensive recipe to
 create an Anemoi dataset based on the atmospheric reanalysis ERA5. Once

anemoi_datasets-0.5.28/docs/howtos/create/04-create-with-filter.rst

@@ -0,0 +1,117 @@
+.. _create-with-filter:
+
+#################################
+ Create a dataset using a filter
+#################################
+
+A ``filter`` is a software component that takes as input the output of a
+source or another filter and can modify the fields and/or their
+metadata. For example, typical filters are interpolations, renaming of
+variables, etc. Filters are available as part of ``anemoi-transform``.
+
+****************
+ Using a filter
+****************
+
+In the example below we see a recipe to create a dataset from MARS data
+in which we perform a rename transform to update ``tp`` to be named
+``tp_era5``. To be able to use the transform we just define it as a
+second step of the pipe, after gathering the data.
+
+.. literalinclude:: yaml/recipe-filter1.yaml
+
+That recipe will generate the following dataset:
+
+.. code:: bash
+
+   Dataset Summary
+   ===============
+
+   📦 Path          : recipe1.zarr
+   🔢 Format version: 0.30.0
+
+   📅 Start      : 2020-12-12 00:00
+   📅 End        : 2020-12-13 12:00
+   ⏰ Frequency  : 12h
+   🚫 Missing    : 0
+   🌎 Resolution : 20.0
+   🌎 Field shape: [9, 18]
+
+   📐 Shape      : 4 × 9 × 1 × 162 (22.8 KiB)
+   💽 Size       : 40.7 KiB (40.7 KiB)
+   📁 Files      : 48
+
+      Index │ Variable │         Min │         Max │         Mean │       Stdev
+      ──────┼──────────┼─────────────┼─────────────┼──────────────┼────────────
+         0 │ 2t       │     226.496 │     309.946 │       278.03 │     19.2561
+         1 │ cp       │           0 │  0.00739765 │   0.00014582 │ 0.000527194
+         2 │ q_100    │ 1.38935e-06 │ 4.20381e-06 │  2.68779e-06 │ 5.59043e-07
+         3 │ q_50     │ 1.26881e-06 │ 3.20919e-06 │  2.74525e-06 │ 4.35595e-07
+         4 │ t_100    │     189.787 │     226.929 │      207.052 │     9.26841
+         5 │ t_50     │      189.14 │      236.51 │       212.79 │      9.5502
+         6 │ tp_era5  │           0 │  0.00823116 │  0.000326814 │  0.00078008
+         7 │ w_100    │  -0.0114685 │   0.0129402 │ -0.000355278 │  0.00448272
+         8 │ w_50     │ -0.00815806 │   0.0126816 │ -0.000267674 │  0.00331866
+      ──────┴──────────┴─────────────┴─────────────┴──────────────┴────────────
+   🔋 Dataset ready, last update 1 minute ago.
+   📊 Statistics ready.
+
+***********************
+ Creating a new filter
+***********************
+
+In order to create a new filter the recommendation is to define it under
+the package ``anemoi-transform``. Available filters can be found in
+``anemoi/transform/filters`` or running the command ``anemoi-transform
+filters list``. For details about how to create a filter please refer to
+the `anemoi-transform
+<https://anemoi.readthedocs.io/projects/transform/en/latest/>`_
+documentation.
+
+************************
+ Using multiple filters
+************************
+
+It's possible to stack multiple filters one after the other. Below you
+can see an updated version of the dataset creation we had where we now
+create a dataset and apply a rename filter and our newly defined
+``VerticalVelocity`` filter.
+
+.. literalinclude:: yaml/recipe-filter2.yaml
+
+That recipe will generate the following dataset:
+
+.. code:: bash
+
+   Dataset Summary
+   ===============
+
+
+   📦 Path          : recipe2.zarr
+   🔢 Format version: 0.30.0
+
+   📅 Start      : 2020-12-12 00:00
+   📅 End        : 2020-12-13 12:00
+   ⏰ Frequency  : 12h
+   🚫 Missing    : 0
+   🌎 Resolution : 20.0
+   🌎 Field shape: [9, 18]
+
+   📐 Shape      : 4 × 9 × 1 × 162 (22.8 KiB)
+   💽 Size       : 41.1 KiB (41.1 KiB)
+   📁 Files      : 48
+
+      Index │ Variable │         Min │         Max │        Mean │       Stdev
+      ──────┼──────────┼─────────────┼─────────────┼─────────────┼────────────
+         0 │ 2t       │     226.496 │     309.946 │      278.03 │     19.2561
+         1 │ cp       │           0 │  0.00739765 │  0.00014582 │ 0.000527194
+         2 │ q_100    │ 1.38935e-06 │ 4.20381e-06 │ 2.68779e-06 │ 5.59043e-07
+         3 │ q_50     │ 1.26881e-06 │ 3.20919e-06 │ 2.74525e-06 │ 4.35595e-07
+         4 │ t_100    │     189.787 │     226.929 │     207.052 │     9.26841
+         5 │ t_50     │      189.14 │      236.51 │      212.79 │      9.5502
+         6 │ tp_era5  │           0 │  0.00823116 │ 0.000326814 │  0.00078008
+         7 │ wz_100   │ -0.00798191 │  0.00721723 │ 0.000224189 │  0.00277693
+         8 │ wz_50    │    -0.01549 │   0.0103844 │ 0.000341309 │  0.00417065
+      ──────┴──────────┴─────────────┴─────────────┴─────────────┴────────────
+   🔋 Dataset ready, last update 11 seconds ago.
+   📊 Statistics ready.

anemoi_datasets-0.5.28/docs/howtos/create/yaml/recipe-filter1.yaml

@@ -0,0 +1,31 @@
+dates:
+  start: 2020-12-12 00:00:00
+  end: 2020-12-14 12:00:00
+  frequency: 12h
+
+input:
+  pipe:
+    - join:
+      - mars:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [2t]
+          levtype: sfc
+      - mars:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [q, t,w]
+          levtype: pl
+          level: [50, 100]
+          stream: oper
+          type: an
+      - accumulations:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [cp, tp]
+    - rename:
+        param:
+          tp: tp_era5

anemoi_datasets-0.5.28/docs/howtos/create/yaml/recipe-filter2.yaml

@@ -0,0 +1,35 @@
+dates:
+  start: 2020-12-12 00:00:00
+  end: 2020-12-14 12:00:00
+  frequency: 12h
+
+input:
+  pipe:
+    - join:
+      - mars:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [2t]
+          levtype: sfc
+      - mars:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [q, t,w]
+          levtype: pl
+          level: [50, 100]
+          stream: oper
+          type: an
+      - accumulations:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [cp, tp]
+    - rename:
+        param:
+          tp: tp_era5
+    - w_to_wz:
+        w_component: w
+        temperature: t
+        humidity: q

{anemoi_datasets-0.5.26 → anemoi_datasets-0.5.28}/docs/howtos/introduction.rst

@@ -13,6 +13,7 @@ good starting point.
 -  :ref:`create-grib-data`
 -  :ref:`create-cf-data`
 -  :ref:`constant-data`
+-  :ref:`create-with-filter`
 
 .. toctree::
    :maxdepth: 1
@@ -34,4 +35,4 @@ good starting point.
    :hidden:
    :caption: Using anemoi-datasets
 
-   usage/*
+   using/*