anemoi-datasets 0.5.25__tar.gz → 0.5.27__tar.gz

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/.github/pull_request_template.md

@@ -11,3 +11,5 @@
 <!-- Include any additional information, caveats, or considerations that the reviewer should be aware of. -->
 
 ***As a contributor to the Anemoi framework, please ensure that your changes include unit tests, updates to any affected dependencies and documentation, and have been tested in a parallel setting  (i.e., with multiple GPUs). As a reviewer, you are also responsible for verifying these aspects and requesting changes if they are not adequately addressed. For guidelines about those please refer to https://anemoi.readthedocs.io/en/latest/***
+
+By opening this pull request, I affirm that all authors agree to the [Contributor License Agreement.](https://github.com/ecmwf/codex/blob/main/Legal/contributor_license_agreement.md)

anemoi_datasets-0.5.27/.github/workflows/pr-label-ats.yml

@@ -0,0 +1,66 @@
+# This workflow checks that the appropriate ATS labels are applied to a pull request:
+# - "ATS Approval Not Needed": pass
+# - "ATS Approved": pass
+# - "ATS Approval Needed": fail
+# - Missing ATS label: fail
+
+name: "[PR] ATS labels"
+
+on:
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - labeled
+      - unlabeled
+      - synchronize
+
+permissions:
+  pull-requests: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get PR details
+        uses: actions/github-script@v7
+        id: check-pr
+        with:
+          script: |
+            const pr = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.payload.pull_request.number,
+            });
+            const labels = pr.data.labels.map(label => label.name);
+            core.setOutput('labels', JSON.stringify(labels));
+            core.setOutput('author', context.payload.pull_request.user.login);
+
+      - name: Evaluate ATS labels
+        run: |
+          AUTHOR='${{ steps.check-pr.outputs.author }}'
+          if [ "$AUTHOR" == "DeployDuck" ] || [ "$AUTHOR" == "pre-commit-ci[bot]" ]; then
+            echo "Bot PR, skipping."
+            exit 0
+          fi
+          LABELS=$(echo '${{ steps.check-pr.outputs.labels }}' | jq -r '.[]')
+          echo "Labels found:"
+          echo -e "$LABELS\n"
+          echo "Result:"
+          if echo "$LABELS" | grep -qi "ATS approval not needed"; then
+            echo "ATS approval not needed. Passing."
+            EXIT_CODE=0
+          elif echo "$LABELS" | grep -qi "ATS approved"; then
+            echo "ATS Approved. Passing."
+            EXIT_CODE=0
+          elif echo "$LABELS" | grep -qi "ATS approval needed"; then
+            echo "ATS Approval Needed. Failing."
+            EXIT_CODE=1
+          else
+            echo "No ATS approval labels found. Please assign the appropriate ATS label. Failing."
+            EXIT_CODE=1
+          fi
+          echo -e "\nFor more information on ATS labels, see:"
+          echo "https://anemoi.readthedocs.io/en/latest/contributing/guidelines.html#labelling-guidelines"
+          exit $EXIT_CODE

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/.github/workflows/python-pull-request.yml

@@ -19,7 +19,7 @@ jobs:
   checks:
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
     uses: ecmwf/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
     with:
       python-version: ${{ matrix.python-version }}

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
     entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
     additional_dependencies: [jupyter]
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
+  rev: v6.0.0
   hooks:
   - id: check-yaml # Check YAML files for syntax errors only
     args: [--unsafe, --allow-multiple-documents]
@@ -41,7 +41,7 @@ repos:
     - --profile black
     - --project anemoi
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.11.12
+  rev: v0.12.11
   hooks:
   - id: ruff
     args:
@@ -69,7 +69,7 @@ repos:
   hooks:
   - id: pyproject-fmt
 -   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
-    rev: v0.69.3
+    rev: v0.71.0
     hooks:
     -   id: docsig
         args:

anemoi_datasets-0.5.27/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.5.27"
+}

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/CHANGELOG.md

@@ -8,6 +8,63 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.5.27](https://github.com/ecmwf/anemoi-datasets/compare/0.5.26...0.5.27) (2025-10-06)
+
+
+### Features
+
+* Add 'model_level_number' to coordinate guesser (ecmwf/anemoi-datasets[#402](https://github.com/ecmwf/anemoi-datasets/issues/402)) ([#403](https://github.com/ecmwf/anemoi-datasets/issues/403)) ([fa0b7e8](https://github.com/ecmwf/anemoi-datasets/commit/fa0b7e809740d918cddb98596a9178181fe900d2))
+* Extend xarray Variable.sel to handle scalar variables ([#381](https://github.com/ecmwf/anemoi-datasets/issues/381)) ([1306d62](https://github.com/ecmwf/anemoi-datasets/commit/1306d62d0c7e2be627460e35ba8a6780b0b26db7))
+* Improve conftest types ([#395](https://github.com/ecmwf/anemoi-datasets/issues/395)) ([bd8143d](https://github.com/ecmwf/anemoi-datasets/commit/bd8143d4baf94a5243c023632e62838226ae8fd4))
+* Remove python 3.9 from pyproject.toml. ([#392](https://github.com/ecmwf/anemoi-datasets/issues/392)) ([7f3a250](https://github.com/ecmwf/anemoi-datasets/commit/7f3a250cddf35bba7c83867933236587a1b48572))
+* **sources:** Add FDB source ([#405](https://github.com/ecmwf/anemoi-datasets/issues/405)) ([9222d59](https://github.com/ecmwf/anemoi-datasets/commit/9222d59280cafaf99f1471a028069fd6043522c9))
+* Validate dataset ([#330](https://github.com/ecmwf/anemoi-datasets/issues/330)) ([34d06df](https://github.com/ecmwf/anemoi-datasets/commit/34d06df5b8ea40568f1a413b30904a103d26acb9))
+
+
+### Bug Fixes
+
+* Allow multiple options to be provided to anemoi-datasets inspect ([#372](https://github.com/ecmwf/anemoi-datasets/issues/372)) ([dea680e](https://github.com/ecmwf/anemoi-datasets/commit/dea680e54a4daa43ce15400ff49adb2a29c39380))
+* Anemoi inspect wrong message ([#429](https://github.com/ecmwf/anemoi-datasets/issues/429)) ([984ccef](https://github.com/ecmwf/anemoi-datasets/commit/984ccef5561245a2a64b6e9b3c0543764b817664))
+* Better handling of xarray metadata ([#322](https://github.com/ecmwf/anemoi-datasets/issues/322)) ([4304cb4](https://github.com/ecmwf/anemoi-datasets/commit/4304cb40a6d6956492b5246a56f4af1a914d7cab))
+* Cleaning _build when building datasets ([#413](https://github.com/ecmwf/anemoi-datasets/issues/413)) ([046a7eb](https://github.com/ecmwf/anemoi-datasets/commit/046a7eb992653df34ba1c7550e38489e3df4d227))
+* Define the missing forwards_subclass_metadata_specific method in the ZipBase class ([#377](https://github.com/ecmwf/anemoi-datasets/issues/377)) ([2816314](https://github.com/ecmwf/anemoi-datasets/commit/28163143a76cec618e74a2b85d86a7d0d31168f5))
+* Remove config_path in Config ([#431](https://github.com/ecmwf/anemoi-datasets/issues/431)) ([2bf674f](https://github.com/ecmwf/anemoi-datasets/commit/2bf674fd966ce2a0c19cb72185173c999ae1c6af))
+* Remove-obsolete-code-and-rename ([#423](https://github.com/ecmwf/anemoi-datasets/issues/423)) ([ab3da59](https://github.com/ecmwf/anemoi-datasets/commit/ab3da591c9d7f809e21da8e354afff07b97cc490))
+* Save_dataset ([#383](https://github.com/ecmwf/anemoi-datasets/issues/383)) ([05b8f51](https://github.com/ecmwf/anemoi-datasets/commit/05b8f51f1b7c60cc3bc2695ccc274c149af2b9cf))
+* Support for sources with already opened datasets ([#417](https://github.com/ecmwf/anemoi-datasets/issues/417)) ([9bc2d17](https://github.com/ecmwf/anemoi-datasets/commit/9bc2d1736d457c09c042e8db45ac587c94e75218))
+
+
+### Documentation
+
+* Reference filters transform ([#389](https://github.com/ecmwf/anemoi-datasets/issues/389)) ([4e5d278](https://github.com/ecmwf/anemoi-datasets/commit/4e5d2785db5bb644d20a90e6cb1d3aa17e412718))
+
+## [0.5.26](https://github.com/ecmwf/anemoi-datasets/compare/0.5.25...0.5.26) (2025-07-11)
+
+
+### Features
+
+* Add planetary planetary source ([#368](https://github.com/ecmwf/anemoi-datasets/issues/368)) ([c85337f](https://github.com/ecmwf/anemoi-datasets/commit/c85337f4b08381dc1e2e8ba71134c998c6258c2d))
+* Extend xarray coordinate guesser to include pressure as a long_name ([#335](https://github.com/ecmwf/anemoi-datasets/issues/335)) ([bcbffaa](https://github.com/ecmwf/anemoi-datasets/commit/bcbffaa0b0ad011db5b1fd5879f9829f0d2e68af))
+* Extented complement nearest functionality to support k>1 ([#371](https://github.com/ecmwf/anemoi-datasets/issues/371)) ([a2c5f92](https://github.com/ecmwf/anemoi-datasets/commit/a2c5f921f57fce60bd3fe728c21dc5e5b7d3a4a1))
+* Point data ([#358](https://github.com/ecmwf/anemoi-datasets/issues/358)) ([78c4f85](https://github.com/ecmwf/anemoi-datasets/commit/78c4f85bf9cd4d6cc7f17268c8c526da32158220))
+* Relaxed compatibility check for horizontal grid. ([#339](https://github.com/ecmwf/anemoi-datasets/issues/339)) ([a7dc859](https://github.com/ecmwf/anemoi-datasets/commit/a7dc859b6ec2fb0bad2ed7b7b8ee3c76034a258b))
+* Sort xarray coordinate with patch ([#356](https://github.com/ecmwf/anemoi-datasets/issues/356)) ([638cbdb](https://github.com/ecmwf/anemoi-datasets/commit/638cbdb013b94b87748654ae338e0d02e69c1900))
+* Use pytest-xdist to run tests in parallel. ([#346](https://github.com/ecmwf/anemoi-datasets/issues/346)) ([1d656ee](https://github.com/ecmwf/anemoi-datasets/commit/1d656ee62c51ee0cbe043943581cef16b2f7d7db))
+
+
+### Bug Fixes
+
+* Deprecated filters in anemoi datasets ([#350](https://github.com/ecmwf/anemoi-datasets/issues/350)) ([d08c6f5](https://github.com/ecmwf/anemoi-datasets/commit/d08c6f55c3d983c930480bd777e74fd5a5bea6eb))
+* **grib-index:** Match against filename, not full path ([#374](https://github.com/ecmwf/anemoi-datasets/issues/374)) ([e71a952](https://github.com/ecmwf/anemoi-datasets/commit/e71a95221493a703515d7a14a17f3dead06c8430))
+* Missing method in xarray flavour ([#369](https://github.com/ecmwf/anemoi-datasets/issues/369)) ([d0b9588](https://github.com/ecmwf/anemoi-datasets/commit/d0b9588f0bff626ae751a74cf4cbbc373a2f67f4))
+* Remove trailing slash from zarr paths.  ([#362](https://github.com/ecmwf/anemoi-datasets/issues/362)) ([9a9bfe5](https://github.com/ecmwf/anemoi-datasets/commit/9a9bfe50406d299f44f7cce7e3befcabef6fefaf))
+
+
+### Documentation
+
+* Fix pre-commit ([#367](https://github.com/ecmwf/anemoi-datasets/issues/367)) ([6e6ebe2](https://github.com/ecmwf/anemoi-datasets/commit/6e6ebe2e81fc735a2af422b8c222b0a2f229d8dc))
+* Zarr dataset creation ([#365](https://github.com/ecmwf/anemoi-datasets/issues/365)) ([622c261](https://github.com/ecmwf/anemoi-datasets/commit/622c26156de44ae4eaa3359fef836741b1d81d67))
+
 ## [0.5.25](https://github.com/ecmwf/anemoi-datasets/compare/0.5.24...0.5.25) (2025-06-11)
 
 

{anemoi_datasets-0.5.25/src/anemoi_datasets.egg-info → anemoi_datasets-0.5.27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-datasets
-Version: 0.5.25
+Version: 0.5.27
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -216,17 +216,16 @@ Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 License-File: LICENSE
 Requires-Dist: anemoi-transform>=0.1.10
-Requires-Dist: anemoi-utils[provenance]>=0.4.21
+Requires-Dist: anemoi-utils[provenance]>=0.4.32
 Requires-Dist: cfunits
 Requires-Dist: numcodecs<0.16
 Requires-Dist: numpy
@@ -262,6 +261,8 @@ Requires-Dist: requests; extra == "remote"
 Provides-Extra: tests
 Requires-Dist: anemoi-datasets[xarray]; extra == "tests"
 Requires-Dist: pytest; extra == "tests"
+Requires-Dist: pytest-skip-slow; extra == "tests"
+Requires-Dist: pytest-xdist; extra == "tests"
 Provides-Extra: xarray
 Requires-Dist: adlfs; extra == "xarray"
 Requires-Dist: gcsfs; extra == "xarray"

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/docs/cli/introduction.rst

@@ -20,4 +20,5 @@ The commands are:
 - :ref:`Inspect Command <Inspect_command>`
 - :ref:`Compare Command <compare_command>`
 - :ref:`Scan Command <scan_command>`
+- :ref:`Validate Command <validate_command>`
 - :ref:`Compare LAM Command <compare_lam_command>`

anemoi_datasets-0.5.27/docs/cli/validate.rst

@@ -0,0 +1,45 @@
+.. _validate_command:
+
+Validate Command
+============
+
+Use this command to validate a zarr dataset, or a class that implements the
+:class:`anemoi.datasets.Dataset` interface.
+
+
+This command has two modes of operation:
+
+1. Validate a local zarr dataset by specifying its path.
+2. Validate a dataset class by providing its fully qualified class name.
+
+Example usage
+-------------
+
+To validate a local zarr dataset:
+
+.. code-block:: bash
+
+    anemoi-datasets validate /path/to/dataset.zarr
+
+To validate a dataset class:
+
+.. code-block:: bash
+
+    anemoi-datasets validate --callable my_package.MyDatasetClass  some-args-relevant-to-the-class
+
+
+In the first case, the command will check the compatibility of the zarr dataset using anemoi-datasets own class, in the second case, it will use the class provided by the user.
+
+When running the command, emojis are used to categorise the different validation results:
+
+* ✅: validation succeeded
+* ⚠️: validation for the attribute/method is not implemented
+* 💣: validation failed because the test does not pass
+* 💥: validation failed because the validation tool is out of date
+
+
+.. argparse::
+    :module: anemoi.datasets.__main__
+    :func: create_parser
+    :prog: anemoi-datasets
+    :path: validate

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/docs/conf.py

@@ -40,9 +40,9 @@ year = datetime.datetime.now().year
 if year == 2024:
     years = "2024"
 else:
-    years = "2024-%s" % (year,)
+    years = f"2024-{year}"
 
-copyright = "%s, Anemoi contributors" % (years,)
+copyright = f"{years}, Anemoi contributors"
 
 
 try:

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/docs/datasets/building/sources/anemoi-dataset.rst

@@ -17,3 +17,9 @@ An anemoi-dataset can be a source for a dataset:
 The parameters are the same as those used in the ``open_dataset``
 function, which allows you to subset and combine datasets. See
 :ref:`opening-datasets` for more information.
+
+In particular, this is how local zarr datasets created with anemoi in a
+can be used as a source, contrary to :ref:`xarray-zarr` :
+
+.. literalinclude:: yaml/anemoi-zarr-dataset.yaml
+   :language: yaml

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/docs/datasets/building/sources/xarray-zarr.rst

@@ -1,3 +1,5 @@
+.. _xarray-zarr:
+
 #############
  xarray-zarr
 #############
@@ -17,4 +19,8 @@ it is necessary to use the :ref:`join <building-join>` operation to join
 separate lists containing 2D variables and 3D variables. If all vertical
 levels are desired, then it is acceptable to specify a single source.
 
+Also, an ``xarray-zarr`` source uses the ``url`` keyword, and cannot be
+used for accessing local datasets. For using local zarr datasets as
+sources, use instead :ref:`anemoi-dataset_source`.
+
 See :ref:`create-cf-data` for more information.

anemoi_datasets-0.5.27/docs/datasets/building/sources/yaml/anemoi-zarr-dataset.yaml

@@ -0,0 +1,3 @@
+input:
+  anemoi-dataset:
+    dataset: path/to/dataset.zarr

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/docs/datasets/using/combining.rst

@@ -235,13 +235,15 @@ variables of `dataset1` and return the result.
        source=dataset2,
        what="variables",
        interpolate="nearest",
+       k=1,
    )
 
 Currently ``what`` can only be ``variables`` and can be omitted.
 
 The value for ``interpolate`` can be one of ``none`` (default) or
 ``nearest``. In the case of ``none``, the grids of the two datasets must
-match.
+match. In case of ``interpolate``, an additional parameter ``k`` can be
+set to specify the number of nearest neighbors to use.
 
 This feature was originally designed to be used in conjunction with
 ``cutout``, where `dataset1` is the lam, and `dataset2` is the global

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/docs/howtos/create/02-cf-data.rst

@@ -46,9 +46,18 @@ can contain patterns. See :ref:`file-pattern` for more information.
  Zarr
 ******
 
+For using remote hosted zarr datasets as sources, use
+:ref:`xarray-zarr`.
+
 .. literalinclude:: yaml/zarr1.yaml
    :language: yaml
 
+For using local zarr datasets (such as anemoi-generated datasets), use
+:ref:`anemoi-dataset_source`.
+
+.. literalinclude:: yaml/zarr2.yaml
+   :language: yaml
+
 *********************************************
  Handling data that is not 100% CF-compliant
 *********************************************

anemoi_datasets-0.5.27/docs/howtos/create/04-create-with-filter.rst

@@ -0,0 +1,117 @@
+.. _create-with-filter:
+
+#################################
+ Create a dataset using a filter
+#################################
+
+A ``filter`` is a software component that takes as input the output of a
+source or another filter and can modify the fields and/or their
+metadata. For example, typical filters are interpolations, renaming of
+variables, etc. Filters are available as part of ``anemoi-transform``.
+
+****************
+ Using a filter
+****************
+
+In the example below we see a recipe to create a dataset from MARS data
+in which we perform a rename transform to update ``tp`` to be named
+``tp_era5``. To be able to use the transform we just define it as a
+second step of the pipe, after gathering the data.
+
+.. literalinclude:: yaml/recipe-filter1.yaml
+
+That recipe will generate the following dataset:
+
+.. code:: bash
+
+   Dataset Summary
+   ===============
+
+   📦 Path          : recipe1.zarr
+   🔢 Format version: 0.30.0
+
+   📅 Start      : 2020-12-12 00:00
+   📅 End        : 2020-12-13 12:00
+   ⏰ Frequency  : 12h
+   🚫 Missing    : 0
+   🌎 Resolution : 20.0
+   🌎 Field shape: [9, 18]
+
+   📐 Shape      : 4 × 9 × 1 × 162 (22.8 KiB)
+   💽 Size       : 40.7 KiB (40.7 KiB)
+   📁 Files      : 48
+
+      Index │ Variable │         Min │         Max │         Mean │       Stdev
+      ──────┼──────────┼─────────────┼─────────────┼──────────────┼────────────
+         0 │ 2t       │     226.496 │     309.946 │       278.03 │     19.2561
+         1 │ cp       │           0 │  0.00739765 │   0.00014582 │ 0.000527194
+         2 │ q_100    │ 1.38935e-06 │ 4.20381e-06 │  2.68779e-06 │ 5.59043e-07
+         3 │ q_50     │ 1.26881e-06 │ 3.20919e-06 │  2.74525e-06 │ 4.35595e-07
+         4 │ t_100    │     189.787 │     226.929 │      207.052 │     9.26841
+         5 │ t_50     │      189.14 │      236.51 │       212.79 │      9.5502
+         6 │ tp_era5  │           0 │  0.00823116 │  0.000326814 │  0.00078008
+         7 │ w_100    │  -0.0114685 │   0.0129402 │ -0.000355278 │  0.00448272
+         8 │ w_50     │ -0.00815806 │   0.0126816 │ -0.000267674 │  0.00331866
+      ──────┴──────────┴─────────────┴─────────────┴──────────────┴────────────
+   🔋 Dataset ready, last update 1 minute ago.
+   📊 Statistics ready.
+
+***********************
+ Creating a new filter
+***********************
+
+In order to create a new filter the recommendation is to define it under
+the package ``anemoi-transform``. Available filters can be found in
+``anemoi/transform/filters`` or running the command ``anemoi-transform
+filters list``. For details about how to create a filter please refer to
+the `anemoi-transform
+<https://anemoi.readthedocs.io/projects/transform/en/latest/>`_
+documentation.
+
+************************
+ Using multiple filters
+************************
+
+It's possible to stack multiple filters one after the other. Below you
+can see an updated version of the dataset creation we had where we now
+create a dataset and apply a rename filter and our newly defined
+``VerticalVelocity`` filter.
+
+.. literalinclude:: yaml/recipe-filter2.yaml
+
+That recipe will generate the following dataset:
+
+.. code:: bash
+
+   Dataset Summary
+   ===============
+
+
+   📦 Path          : recipe2.zarr
+   🔢 Format version: 0.30.0
+
+   📅 Start      : 2020-12-12 00:00
+   📅 End        : 2020-12-13 12:00
+   ⏰ Frequency  : 12h
+   🚫 Missing    : 0
+   🌎 Resolution : 20.0
+   🌎 Field shape: [9, 18]
+
+   📐 Shape      : 4 × 9 × 1 × 162 (22.8 KiB)
+   💽 Size       : 41.1 KiB (41.1 KiB)
+   📁 Files      : 48
+
+      Index │ Variable │         Min │         Max │        Mean │       Stdev
+      ──────┼──────────┼─────────────┼─────────────┼─────────────┼────────────
+         0 │ 2t       │     226.496 │     309.946 │      278.03 │     19.2561
+         1 │ cp       │           0 │  0.00739765 │  0.00014582 │ 0.000527194
+         2 │ q_100    │ 1.38935e-06 │ 4.20381e-06 │ 2.68779e-06 │ 5.59043e-07
+         3 │ q_50     │ 1.26881e-06 │ 3.20919e-06 │ 2.74525e-06 │ 4.35595e-07
+         4 │ t_100    │     189.787 │     226.929 │     207.052 │     9.26841
+         5 │ t_50     │      189.14 │      236.51 │      212.79 │      9.5502
+         6 │ tp_era5  │           0 │  0.00823116 │ 0.000326814 │  0.00078008
+         7 │ wz_100   │ -0.00798191 │  0.00721723 │ 0.000224189 │  0.00277693
+         8 │ wz_50    │    -0.01549 │   0.0103844 │ 0.000341309 │  0.00417065
+      ──────┴──────────┴─────────────┴─────────────┴─────────────┴────────────
+   🔋 Dataset ready, last update 11 seconds ago.
+   📊 Statistics ready.

anemoi_datasets-0.5.27/docs/howtos/create/yaml/recipe-filter1.yaml

@@ -0,0 +1,31 @@
+dates:
+  start: 2020-12-12 00:00:00
+  end: 2020-12-14 12:00:00
+  frequency: 12h
+
+input:
+  pipe:
+    - join:
+      - mars:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [2t]
+          levtype: sfc
+      - mars:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [q, t,w]
+          levtype: pl
+          level: [50, 100]
+          stream: oper
+          type: an
+      - accumulations:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [cp, tp]
+    - rename:
+        param:
+          tp: tp_era5

anemoi_datasets-0.5.27/docs/howtos/create/yaml/recipe-filter2.yaml

@@ -0,0 +1,35 @@
+dates:
+  start: 2020-12-12 00:00:00
+  end: 2020-12-14 12:00:00
+  frequency: 12h
+
+input:
+  pipe:
+    - join:
+      - mars:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [2t]
+          levtype: sfc
+      - mars:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [q, t,w]
+          levtype: pl
+          level: [50, 100]
+          stream: oper
+          type: an
+      - accumulations:
+          expver: "0001"
+          class: ea
+          grid: 20./20.
+          param: [cp, tp]
+    - rename:
+        param:
+          tp: tp_era5
+    - w_to_wz:
+        w_component: w
+        temperature: t
+        humidity: q

anemoi_datasets-0.5.27/docs/howtos/create/yaml/zarr2.yaml

@@ -0,0 +1,8 @@
+dates:
+  start: 2023-01-01T00:00:00
+  end: 2023-01-02T18:00:00
+  frequency: 6h
+
+input:
+  anemoi-dataset:
+    dataset: /path/to/input.zarr

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/docs/howtos/introduction.rst

@@ -13,6 +13,7 @@ good starting point.
 -  :ref:`create-grib-data`
 -  :ref:`create-cf-data`
 -  :ref:`constant-data`
+-  :ref:`create-with-filter`
 
 .. toctree::
    :maxdepth: 1

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/docs/howtos/usage/code/cutout-complement1.py

@@ -14,4 +14,5 @@ ds = open_dataset(
     },
     source="global-dataset",
     interpolation="nearest",
+    k=1,
 )

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/docs/howtos/usage/yaml/cutout-complement1.yaml

@@ -9,3 +9,4 @@ dataset:
          adjust: dates
    source: global-dataset
    interpolation: nearest
+   k: 1

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/docs/index.rst

@@ -164,6 +164,7 @@ You may also have to install pandoc on macOS:
    cli/scan
    cli/patch
    cli/compare-lam
+   cli/validate
 
 .. toctree::
    :maxdepth: 1

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/pyproject.toml

@@ -28,7 +28,7 @@ authors = [
   { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" },
 ]
 
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 
 classifiers = [
   "Development Status :: 4 - Beta",
@@ -36,7 +36,6 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Operating System :: OS Independent",
   "Programming Language :: Python :: 3 :: Only",
-  "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
@@ -50,7 +49,7 @@ dynamic = [
 ]
 dependencies = [
   "anemoi-transform>=0.1.10",
-  "anemoi-utils[provenance]>=0.4.21",
+  "anemoi-utils[provenance]>=0.4.32",
   "cfunits",
   "numcodecs<0.16",                   # Until we move to zarr3
   "numpy",
@@ -100,6 +99,8 @@ optional-dependencies.remote = [
 optional-dependencies.tests = [
   "anemoi-datasets[xarray]",
   "pytest",
+  "pytest-skip-slow",
+  "pytest-xdist",
 ]
 
 optional-dependencies.xarray = [
@@ -131,6 +132,14 @@ version_file = "src/anemoi/datasets/_version.py"
 [tool.isort]
 profile = "black"
 
+[tool.pytest.ini_options]
+testpaths = "tests"
+addopts = [
+  "--numprocesses=auto",
+  "--maxprocesses=16",
+  "--strict-config",
+]
+
 [tool.mypy]
 strict = false
 exclude = [

{anemoi_datasets-0.5.25 → anemoi_datasets-0.5.27}/src/anemoi/datasets/__init__.py

@@ -7,7 +7,6 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-from typing import List
 
 from .data import MissingDateError
 from .data import add_dataset_path
@@ -23,7 +22,7 @@ except ImportError:  # pragma: no cover
     # Local copy or not installed with setuptools
     __version__ = "999"
 
-__all__: List[str] = [
+__all__: list[str] = [
     "add_dataset_path",
     "add_named_dataset",
     "list_dataset_names",