anemoi-datasets 0.4.4__tar.gz → 0.4.5__tar.gz

anemoi_datasets-0.4.5/.gitattributes

@@ -0,0 +1 @@
+CHANGELOG.md merge=union

anemoi_datasets-0.4.5/.github/workflows/changelog-release-update.yml

@@ -0,0 +1,34 @@
+# .github/workflows/update-changelog.yaml
+name: "Update Changelog"
+
+on:
+  release:
+    types: [released]
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.release.target_commitish }}
+
+    - name: Update Changelog
+      uses: stefanzweifel/changelog-updater-action@v1
+      with:
+        latest-version: ${{ github.event.release.tag_name }}
+        heading-text: ${{ github.event.release.name }}
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v6
+      with:
+        branch: docs/changelog-update-${{ github.event.release.tag_name }}
+        title: '[Changelog] Update to ${{ github.event.release.tag_name }}'
+        add-paths: |
+          CHANGELOG.md

anemoi_datasets-0.4.5/.github/workflows/python-publish.yml

@@ -0,0 +1,22 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
+
+  checks:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+
+  deploy:
+    needs: [checks, quality]
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit

anemoi_datasets-0.4.5/.github/workflows/python-pull-request.yml

@@ -0,0 +1,23 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Code Quality checks for PRs
+
+on:
+  push:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
+
+  checks:
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+    with:
+      python-version: ${{ matrix.python-version }}

{anemoi_datasets-0.4.4 → anemoi_datasets-0.4.5}/.gitignore

@@ -188,3 +188,8 @@ _build/
 *.sync
 *.dot
 _dev/
+*.to_upload
+*.tmp
+test.ipynb
+*tmp_data/
+tempCodeRunnerFile.python

{anemoi_datasets-0.4.4 → anemoi_datasets-0.4.5}/.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.4.2
+  rev: 24.8.0
   hooks:
   - id: black
     args: [--line-length=120]
@@ -34,7 +34,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.6
+  rev: v0.6.4
   hooks:
   - id: ruff
     # Next line is to exclude for documentation code snippets
@@ -65,6 +65,6 @@ repos:
   - id: optional-dependencies-all
     args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "2.1.3"
+  rev: "2.2.3"
   hooks:
   - id: pyproject-fmt

{anemoi_datasets-0.4.4 → anemoi_datasets-0.4.5}/CHANGELOG.md

@@ -11,9 +11,29 @@ Keep it human-readable, your future self will thank you!
 ## [Unreleased]
 
 ### Added
+### Changed
+
+- Added incremental building of datasets
+
+### Removed
+
+## [0.4.5]
+
+### Added
+
+- New `interpolate_frequency` keyword in `open_dataset`
+- CI workflow to update the changelog on release
+- adds the reusable cd pypi workflow
+- merge strategy for changelog in .gitattributes #25
 
 ### Changed
 
+- update CI to reusable workflows for PRs and releases
+- Support sub-hourly datasets.
+- Change negative variance detection to make it less restrictive
+- Fix cutout bug that left some global grid points in the lam part
+- Fix bug in computing missing dates in cutout option
+
 ### Removed
 
 ## [0.4.4] Bug fixes

{anemoi_datasets-0.4.4 → anemoi_datasets-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.4.4
+Version: 0.4.5
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -223,7 +223,7 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: anemoi-utils[provenance]>=0.3.13
+Requires-Dist: anemoi-utils[provenance]>=0.3.15
 Requires-Dist: numpy
 Requires-Dist: pyyaml
 Requires-Dist: semantic-version
@@ -235,7 +235,7 @@ Requires-Dist: boto3; extra == "all"
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "all"
 Requires-Dist: earthkit-geo>=0.2; extra == "all"
 Requires-Dist: earthkit-meteo; extra == "all"
-Requires-Dist: ecmwflibs>=0.6.3; extra == "all"
+Requires-Dist: eccodes>=2.37; extra == "all"
 Requires-Dist: entrypoints; extra == "all"
 Requires-Dist: gcsfs; extra == "all"
 Requires-Dist: kerchunk; extra == "all"
@@ -246,7 +246,7 @@ Provides-Extra: create
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "create"
 Requires-Dist: earthkit-geo>=0.2; extra == "create"
 Requires-Dist: earthkit-meteo; extra == "create"
-Requires-Dist: ecmwflibs>=0.6.3; extra == "create"
+Requires-Dist: eccodes>=2.37; extra == "create"
 Requires-Dist: entrypoints; extra == "create"
 Requires-Dist: pyproj; extra == "create"
 Provides-Extra: dev
@@ -255,7 +255,7 @@ Requires-Dist: boto3; extra == "dev"
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "dev"
 Requires-Dist: earthkit-geo>=0.2; extra == "dev"
 Requires-Dist: earthkit-meteo; extra == "dev"
-Requires-Dist: ecmwflibs>=0.6.3; extra == "dev"
+Requires-Dist: eccodes>=2.37; extra == "dev"
 Requires-Dist: entrypoints; extra == "dev"
 Requires-Dist: gcsfs; extra == "dev"
 Requires-Dist: kerchunk; extra == "dev"

anemoi_datasets-0.4.5/docs/building/incremental.rst

@@ -0,0 +1,120 @@
+.. _create-incremental:
+
+##################################
+ Creating a dataset incrementally
+##################################
+
+This guide shows how to create a dataset incrementally. This is useful
+when you have a large dataset that you want to load in parts, to avoid
+running out of memory. Because parts can be loaded in parallel, this can
+also speed up the process.
+
+**********************
+ Building the dataset
+**********************
+
+You first need to create an empty dataset with the `init` command, which
+is similar to the `create` command. If there is already a dataset with
+the same name, you can use the `--overwrite` flag to replace it. The
+`init` command requires a YAML file with the dataset configuration and a
+name for the dataset. The content of the YAML file will be copied into
+the dataset, so it will not be needed by following commands.
+
+.. code:: bash
+
+   anemoi-datasets init dataset.yaml dataset.zarr --overwrite
+
+You can then load the dataset in parts with the `load` command. You just
+pass which part you want to load with the `--part` flag.
+
+.. note::
+
+   Parts are numbered from 1 to N, where N is the total number of parts
+   (unlike Python, where they would start at zero). This is to make it
+   easier to use the `seq(1)` command in bash.
+
+You can load multiple parts in any order and in parallel by running the
+`load` command in different terminals, slurm jobs or any other
+parallelisation tool. The library relies on the `zarr` library to handle
+concurrent writes.
+
+.. code:: bash
+
+   anemoi-datasets load dataset.zarr --part 1/20
+
+.. code:: bash
+
+   anemoi-datasets load dataset.zarr --part 2/20
+
+... and so on ... until:
+
+.. code:: bash
+
+   anemoi-datasets load dataset.zarr --part 20/20
+
+Once you have loaded all the parts, you can finalise the dataset with
+the `finalise` command. This will write the metadata and the attributes
+to the dataset, and consolidate the statistics and cleanup some
+temporary files.
+
+.. code:: bash
+
+   anemoi-datasets finalise dataset.zarr
+
+You can follow the progress of the dataset creation with the `inspect`
+command. This will show you the percentage of parts loaded.
+
+.. code:: bash
+
+   anemoi-datasets inspect dataset.zarr
+
+It is possible that some temporary files are left behind at the end of
+the process. You can clean them up with the `cleanup` command.
+
+.. code:: bash
+
+   anemoi-datasets cleanup dataset.zarr
+
+***********************
+ Additional statistics
+***********************
+
+`anemoi-datasets` can compute additional statistics for the dataset,
+mostly statistics of the increments between two dates (e.g. 6h or 12h).
+
+To add statistics for 6h increments:
+
+.. code:: bash
+
+   anemoi-datasets init-additions dataset.zarr --delta 6h anemoi-datasets
+   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 6h anemoi-datasets
+   anemoi-datasets load-additions dataset.zarr --part 2/2 --delta 6h
+   anemoi-datasets finalise-additions dataset.zarr --delta 6h
+
+To add statistics for 12h increments:
+
+.. code:: bash
+
+   anemoi-datasets init-additions dataset.zarr --delta 12h
+   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 12h anemoi-datasets
+   anemoi-datasets load-additions dataset.zarr --part 2/2 --delta 12h
+   anemoi-datasets finalise-additions dataset.zarr --delta 12h
+
+If this process leaves temporary files behind, you can clean them up
+with the `cleanup` command.
+
+.. code:: bash
+
+   anemoi-datasets cleanup dataset.zarr
+
+********************************
+ Patching the dataset metadata:
+********************************
+
+The following command will patch the dataset metadata. In particular, it
+will remove any references to the YAML file used to initialise the
+dataset.
+
+.. code:: bash
+
+   anemoi-datasets patch dataset.zarr

{anemoi_datasets-0.4.4 → anemoi_datasets-0.4.5}/docs/index.rst

@@ -44,7 +44,9 @@ datasets <building-introduction>`.
 -  :doc:`using/combining`
 -  :doc:`using/selecting`
 -  :doc:`using/grids`
+-  :doc:`using/zip`
 -  :doc:`using/statistics`
+-  :doc:`using/missing`
 -  :doc:`using/other`
 -  :doc:`using/matching`
 -  :doc:`using/miscellaneous`
@@ -62,7 +64,9 @@ datasets <building-introduction>`.
    using/combining
    using/selecting
    using/grids
+   using/zip
    using/statistics
+   using/missing
    using/other
    using/matching
    using/miscellaneous
@@ -75,6 +79,7 @@ datasets <building-introduction>`.
 -  :doc:`building/sources`
 -  :doc:`building/filters`
 -  :doc:`building/statistics`
+-  :doc:`building/incremental`
 
 .. toctree::
    :maxdepth: 1
@@ -89,6 +94,7 @@ datasets <building-introduction>`.
    building/handling-missing-dates
    building/handling-missing-values
    building/statistics
+   building/incremental
    building/advanced-options
 
 **Command line tool**

anemoi_datasets-0.4.5/docs/using/code/fill_missing_gaps_.py

@@ -0,0 +1 @@
+ds = open_dataset(concat=[dataset1, dataset2, ...], fill_missing_gaps=True)

anemoi_datasets-0.4.5/docs/using/code/frequency2_.py

@@ -0,0 +1 @@
+ds = open_dataset(dataset, interpolate_frequency="10m")

anemoi_datasets-0.4.5/docs/using/code/iterating_missing_dates1_.py

@@ -0,0 +1,4 @@
+ds = open_dataset(dataset)
+
+for i in range(len(ds) - 1):
+    ds = ds[i + 1] - ds[i]

anemoi_datasets-0.4.5/docs/using/code/iterating_missing_dates2_.py

@@ -0,0 +1,9 @@
+ds = open_dataset(
+    dataset,
+    skip_missing_dates=True,
+    expected_access=slice(0, 2),
+)
+
+for i in range(len(ds)):
+    xi, xi_1 = ds[i]
+    dx = xi_1 - xi

anemoi_datasets-0.4.5/docs/using/code/iterating_missing_dates3_.py

@@ -0,0 +1,2 @@
+expected_access = slice(0, 2)
+expected_access = 2

anemoi_datasets-0.4.5/docs/using/code/missing_dates_.py

@@ -0,0 +1 @@
+ds = open_dataset(dataset, missing_dates=["2010-01-01T12:00:00", "2010-02-01T12:00:00"])

anemoi_datasets-0.4.5/docs/using/code/xy1_.py

@@ -0,0 +1,5 @@
+ds = open_dataset(x=dataset1, y=dataset2)
+
+# or
+
+ds = open_dataset(xy=[dataset1, dataset2])

anemoi_datasets-0.4.5/docs/using/code/xy2_.py

@@ -0,0 +1,4 @@
+for x, y in ds:
+    y_hat = model(x)
+    loss = criterion(y_hat, y)
+    loss.backward()

anemoi_datasets-0.4.5/docs/using/code/zip_xy_.py

@@ -0,0 +1,9 @@
+input = open_dataset(zip=[low_res_dataset, high_res_orography_dataset])
+output = open_dataset(high_res_dataset)
+
+ds = open_dataset(x=input, y=output)
+
+for (x, orography), y in ds:
+    y_hat = model(x, orography)
+    loss = criterion(y_hat, y)
+    loss.backward()

{anemoi_datasets-0.4.4 → anemoi_datasets-0.4.5}/docs/using/combining.rst

@@ -48,7 +48,9 @@ attempt to combine them:
 You can concatenate two or more datasets along the dates dimension. The
 package will check that all datasets are compatible (same resolution,
 same variables, etc.). Currently, the datasets must be given in
-chronological order with no gaps between them.
+chronological order with no gaps between them. If you want to
+concatenate datasets that have gaps between them, check the
+:ref:`fill_missing_gaps <fill_missing_gaps>` option.
 
 .. literalinclude:: code/concat1.py
    :language: python
@@ -155,3 +157,28 @@ cutout:
    :width: 75%
    :align: center
    :alt: Cutout
+
+You can also pass a `min_distance_km` parameter to the `cutout`
+function. Any grid points in the global dataset that are closer than
+this distance to a grid point in the LAM dataset will be removed. This
+can be useful to control the behaviour of the algorithm at the edge of
+the cutout area. If no value is provided, the algorithm will compute its
+value as the smallest distance between two grid points in the global
+dataset over the cutout area. If you do not want to use this feature,
+you can set `min_distance_km=0`, or provide your own value.
+
+The plots below illustrate how the cutout differs if `min_distance_km`
+is not given (top) or if `min_distance_km` is is set to `0` (bottom).
+The difference can be seen at the boundary between the two grids:
+
+.. image:: images/cutout-5.png
+   :align: center
+   :alt: Cutout
+
+.. image:: images/cutout-6.png
+   :align: center
+   :alt: Cutout
+
+To debug the combination, you can pass `plot=True` to the `cutout`
+function (when running from a Notebook), of use `plot="prefix"` to save
+the plots to series of PNG files in the current directory.

anemoi_datasets-0.4.5/docs/using/missing.rst

@@ -0,0 +1,78 @@
+.. _selecting-missing:
+
+########################
+ Managing missing dates
+########################
+
+************************************************
+ Skipping missing when iterating over a dataset
+************************************************
+
+If you iterate over a dataset that has missing dates, the library will
+raise a ``MissingDatesError`` exception if you attempt to access a
+missing date.
+
+This code below will throw an exception if ``ds[i]`` or ``ds[i+1]`` are
+missing dates. Because we iterate over the whole dataset, we are
+guaranteed to fail if there are any missing dates.
+
+.. literalinclude:: code/iterating_missing_dates1_.py
+
+You can skip missing dates by setting the ``skip_missing_dates`` option
+to ``True``. You will have to also provide a hint about how you intent
+to iterate over the dataset. The hint is given using the parameter
+``expected_access`` which takes a ``slice`` as argument.
+
+The library will check the slice against the missing dates and insure
+that, when iterating over the dataset with that slice, no missing dates
+are accessed.
+
+The algorithm is illustrated in the picture below. The cells represents
+the dates in the dataset, and the red cells are the missing dates. Given
+``expected_access=slice(0, 2)``, the library will consider each group of
+matching dates that are not missing (in blue). The interval between each
+dates of a group is guaranteed to be constant across all groups.
+
+.. image:: images/skip-missing.png
+   :align: center
+
+.. literalinclude:: code/iterating_missing_dates2_.py
+
+The code above will not raise an exception, even if there are missing
+dates. The ``slice(0, 2)`` represents the ``i`` and ``i+1`` indices in
+the loop (the Python :py:class:`slice` is similar to Python's
+:py:class:`range`, as the first bound in included while the last bound
+is excluded).
+
+You can also provide a single integer to the ```expected_access``
+parameter. The two forms below are identical:
+
+.. literalinclude:: code/iterating_missing_dates3_.py
+
+.. _fill_missing_gaps:
+
+***********************************************
+ Concatenating datasets with gaps between them
+***********************************************
+
+When you concatenate two or more datasets, the library will check that
+the dates are contiguous, i.e. that the last date of a dataset is one
+`frequency` before the first date of the next dataset.
+
+If the dates are not contiguous, the library will raise an error. You
+can force the concatenation by setting the ``fill_missing_gaps`` option:
+
+.. literalinclude:: code/fill_missing_gaps_.py
+
+If there is a gap between the datasets, the library will fill the gap by
+creating a virtual dataset with only missing dates, and add it between
+the datasets to make the dates contiguous.
+
+***********
+ Debugging
+***********
+
+You can set missing dates using the ``missing_dates`` option. This
+option is for debugging purposes only.
+
+.. literalinclude:: code/missing_dates_.py

{anemoi_datasets-0.4.4 → anemoi_datasets-0.4.5}/docs/using/other.rst

@@ -9,22 +9,6 @@
    The operations described in this section do not check that their
    inputs are compatible.
 
-*****
- zip
-*****
-
-The `zip` operation is used to combine multiple datasets into a single
-dataset.
-
-.. literalinclude:: code/zip1_.py
-   :language: python
-
-This operation is similar to Python's :py:func:`zip` function, but it
-returns tuples of the selected indices instead of the values:
-
-.. literalinclude:: code/zip2_.py
-   :language: python
-
 *******
  chain
 *******

{anemoi_datasets-0.4.4 → anemoi_datasets-0.4.5}/docs/using/subsetting.rst

@@ -52,5 +52,15 @@ with a ``frequency`` of 6 hours is equivalent to ``end="2020-12-31
 
 You can change the frequency of the dataset by passing a string with:
 
-.. literalinclude:: code/frequency_.py
+.. literalinclude:: code/frequency1_.py
+   :language: python
+
+The new frequency must be a multiple of the original frequency.
+
+To artificially increase the frequency, you can use the
+``interpolate_frequency`` option. This will create new dates in the
+dataset by linearly interpolating the data values between the original
+dates.
+
+.. literalinclude:: code/frequency2_.py
    :language: python

anemoi_datasets-0.4.5/docs/using/zip.rst

@@ -0,0 +1,64 @@
+.. _selecting-zip:
+
+#################################
+ Iterating over several datasets
+#################################
+
+Sometimes you need to iterate over several datasets at the same time.
+The following functions will help you with that.
+
+.. warning::
+
+   When iterating over several datasets, most of the functions and
+   properties of the dataset object returned by :py:func:`open_dataset`
+   will not return a :py:class:`tuple` instead of a single value. The
+   tuple will contain the values of the corresponding datasets. There
+   are a few exceptions, such as `dates` or `missing`.
+
+**************
+ x=..., y=...
+**************
+
+In machine learning, you often need to iterate over two datasets at the
+same time. One representing the input ``x`` and the other the output
+``y``.
+
+.. literalinclude:: code/xy1_.py
+
+You will then be able to iterate over the datasets as follows:
+
+.. literalinclude:: code/xy2_.py
+
+**Note:** `xy` is currently a shortcut for `zip` below, and is intended
+to make the code more readable.
+
+*****
+ zip
+*****
+
+The `zip` option is used to combine multiple datasets into a single
+dataset, that can be iterated over or indexed simultaneously.
+
+.. literalinclude:: code/zip1_.py
+
+The dataset can then be indexed as follows:
+
+.. literalinclude:: code/zip2_.py
+
+.. note::
+
+   The `zip` option is similar to Python's :py:func:`zip` function. The
+   main difference the datasets are checked to be compatible before
+   being combined (same ranges of dates, same frequency, etc.). Also,
+   Python's :py:func:`zip` only allows iteration, while the `zip` option
+   allows indexing as well.
+
+****************
+ Combining both
+****************
+
+Both options can be combined. The example below is based on a model that
+is trained to upscale a dataset. The input is the low resolution and a
+high resolution orography. The output is the high resolution dataset.
+
+.. literalinclude:: code/zip_xy_.py

{anemoi_datasets-0.4.4 → anemoi_datasets-0.4.5}/pyproject.toml

@@ -50,7 +50,7 @@ dynamic = [
   "version",
 ]
 dependencies = [
-  "anemoi-utils[provenance]>=0.3.13",
+  "anemoi-utils[provenance]>=0.3.15",
   "numpy",
   "pyyaml",
   "semantic-version",
@@ -64,7 +64,7 @@ optional-dependencies.all = [
   "earthkit-data[mars]>=0.9",
   "earthkit-geo>=0.2",
   "earthkit-meteo",
-  "ecmwflibs>=0.6.3",
+  "eccodes>=2.37",
   "entrypoints",
   "gcsfs",
   "kerchunk",
@@ -77,7 +77,7 @@ optional-dependencies.create = [
   "earthkit-data[mars]>=0.9",
   "earthkit-geo>=0.2",
   "earthkit-meteo",
-  "ecmwflibs>=0.6.3",
+  "eccodes>=2.37",
   "entrypoints",
   "pyproj",
 ]
@@ -88,7 +88,7 @@ optional-dependencies.dev = [
   "earthkit-data[mars]>=0.9",
   "earthkit-geo>=0.2",
   "earthkit-meteo",
-  "ecmwflibs>=0.6.3",
+  "eccodes>=2.37",
   "entrypoints",
   "gcsfs",
   "kerchunk",