anemoi-datasets 0.4.4__tar.gz → 0.5.0__tar.gz

anemoi_datasets-0.5.0/.gitattributes

@@ -0,0 +1 @@
+CHANGELOG.md merge=union

anemoi_datasets-0.5.0/.github/CODEOWNERS

@@ -0,0 +1,6 @@
+# CODEOWNERS file
+
+# Protect workflow files
+/.github/ @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb
+/.pre-commit-config.yaml @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb
+/pyproject.toml @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb

anemoi_datasets-0.5.0/.github/ci-hpc-config.yml

@@ -0,0 +1,18 @@
+build:
+  python: '3.10'
+  modules:
+    - ninja
+  dependencies:
+    - ecmwf/ecbuild@develop
+    - ecmwf/eccodes@develop
+    - ecmwf/eckit@develop
+    - ecmwf/odc@develop
+  python_dependencies:
+    - ecmwf/anemoi-utils@develop
+    - ecmwf/earthkit-data@develop
+    - ecmwf/earthkit-meteo@develop
+    - ecmwf/earthkit-geo@develop
+  parallel: 64
+
+  pytest_cmd: |
+    python -m pytest -vv -m 'not notebook and not no_cache_init' --cov=. --cov-report=xml

{anemoi_datasets-0.4.4 → anemoi_datasets-0.5.0}/.github/workflows/changelog-pr-update.yml

@@ -5,6 +5,9 @@ on:
     branches:
       - main
       - develop
+    paths-ignore:
+      - .pre-commit-config.yaml
+      - .readthedocs.yaml
 jobs:
   Check-Changelog:
     name: Check Changelog Action

anemoi_datasets-0.5.0/.github/workflows/changelog-release-update.yml

@@ -0,0 +1,35 @@
+# .github/workflows/update-changelog.yaml
+name: "Update Changelog"
+
+on:
+  release:
+    types: [released]
+  workflow_dispatch: ~
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.release.target_commitish }}
+
+    - name: Update Changelog
+      uses: stefanzweifel/changelog-updater-action@v1
+      with:
+        latest-version: ${{ github.event.release.tag_name }}
+        heading-text: ${{ github.event.release.name }}
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v6
+      with:
+        branch: docs/changelog-update-${{ github.event.release.tag_name }}
+        title: '[Changelog] Update to ${{ github.event.release.tag_name }}'
+        add-paths: |
+          CHANGELOG.md

{anemoi_datasets-0.4.4 → anemoi_datasets-0.5.0}/.github/workflows/ci.yml

@@ -8,9 +8,17 @@ on:
       - 'develop'
     tags-ignore:
       - '**'
+    paths-ignore:
+      - "docs/**"
+      - "CHANGELOG.md"
+      - "README.md"
 
   # Trigger the workflow on pull request
-  pull_request: ~
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "CHANGELOG.md"
+      - "README.md"
 
   # Trigger the workflow manually
   workflow_dispatch: ~
@@ -18,6 +26,11 @@ on:
   # Trigger after public PR approved for CI
   pull_request_target:
     types: [labeled]
+    paths-ignore:
+      - "docs/**"
+      - "CHANGELOG.md"
+      - "README.md"
+
 
 jobs:
   # Run CI including downstream packages on self-hosted runners
@@ -34,7 +47,7 @@ jobs:
   downstream-ci-hpc:
     name: downstream-ci-hpc
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
     with:
       anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
     secrets: inherit

anemoi_datasets-0.5.0/.github/workflows/python-publish.yml

@@ -0,0 +1,24 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
+
+  checks:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+    with:
+      python-version: "3.9"
+
+  deploy:
+    needs: [checks, quality]
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit

anemoi_datasets-0.5.0/.github/workflows/python-pull-request.yml

@@ -0,0 +1,23 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Code Quality checks for PRs
+
+on:
+  push:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
+
+  checks:
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+    with:
+      python-version: ${{ matrix.python-version }}

{anemoi_datasets-0.4.4 → anemoi_datasets-0.5.0}/.gitignore

@@ -120,6 +120,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
+.envrc
 .env
 .venv
 env/
@@ -188,3 +189,8 @@ _build/
 *.sync
 *.dot
 _dev/
+*.to_upload
+*.tmp
+test.ipynb
+*tmp_data/
+tempCodeRunnerFile.python

{anemoi_datasets-0.4.4 → anemoi_datasets-0.5.0}/.pre-commit-config.yaml

@@ -20,8 +20,14 @@ repos:
   - id: no-commit-to-branch # Prevent committing to main / master
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0  # Use the ref you want to point at
+    hooks:
+    -   id: python-use-type-annotations # Check for missing type annotations
+    -   id: python-check-blanket-noqa # Check for # noqa: all
+    -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.4.2
+  rev: 24.8.0
   hooks:
   - id: black
     args: [--line-length=120]
@@ -34,18 +40,18 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.6
+  rev: v0.6.4
   hooks:
   - id: ruff
-    # Next line is to exclude for documentation code snippets
-    exclude: 'docs/(.*/)?[a-z]\w+_.py$'
+    # Next line if for documenation cod snippets
+    exclude: '^[^_].*_\.py$'
     args:
     - --line-length=120
     - --fix
     - --exit-non-zero-on-fix
     - --preview
 - repo: https://github.com/sphinx-contrib/sphinx-lint
-  rev: v0.9.1
+  rev: v1.0.0
   hooks:
   - id: sphinx-lint
 # For now, we use it. But it does not support a lot of sphinx features
@@ -59,12 +65,10 @@ repos:
   hooks:
   - id: docconvert
     args: ["numpy"]
-- repo: https://github.com/b8raoult/optional-dependencies-all
-  rev: "0.0.6"
-  hooks:
-  - id: optional-dependencies-all
-    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "2.1.3"
+  rev: "2.2.3"
   hooks:
   - id: pyproject-fmt
+
+ci:
+  autoupdate_schedule: monthly

{anemoi_datasets-0.4.4 → anemoi_datasets-0.5.0}/CHANGELOG.md

@@ -12,8 +12,39 @@ Keep it human-readable, your future self will thank you!
 
 ### Added
 
+- New `rescale` keyword in `open_dataset` to change units of variables #36
+- Simplify imports
+
 ### Changed
 
+- Added incremental building of datasets
+- Add missing dependency for documentation building
+- Fix failing test due to previous merge
+- Bug fix when creating dataset from zarr
+- Bug fix with area selection in cutout operation
+- add paths-ignore to ci workflow
+
+### Removed
+- pytest for notebooks
+
+## [0.4.5]
+
+### Added
+
+- New `interpolate_frequency` keyword in `open_dataset`
+- CI workflow to update the changelog on release
+- adds the reusable cd pypi workflow
+- merge strategy for changelog in .gitattributes #25
+- adds ci hpc config (#43)
+
+### Changed
+
+- update CI to reusable workflows for PRs and releases
+- Support sub-hourly datasets.
+- Change negative variance detection to make it less restrictive
+- Fix cutout bug that left some global grid points in the lam part
+- Fix bug in computing missing dates in cutout option
+
 ### Removed
 
 ## [0.4.4] Bug fixes
@@ -54,6 +85,9 @@ Keep it human-readable, your future self will thank you!
 - combine datasets
 
 ## Git Diffs:
+[Unreleased]: https://github.com/ecmwf/anemoi-datasets/compare/0.4.5...HEAD
+[0.4.5]: https://github.com/ecmwf/anemoi-datasets/compare/0.4.4...0.4.5
+[0.4.4]: https://github.com/ecmwf/anemoi-datasets/compare/0.4.0...0.4.4
 [0.4.0]: https://github.com/ecmwf/anemoi-datasets/compare/0.3.0...0.4.0
 [0.3.0]: https://github.com/ecmwf/anemoi-datasets/compare/0.2.0...0.3.0
 [0.2.0]: https://github.com/ecmwf/anemoi-datasets/compare/0.1.0...0.2.0

{anemoi_datasets-0.4.4 → anemoi_datasets-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.4.4
+Version: 0.5.0
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -223,14 +223,14 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: anemoi-utils[provenance]>=0.3.13
+Requires-Dist: anemoi-utils[provenance]>=0.3.15
+Requires-Dist: cfunits
 Requires-Dist: numpy
 Requires-Dist: pyyaml
 Requires-Dist: semantic-version
 Requires-Dist: tqdm
-Requires-Dist: zarr
+Requires-Dist: zarr<=2.17
 Provides-Extra: all
-Requires-Dist: aiohttp; extra == "all"
 Requires-Dist: boto3; extra == "all"
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "all"
 Requires-Dist: earthkit-geo>=0.2; extra == "all"
@@ -241,7 +241,6 @@ Requires-Dist: gcsfs; extra == "all"
 Requires-Dist: kerchunk; extra == "all"
 Requires-Dist: pyproj; extra == "all"
 Requires-Dist: requests; extra == "all"
-Requires-Dist: s3fs; extra == "all"
 Provides-Extra: create
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "create"
 Requires-Dist: earthkit-geo>=0.2; extra == "create"
@@ -250,7 +249,6 @@ Requires-Dist: ecmwflibs>=0.6.3; extra == "create"
 Requires-Dist: entrypoints; extra == "create"
 Requires-Dist: pyproj; extra == "create"
 Provides-Extra: dev
-Requires-Dist: aiohttp; extra == "dev"
 Requires-Dist: boto3; extra == "dev"
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "dev"
 Requires-Dist: earthkit-geo>=0.2; extra == "dev"
@@ -264,26 +262,19 @@ Requires-Dist: pandoc; extra == "dev"
 Requires-Dist: pyproj; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: requests; extra == "dev"
-Requires-Dist: rstfmt; extra == "dev"
-Requires-Dist: s3fs; extra == "dev"
 Requires-Dist: sphinx; extra == "dev"
-Requires-Dist: sphinx-argparse<0.5; extra == "dev"
 Requires-Dist: sphinx-rtd-theme; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: nbsphinx; extra == "docs"
 Requires-Dist: pandoc; extra == "docs"
-Requires-Dist: rstfmt; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
-Requires-Dist: sphinx-argparse<0.5; extra == "docs"
+Requires-Dist: sphinx-argparse; extra == "docs"
 Requires-Dist: sphinx-rtd-theme; extra == "docs"
-Provides-Extra: kerchunk
-Requires-Dist: gcsfs; extra == "kerchunk"
-Requires-Dist: kerchunk; extra == "kerchunk"
-Requires-Dist: s3fs; extra == "kerchunk"
 Provides-Extra: remote
-Requires-Dist: aiohttp; extra == "remote"
 Requires-Dist: boto3; extra == "remote"
 Requires-Dist: requests; extra == "remote"
-Requires-Dist: s3fs; extra == "remote"
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
+Provides-Extra: xarray
+Requires-Dist: gcsfs; extra == "xarray"
+Requires-Dist: kerchunk; extra == "xarray"

anemoi_datasets-0.5.0/docs/building/incremental.rst

@@ -0,0 +1,120 @@
+.. _create-incremental:
+
+##################################
+ Creating a dataset incrementally
+##################################
+
+This guide shows how to create a dataset incrementally. This is useful
+when you have a large dataset that you want to load in parts, to avoid
+running out of memory. Because parts can be loaded in parallel, this can
+also speed up the process.
+
+**********************
+ Building the dataset
+**********************
+
+You first need to create an empty dataset with the `init` command, which
+is similar to the `create` command. If there is already a dataset with
+the same name, you can use the `--overwrite` flag to replace it. The
+`init` command requires a YAML file with the dataset configuration and a
+name for the dataset. The content of the YAML file will be copied into
+the dataset, so it will not be needed by following commands.
+
+.. code:: bash
+
+   anemoi-datasets init dataset.yaml dataset.zarr --overwrite
+
+You can then load the dataset in parts with the `load` command. You just
+pass which part you want to load with the `--part` flag.
+
+.. note::
+
+   Parts are numbered from 1 to N, where N is the total number of parts
+   (unlike Python, where they would start at zero). This is to make it
+   easier to use the `seq(1)` command in bash.
+
+You can load multiple parts in any order and in parallel by running the
+`load` command in different terminals, slurm jobs or any other
+parallelisation tool. The library relies on the `zarr` library to handle
+concurrent writes.
+
+.. code:: bash
+
+   anemoi-datasets load dataset.zarr --part 1/20
+
+.. code:: bash
+
+   anemoi-datasets load dataset.zarr --part 2/20
+
+... and so on ... until:
+
+.. code:: bash
+
+   anemoi-datasets load dataset.zarr --part 20/20
+
+Once you have loaded all the parts, you can finalise the dataset with
+the `finalise` command. This will write the metadata and the attributes
+to the dataset, and consolidate the statistics and cleanup some
+temporary files.
+
+.. code:: bash
+
+   anemoi-datasets finalise dataset.zarr
+
+You can follow the progress of the dataset creation with the `inspect`
+command. This will show you the percentage of parts loaded.
+
+.. code:: bash
+
+   anemoi-datasets inspect dataset.zarr
+
+It is possible that some temporary files are left behind at the end of
+the process. You can clean them up with the `cleanup` command.
+
+.. code:: bash
+
+   anemoi-datasets cleanup dataset.zarr
+
+***********************
+ Additional statistics
+***********************
+
+`anemoi-datasets` can compute additional statistics for the dataset,
+mostly statistics of the increments between two dates (e.g. 6h or 12h).
+
+To add statistics for 6h increments:
+
+.. code:: bash
+
+   anemoi-datasets init-additions dataset.zarr --delta 6h
+   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 6h
+   anemoi-datasets load-additions dataset.zarr --part 2/2 --delta 6h
+   anemoi-datasets finalise-additions dataset.zarr --delta 6h
+
+To add statistics for 12h increments:
+
+.. code:: bash
+
+   anemoi-datasets init-additions dataset.zarr --delta 12h
+   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 12h
+   anemoi-datasets load-additions dataset.zarr --part 2/2 --delta 12h
+   anemoi-datasets finalise-additions dataset.zarr --delta 12h
+
+If this process leaves temporary files behind, you can clean them up
+with the `cleanup` command.
+
+.. code:: bash
+
+   anemoi-datasets cleanup dataset.zarr
+
+********************************
+ Patching the dataset metadata:
+********************************
+
+The following command will patch the dataset metadata. In particular, it
+will remove any references to the YAML file used to initialise the
+dataset.
+
+.. code:: bash
+
+   anemoi-datasets patch dataset.zarr

{anemoi_datasets-0.4.4 → anemoi_datasets-0.5.0}/docs/index.rst

@@ -44,7 +44,9 @@ datasets <building-introduction>`.
 -  :doc:`using/combining`
 -  :doc:`using/selecting`
 -  :doc:`using/grids`
+-  :doc:`using/zip`
 -  :doc:`using/statistics`
+-  :doc:`using/missing`
 -  :doc:`using/other`
 -  :doc:`using/matching`
 -  :doc:`using/miscellaneous`
@@ -62,7 +64,9 @@ datasets <building-introduction>`.
    using/combining
    using/selecting
    using/grids
+   using/zip
    using/statistics
+   using/missing
    using/other
    using/matching
    using/miscellaneous
@@ -75,6 +79,7 @@ datasets <building-introduction>`.
 -  :doc:`building/sources`
 -  :doc:`building/filters`
 -  :doc:`building/statistics`
+-  :doc:`building/incremental`
 
 .. toctree::
    :maxdepth: 1
@@ -89,6 +94,7 @@ datasets <building-introduction>`.
    building/handling-missing-dates
    building/handling-missing-values
    building/statistics
+   building/incremental
    building/advanced-options
 
 **Command line tool**

anemoi_datasets-0.5.0/docs/using/code/fill_missing_gaps_.py

@@ -0,0 +1 @@
+ds = open_dataset(concat=[dataset1, dataset2, ...], fill_missing_gaps=True)

anemoi_datasets-0.5.0/docs/using/code/frequency2_.py

@@ -0,0 +1 @@
+ds = open_dataset(dataset, interpolate_frequency="10m")

anemoi_datasets-0.5.0/docs/using/code/iterating_missing_dates1_.py

@@ -0,0 +1,4 @@
+ds = open_dataset(dataset)
+
+for i in range(len(ds) - 1):
+    ds = ds[i + 1] - ds[i]

anemoi_datasets-0.5.0/docs/using/code/iterating_missing_dates2_.py

@@ -0,0 +1,9 @@
+ds = open_dataset(
+    dataset,
+    skip_missing_dates=True,
+    expected_access=slice(0, 2),
+)
+
+for i in range(len(ds)):
+    xi, xi_1 = ds[i]
+    dx = xi_1 - xi

anemoi_datasets-0.5.0/docs/using/code/iterating_missing_dates3_.py

@@ -0,0 +1,2 @@
+expected_access = slice(0, 2)
+expected_access = 2

anemoi_datasets-0.5.0/docs/using/code/missing_dates_.py

@@ -0,0 +1 @@
+ds = open_dataset(dataset, missing_dates=["2010-01-01T12:00:00", "2010-02-01T12:00:00"])

anemoi_datasets-0.5.0/docs/using/code/rescale_.py

@@ -0,0 +1,30 @@
+# Scale and offset can be passed as a dictionnary...
+
+ds = open_dataset(
+    dataset,
+    rescale={"2t": {"scale": 1.0, "offset": -273.15}},
+)
+
+# ... a tuple of floating points ....
+
+ds = open_dataset(
+    dataset,
+    rescale={"2t": (1.0, -273.15)},
+)
+
+# ... or a tuple of strings representing units.
+
+ds = open_dataset(
+    dataset,
+    rescale={"2t": ("K", "degC")},
+)
+
+# Several variables can be rescaled at once.
+
+ds = open_dataset(
+    dataset,
+    rescale={
+        "2t": ("K", "degC"),
+        "tp": ("m", "mm"),
+    },
+)

anemoi_datasets-0.5.0/docs/using/code/xy1_.py

@@ -0,0 +1,5 @@
+ds = open_dataset(x=dataset1, y=dataset2)
+
+# or
+
+ds = open_dataset(xy=[dataset1, dataset2])

anemoi_datasets-0.5.0/docs/using/code/xy2_.py

@@ -0,0 +1,4 @@
+for x, y in ds:
+    y_hat = model(x)
+    loss = criterion(y_hat, y)
+    loss.backward()

anemoi_datasets-0.5.0/docs/using/code/zip_xy_.py

@@ -0,0 +1,9 @@
+input = open_dataset(zip=[low_res_dataset, high_res_orography_dataset])
+output = open_dataset(high_res_dataset)
+
+ds = open_dataset(x=input, y=output)
+
+for (x, orography), y in ds:
+    y_hat = model(x, orography)
+    loss = criterion(y_hat, y)
+    loss.backward()

{anemoi_datasets-0.4.4 → anemoi_datasets-0.5.0}/docs/using/combining.rst

@@ -48,7 +48,9 @@ attempt to combine them:
 You can concatenate two or more datasets along the dates dimension. The
 package will check that all datasets are compatible (same resolution,
 same variables, etc.). Currently, the datasets must be given in
-chronological order with no gaps between them.
+chronological order with no gaps between them. If you want to
+concatenate datasets that have gaps between them, check the
+:ref:`fill_missing_gaps <fill_missing_gaps>` option.
 
 .. literalinclude:: code/concat1.py
    :language: python
@@ -155,3 +157,28 @@ cutout:
    :width: 75%
    :align: center
    :alt: Cutout
+
+You can also pass a `min_distance_km` parameter to the `cutout`
+function. Any grid points in the global dataset that are closer than
+this distance to a grid point in the LAM dataset will be removed. This
+can be useful to control the behaviour of the algorithm at the edge of
+the cutout area. If no value is provided, the algorithm will compute its
+value as the smallest distance between two grid points in the global
+dataset over the cutout area. If you do not want to use this feature,
+you can set `min_distance_km=0`, or provide your own value.
+
+The plots below illustrate how the cutout differs if `min_distance_km`
+is not given (top) or if `min_distance_km` is is set to `0` (bottom).
+The difference can be seen at the boundary between the two grids:
+
+.. image:: images/cutout-5.png
+   :align: center
+   :alt: Cutout
+
+.. image:: images/cutout-6.png
+   :align: center
+   :alt: Cutout
+
+To debug the combination, you can pass `plot=True` to the `cutout`
+function (when running from a Notebook), of use `plot="prefix"` to save
+the plots to series of PNG files in the current directory.