anemoi-datasets 0.4.3__tar.gz → 0.4.5__tar.gz

anemoi_datasets-0.4.5/.gitattributes

@@ -0,0 +1 @@
+CHANGELOG.md merge=union

anemoi_datasets-0.4.5/.github/ci-config.yml

@@ -0,0 +1,9 @@
+dependencies: |
+  ecmwf/ecbuild
+  MathisRosenhauer/libaec@master
+  ecmwf/eccodes
+  ecmwf/eckit
+  ecmwf/odc
+dependency_branch: develop
+parallelism_factor: 8
+self_build: false # Only for python packages

anemoi_datasets-0.4.5/.github/workflows/changelog-pr-update.yml

@@ -0,0 +1,15 @@
+name: Check Changelog Update on PR
+on:
+  pull_request:
+    types: [assigned, opened, synchronize, reopened, labeled, unlabeled]
+    branches:
+      - main
+      - develop
+jobs:
+  Check-Changelog:
+    name: Check Changelog Action
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: tarides/changelog-check-action@v2
+        with:
+          changelog: CHANGELOG.md

anemoi_datasets-0.4.5/.github/workflows/changelog-release-update.yml

@@ -0,0 +1,34 @@
+# .github/workflows/update-changelog.yaml
+name: "Update Changelog"
+
+on:
+  release:
+    types: [released]
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.release.target_commitish }}
+
+    - name: Update Changelog
+      uses: stefanzweifel/changelog-updater-action@v1
+      with:
+        latest-version: ${{ github.event.release.tag_name }}
+        heading-text: ${{ github.event.release.name }}
+
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@v6
+      with:
+        branch: docs/changelog-update-${{ github.event.release.tag_name }}
+        title: '[Changelog] Update to ${{ github.event.release.tag_name }}'
+        add-paths: |
+          CHANGELOG.md

anemoi_datasets-0.4.5/.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: ci
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - 'main'
+      - 'develop'
+    tags-ignore:
+      - '**'
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+    secrets: inherit
+
+   # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit

anemoi_datasets-0.4.5/.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

anemoi_datasets-0.4.5/.github/workflows/python-publish.yml

@@ -0,0 +1,22 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
+
+  checks:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+
+  deploy:
+    needs: [checks, quality]
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
+    secrets: inherit

anemoi_datasets-0.4.5/.github/workflows/python-pull-request.yml

@@ -0,0 +1,23 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Code Quality checks for PRs
+
+on:
+  push:
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  quality:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
+    with:
+      skip-hooks: "no-commit-to-branch"
+
+  checks:
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+    with:
+      python-version: ${{ matrix.python-version }}

anemoi_datasets-0.4.5/.github/workflows/readthedocs-pr-update.yml

@@ -0,0 +1,22 @@
+name: Read the Docs PR Preview
+on:
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    # Execute this action only on PRs that touch
+    # documentation files.
+    paths:
+      - "docs/**"
+
+permissions:
+  pull-requests: write
+
+jobs:
+  documentation-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/actions/preview@v1
+        with:
+          project-slug: "anemoi-datasets"

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.5}/.gitignore

@@ -188,3 +188,8 @@ _build/
 *.sync
 *.dot
 _dev/
+*.to_upload
+*.tmp
+test.ipynb
+*tmp_data/
+tempCodeRunnerFile.python

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.5}/.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 24.4.2
+  rev: 24.8.0
   hooks:
   - id: black
     args: [--line-length=120]
@@ -34,7 +34,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.4.6
+  rev: v0.6.4
   hooks:
   - id: ruff
     # Next line is to exclude for documentation code snippets
@@ -65,6 +65,6 @@ repos:
   - id: optional-dependencies-all
     args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "2.1.3"
+  rev: "2.2.3"
   hooks:
   - id: pyproject-fmt

anemoi_datasets-0.4.5/CHANGELOG.md

@@ -0,0 +1,80 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Please add your functional changes to the appropriate section in the PR.
+Keep it human-readable, your future self will thank you!
+
+## [Unreleased]
+
+### Added
+### Changed
+
+- Added incremental building of datasets
+
+### Removed
+
+## [0.4.5]
+
+### Added
+
+- New `interpolate_frequency` keyword in `open_dataset`
+- CI workflow to update the changelog on release
+- adds the reusable cd pypi workflow
+- merge strategy for changelog in .gitattributes #25
+
+### Changed
+
+- update CI to reusable workflows for PRs and releases
+- Support sub-hourly datasets.
+- Change negative variance detection to make it less restrictive
+- Fix cutout bug that left some global grid points in the lam part
+- Fix bug in computing missing dates in cutout option
+
+### Removed
+
+## [0.4.4] Bug fixes
+
+- Bug fix in accumulations() that did not work with webmars
+- Bug fix when using hindcasts input (missing dates on leap years)
+
+## [0.4.0] Minor Release
+
+### Added
+- earthkit-data replaces climetlab
+
+### Removed
+- climetlab
+
+## [0.3.0] Minor Release
+
+### Added
+- hindcast source
+
+### Changed
+- updated documentation
+
+## [0.2.0] Minor Release
+
+### Added
+- statistics tendencies
+
+### Removed
+- CubesFilter
+
+## [0.1.0] Initial Release
+
+### Added
+- Documentation
+- Initial code release for anemoi-datasets: create datasets for data-driven weather-models
+- open datasets
+- combine datasets
+
+## Git Diffs:
+[0.4.0]: https://github.com/ecmwf/anemoi-datasets/compare/0.3.0...0.4.0
+[0.3.0]: https://github.com/ecmwf/anemoi-datasets/compare/0.2.0...0.3.0
+[0.2.0]: https://github.com/ecmwf/anemoi-datasets/compare/0.1.0...0.2.0
+[0.1.0]: https://github.com/ecmwf/anemoi-models/releases/tag/0.1.0

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.4.3
+Version: 0.4.5
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -223,7 +223,7 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: anemoi-utils[provenance]>=0.3.13
+Requires-Dist: anemoi-utils[provenance]>=0.3.15
 Requires-Dist: numpy
 Requires-Dist: pyyaml
 Requires-Dist: semantic-version
@@ -235,7 +235,7 @@ Requires-Dist: boto3; extra == "all"
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "all"
 Requires-Dist: earthkit-geo>=0.2; extra == "all"
 Requires-Dist: earthkit-meteo; extra == "all"
-Requires-Dist: ecmwflibs>=0.6.3; extra == "all"
+Requires-Dist: eccodes>=2.37; extra == "all"
 Requires-Dist: entrypoints; extra == "all"
 Requires-Dist: gcsfs; extra == "all"
 Requires-Dist: kerchunk; extra == "all"
@@ -246,7 +246,7 @@ Provides-Extra: create
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "create"
 Requires-Dist: earthkit-geo>=0.2; extra == "create"
 Requires-Dist: earthkit-meteo; extra == "create"
-Requires-Dist: ecmwflibs>=0.6.3; extra == "create"
+Requires-Dist: eccodes>=2.37; extra == "create"
 Requires-Dist: entrypoints; extra == "create"
 Requires-Dist: pyproj; extra == "create"
 Provides-Extra: dev
@@ -255,7 +255,7 @@ Requires-Dist: boto3; extra == "dev"
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "dev"
 Requires-Dist: earthkit-geo>=0.2; extra == "dev"
 Requires-Dist: earthkit-meteo; extra == "dev"
-Requires-Dist: ecmwflibs>=0.6.3; extra == "dev"
+Requires-Dist: eccodes>=2.37; extra == "dev"
 Requires-Dist: entrypoints; extra == "dev"
 Requires-Dist: gcsfs; extra == "dev"
 Requires-Dist: kerchunk; extra == "dev"

anemoi_datasets-0.4.5/docs/building/incremental.rst

@@ -0,0 +1,120 @@
+.. _create-incremental:
+
+##################################
+ Creating a dataset incrementally
+##################################
+
+This guide shows how to create a dataset incrementally. This is useful
+when you have a large dataset that you want to load in parts, to avoid
+running out of memory. Because parts can be loaded in parallel, this can
+also speed up the process.
+
+**********************
+ Building the dataset
+**********************
+
+You first need to create an empty dataset with the `init` command, which
+is similar to the `create` command. If there is already a dataset with
+the same name, you can use the `--overwrite` flag to replace it. The
+`init` command requires a YAML file with the dataset configuration and a
+name for the dataset. The content of the YAML file will be copied into
+the dataset, so it will not be needed by following commands.
+
+.. code:: bash
+
+   anemoi-datasets init dataset.yaml dataset.zarr --overwrite
+
+You can then load the dataset in parts with the `load` command. You just
+pass which part you want to load with the `--part` flag.
+
+.. note::
+
+   Parts are numbered from 1 to N, where N is the total number of parts
+   (unlike Python, where they would start at zero). This is to make it
+   easier to use the `seq(1)` command in bash.
+
+You can load multiple parts in any order and in parallel by running the
+`load` command in different terminals, slurm jobs or any other
+parallelisation tool. The library relies on the `zarr` library to handle
+concurrent writes.
+
+.. code:: bash
+
+   anemoi-datasets load dataset.zarr --part 1/20
+
+.. code:: bash
+
+   anemoi-datasets load dataset.zarr --part 2/20
+
+... and so on ... until:
+
+.. code:: bash
+
+   anemoi-datasets load dataset.zarr --part 20/20
+
+Once you have loaded all the parts, you can finalise the dataset with
+the `finalise` command. This will write the metadata and the attributes
+to the dataset, and consolidate the statistics and cleanup some
+temporary files.
+
+.. code:: bash
+
+   anemoi-datasets finalise dataset.zarr
+
+You can follow the progress of the dataset creation with the `inspect`
+command. This will show you the percentage of parts loaded.
+
+.. code:: bash
+
+   anemoi-datasets inspect dataset.zarr
+
+It is possible that some temporary files are left behind at the end of
+the process. You can clean them up with the `cleanup` command.
+
+.. code:: bash
+
+   anemoi-datasets cleanup dataset.zarr
+
+***********************
+ Additional statistics
+***********************
+
+`anemoi-datasets` can compute additional statistics for the dataset,
+mostly statistics of the increments between two dates (e.g. 6h or 12h).
+
+To add statistics for 6h increments:
+
+.. code:: bash
+
+   anemoi-datasets init-additions dataset.zarr --delta 6h anemoi-datasets
+   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 6h anemoi-datasets
+   anemoi-datasets load-additions dataset.zarr --part 2/2 --delta 6h
+   anemoi-datasets finalise-additions dataset.zarr --delta 6h
+
+To add statistics for 12h increments:
+
+.. code:: bash
+
+   anemoi-datasets init-additions dataset.zarr --delta 12h
+   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 12h anemoi-datasets
+   anemoi-datasets load-additions dataset.zarr --part 2/2 --delta 12h
+   anemoi-datasets finalise-additions dataset.zarr --delta 12h
+
+If this process leaves temporary files behind, you can clean them up
+with the `cleanup` command.
+
+.. code:: bash
+
+   anemoi-datasets cleanup dataset.zarr
+
+********************************
+ Patching the dataset metadata:
+********************************
+
+The following command will patch the dataset metadata. In particular, it
+will remove any references to the YAML file used to initialise the
+dataset.
+
+.. code:: bash
+
+   anemoi-datasets patch dataset.zarr

anemoi_datasets-0.4.5/docs/building/sources/xarray-kerchunk.py

@@ -0,0 +1,30 @@
+import json
+
+import fsspec
+import tqdm
+from kerchunk.combine import MultiZarrToZarr
+from kerchunk.hdf import SingleHdf5ToZarr
+
+fs = fsspec.filesystem("s3", anon=True)
+
+pattern = "s3://nsf-ncar-era5/e5.oper.an.pl/202403/e5.oper.an.pl.*.ll025sc.2024????00_2024????23.nc"
+
+
+jsons = []
+
+for file in tqdm.tqdm(fs.glob(pattern)):
+    with fs.open(file, "rb", anon=True) as f:
+        h5chunks = SingleHdf5ToZarr(f, file)
+        jsons.append(h5chunks.translate())
+
+
+mzz = MultiZarrToZarr(
+    jsons,
+    remote_protocol="s3",
+    remote_options={"anon": True},
+    concat_dims=["time"],
+    identical_dims=["latitude", "longitude"],
+)
+
+with open("combined.json", "w") as f:
+    json.dump(mzz.translate(), f)

anemoi_datasets-0.4.5/docs/building/sources/xarray-kerchunk.rst

@@ -0,0 +1,17 @@
+#################
+ xarray-kerchunk
+#################
+
+.. literalinclude:: xarray-kerchunk.yaml
+   :language: yaml
+
+The code below is inspired by the `kerchunk tutorial`_, and makes use of
+a subset of the `ERA5 dataset available on AWS`_. You may need to
+install the relevant packages before running the code below.
+
+.. literalinclude:: xarray-kerchunk.py
+   :language: python
+
+.. _era5 dataset available on aws: https://registry.opendata.aws/ecmwf-era5/
+
+.. _kerchunk tutorial: https://fsspec.github.io/kerchunk/tutorial.html

anemoi_datasets-0.4.5/docs/building/sources/xarray-kerchunk.yaml

@@ -0,0 +1,10 @@
+dates:
+  start: 2024-03-24T00:00:00
+  end: 2024-03-24T03:00:00
+  frequency: 1h
+
+input:
+  xarray-kerchunk:
+    json: combined.json
+    param: T
+    level: [1000, 50]

anemoi_datasets-0.4.5/docs/building/sources/xarray-zarr.rst

@@ -0,0 +1,6 @@
+#############
+ xarray-zarr
+#############
+
+.. literalinclude:: xarray-zarr.yaml
+   :language: yaml

anemoi_datasets-0.4.3/docs/building/sources/xarray.yaml → anemoi_datasets-0.4.5/docs/building/sources/xarray-zarr.yaml

@@ -1,3 +1,3 @@
 input:
-  xarray:
+  xarray-zarr:
     url: https://...

anemoi_datasets-0.4.5/docs/building/sources/zenodo.rst

@@ -0,0 +1,8 @@
+########
+ zenodo
+########
+
+https://zenodo.org/records/6470725
+
+.. literalinclude:: zenodo.yaml
+   :language: yaml

anemoi_datasets-0.4.5/docs/building/sources/zenodo.yaml

@@ -0,0 +1,9 @@
+dates:
+  start: 2021-06-28
+  end: 2021-06-29
+
+input:
+  zenodo:
+    record_id: 6470725
+    file_key: "{param}_INCA_202106280700.nc"
+    param: [RR,PN, PT]

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.5}/docs/building/sources.rst

@@ -20,11 +20,14 @@ The following `sources` are currently available:
 .. toctree::
    :maxdepth: 1
 
-   sources/mars
+   sources/accumulations
+   sources/forcings
    sources/grib
+   sources/hindcasts
+   sources/mars
    sources/netcdf
-   sources/xarray
    sources/opendap
-   sources/forcings
-   sources/accumulations
    sources/recentre
+   sources/xarray-kerchunk
+   sources/xarray-zarr
+   sources/zenodo

{anemoi_datasets-0.4.3 → anemoi_datasets-0.4.5}/docs/index.rst

@@ -44,7 +44,9 @@ datasets <building-introduction>`.
 -  :doc:`using/combining`
 -  :doc:`using/selecting`
 -  :doc:`using/grids`
+-  :doc:`using/zip`
 -  :doc:`using/statistics`
+-  :doc:`using/missing`
 -  :doc:`using/other`
 -  :doc:`using/matching`
 -  :doc:`using/miscellaneous`
@@ -62,7 +64,9 @@ datasets <building-introduction>`.
    using/combining
    using/selecting
    using/grids
+   using/zip
    using/statistics
+   using/missing
    using/other
    using/matching
    using/miscellaneous
@@ -75,6 +79,7 @@ datasets <building-introduction>`.
 -  :doc:`building/sources`
 -  :doc:`building/filters`
 -  :doc:`building/statistics`
+-  :doc:`building/incremental`
 
 .. toctree::
    :maxdepth: 1
@@ -89,6 +94,7 @@ datasets <building-introduction>`.
    building/handling-missing-dates
    building/handling-missing-values
    building/statistics
+   building/incremental
    building/advanced-options
 
 **Command line tool**

anemoi_datasets-0.4.5/docs/using/code/fill_missing_gaps_.py

@@ -0,0 +1 @@
+ds = open_dataset(concat=[dataset1, dataset2, ...], fill_missing_gaps=True)

anemoi_datasets-0.4.5/docs/using/code/frequency2_.py

@@ -0,0 +1 @@
+ds = open_dataset(dataset, interpolate_frequency="10m")

anemoi_datasets-0.4.5/docs/using/code/iterating_missing_dates1_.py

@@ -0,0 +1,4 @@
+ds = open_dataset(dataset)
+
+for i in range(len(ds) - 1):
+    ds = ds[i + 1] - ds[i]

anemoi_datasets-0.4.5/docs/using/code/iterating_missing_dates2_.py

@@ -0,0 +1,9 @@
+ds = open_dataset(
+    dataset,
+    skip_missing_dates=True,
+    expected_access=slice(0, 2),
+)
+
+for i in range(len(ds)):
+    xi, xi_1 = ds[i]
+    dx = xi_1 - xi

anemoi_datasets-0.4.5/docs/using/code/iterating_missing_dates3_.py

@@ -0,0 +1,2 @@
+expected_access = slice(0, 2)
+expected_access = 2

anemoi_datasets-0.4.5/docs/using/code/missing_dates_.py

@@ -0,0 +1 @@
+ds = open_dataset(dataset, missing_dates=["2010-01-01T12:00:00", "2010-02-01T12:00:00"])

anemoi_datasets-0.4.5/docs/using/code/xy1_.py

@@ -0,0 +1,5 @@
+ds = open_dataset(x=dataset1, y=dataset2)
+
+# or
+
+ds = open_dataset(xy=[dataset1, dataset2])

anemoi_datasets-0.4.5/docs/using/code/xy2_.py

@@ -0,0 +1,4 @@
+for x, y in ds:
+    y_hat = model(x)
+    loss = criterion(y_hat, y)
+    loss.backward()

anemoi_datasets-0.4.5/docs/using/code/zip_xy_.py

@@ -0,0 +1,9 @@
+input = open_dataset(zip=[low_res_dataset, high_res_orography_dataset])
+output = open_dataset(high_res_dataset)
+
+ds = open_dataset(x=input, y=output)
+
+for (x, orography), y in ds:
+    y_hat = model(x, orography)
+    loss = criterion(y_hat, y)
+    loss.backward()