anemoi-datasets 0.4.5__tar.gz → 0.5.5__tar.gz

anemoi_datasets-0.5.5/.github/CODEOWNERS

@@ -0,0 +1,6 @@
+# CODEOWNERS file
+
+# Protect workflow files
+/.github/ @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb
+/.pre-commit-config.yaml @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb
+/pyproject.toml @theissenhelen @jesperdramsch @gmertes @b8raoult @floriankrb

anemoi_datasets-0.5.5/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,40 @@
+---
+name: Bug report
+about: Create a report to help us improve. **❗ IMPORTANT:** If you have difficulties creating a dataset from any source, please provide us with a sample data file or a URL to the source data.
+title: ''
+labels: ''
+assignees: ''
+
+---
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**URL to sample input data**
+Provide a URL to a sample input data, or attach a file to that report if it is small enough.
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Smartphone (please complete the following information):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - Browser [e.g. stock browser, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

anemoi_datasets-0.5.5/.github/ci-hpc-config.yml

@@ -0,0 +1,18 @@
+build:
+  python: '3.10'
+  modules:
+    - ninja
+  dependencies:
+    - ecmwf/ecbuild@develop
+    - ecmwf/eccodes@develop
+    - ecmwf/eckit@develop
+    - ecmwf/odc@develop
+  python_dependencies:
+    - ecmwf/anemoi-utils@develop
+    - ecmwf/earthkit-data@develop
+    - ecmwf/earthkit-meteo@develop
+    - ecmwf/earthkit-geo@develop
+  parallel: 64
+
+  pytest_cmd: |
+    python -m pytest -vv -m 'not notebook and not no_cache_init' --cov=. --cov-report=xml

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/.github/workflows/changelog-pr-update.yml

@@ -5,6 +5,9 @@ on:
     branches:
       - main
       - develop
+    paths-ignore:
+      - .pre-commit-config.yaml
+      - .readthedocs.yaml
 jobs:
   Check-Changelog:
     name: Check Changelog Action

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/.github/workflows/changelog-release-update.yml

@@ -4,6 +4,7 @@ name: "Update Changelog"
 on:
   release:
     types: [released]
+  workflow_dispatch: ~
 
 permissions:
   pull-requests: write

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/.github/workflows/ci.yml

@@ -8,9 +8,17 @@ on:
       - 'develop'
     tags-ignore:
       - '**'
+    paths-ignore:
+      - "docs/**"
+      - "CHANGELOG.md"
+      - "README.md"
 
   # Trigger the workflow on pull request
-  pull_request: ~
+  pull_request:
+    paths-ignore:
+      - "docs/**"
+      - "CHANGELOG.md"
+      - "README.md"
 
   # Trigger the workflow manually
   workflow_dispatch: ~
@@ -18,6 +26,11 @@ on:
   # Trigger after public PR approved for CI
   pull_request_target:
     types: [labeled]
+    paths-ignore:
+      - "docs/**"
+      - "CHANGELOG.md"
+      - "README.md"
+
 
 jobs:
   # Run CI including downstream packages on self-hosted runners
@@ -34,7 +47,7 @@ jobs:
   downstream-ci-hpc:
     name: downstream-ci-hpc
     if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
     with:
       anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
     secrets: inherit

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/.github/workflows/python-publish.yml

@@ -15,6 +15,8 @@ jobs:
 
   checks:
     uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
+    with:
+      python-version: "3.9"
 
   deploy:
     needs: [checks, quality]

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/.github/workflows/python-pull-request.yml

@@ -5,7 +5,7 @@ name: Code Quality checks for PRs
 
 on:
   push:
-  pull_request_target:
+  pull_request:
     types: [opened, synchronize, reopened]
 
 jobs:

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/.gitignore

@@ -120,6 +120,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
+.envrc
 .env
 .venv
 env/
@@ -193,3 +194,4 @@ _dev/
 test.ipynb
 *tmp_data/
 tempCodeRunnerFile.python
+Untitled-*.py

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/.pre-commit-config.yaml

@@ -20,6 +20,12 @@ repos:
   - id: no-commit-to-branch # Prevent committing to main / master
   - id: check-added-large-files # Check for large files added to git
   - id: check-merge-conflict # Check for files that contain merge conflict
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0  # Use the ref you want to point at
+    hooks:
+    -   id: python-use-type-annotations # Check for missing type annotations
+    -   id: python-check-blanket-noqa # Check for # noqa: all
+    -   id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
   rev: 24.8.0
   hooks:
@@ -37,15 +43,15 @@ repos:
   rev: v0.6.4
   hooks:
   - id: ruff
-    # Next line is to exclude for documentation code snippets
-    exclude: 'docs/(.*/)?[a-z]\w+_.py$'
+    # Next line if for documenation cod snippets
+    exclude: '.*/[^_].*_\.py$'
     args:
     - --line-length=120
     - --fix
     - --exit-non-zero-on-fix
     - --preview
 - repo: https://github.com/sphinx-contrib/sphinx-lint
-  rev: v0.9.1
+  rev: v1.0.0
   hooks:
   - id: sphinx-lint
 # For now, we use it. But it does not support a lot of sphinx features
@@ -59,12 +65,10 @@ repos:
   hooks:
   - id: docconvert
     args: ["numpy"]
-- repo: https://github.com/b8raoult/optional-dependencies-all
-  rev: "0.0.6"
-  hooks:
-  - id: optional-dependencies-all
-    args: ["--inplace", "--exclude-keys=dev,docs,tests", "--group=dev=all,docs,tests"]
 - repo: https://github.com/tox-dev/pyproject-fmt
   rev: "2.2.3"
   hooks:
   - id: pyproject-fmt
+
+ci:
+  autoupdate_schedule: monthly

anemoi_datasets-0.5.5/CHANGELOG.md

@@ -0,0 +1,116 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+Please add your functional changes to the appropriate section in the PR.
+Keep it human-readable, your future self will thank you!
+
+
+## [Unreleased](https://github.com/ecmwf/anemoi-datasets/compare/0.5.4...HEAD)
+
+## [Add support for (semi-)constant fields](https://github.com/ecmwf/anemoi-datasets/compare/0.5.1...0.5.2) - 2024-10-03
+
+### Changed
+
+- Fix failing zarr lat/lon tests
+
+## [Bug fixes](https://github.com/ecmwf/anemoi-datasets/compare/0.5.0...0.5.1) - 2024-10-01
+
+### Added
+
+- Adding the user recipe in the dataset PR #59.
+- Add `repeated_dates` action in create.
+
+### Changed
+
+- Bug fix in create/rename
+
+## [0.5.0 - Incremental builds and Rescaling](https://github.com/ecmwf/anemoi-datasets/compare/0.4.5...0.5.0) - 2024-09-25
+
+### Added
+
+- New `rescale` keyword in `open_dataset` to change units of variables #36
+- Add support for constant fields when creating datasets
+- Simplify imports
+
+### Changed
+
+- Added incremental building of datasets
+- Add missing dependency for documentation building
+- Fix failing test due to previous merge
+- Bug fix when creating dataset from zarr
+- Bug fix with area selection in cutout operation
+- add paths-ignore to ci workflow
+
+### Removed
+
+- pytest for notebooks
+
+## [0.4.5](https://github.com/ecmwf/anemoi-datasets/compare/0.4.4...0.4.5)
+
+### Added
+
+- New `interpolate_frequency` keyword in `open_dataset`
+- CI workflow to update the changelog on release
+- adds the reusable cd pypi workflow
+- merge strategy for changelog in .gitattributes #25
+- adds ci hpc config (#43)
+
+### Changed
+
+- update CI to reusable workflows for PRs and releases
+- Support sub-hourly datasets.
+- Change negative variance detection to make it less restrictive
+- Fix cutout bug that left some global grid points in the lam part
+- Fix bug in computing missing dates in cutout option
+
+### Removed
+
+## [0.4.4](https://github.com/ecmwf/anemoi-datasets/compare/0.4.0...0.4.4) Bug fixes
+
+- Bug fix in accumulations() that did not work with webmars
+- Bug fix when using hindcasts input (missing dates on leap years)
+
+## [0.4.0](https://github.com/ecmwf/anemoi-datasets/compare/0.3.0...0.4.0) Minor Release
+
+### Added
+
+- earthkit-data replaces climetlab
+
+### Removed
+
+- climetlab
+
+## [0.3.0](https://github.com/ecmwf/anemoi-datasets/compare/0.2.0...0.3.0) Minor Release
+
+### Added
+
+- hindcast source
+
+### Changed
+
+- updated documentation
+
+## [0.2.0](https://github.com/ecmwf/anemoi-datasets/compare/0.1.0...0.2.0) Minor Release
+
+### Added
+
+- statistics tendencies
+
+### Removed
+
+- CubesFilter
+
+## [0.1.0](https://github.com/ecmwf/anemoi-models/releases/tag/0.1.0) Initial Release
+
+### Added
+
+- Documentation
+- Initial code release for anemoi-datasets: create datasets for data-driven weather-models
+- open datasets
+- combine datasets
+
+## Git Diffs:

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.4.5
+Version: 0.5.5
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -224,38 +224,36 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: anemoi-utils[provenance]>=0.3.15
+Requires-Dist: cfunits
 Requires-Dist: numpy
 Requires-Dist: pyyaml
 Requires-Dist: semantic-version
 Requires-Dist: tqdm
-Requires-Dist: zarr
+Requires-Dist: zarr<=2.17
 Provides-Extra: all
-Requires-Dist: aiohttp; extra == "all"
 Requires-Dist: boto3; extra == "all"
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "all"
 Requires-Dist: earthkit-geo>=0.2; extra == "all"
 Requires-Dist: earthkit-meteo; extra == "all"
-Requires-Dist: eccodes>=2.37; extra == "all"
+Requires-Dist: ecmwflibs>=0.6.3; extra == "all"
 Requires-Dist: entrypoints; extra == "all"
 Requires-Dist: gcsfs; extra == "all"
 Requires-Dist: kerchunk; extra == "all"
 Requires-Dist: pyproj; extra == "all"
 Requires-Dist: requests; extra == "all"
-Requires-Dist: s3fs; extra == "all"
 Provides-Extra: create
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "create"
 Requires-Dist: earthkit-geo>=0.2; extra == "create"
 Requires-Dist: earthkit-meteo; extra == "create"
-Requires-Dist: eccodes>=2.37; extra == "create"
+Requires-Dist: ecmwflibs>=0.6.3; extra == "create"
 Requires-Dist: entrypoints; extra == "create"
 Requires-Dist: pyproj; extra == "create"
 Provides-Extra: dev
-Requires-Dist: aiohttp; extra == "dev"
 Requires-Dist: boto3; extra == "dev"
 Requires-Dist: earthkit-data[mars]>=0.9; extra == "dev"
 Requires-Dist: earthkit-geo>=0.2; extra == "dev"
 Requires-Dist: earthkit-meteo; extra == "dev"
-Requires-Dist: eccodes>=2.37; extra == "dev"
+Requires-Dist: ecmwflibs>=0.6.3; extra == "dev"
 Requires-Dist: entrypoints; extra == "dev"
 Requires-Dist: gcsfs; extra == "dev"
 Requires-Dist: kerchunk; extra == "dev"
@@ -264,26 +262,19 @@ Requires-Dist: pandoc; extra == "dev"
 Requires-Dist: pyproj; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
 Requires-Dist: requests; extra == "dev"
-Requires-Dist: rstfmt; extra == "dev"
-Requires-Dist: s3fs; extra == "dev"
 Requires-Dist: sphinx; extra == "dev"
-Requires-Dist: sphinx-argparse<0.5; extra == "dev"
 Requires-Dist: sphinx-rtd-theme; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: nbsphinx; extra == "docs"
 Requires-Dist: pandoc; extra == "docs"
-Requires-Dist: rstfmt; extra == "docs"
 Requires-Dist: sphinx; extra == "docs"
-Requires-Dist: sphinx-argparse<0.5; extra == "docs"
+Requires-Dist: sphinx-argparse; extra == "docs"
 Requires-Dist: sphinx-rtd-theme; extra == "docs"
-Provides-Extra: kerchunk
-Requires-Dist: gcsfs; extra == "kerchunk"
-Requires-Dist: kerchunk; extra == "kerchunk"
-Requires-Dist: s3fs; extra == "kerchunk"
 Provides-Extra: remote
-Requires-Dist: aiohttp; extra == "remote"
 Requires-Dist: boto3; extra == "remote"
 Requires-Dist: requests; extra == "remote"
-Requires-Dist: s3fs; extra == "remote"
 Provides-Extra: tests
 Requires-Dist: pytest; extra == "tests"
+Provides-Extra: xarray
+Requires-Dist: gcsfs; extra == "xarray"
+Requires-Dist: kerchunk; extra == "xarray"

anemoi_datasets-0.5.5/docs/building/filters/rename.rst

@@ -0,0 +1,6 @@
+########
+ rename
+########
+
+.. literalinclude:: yaml/rename.yaml
+   :language: yaml

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/building/incremental.rst

@@ -86,8 +86,8 @@ To add statistics for 6h increments:
 
 .. code:: bash
 
-   anemoi-datasets init-additions dataset.zarr --delta 6h anemoi-datasets
-   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 6h anemoi-datasets
+   anemoi-datasets init-additions dataset.zarr --delta 6h
+   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 6h
    anemoi-datasets load-additions dataset.zarr --part 2/2 --delta 6h
    anemoi-datasets finalise-additions dataset.zarr --delta 6h
 
@@ -96,7 +96,7 @@ To add statistics for 12h increments:
 .. code:: bash
 
    anemoi-datasets init-additions dataset.zarr --delta 12h
-   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 12h anemoi-datasets
+   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 12h
    anemoi-datasets load-additions dataset.zarr --part 2/2 --delta 12h
    anemoi-datasets finalise-additions dataset.zarr --delta 12h
 

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/building/sources/accumulations.rst

@@ -54,12 +54,12 @@ accumulation between step 1h and step 7h if it is the most appropriate
 data available, while in the second case, the source will always return
 the accumulation between step 0h and step 6h, if available.
 
-.. literalinclude:: accumulations1.yaml
+.. literalinclude:: yaml/accumulations1.yaml
    :language: yaml
 
 or:
 
-.. literalinclude:: accumulations2.yaml
+.. literalinclude:: yaml/accumulations2.yaml
    :language: yaml
 
 .. [1]

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/building/sources/forcings.rst

@@ -11,7 +11,7 @@ Because the source needs to generate fields on the same grids as the
 other, it requires a template field. This is provided in the recipe with
 the ``template`` keyword:
 
-.. literalinclude:: forcings.yaml
+.. literalinclude:: yaml/forcings.yaml
    :language: yaml
 
 .. _yaml-reference:

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/building/sources/mars.rst

@@ -13,13 +13,13 @@ specification`_, with the exception of the ``date``, ``time``` and
 The missing keys will be filled with the default values, as defined in
 the MARS language specification.
 
-.. literalinclude:: mars1.yaml
+.. literalinclude:: yaml/mars1.yaml
    :language: yaml
 
 Data from several levels types must be requested in separate requests,
 with the ``join`` command.
 
-.. literalinclude:: mars2.yaml
+.. literalinclude:: yaml/mars2.yaml
    :language: yaml
 
 See :ref:`naming-variables` for information on how to name the variables

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/building/sources/netcdf.rst

@@ -2,5 +2,5 @@
  netcdf
 ########
 
-.. literalinclude:: netcdf.yaml
+.. literalinclude:: yaml/netcdf.yaml
    :language: yaml

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/building/sources/opendap.rst

@@ -2,5 +2,5 @@
  opendap
 #########
 
-.. literalinclude:: opendap.yaml
+.. literalinclude:: yaml/opendap.yaml
    :language: yaml

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/building/sources/recentre.rst

@@ -19,7 +19,7 @@ formula:
 
 Additionally, some variables must be non-negative to have a physical
 meaning (e.g. accumulated variables or `specific humidity`). To ensure
-this, positive clipping is performed using the alternative fomula :
+this, positive clipping is performed using the alternative formula:
 
 .. math::
 

anemoi_datasets-0.5.5/docs/building/sources/repeated_dates.rst

@@ -0,0 +1,29 @@
+################
+ repeated_dates
+################
+
+The generale format of the `repeated_dates` source is:
+
+.. literalinclude:: yaml/repeated_dates1.yaml
+   :language: yaml
+
+**********
+ constant
+**********
+
+.. literalinclude:: yaml/repeated_dates2.yaml
+   :language: yaml
+
+*************
+ climatology
+*************
+
+.. literalinclude:: yaml/repeated_dates3.yaml
+   :language: yaml
+
+*********
+ closest
+*********
+
+.. literalinclude:: yaml/repeated_dates4.yaml
+   :language: yaml

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/building/sources/xarray-zarr.rst

@@ -2,5 +2,5 @@
  xarray-zarr
 #############
 
-.. literalinclude:: xarray-zarr.yaml
+.. literalinclude:: yaml/xarray-zarr.yaml
    :language: yaml

anemoi_datasets-0.5.5/docs/building/sources/yaml/hindcasts.yaml

@@ -0,0 +1,3 @@
+---
+input:
+   hincasts:

anemoi_datasets-0.5.5/docs/building/sources/yaml/repeated_dates1.yaml

@@ -0,0 +1,6 @@
+
+repeated_dates:
+    mode: mode
+    # ... parameters related to the mode ...
+    source:
+      # ... a source definition ...

anemoi_datasets-0.5.5/docs/building/sources/yaml/repeated_dates2.yaml

@@ -0,0 +1,6 @@
+repeated_dates:
+  mode: constant
+  source:
+      xarray-zarr:
+        url: dem.zarr
+        variable: dem

anemoi_datasets-0.5.5/docs/building/sources/yaml/repeated_dates3.yaml

@@ -0,0 +1,8 @@
+repeated_dates:
+    mode: climatology
+    year: 2019
+    day: 15
+    source:
+        grib:
+          path: some/path/to/data.grib
+          param: [some_param]

anemoi_datasets-0.5.5/docs/building/sources/yaml/repeated_dates4.yaml

@@ -0,0 +1,9 @@
+repeated_dates:
+    mode: closest
+    frequency: 24h
+    maximum: 30d
+    skip_all_nans: true
+    source:
+      grib:
+          path: path/to/data.grib
+          param: [some_param]

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/building/sources/zenodo.rst

@@ -4,5 +4,5 @@
 
 https://zenodo.org/records/6470725
 
-.. literalinclude:: zenodo.yaml
+.. literalinclude:: yaml/zenodo.yaml
    :language: yaml

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/building/sources.rst

@@ -28,6 +28,7 @@ The following `sources` are currently available:
    sources/netcdf
    sources/opendap
    sources/recentre
+   sources/repeated_dates
    sources/xarray-kerchunk
    sources/xarray-zarr
    sources/zenodo

{anemoi_datasets-0.4.5 → anemoi_datasets-0.5.5}/docs/index.rst

@@ -1,8 +1,8 @@
 .. _index-page:
 
-####################################
- Welcome to Anemoi's documentation!
-####################################
+#############################################
+ Welcome to `anemoi-datasets` documentation!
+#############################################
 
 .. warning::
 

anemoi_datasets-0.5.5/docs/using/code/rescale_.py

@@ -0,0 +1,30 @@
+# Scale and offset can be passed as a dictionnary...
+
+ds = open_dataset(
+    dataset,
+    rescale={"2t": {"scale": 1.0, "offset": -273.15}},
+)
+
+# ... a tuple of floating points ....
+
+ds = open_dataset(
+    dataset,
+    rescale={"2t": (1.0, -273.15)},
+)
+
+# ... or a tuple of strings representing units.
+
+ds = open_dataset(
+    dataset,
+    rescale={"2t": ("K", "degC")},
+)
+
+# Several variables can be rescaled at once.
+
+ds = open_dataset(
+    dataset,
+    rescale={
+        "2t": ("K", "degC"),
+        "tp": ("m", "mm"),
+    },
+)