anemoi-datasets 0.5.29__tar.gz → 0.5.30__tar.gz

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/.github/workflows/python-pull-request.yml

@@ -19,7 +19,7 @@ jobs:
   checks:
     strategy:
       matrix:
-        python-version: ["3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
     uses: ecmwf/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
     with:
       python-version: ${{ matrix.python-version }}

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
   - id: python-check-blanket-noqa # Check for # noqa: all
   - id: python-no-log-warn # Check for log.warn
 - repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 25.12.0
+  rev: 26.1.0
   hooks:
   - id: black
     args: [--line-length=120]
@@ -41,7 +41,7 @@ repos:
     - --profile black
     - --project anemoi
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.14.10
+  rev: v0.14.14
   hooks:
   - id: ruff
     args:
@@ -59,12 +59,12 @@ repos:
   - id: docconvert
     args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "v2.11.1"
+  rev: "v2.12.1"
   hooks:
   - id: pyproject-fmt
     args: ["--max-supported-python", "3.12"]
 - repo: https://github.com/jshwi/docsig # Check docstrings against function sig
-  rev: v0.74.0
+  rev: v0.77.0
   hooks:
   - id: docsig
     args:

anemoi_datasets-0.5.30/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.5.30"
+}

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/CHANGELOG.md

@@ -8,6 +8,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.5.30](https://github.com/ecmwf/anemoi-datasets/compare/0.5.29...0.5.30) (2026-02-03)
+
+
+### Features
+
+* Add optional patch for accumulation when metadata is wrong ([#525](https://github.com/ecmwf/anemoi-datasets/issues/525)) ([8016551](https://github.com/ecmwf/anemoi-datasets/commit/8016551af7558ca299540f8da6e629ce2338e3c3))
+
+
+### Documentation
+
+* Correct typo of part to parts in incremental build [#518](https://github.com/ecmwf/anemoi-datasets/issues/518) ([#519](https://github.com/ecmwf/anemoi-datasets/issues/519)) ([78e8d75](https://github.com/ecmwf/anemoi-datasets/commit/78e8d75c9fa370a8b6c6ba2feaff437699e7bc90))
+
 ## [0.5.29](https://github.com/ecmwf/anemoi-datasets/compare/0.5.28...0.5.29) (2026-01-28)
 
 

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-datasets
-Version: 0.5.29
+Version: 0.5.30
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -218,9 +218,10 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.11
+Requires-Python: <3.14,>=3.11
 License-File: LICENSE
 Requires-Dist: anemoi-transform>=0.1.12
 Requires-Dist: anemoi-utils>=0.4.26
@@ -235,7 +236,7 @@ Requires-Dist: pyyaml
 Requires-Dist: ruamel-yaml>=0.16
 Requires-Dist: semantic-version
 Requires-Dist: tqdm
-Requires-Dist: zarr<=2.18.4
+Requires-Dist: zarr<=2.18.7
 Provides-Extra: all
 Requires-Dist: anemoi-datasets[comparelam,create,remote,xarray]; extra == "all"
 Provides-Extra: comparelam

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/docs/building/incremental.rst

@@ -25,7 +25,7 @@ the dataset, so it will not be needed by following commands.
    anemoi-datasets init dataset.yaml dataset.zarr --overwrite
 
 You can then load the dataset in parts with the `load` command. You just
-pass which part you want to load with the `--part` flag.
+pass which part you want to load with the `--parts` flag.
 
 .. note::
 
@@ -40,17 +40,17 @@ concurrent writes.
 
 .. code:: bash
 
-   anemoi-datasets load dataset.zarr --part 1/20
+   anemoi-datasets load dataset.zarr --parts 1/20
 
 .. code:: bash
 
-   anemoi-datasets load dataset.zarr --part 2/20
+   anemoi-datasets load dataset.zarr --parts 2/20
 
 ... and so on ... until:
 
 .. code:: bash
 
-   anemoi-datasets load dataset.zarr --part 20/20
+   anemoi-datasets load dataset.zarr --parts 20/20
 
 Once you have loaded all the parts, you can finalise the dataset with
 the `finalise` command. This will write the metadata and the attributes
@@ -87,8 +87,8 @@ To add statistics for 6h increments:
 .. code:: bash
 
    anemoi-datasets init-additions dataset.zarr --delta 6h
-   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 6h
-   anemoi-datasets load-additions dataset.zarr --part 2/2 --delta 6h
+   anemoi-datasets load-additions dataset.zarr --parts 1/2 --delta 6h
+   anemoi-datasets load-additions dataset.zarr --parts 2/2 --delta 6h
    anemoi-datasets finalise-additions dataset.zarr --delta 6h
 
 To add statistics for 12h increments:
@@ -96,8 +96,8 @@ To add statistics for 12h increments:
 .. code:: bash
 
    anemoi-datasets init-additions dataset.zarr --delta 12h
-   anemoi-datasets load-additions dataset.zarr --part 1/2 --delta 12h
-   anemoi-datasets load-additions dataset.zarr --part 2/2 --delta 12h
+   anemoi-datasets load-additions dataset.zarr --parts 1/2 --delta 12h
+   anemoi-datasets load-additions dataset.zarr --parts 2/2 --delta 12h
    anemoi-datasets finalise-additions dataset.zarr --delta 12h
 
 If this process leaves temporary files behind, you can clean them up

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/pyproject.toml

@@ -28,7 +28,7 @@ authors = [
   { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" },
 ]
 
-requires-python = ">=3.11"
+requires-python = ">=3.11,<3.14"
 
 classifiers = [
   "Development Status :: 4 - Beta",
@@ -38,6 +38,7 @@ classifiers = [
   "Programming Language :: Python :: 3 :: Only",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
@@ -59,7 +60,7 @@ dependencies = [
   "ruamel-yaml>=0.16",
   "semantic-version",
   "tqdm",
-  "zarr<=2.18.4",
+  "zarr<=2.18.7",
 ]
 
 optional-dependencies.all = [

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/src/anemoi/datasets/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.5.29'
-__version_tuple__ = version_tuple = (0, 5, 29)
+__version__ = version = '0.5.30'
+__version_tuple__ = version_tuple = (0, 5, 30)
 
-__commit_id__ = commit_id = 'g63e10cd28'
+__commit_id__ = commit_id = 'g1df12fc54'

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/src/anemoi/datasets/create/sources/accumulate_utils/field_to_interval.py

@@ -32,6 +32,7 @@ class FieldToInterval:
                 "start_step_is_zero",
                 "start_step_is_end_step",
                 "start_step_greater_than_end_step",
+                "set_start_step_to_zero",
             ):
                 raise ValueError(f"Unknown patch key: {key}")
 
@@ -45,6 +46,9 @@ class FieldToInterval:
 
         LOG.debug(f"    field before patching: {startStep=}, {endStep=}")
 
+        if self.patches.get("set_start_step_to_zero", False):
+            startStep, endStep = 0, endStep
+
         if startStep > endStep:
             startStep, endStep = self.start_step_greater_than_end_step(startStep, endStep, field=field)
         elif startStep == endStep:

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/src/anemoi/datasets/create/sources/grib_index.py

@@ -106,21 +106,18 @@ class GribIndex:
         """Create the necessary tables in the database."""
         assert self.update
 
-        self.cursor.execute(
-            """
+        self.cursor.execute("""
         CREATE TABLE IF NOT EXISTS paths (
             id INTEGER PRIMARY KEY,
             path TEXT not null
         )
-        """
-        )
+        """)
 
         columns = ("valid_datetime",)
         # We don't use NULL as a default because NULL is considered a different value
         # in UNIQUE INDEX constraints (https://www.sqlite.org/lang_createindex.html)
 
-        self.cursor.execute(
-            f"""
+        self.cursor.execute(f"""
         CREATE TABLE IF NOT EXISTS grib_index (
             _id INTEGER PRIMARY KEY,
             _path_id INTEGER not null,
@@ -128,30 +125,23 @@ class GribIndex:
             _length INTEGER not null,
             {', '.join(f"{key} TEXT not null default ''" for key in columns)},
             FOREIGN KEY(_path_id) REFERENCES paths(id))
-        """
-        )  # ,
+        """)  # ,
 
-        self.cursor.execute(
-            """
+        self.cursor.execute("""
         CREATE UNIQUE INDEX IF NOT EXISTS idx_grib_index_path_offset
         ON grib_index (_path_id, _offset)
-        """
-        )
+        """)
 
-        self.cursor.execute(
-            f"""
+        self.cursor.execute(f"""
         CREATE UNIQUE INDEX IF NOT EXISTS idx_grib_index_all_keys
         ON grib_index ({', '.join(columns)})
-        """
-        )
+        """)
 
         for key in columns:
-            self.cursor.execute(
-                f"""
+            self.cursor.execute(f"""
             CREATE INDEX IF NOT EXISTS idx_grib_index_{key}
             ON grib_index ({key})
-            """
-            )
+            """)
 
         self._commit()
 
@@ -267,20 +257,16 @@ class GribIndex:
         self.cursor.execute("""DROP INDEX IF EXISTS idx_grib_index_all_keys""")
         all_columns = self._all_columns()
 
-        self.cursor.execute(
-            f"""
+        self.cursor.execute(f"""
         CREATE UNIQUE INDEX IF NOT EXISTS idx_grib_index_all_keys
         ON grib_index ({', '.join(all_columns)})
-        """
-        )
+        """)
 
         for key in all_columns:
-            self.cursor.execute(
-                f"""
+            self.cursor.execute(f"""
             CREATE INDEX IF NOT EXISTS idx_grib_index_{key}
             ON grib_index ({key})
-            """
-            )
+            """)
 
     def add_grib_file(self, path: str) -> None:
         """Add a GRIB file to the database.
@@ -542,9 +528,7 @@ class GribIndex:
         dates = [d.isoformat() for d in dates]
 
         query = """SELECT _path_id, _offset, _length
-                   FROM grib_index WHERE valid_datetime IN ({})""".format(
-            ", ".join("?" for _ in dates)
-        )
+                   FROM grib_index WHERE valid_datetime IN ({})""".format(", ".join("?" for _ in dates))
         params = dates
 
         for k, v in kwargs.items():

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/src/anemoi_datasets.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-datasets
-Version: 0.5.29
+Version: 0.5.30
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -218,9 +218,10 @@ Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.11
+Requires-Python: <3.14,>=3.11
 License-File: LICENSE
 Requires-Dist: anemoi-transform>=0.1.12
 Requires-Dist: anemoi-utils>=0.4.26
@@ -235,7 +236,7 @@ Requires-Dist: pyyaml
 Requires-Dist: ruamel-yaml>=0.16
 Requires-Dist: semantic-version
 Requires-Dist: tqdm
-Requires-Dist: zarr<=2.18.4
+Requires-Dist: zarr<=2.18.7
 Provides-Extra: all
 Requires-Dist: anemoi-datasets[comparelam,create,remote,xarray]; extra == "all"
 Provides-Extra: comparelam

{anemoi_datasets-0.5.29 → anemoi_datasets-0.5.30}/src/anemoi_datasets.egg-info/requires.txt

@@ -11,7 +11,7 @@ pyyaml
 ruamel-yaml>=0.16
 semantic-version
 tqdm
-zarr<=2.18.4
+zarr<=2.18.7
 
 [all]
 anemoi-datasets[comparelam,create,remote,xarray]

anemoi_datasets-0.5.29/.release-please-manifest.json

@@ -1,3 +0,0 @@
-{
-    ".": "0.5.29"
-}