anemoi-datasets 0.5.18__tar.gz → 0.5.19__tar.gz

anemoi_datasets-0.5.19/.release-please-manifest.json

@@ -0,0 +1,3 @@
+{
+    ".": "0.5.19"
+}

{anemoi_datasets-0.5.18 → anemoi_datasets-0.5.19}/CHANGELOG.md

@@ -8,6 +8,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [0.5.19](https://github.com/ecmwf/anemoi-datasets/compare/0.5.18...0.5.19) (2025-03-27)
+
+
+### Features
+
+* 24h accumulations for era5 ([#266](https://github.com/ecmwf/anemoi-datasets/issues/266)) ([33f7919](https://github.com/ecmwf/anemoi-datasets/commit/33f791961649de287f7900fb5c7b340ebc6fb32c))
+* add environment variables in recipe ([#228](https://github.com/ecmwf/anemoi-datasets/issues/228)) ([6d898aa](https://github.com/ecmwf/anemoi-datasets/commit/6d898aa681b052bb3444614f643ab14eaf6fdc62))
+
+
+### Bug Fixes
+
+* broken references in recipes ([#269](https://github.com/ecmwf/anemoi-datasets/issues/269)) ([bbdf339](https://github.com/ecmwf/anemoi-datasets/commit/bbdf3393dfd07755fbc424079e0a9072d823362b))
+* fix typo ([#268](https://github.com/ecmwf/anemoi-datasets/issues/268)) ([a12f58f](https://github.com/ecmwf/anemoi-datasets/commit/a12f58fb7e379b75baf8063a02f3938ff29303ba))
+* swap meshgrid dimension ordering in xarray grid creation ([#249](https://github.com/ecmwf/anemoi-datasets/issues/249)) ([938f3c9](https://github.com/ecmwf/anemoi-datasets/commit/938f3c926e6d8083db5e00a515301f79bac5637d))
+
+
+### Documentation
+
+* add reference to anemoi contributing guidelines  ([#265](https://github.com/ecmwf/anemoi-datasets/issues/265)) ([7322e8a](https://github.com/ecmwf/anemoi-datasets/commit/7322e8a04b631e15c0d424f4cf73e6ee8b7bc199))
+
 ## [0.5.18](https://github.com/ecmwf/anemoi-datasets/compare/0.5.17...0.5.18) (2025-03-25)
 
 

{anemoi_datasets-0.5.18/src/anemoi_datasets.egg-info → anemoi_datasets-0.5.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-datasets
-Version: 0.5.18
+Version: 0.5.19
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_datasets-0.5.18 → anemoi_datasets-0.5.19}/docs/conf.py

@@ -79,6 +79,10 @@ exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "'**.ipynb_checkpoints'"
 
 intersphinx_mapping = {
     "python": ("https://python.readthedocs.io/en/latest", None),
+    "anemoi-docs": (
+        "https://anemoi.readthedocs.io/en/latest/",
+        ("../../../anemoi-docs/docs/_build/html/objects.inv", None),
+    ),
     "anemoi-utils": (
         "https://anemoi-utils.readthedocs.io/en/latest/",
         ("../../anemoi-utils/docs/_build/html/objects.inv", None),

anemoi_datasets-0.5.19/docs/dev/contributing.rst

@@ -0,0 +1,22 @@
+.. _dev-contributing:
+
+####################
+ General guidelines
+####################
+
+Thank you for your interest in Anemoi Datasets! Please follow the
+:ref:`general Anemoi contributing guidelines
+<anemoi-docs:contributing>`.
+
+These include general guidelines for contributions to Anemoi,
+instructions on setting up a development environment, and guidelines on
+collaboration on GitHub, writing documentation, testing, and code style.
+
+************
+ Unit tests
+************
+
+Anemoi-datasets include unit tests that can be executed locally using
+pytest. For more information on testing, please refer to the
+:ref:`general Anemoi testing guidelines
+<anemoi-docs:testing-guidelines>`.

{anemoi_datasets-0.5.18 → anemoi_datasets-0.5.19}/docs/index.rst

@@ -11,8 +11,8 @@
 An *Anemoi dataset* is a thin wrapper around a zarr_ store that is
 optimised for training data-driven weather forecasting models.
 anemoi-datasets are organised in such a way that I/O operations are
-minimised. It is one of the packages within the `anemoi framework
-<https://anemoi-docs.readthedocs.io/en/latest/>`_.
+minimised. It is one of the packages within the :ref:`anemoi framework
+<anemoi-docs:index>`.
 
 **************
  About Anemoi
@@ -167,8 +167,6 @@ You may also have to install pandoc on MacOS:
 .. toctree::
    :maxdepth: 1
    :hidden:
-   :caption: Developing Anemoi Datasets
+   :caption: Contributing
 
    dev/contributing
-   dev/code_structure
-   dev/testing

{anemoi_datasets-0.5.18 → anemoi_datasets-0.5.19}/src/anemoi/datasets/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.5.18'
-__version_tuple__ = version_tuple = (0, 5, 18)
+__version__ = version = '0.5.19'
+__version_tuple__ = version_tuple = (0, 5, 19)

{anemoi_datasets-0.5.18 → anemoi_datasets-0.5.19}/src/anemoi/datasets/create/__init__.py

@@ -294,7 +294,14 @@ class Dataset:
         import zarr
 
         z = zarr.open(self.path, mode="r")
-        return loader_config(z.attrs.get("_create_yaml_config"))
+        config = loader_config(z.attrs.get("_create_yaml_config"))
+
+        if "env" in config:
+            for k, v in config["env"].items():
+                LOG.info(f"Setting env variable {k}={v}")
+                os.environ[k] = str(v)
+
+        return config
 
 
 class WritableDataset(Dataset):

{anemoi_datasets-0.5.18 → anemoi_datasets-0.5.19}/src/anemoi/datasets/create/config.py

@@ -420,6 +420,11 @@ def loader_config(config: dict, is_test: bool = False) -> LoadersConfig:
         print(b)
         raise ValueError("Serialisation failed")
 
+    if "env" in copy:
+        for k, v in copy["env"].items():
+            LOG.info(f"Setting env variable {k}={v}")
+            os.environ[k] = str(v)
+
     return copy
 
 

{anemoi_datasets-0.5.18 → anemoi_datasets-0.5.19}/src/anemoi/datasets/create/filters/transform.py

@@ -33,15 +33,13 @@ class TransformFilter(Filter):
         from anemoi.transform.filters import create_filter
 
         self.name = name
-        self.transform_filter = create_filter(self, config)
+        self.transform_filter = create_filter(context, config)
 
     def execute(self, input: ekd.FieldList) -> ekd.FieldList:
         """Execute the transformation filter.
 
         Parameters
         ----------
-        context : Any
-            The context in which the execution occurs.
         input : ekd.FieldList
             The input data to be transformed.
 

{anemoi_datasets-0.5.18 → anemoi_datasets-0.5.19}/src/anemoi/datasets/create/input/action.py

@@ -17,6 +17,7 @@ from earthkit.data.core.order import build_remapping
 
 from ...dates.groups import GroupOfDates
 from .context import Context
+from .template import substitute
 
 LOG = logging.getLogger(__name__)
 
@@ -248,7 +249,7 @@ def action_factory(config: Dict[str, Any], context: ActionContext, action_path:
     if cls is None:
         from ..sources import create_source
 
-        source = create_source(None, config)
+        source = create_source(None, substitute(context, config))
         return FunctionAction(context, action_path + [key], key, source)
 
     return cls(context, action_path + [key], *args, **kwargs)

{anemoi_datasets-0.5.18 → anemoi_datasets-0.5.19}/src/anemoi/datasets/create/input/function.py

@@ -20,7 +20,6 @@ from .misc import _tidy
 from .misc import assert_fieldlist
 from .result import Result
 from .template import notify_result
-from .template import resolve
 from .template import substitute
 from .trace import trace
 from .trace import trace_datasource
@@ -79,6 +78,9 @@ class FunctionContext:
         """Returns whether partial results are acceptable."""
         return self.owner.group_of_dates.partial_ok
 
+    def get_result(self, *args, **kwargs) -> Any:
+        return self.owner.context.get_result(*args, **kwargs)
+
 
 class FunctionAction(Action):
     """Represents an action that executes a function.
@@ -203,14 +205,12 @@ class FunctionResult(Result):
     @trace_datasource
     def datasource(self) -> FieldList:
         """Returns the datasource for the function result."""
-        args, kwargs = resolve(self.context, (self.args, self.kwargs))
+        # args, kwargs = resolve(self.context, (self.args, self.kwargs))
         self.action.source.context = FunctionContext(self)
 
         return _tidy(
             self.action.source.execute(
-                self.group_of_dates,  # Will provide a list of datetime objects
-                *args,
-                **kwargs,
+                list(self.group_of_dates),  # Will provide a list of datetime objects
             )
         )
 

{anemoi_datasets-0.5.18 → anemoi_datasets-0.5.19}/src/anemoi/datasets/create/sources/accumulations.py

@@ -636,6 +636,19 @@ def accumulations(
     Any
         The computed accumulations.
     """
+
+    if (
+        request.get("class") == "ea"
+        and request.get("stream", "oper") == "oper"
+        and request.get("accumulation_period") == 24
+    ):
+        from .accumulations2 import accumulations as accumulations2
+
+        LOG.warning(
+            "🧪️ Experimental features: Using accumulations2, because class=ea stream=oper and accumulation_period=24"
+        )
+        return accumulations2(context, dates, **request)
+
     _to_list(request["param"])
     class_ = request.get("class", "od")
     stream = request.get("stream", "oper")