anemoi-datasets 0.5.5__tar.gz → 0.5.7__tar.gz

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/.github/workflows/ci.yml

@@ -35,8 +35,9 @@ on:
 jobs:
   # Run CI including downstream packages on self-hosted runners
   downstream-ci:
+
     name: downstream-ci
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    if: ${{ !contains(github.repository, 'private') && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
     uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
     with:
       anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}
@@ -46,7 +47,7 @@ jobs:
    # Build downstream packages on HPC
   downstream-ci-hpc:
     name: downstream-ci-hpc
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    if: ${{ !contains(github.repository, 'private') && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
     uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
     with:
       anemoi-datasets: ecmwf/anemoi-datasets@${{ github.event.pull_request.head.sha || github.sha }}

anemoi_datasets-0.5.7/.github/workflows/push-to-private.yml

@@ -0,0 +1,33 @@
+name: Push to private repository
+
+on:
+  push:
+    branches:
+      - develop
+
+jobs:
+  push_changes:
+    if: ${{ !contains(github.repository, 'private') }}
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout source repository
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - name: Set up Git configuration
+      run: |
+        git config user.name "github-actions[bot]"
+        git config user.email "github-actions[bot]@users.noreply.github.com"
+
+    - name: Setup SSH key
+      uses: webfactory/ssh-agent@v0.5.0
+      with:
+        ssh-private-key: ${{ secrets.KEY_TO_PRIVATE }}
+
+    - name: Push changes to private repository
+      run: |
+        git remote add private git@github.com:${{ github.repository }}-private.git
+        git push --set-upstream private develop

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/.github/workflows/python-publish.yml

@@ -9,6 +9,7 @@ on:
 
 jobs:
   quality:
+    if: ${{ !contains(github.repository, 'private') }}
     uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2
     with:
       skip-hooks: "no-commit-to-branch"

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/.pre-commit-config.yaml

@@ -5,12 +5,12 @@ repos:
   - id: clear-notebooks-output
     name: clear-notebooks-output
     files: tools/.*\.ipynb$
-    stages: [commit]
+    stages: [pre-commit]
     language: python
     entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
     additional_dependencies: [jupyter]
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.6.0
+  rev: v5.0.0
   hooks:
   - id: check-yaml # Check YAML files for syntax errors only
     args: [--unsafe, --allow-multiple-documents]
@@ -40,7 +40,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.4
+  rev: v0.6.9
   hooks:
   - id: ruff
     # Next line if for documenation cod snippets
@@ -66,7 +66,7 @@ repos:
   - id: docconvert
     args: ["numpy"]
 - repo: https://github.com/tox-dev/pyproject-fmt
-  rev: "2.2.3"
+  rev: "2.2.4"
   hooks:
   - id: pyproject-fmt
 

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/CHANGELOG.md

@@ -8,8 +8,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 Please add your functional changes to the appropriate section in the PR.
 Keep it human-readable, your future self will thank you!
 
+## [Unreleased](https://github.com/ecmwf/anemoi-datasets/compare/0.5.6...HEAD)
 
-## [Unreleased](https://github.com/ecmwf/anemoi-datasets/compare/0.5.4...HEAD)
+## [Allow for unknown CF coordinates](https://github.com/ecmwf/anemoi-datasets/compare/0.5.5...0.5.6) - 2024-10-04
+
+- Update documentation
+
+### Changed
+
+- Add `variables_metadata` entry in the dataset metadata
+
+### Changed
+
+- Add `variables_metadata` entry in the dataset metadata
+
+## [0.5.5](https://github.com/ecmwf/anemoi-datasets/compare/0.5.4...0.5.5) - 2024-10-04
+
+### Changed
+
+- Allow for unknown coordinates when parsing CF input
 
 ## [Add support for (semi-)constant fields](https://github.com/ecmwf/anemoi-datasets/compare/0.5.1...0.5.2) - 2024-10-03
 
@@ -44,6 +61,7 @@ Keep it human-readable, your future self will thank you!
 - Bug fix when creating dataset from zarr
 - Bug fix with area selection in cutout operation
 - add paths-ignore to ci workflow
+- call provenance less often
 
 ### Removed
 

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.5.5
+Version: 0.5.7
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

anemoi_datasets-0.5.7/docs/building/filters/rename.rst

@@ -0,0 +1,20 @@
+########
+ rename
+########
+
+When combining several sources, it is common to have different values
+for a given attribute to represents the same concept. For example,
+``temperature_850hPa`` and ``t_850`` are two different ways to represent
+the temperature at 850 hPa. The ``rename`` filter allows to rename a key
+to another key. It is a :ref:`filter <filters>` that needs to follow a
+:ref:`source <sources>` or another filter in a :ref:`building-pipe`
+operation.
+
+.. literalinclude:: yaml/rename.yaml
+   :language: yaml
+
+.. note::
+
+   The ``rename`` filter was mostly designed to rename the ``param``
+   attribute, but any key can be renamed. The ``rename`` filter can take
+   several renaming keys.

anemoi_datasets-0.5.7/docs/building/filters/yaml/rename.yaml

@@ -0,0 +1,11 @@
+input:
+  pipe:
+    - source: # mars, grib, netcdf, etc.
+      # source attributes here
+      # ...
+
+    - rename:
+        param: # Map old `param` names to new ones
+          temperature_2m: 2t
+          temperature_850hPa: t_850
+          # ...

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/docs/building/sources/netcdf.rst

@@ -4,3 +4,5 @@
 
 .. literalinclude:: yaml/netcdf.yaml
    :language: yaml
+
+.. include:: xarray-based.rst

anemoi_datasets-0.5.7/docs/building/sources/repeated_dates.rst

@@ -0,0 +1,39 @@
+################
+ repeated_dates
+################
+
+The `repeated_dates` source is used to repeat a single source multiple
+times, so that its data is present at multiple dates. A simple example
+of this is when you have source that contains a constant field, such as
+orography or bathymetry, that you want to have repeated at all the dates
+of the dataset.
+
+The generale format of the `repeated_dates` source is:
+
+.. literalinclude:: yaml/repeated_dates1.yaml
+   :language: yaml
+
+where ``source`` is any of the :ref:`operations <operations>` or
+:ref:`sources <sources>` described in the previous sections. The
+``mode`` parameter can be one of the following:
+
+**********
+ constant
+**********
+
+.. literalinclude:: yaml/repeated_dates2.yaml
+   :language: yaml
+
+*************
+ climatology
+*************
+
+.. literalinclude:: yaml/repeated_dates3.yaml
+   :language: yaml
+
+*********
+ closest
+*********
+
+.. literalinclude:: yaml/repeated_dates4.yaml
+   :language: yaml

anemoi_datasets-0.5.7/docs/building/sources/xarray-based.rst

@@ -0,0 +1,13 @@
+This source uses Xarray_ internally to access the data, and assumes that
+it follows the `CF conventions`_.
+
+You specify any valid xarray.open_dataset_ arguments in the source.
+
+.. literalinclude:: yaml/xarray-based.yaml
+   :language: yaml
+
+.. _cf conventions: http://cfconventions.org/
+
+.. _xarray: https://docs.xarray.dev/en/stable/index.html
+
+.. _xarray.open_dataset: https://docs.xarray.dev/en/stable/generated/xarray.open_dataset.html

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/docs/building/sources/xarray-kerchunk.rst

@@ -2,7 +2,7 @@
  xarray-kerchunk
 #################
 
-.. literalinclude:: xarray-kerchunk.yaml
+.. literalinclude:: yaml/xarray-kerchunk.yaml
    :language: yaml
 
 The code below is inspired by the `kerchunk tutorial`_, and makes use of

anemoi_datasets-0.5.7/docs/building/sources/yaml/xarray-based.yaml

@@ -0,0 +1,3 @@
+input:
+   xarray-based-source: # netcdf, zarr, opendap, etc.
+      group: mygroup

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.5.5'
-__version_tuple__ = version_tuple = (0, 5, 5)
+__version__ = version = '0.5.7'
+__version_tuple__ = version_tuple = (0, 5, 7)

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/commands/inspect.py

@@ -506,6 +506,10 @@ class Version0_6(Version):
     def variables(self):
         return self.metadata["variables"]
 
+    @property
+    def variables_metadata(self):
+        return self.metadata.get("variables_metadata", {})
+
 
 class Version0_12(Version0_6):
     def details(self):

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/create/__init__.py

@@ -25,6 +25,7 @@ from anemoi.utils.dates import frequency_to_string
 from anemoi.utils.dates import frequency_to_timedelta
 from anemoi.utils.humanize import compress_dates
 from anemoi.utils.humanize import seconds_to_human
+from anemoi.utils.sanitise import sanitise
 from earthkit.data.core.order import build_remapping
 
 from anemoi.datasets import MissingDateError
@@ -52,7 +53,7 @@ from .writer import ViewCacheArray
 
 LOG = logging.getLogger(__name__)
 
-VERSION = "0.20"
+VERSION = "0.30"
 
 
 def json_tidy(o):
@@ -325,43 +326,6 @@ def build_input_(main_config, output_config):
     return builder
 
 
-def tidy_recipe(config: object):
-    """Remove potentially private information in the config"""
-    config = deepcopy(config)
-    if isinstance(config, (tuple, list)):
-        return [tidy_recipe(_) for _ in config]
-    if isinstance(config, (dict, DotDict)):
-        for k, v in config.items():
-            if k.startswith("_"):
-                config[k] = "*** REMOVED FOR SECURITY ***"
-            else:
-                config[k] = tidy_recipe(v)
-    if isinstance(config, str):
-        if config.startswith("_"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("s3://"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("gs://"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("http"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("ftp"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("file"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("ssh"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("scp"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("rsync"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("/"):
-            return "*** REMOVED FOR SECURITY ***"
-        if "@" in config:
-            return "*** REMOVED FOR SECURITY ***"
-    return config
-
-
 class Init(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixin):
     dataset_class = NewDataset
     def __init__(self, path, config, check_name=False, overwrite=False, use_threads=False, statistics_temp_dir=None, progress=None, test=False, cache=None, **kwargs):  # fmt: skip
@@ -448,7 +412,7 @@ class Init(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixi
         metadata.update(self.main_config.get("add_metadata", {}))
 
         metadata["_create_yaml_config"] = self.main_config.get_serialisable_dict()
-        metadata["recipe"] = tidy_recipe(self.main_config.get_serialisable_dict())
+        metadata["recipe"] = sanitise(self.main_config.get_serialisable_dict())
 
         metadata["description"] = self.main_config.description
         metadata["licence"] = self.main_config["licence"]
@@ -467,6 +431,7 @@ class Init(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixi
         metadata["data_request"] = self.minimal_input.data_request
         metadata["field_shape"] = self.minimal_input.field_shape
         metadata["proj_string"] = self.minimal_input.proj_string
+        metadata["variables_metadata"] = self.minimal_input.variables_metadata
 
         metadata["start_date"] = dates[0].isoformat()
         metadata["end_date"] = dates[-1].isoformat()

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/create/functions/sources/xarray/__init__.py

@@ -42,7 +42,7 @@ def load_one(emoji, context, dates, dataset, options={}, flavour=None, **kwargs)
     We have seen this bug triggered when we run many clients in parallel, for example, when we create a new dataset using `xarray-zarr`.
     """
 
-    context.trace(emoji, dataset, options)
+    context.trace(emoji, dataset, options, kwargs)
 
     if isinstance(dataset, str) and ".zarr" in dataset:
         data = xr.open_zarr(name_to_zarr_store(dataset), **options)

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/create/functions/sources/xarray/coordinates.py

@@ -30,6 +30,8 @@ def extract_single_value(variable):
     if np.issubdtype(variable.values.dtype, np.datetime64):
         if len(shape) == 0:
             return to_datetime(variable.values)  # Convert to python datetime
+        if shape == (1,):
+            return to_datetime(variable.values[0])
         assert False, (shape, variable.values[:2])
 
     if np.issubdtype(variable.values.dtype, np.timedelta64):
@@ -244,3 +246,9 @@ class ScalarCoordinate(Coordinate):
     @property
     def mars_names(self):
         return (self.variable.name,)
+
+
+class UnsupportedCoordinate(Coordinate):
+    @property
+    def mars_names(self):
+        return (self.variable.name,)

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/create/functions/sources/xarray/flavour.py

@@ -18,6 +18,7 @@ from .coordinates import LongitudeCoordinate
 from .coordinates import ScalarCoordinate
 from .coordinates import StepCoordinate
 from .coordinates import TimeCoordinate
+from .coordinates import UnsupportedCoordinate
 from .coordinates import XCoordinate
 from .coordinates import YCoordinate
 from .coordinates import is_scalar
@@ -157,11 +158,13 @@ class CoordinateGuesser:
         if c.shape in ((1,), tuple()):
             return ScalarCoordinate(c)
 
-        raise NotImplementedError(
+        LOG.warning(
             f"Coordinate {coord} not supported\n{axis=}, {name=},"
             f" {long_name=}, {standard_name=}, units\n\n{c}\n\n{type(c.values)} {c.shape}"
         )
 
+        return UnsupportedCoordinate(c)
+
     def grid(self, coordinates, variable):
         lat = [c for c in coordinates if c.is_lat]
         lon = [c for c in coordinates if c.is_lon]
@@ -225,20 +228,15 @@ class CoordinateGuesser:
         x = x[0]
         y = y[0]
 
-        _, unstructured = self._check_dims(variable, x, y)
-
-        if x.variable.dims != y.variable.dims:
-            raise ValueError(f"Dimensions do not match {x.name}{x.variable.dims} != {y.name}{y.variable.dims}")
-
-        if (x.name, y.name) in self._cache:
-            return self._cache[(x.name, y.name)]
+        dim_vars, unstructured = self._check_dims(variable, x, y)
 
-        if (x.name, y.name) in self._cache:
-            return self._cache[(x.name, y.name)]
-
-        assert len(x.variable.shape) == len(y.variable.shape), (x.variable.shape, y.variable.shape)
+        if (x.name, y.name, dim_vars) in self._cache:
+            return self._cache[(x.name, y.name, dim_vars)]
 
         grid_mapping = variable.attrs.get("grid_mapping", None)
+        if grid_mapping is not None:
+            print(f"grid_mapping: {grid_mapping}")
+            print(self.ds[grid_mapping])
 
         if grid_mapping is None:
             LOG.warning(f"No 'grid_mapping' attribute provided for '{variable.name}'")
@@ -285,11 +283,19 @@ class CoordinateGuesser:
                 grid_mapping = self.ds.attrs["crs"]
                 LOG.warning(f"Using CRS {grid_mapping} from global attributes")
 
+        grid = None
         if grid_mapping is not None:
+
+            grid_mapping = dict(self.ds[grid_mapping].attrs)
+
             if unstructured:
-                return UnstructuredProjectionGrid(x, y, grid_mapping)
+                grid = UnstructuredProjectionGrid(x, y, grid_mapping)
             else:
-                return MeshProjectionGrid(x, y, grid_mapping)
+                grid = MeshProjectionGrid(x, y, grid_mapping)
+
+        if grid is not None:
+            self._cache[(x.name, y.name, dim_vars)] = grid
+            return grid
 
         LOG.error("Could not fine a candidate for 'grid_mapping'")
         raise NotImplementedError(f"Unstructured grid {x.name} {y.name}")
@@ -337,12 +343,14 @@ class DefaultCoordinateGuesser(CoordinateGuesser):
         if standard_name == "time":
             return TimeCoordinate(c)
 
-        if name == "time":
+        # That is the output of `cfgrib` for forecasts
+        if name == "time" and standard_name != "forecast_reference_time":
             return TimeCoordinate(c)
 
     def _is_date(self, c, *, axis, name, long_name, standard_name, units):
         if standard_name == "forecast_reference_time":
             return DateCoordinate(c)
+
         if name == "forecast_reference_time":
             return DateCoordinate(c)
 

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/create/functions/sources/xarray/grid.py

@@ -100,6 +100,7 @@ class ProjectionGrid(XYGrid):
             data_crs = CRS.from_cf(self.projection)
         else:
             data_crs = self.projection
+
         wgs84_crs = CRS.from_epsg(4326)  # WGS84
 
         return Transformer.from_crs(data_crs, wgs84_crs, always_xy=True)

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/create/functions/sources/xarray/time.py

@@ -9,9 +9,12 @@
 
 
 import datetime
+import logging
 
 from anemoi.utils.dates import as_datetime
 
+LOG = logging.getLogger(__name__)
+
 
 class Time:
 
@@ -36,7 +39,28 @@ class Time:
         if len(date_coordinate) == 1 and len(time_coordinate) == 0 and len(step_coordinate) == 1:
             return ForecastFromBaseTimeAndDate(date_coordinate[0], step_coordinate[0])
 
-        raise NotImplementedError(f"{date_coordinate=} {time_coordinate=} {step_coordinate=}")
+        if len(date_coordinate) == 1 and len(time_coordinate) == 1 and len(step_coordinate) == 1:
+            return ForecastFromValidTimeAndStep(time_coordinate[0], step_coordinate[0], date_coordinate[0])
+
+        LOG.error("")
+        LOG.error(f"{len(date_coordinate)} date_coordinate")
+        for c in date_coordinate:
+            LOG.error("    %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
+            # LOG.error('    %s', c.variable)
+
+        LOG.error("")
+        LOG.error(f"{len(time_coordinate)} time_coordinate")
+        for c in time_coordinate:
+            LOG.error("    %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
+            # LOG.error('    %s', c.variable)
+
+        LOG.error("")
+        LOG.error(f"{len(step_coordinate)} step_coordinate")
+        for c in step_coordinate:
+            LOG.error("    %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
+            # LOG.error('    %s', c.variable)
+
+        raise NotImplementedError(f"{len(date_coordinate)=} {len(time_coordinate)=} {len(step_coordinate)=}")
 
 
 class Constant(Time):
@@ -62,9 +86,10 @@ class Analysis(Time):
 
 class ForecastFromValidTimeAndStep(Time):
 
-    def __init__(self, time_coordinate, step_coordinate):
+    def __init__(self, time_coordinate, step_coordinate, date_coordinate=None):
         self.time_coordinate_name = time_coordinate.variable.name
         self.step_coordinate_name = step_coordinate.variable.name
+        self.date_coordinate_name = date_coordinate.variable.name if date_coordinate else None
 
     def fill_time_metadata(self, coords_values, metadata):
         valid_datetime = coords_values[self.time_coordinate_name]
@@ -79,6 +104,16 @@ class ForecastFromValidTimeAndStep(Time):
         metadata["date"] = as_datetime(base_datetime).strftime("%Y%m%d")
         metadata["time"] = as_datetime(base_datetime).strftime("%H%M")
         metadata["step"] = int(hours)
+
+        # When date is present, it should be compatible with time and step
+
+        if self.date_coordinate_name is not None:
+            # Not sure that this is the correct assumption
+            assert coords_values[self.date_coordinate_name] == base_datetime, (
+                coords_values[self.date_coordinate_name],
+                base_datetime,
+            )
+
         return valid_datetime
 
 

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/create/input/result.py

@@ -30,6 +30,31 @@ from .trace import trace_datasource
 LOG = logging.getLogger(__name__)
 
 
+def _fields_metatata(variables, cube):
+    assert isinstance(variables, tuple), variables
+
+    result = {}
+    for i, c in enumerate(cube.iterate_cubelets()):
+        assert c._coords_names[1] == variables[i], (c._coords_names[1], variables[i])
+        f = cube[c.coords]
+        md = f.metadata(namespace="mars")
+        if not md:
+            md = f.metadata(namespace="default")
+
+        if md.get("param") == "~":
+            md["param"] = f.metadata("param")
+            assert md["param"] not in ("~", "unknown"), (md, f.metadata("param"))
+
+        if md.get("param") == "unknown":
+            md["param"] = str(f.metadata("paramId", default="unknown"))
+            # assert md['param'] != 'unknown', (md, f.metadata('param'))
+
+        result[variables[i]] = md
+
+    assert i + 1 == len(variables), (i + 1, len(variables))
+    return result
+
+
 def _data_request(data):
     date = None
     params_levels = defaultdict(set)
@@ -312,7 +337,10 @@ class Result:
     def build_coords(self):
         if self._coords_already_built:
             return
-        from_data = self.get_cube().user_coords
+
+        cube = self.get_cube()
+
+        from_data = cube.user_coords
         from_config = self.context.order_by
 
         keys_from_config = list(from_config.keys())
@@ -359,11 +387,19 @@ class Result:
         self._field_shape = first_field.shape
         self._proj_string = first_field.proj_string if hasattr(first_field, "proj_string") else None
 
+        self._cube = cube
+
+        self._coords_already_built = True
+
     @property
     def variables(self):
         self.build_coords()
         return self._variables
 
+    @property
+    def variables_metadata(self):
+        return _fields_metatata(self.variables, self._cube)
+
     @property
     def ensembles(self):
         self.build_coords()

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/create/persistent.py

@@ -56,8 +56,11 @@ class PersistentDict:
                 yield pickle.load(f)
 
     def add_provenance(self, **kwargs):
+        path = os.path.join(self.dirname, "provenance.json")
+        if os.path.exists(path):
+            return
         out = dict(provenance=gather_provenance_info(), **kwargs)
-        with open(os.path.join(self.dirname, "provenance.json"), "w") as f:
+        with open(path, "w") as f:
             json.dump(out, f)
 
     def add(self, elt, *, key):

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/create/statistics/__init__.py

@@ -187,8 +187,11 @@ class TmpStatistics:
 
     def add_provenance(self, **kwargs):
         self.create(exist_ok=True)
+        path = os.path.join(self.dirname, "provenance.json")
+        if os.path.exists(path):
+            return
         out = dict(provenance=gather_provenance_info(), **kwargs)
-        with open(os.path.join(self.dirname, "provenance.json"), "w") as f:
+        with open(path, "w") as f:
             json.dump(out, f)
 
     def create(self, exist_ok):

{anemoi_datasets-0.5.5 → anemoi_datasets-0.5.7}/src/anemoi/datasets/create/zarr.py

@@ -168,7 +168,11 @@ class ZarrBuiltRegistry:
         return self.create(lengths, overwrite=True)
 
     def add_provenance(self, name):
+        z = self._open_write()
+
+        if name in z.attrs:
+            return
+
         from anemoi.utils.provenance import gather_provenance_info
 
-        z = self._open_write()
         z.attrs[name] = gather_provenance_info()