anemoi-datasets 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.5.5'
-__version_tuple__ = version_tuple = (0, 5, 5)
+__version__ = version = '0.5.7'
+__version_tuple__ = version_tuple = (0, 5, 7)

anemoi/datasets/commands/inspect.py

@@ -506,6 +506,10 @@ class Version0_6(Version):
     def variables(self):
         return self.metadata["variables"]
 
+    @property
+    def variables_metadata(self):
+        return self.metadata.get("variables_metadata", {})
+
 
 class Version0_12(Version0_6):
     def details(self):

anemoi/datasets/create/__init__.py

@@ -25,6 +25,7 @@ from anemoi.utils.dates import frequency_to_string
 from anemoi.utils.dates import frequency_to_timedelta
 from anemoi.utils.humanize import compress_dates
 from anemoi.utils.humanize import seconds_to_human
+from anemoi.utils.sanitise import sanitise
 from earthkit.data.core.order import build_remapping
 
 from anemoi.datasets import MissingDateError
@@ -52,7 +53,7 @@ from .writer import ViewCacheArray
 
 LOG = logging.getLogger(__name__)
 
-VERSION = "0.20"
+VERSION = "0.30"
 
 
 def json_tidy(o):
@@ -325,43 +326,6 @@ def build_input_(main_config, output_config):
     return builder
 
 
-def tidy_recipe(config: object):
-    """Remove potentially private information in the config"""
-    config = deepcopy(config)
-    if isinstance(config, (tuple, list)):
-        return [tidy_recipe(_) for _ in config]
-    if isinstance(config, (dict, DotDict)):
-        for k, v in config.items():
-            if k.startswith("_"):
-                config[k] = "*** REMOVED FOR SECURITY ***"
-            else:
-                config[k] = tidy_recipe(v)
-    if isinstance(config, str):
-        if config.startswith("_"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("s3://"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("gs://"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("http"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("ftp"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("file"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("ssh"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("scp"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("rsync"):
-            return "*** REMOVED FOR SECURITY ***"
-        if config.startswith("/"):
-            return "*** REMOVED FOR SECURITY ***"
-        if "@" in config:
-            return "*** REMOVED FOR SECURITY ***"
-    return config
-
-
 class Init(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixin):
     dataset_class = NewDataset
     def __init__(self, path, config, check_name=False, overwrite=False, use_threads=False, statistics_temp_dir=None, progress=None, test=False, cache=None, **kwargs):  # fmt: skip
@@ -448,7 +412,7 @@ class Init(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixi
         metadata.update(self.main_config.get("add_metadata", {}))
 
         metadata["_create_yaml_config"] = self.main_config.get_serialisable_dict()
-        metadata["recipe"] = tidy_recipe(self.main_config.get_serialisable_dict())
+        metadata["recipe"] = sanitise(self.main_config.get_serialisable_dict())
 
         metadata["description"] = self.main_config.description
         metadata["licence"] = self.main_config["licence"]
@@ -467,6 +431,7 @@ class Init(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixi
         metadata["data_request"] = self.minimal_input.data_request
         metadata["field_shape"] = self.minimal_input.field_shape
         metadata["proj_string"] = self.minimal_input.proj_string
+        metadata["variables_metadata"] = self.minimal_input.variables_metadata
 
         metadata["start_date"] = dates[0].isoformat()
         metadata["end_date"] = dates[-1].isoformat()

anemoi/datasets/create/functions/sources/xarray/__init__.py

@@ -42,7 +42,7 @@ def load_one(emoji, context, dates, dataset, options={}, flavour=None, **kwargs)
     We have seen this bug triggered when we run many clients in parallel, for example, when we create a new dataset using `xarray-zarr`.
     """
 
-    context.trace(emoji, dataset, options)
+    context.trace(emoji, dataset, options, kwargs)
 
     if isinstance(dataset, str) and ".zarr" in dataset:
         data = xr.open_zarr(name_to_zarr_store(dataset), **options)

anemoi/datasets/create/functions/sources/xarray/coordinates.py

@@ -30,6 +30,8 @@ def extract_single_value(variable):
     if np.issubdtype(variable.values.dtype, np.datetime64):
         if len(shape) == 0:
             return to_datetime(variable.values)  # Convert to python datetime
+        if shape == (1,):
+            return to_datetime(variable.values[0])
         assert False, (shape, variable.values[:2])
 
     if np.issubdtype(variable.values.dtype, np.timedelta64):
@@ -244,3 +246,9 @@ class ScalarCoordinate(Coordinate):
     @property
     def mars_names(self):
         return (self.variable.name,)
+
+
+class UnsupportedCoordinate(Coordinate):
+    @property
+    def mars_names(self):
+        return (self.variable.name,)

anemoi/datasets/create/functions/sources/xarray/flavour.py

@@ -18,6 +18,7 @@ from .coordinates import LongitudeCoordinate
 from .coordinates import ScalarCoordinate
 from .coordinates import StepCoordinate
 from .coordinates import TimeCoordinate
+from .coordinates import UnsupportedCoordinate
 from .coordinates import XCoordinate
 from .coordinates import YCoordinate
 from .coordinates import is_scalar
@@ -157,11 +158,13 @@ class CoordinateGuesser:
         if c.shape in ((1,), tuple()):
             return ScalarCoordinate(c)
 
-        raise NotImplementedError(
+        LOG.warning(
             f"Coordinate {coord} not supported\n{axis=}, {name=},"
             f" {long_name=}, {standard_name=}, units\n\n{c}\n\n{type(c.values)} {c.shape}"
         )
 
+        return UnsupportedCoordinate(c)
+
     def grid(self, coordinates, variable):
         lat = [c for c in coordinates if c.is_lat]
         lon = [c for c in coordinates if c.is_lon]
@@ -225,20 +228,15 @@ class CoordinateGuesser:
         x = x[0]
         y = y[0]
 
-        _, unstructured = self._check_dims(variable, x, y)
-
-        if x.variable.dims != y.variable.dims:
-            raise ValueError(f"Dimensions do not match {x.name}{x.variable.dims} != {y.name}{y.variable.dims}")
-
-        if (x.name, y.name) in self._cache:
-            return self._cache[(x.name, y.name)]
+        dim_vars, unstructured = self._check_dims(variable, x, y)
 
-        if (x.name, y.name) in self._cache:
-            return self._cache[(x.name, y.name)]
-
-        assert len(x.variable.shape) == len(y.variable.shape), (x.variable.shape, y.variable.shape)
+        if (x.name, y.name, dim_vars) in self._cache:
+            return self._cache[(x.name, y.name, dim_vars)]
 
         grid_mapping = variable.attrs.get("grid_mapping", None)
+        if grid_mapping is not None:
+            print(f"grid_mapping: {grid_mapping}")
+            print(self.ds[grid_mapping])
 
         if grid_mapping is None:
             LOG.warning(f"No 'grid_mapping' attribute provided for '{variable.name}'")
@@ -285,11 +283,19 @@ class CoordinateGuesser:
                 grid_mapping = self.ds.attrs["crs"]
                 LOG.warning(f"Using CRS {grid_mapping} from global attributes")
 
+        grid = None
         if grid_mapping is not None:
+
+            grid_mapping = dict(self.ds[grid_mapping].attrs)
+
             if unstructured:
-                return UnstructuredProjectionGrid(x, y, grid_mapping)
+                grid = UnstructuredProjectionGrid(x, y, grid_mapping)
             else:
-                return MeshProjectionGrid(x, y, grid_mapping)
+                grid = MeshProjectionGrid(x, y, grid_mapping)
+
+        if grid is not None:
+            self._cache[(x.name, y.name, dim_vars)] = grid
+            return grid
 
         LOG.error("Could not fine a candidate for 'grid_mapping'")
         raise NotImplementedError(f"Unstructured grid {x.name} {y.name}")
@@ -337,12 +343,14 @@ class DefaultCoordinateGuesser(CoordinateGuesser):
         if standard_name == "time":
             return TimeCoordinate(c)
 
-        if name == "time":
+        # That is the output of `cfgrib` for forecasts
+        if name == "time" and standard_name != "forecast_reference_time":
             return TimeCoordinate(c)
 
     def _is_date(self, c, *, axis, name, long_name, standard_name, units):
         if standard_name == "forecast_reference_time":
             return DateCoordinate(c)
+
         if name == "forecast_reference_time":
             return DateCoordinate(c)
 

anemoi/datasets/create/functions/sources/xarray/grid.py

@@ -100,6 +100,7 @@ class ProjectionGrid(XYGrid):
             data_crs = CRS.from_cf(self.projection)
         else:
             data_crs = self.projection
+
         wgs84_crs = CRS.from_epsg(4326)  # WGS84
 
         return Transformer.from_crs(data_crs, wgs84_crs, always_xy=True)

anemoi/datasets/create/functions/sources/xarray/time.py

@@ -9,9 +9,12 @@
 
 
 import datetime
+import logging
 
 from anemoi.utils.dates import as_datetime
 
+LOG = logging.getLogger(__name__)
+
 
 class Time:
 
@@ -36,7 +39,28 @@ class Time:
         if len(date_coordinate) == 1 and len(time_coordinate) == 0 and len(step_coordinate) == 1:
             return ForecastFromBaseTimeAndDate(date_coordinate[0], step_coordinate[0])
 
-        raise NotImplementedError(f"{date_coordinate=} {time_coordinate=} {step_coordinate=}")
+        if len(date_coordinate) == 1 and len(time_coordinate) == 1 and len(step_coordinate) == 1:
+            return ForecastFromValidTimeAndStep(time_coordinate[0], step_coordinate[0], date_coordinate[0])
+
+        LOG.error("")
+        LOG.error(f"{len(date_coordinate)} date_coordinate")
+        for c in date_coordinate:
+            LOG.error("    %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
+            # LOG.error('    %s', c.variable)
+
+        LOG.error("")
+        LOG.error(f"{len(time_coordinate)} time_coordinate")
+        for c in time_coordinate:
+            LOG.error("    %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
+            # LOG.error('    %s', c.variable)
+
+        LOG.error("")
+        LOG.error(f"{len(step_coordinate)} step_coordinate")
+        for c in step_coordinate:
+            LOG.error("    %s %s %s %s", c, c.is_date, c.is_time, c.is_step)
+            # LOG.error('    %s', c.variable)
+
+        raise NotImplementedError(f"{len(date_coordinate)=} {len(time_coordinate)=} {len(step_coordinate)=}")
 
 
 class Constant(Time):
@@ -62,9 +86,10 @@ class Analysis(Time):
 
 class ForecastFromValidTimeAndStep(Time):
 
-    def __init__(self, time_coordinate, step_coordinate):
+    def __init__(self, time_coordinate, step_coordinate, date_coordinate=None):
         self.time_coordinate_name = time_coordinate.variable.name
         self.step_coordinate_name = step_coordinate.variable.name
+        self.date_coordinate_name = date_coordinate.variable.name if date_coordinate else None
 
     def fill_time_metadata(self, coords_values, metadata):
         valid_datetime = coords_values[self.time_coordinate_name]
@@ -79,6 +104,16 @@ class ForecastFromValidTimeAndStep(Time):
         metadata["date"] = as_datetime(base_datetime).strftime("%Y%m%d")
         metadata["time"] = as_datetime(base_datetime).strftime("%H%M")
         metadata["step"] = int(hours)
+
+        # When date is present, it should be compatible with time and step
+
+        if self.date_coordinate_name is not None:
+            # Not sure that this is the correct assumption
+            assert coords_values[self.date_coordinate_name] == base_datetime, (
+                coords_values[self.date_coordinate_name],
+                base_datetime,
+            )
+
         return valid_datetime
 
 

anemoi/datasets/create/input/result.py

@@ -30,6 +30,31 @@ from .trace import trace_datasource
 LOG = logging.getLogger(__name__)
 
 
+def _fields_metatata(variables, cube):
+    assert isinstance(variables, tuple), variables
+
+    result = {}
+    for i, c in enumerate(cube.iterate_cubelets()):
+        assert c._coords_names[1] == variables[i], (c._coords_names[1], variables[i])
+        f = cube[c.coords]
+        md = f.metadata(namespace="mars")
+        if not md:
+            md = f.metadata(namespace="default")
+
+        if md.get("param") == "~":
+            md["param"] = f.metadata("param")
+            assert md["param"] not in ("~", "unknown"), (md, f.metadata("param"))
+
+        if md.get("param") == "unknown":
+            md["param"] = str(f.metadata("paramId", default="unknown"))
+            # assert md['param'] != 'unknown', (md, f.metadata('param'))
+
+        result[variables[i]] = md
+
+    assert i + 1 == len(variables), (i + 1, len(variables))
+    return result
+
+
 def _data_request(data):
     date = None
     params_levels = defaultdict(set)
@@ -312,7 +337,10 @@ class Result:
     def build_coords(self):
         if self._coords_already_built:
             return
-        from_data = self.get_cube().user_coords
+
+        cube = self.get_cube()
+
+        from_data = cube.user_coords
         from_config = self.context.order_by
 
         keys_from_config = list(from_config.keys())
@@ -359,11 +387,19 @@ class Result:
         self._field_shape = first_field.shape
         self._proj_string = first_field.proj_string if hasattr(first_field, "proj_string") else None
 
+        self._cube = cube
+
+        self._coords_already_built = True
+
     @property
     def variables(self):
         self.build_coords()
         return self._variables
 
+    @property
+    def variables_metadata(self):
+        return _fields_metatata(self.variables, self._cube)
+
     @property
     def ensembles(self):
         self.build_coords()

anemoi/datasets/create/persistent.py

@@ -56,8 +56,11 @@ class PersistentDict:
                 yield pickle.load(f)
 
     def add_provenance(self, **kwargs):
+        path = os.path.join(self.dirname, "provenance.json")
+        if os.path.exists(path):
+            return
         out = dict(provenance=gather_provenance_info(), **kwargs)
-        with open(os.path.join(self.dirname, "provenance.json"), "w") as f:
+        with open(path, "w") as f:
             json.dump(out, f)
 
     def add(self, elt, *, key):

anemoi/datasets/create/statistics/__init__.py

@@ -187,8 +187,11 @@ class TmpStatistics:
 
     def add_provenance(self, **kwargs):
         self.create(exist_ok=True)
+        path = os.path.join(self.dirname, "provenance.json")
+        if os.path.exists(path):
+            return
         out = dict(provenance=gather_provenance_info(), **kwargs)
-        with open(os.path.join(self.dirname, "provenance.json"), "w") as f:
+        with open(path, "w") as f:
             json.dump(out, f)
 
     def create(self, exist_ok):

anemoi/datasets/create/zarr.py

@@ -168,7 +168,11 @@ class ZarrBuiltRegistry:
         return self.create(lengths, overwrite=True)
 
     def add_provenance(self, name):
+        z = self._open_write()
+
+        if name in z.attrs:
+            return
+
         from anemoi.utils.provenance import gather_provenance_info
 
-        z = self._open_write()
         z.attrs[name] = gather_provenance_info()

anemoi/datasets/data/forwards.py

@@ -63,6 +63,10 @@ class Forwards(Dataset):
     def variables(self):
         return self.forward.variables
 
+    @property
+    def variables_metadata(self):
+        return self.forward.variables_metadata
+
     @property
     def statistics(self):
         return self.forward.statistics
@@ -253,7 +257,6 @@ class GivenAxis(Combined):
         offset = 0
         result = set()
         for d in self.datasets:
-            print("--->", d.missing, d)
             result.update(offset + m for m in d.missing)
             if self.axis == 0:  # Advance if axis is time
                 offset += len(d)

anemoi/datasets/data/join.py

@@ -111,6 +111,19 @@ class Join(Combined):
 
         return result
 
+    @cached_property
+    def variables_metadata(self):
+        seen = set()
+        result = {}
+        for d in reversed(self.datasets):
+            for v in reversed(d.variables):
+                while v in seen:
+                    v = f"({v})"
+                seen.add(v)
+                result[v] = d.variables_metadata[v]
+
+        return result
+
     @cached_property
     def name_to_index(self):
         return {k: i for i, k in enumerate(self.variables)}

anemoi/datasets/data/select.py

@@ -76,6 +76,10 @@ class Select(Forwards):
     def variables(self):
         return [self.dataset.variables[i] for i in self.indices]
 
+    @cached_property
+    def variables_metadata(self):
+        return {k: v for k, v in self.dataset.variables_metadata.items() if k in self.variables}
+
     @cached_property
     def name_to_index(self):
         return {k: i for i, k in enumerate(self.variables)}
@@ -108,13 +112,20 @@ class Rename(Forwards):
         super().__init__(dataset)
         for n in rename:
             assert n in dataset.variables, n
+
         self._variables = [rename.get(v, v) for v in dataset.variables]
+        self._variables_metadata = {rename.get(k, k): v for k, v in dataset.variables_metadata.items()}
+
         self.rename = rename
 
     @property
     def variables(self):
         return self._variables
 
+    @property
+    def variables_metadata(self):
+        return self._variables_metadata
+
     @cached_property
     def name_to_index(self):
         return {k: i for i, k in enumerate(self.variables)}

anemoi/datasets/data/stores.py

@@ -302,6 +302,10 @@ class Zarr(Dataset):
             )
         ]
 
+    @property
+    def variables_metadata(self):
+        return self.z.attrs.get("variables_metadata", {})
+
     def __repr__(self):
         return self.path
 

anemoi/datasets/data/unchecked.py

@@ -91,6 +91,11 @@ class Unchecked(Combined):
     def variables(self):
         raise NotImplementedError()
 
+    @property
+    @check("check_same_variables")
+    def variables_metadata(self):
+        raise NotImplementedError()
+
     @property
     @check("check_same_variables")
     def statistics(self):

anemoi/datasets/testing.py

@@ -0,0 +1,59 @@
+# (C) Copyright 2024 ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+#
+
+# A collection of functions to support pytest testing
+
+import logging
+
+LOG = logging.getLogger(__name__)
+
+
+def assert_field_list(fs, size=None, start=None, end=None, constant=False, skip=None):
+    import numpy as np
+
+    if size is None:
+        assert len(fs) > 0, fs
+    else:
+        assert len(fs) == size, (len(fs), size)
+
+    first = fs[0]
+    last = fs[-1]
+
+    if constant:
+        # TODO: add a check for constant fields
+        pass
+    else:
+        assert start is None or first.metadata("valid_datetime") == start, (first.metadata("valid_datetime"), start)
+        assert end is None or last.metadata("valid_datetime") == end, (last.metadata("valid_datetime"), end)
+        print(first.datetime())
+
+    print(last.metadata())
+
+    first = first
+    latitudes, longitudes = first.grid_points()
+
+    assert len(latitudes.shape) == 1, latitudes.shape
+    assert len(longitudes.shape) == 1, longitudes.shape
+
+    assert len(latitudes) == len(longitudes), (len(latitudes), len(longitudes))
+    data = first.to_numpy(flatten=True)
+
+    assert len(data) == len(latitudes), (len(data), len(latitudes))
+
+    north = np.max(latitudes)
+    south = np.min(latitudes)
+    east = np.max(longitudes)
+    west = np.min(longitudes)
+
+    assert north >= south, (north, south)
+    assert east >= west, (east, west)
+    assert north <= 90, north
+    assert south >= -90, south
+    assert east <= 360, east
+    assert west >= -180, west

{anemoi_datasets-0.5.5.dist-info → anemoi_datasets-0.5.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.5.5
+Version: 0.5.7
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License: Apache License

{anemoi_datasets-0.5.5.dist-info → anemoi_datasets-0.5.7.dist-info}/RECORD

@@ -1,8 +1,9 @@
 anemoi/datasets/__init__.py,sha256=Z1gqZWhecLcT0RZQqYBLlz01MUlUZd0kWEj_RavbITM,782
 anemoi/datasets/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
-anemoi/datasets/_version.py,sha256=q2mmmMeZBj38-RlMPxNZk3zblQ8jPjzN_oIC744dsOc,411
+anemoi/datasets/_version.py,sha256=WG2gUkVRG4TIqG0-naezn012emNIbgERkkZYUI1CHww,411
 anemoi/datasets/fields.py,sha256=95TYAfqS_kZp1bzj84GngurVfyVvziVBy3zpUyoturI,1968
 anemoi/datasets/grids.py,sha256=xgvIbpMGuN2GKi2wIBhOLEMzj940nY9PH-toD0rCmPo,8980
+anemoi/datasets/testing.py,sha256=xtJBYywbsciaMp9NZ4jkjI4TdddCSkqqCCwSEHNwaNM,1810
 anemoi/datasets/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
 anemoi/datasets/commands/cleanup.py,sha256=_BkzGPUgvSqnuleymBfBx-sIyIM55hjK61m-v7yK0T8,1062
 anemoi/datasets/commands/compare.py,sha256=svEhyR7pOS1847_RJr1I6vF7ZDPB9AVlcrhy_gxQVms,3263
@@ -12,7 +13,7 @@ anemoi/datasets/commands/finalise-additions.py,sha256=876K37hVjslHJiu9VXZfre4YJh
 anemoi/datasets/commands/finalise.py,sha256=oPul5za7E__eJqkf5fRwvdL_0n2nG5Xk3JraRZQe64k,913
 anemoi/datasets/commands/init-additions.py,sha256=K3eKH5V_6ERiBKKyqIUuI1cfvAsjWV9ZAFzbtjIDyjs,1146
 anemoi/datasets/commands/init.py,sha256=7BpWaZ4gzMMiRqD9djHRciQNa0W2R4cOzoy5R-UJ0ck,2033
-anemoi/datasets/commands/inspect.py,sha256=kaEQhj_PcrcQwbz5uRhDblaa_IwZmKpGaboSrsQNi0Y,18714
+anemoi/datasets/commands/inspect.py,sha256=RFOkw0rB72kR3CGb-JRyb0-oh9zKois8ddeQwDWWKhQ,18822
 anemoi/datasets/commands/load-additions.py,sha256=Tg4FX0xealmwyB--zv03ZBrqTAGddkT0rKrHlcu4VW0,1234
 anemoi/datasets/commands/load.py,sha256=3jhL3DKUg07C8D6L8RsRWciq8korYsJ40_9atp-p68U,1304
 anemoi/datasets/commands/patch.py,sha256=gHY16r46GHxARItaAUHVjKSOm0Q7ZyZvSZmXqg1Wnig,841
@@ -20,16 +21,16 @@ anemoi/datasets/commands/publish.py,sha256=HV8W4wFOD7RB-Fn0qzkwyVoyZyaGsx8MUGFa5
 anemoi/datasets/commands/scan.py,sha256=MaTdne4JrtlqO3LhOUr43DZhZ6O-RZwC7uQ7C6PG7Os,2910
 anemoi/datasets/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 anemoi/datasets/compute/recentre.py,sha256=CMvRxTMv2H1hF1KqaZHT3Maa9P01GtsgAxr1YHbvH_o,4939
-anemoi/datasets/create/__init__.py,sha256=e5DaAuUdqUGeTaPnF-6vmV-_Rl2xWTt3elyB36E7W24,36554
+anemoi/datasets/create/__init__.py,sha256=rJjN17VUExSeGXPN1UwU005a1x_ERfWMeZGZje0BJfI,35221
 anemoi/datasets/create/check.py,sha256=j0vaA7y7dEYsgvAJVGhTZgjhWgvu7DDF1Bl-JkKxFcU,6128
 anemoi/datasets/create/chunks.py,sha256=1Inh3sIBn-2sNguErb-BsLns6W_HtDiOJAjIb29lp-U,2442
 anemoi/datasets/create/config.py,sha256=IqWsW_AVcp0uA2M1IPB-TT0IWjquvZL2R7PeiviOb4Y,8882
 anemoi/datasets/create/patch.py,sha256=c8cWoqxFzcY9mKREosLjuQCUeJMJL6sbNNkoVvhCZDA,3800
-anemoi/datasets/create/persistent.py,sha256=isENKsd6OievmNfe1rF6_3mY7X4HjaXGIrVIfH82JKY,4294
+anemoi/datasets/create/persistent.py,sha256=ts4p4ouQdin9Mu9mZMQDgENjFTjUgmzXogeRcqF6EZ0,4366
 anemoi/datasets/create/size.py,sha256=k4COEjs3wud0oKHH5P3n8Fap35xddXs002ucPjpBC88,1040
 anemoi/datasets/create/utils.py,sha256=v0dyQUk3hWOVQeZ82pnH3GmRwENXIHSwlwB7qqnomuU,2708
 anemoi/datasets/create/writer.py,sha256=G1qAPvdn8anGnpWYhvSSP4u3Km_tHKPdMXm0G4skKSk,1379
-anemoi/datasets/create/zarr.py,sha256=5m6YdC2Bj9iZObPohDcR1MacFQ9v1wA0OV4kSUwZtj8,5442
+anemoi/datasets/create/zarr.py,sha256=eNhopqcNSWtuJ_ltQIGdHVL2bGFRhS4Q6KkxAZ9s-Jw,5491
 anemoi/datasets/create/functions/__init__.py,sha256=5HmelLkXDjFOhNhX0Z78aV3ZlW2txiJliJwT4jfLEN4,945
 anemoi/datasets/create/functions/filters/__init__.py,sha256=Xe9G54CKvCI3ji-7k0R5l0WZZdhlydRgawsXuBcX_hg,379
 anemoi/datasets/create/functions/filters/empty.py,sha256=QGj7YEfbo3gwlmwHi1lPATjST0332TH2-uc6_wKENjI,621
@@ -61,14 +62,14 @@ anemoi/datasets/create/functions/sources/tendencies.py,sha256=4hFPGU51KnxXp-KteS
 anemoi/datasets/create/functions/sources/xarray_kerchunk.py,sha256=JsuGjTs5BQdJIPRI8TuUNde680UQkT4GbUhOt6wYy38,1432
 anemoi/datasets/create/functions/sources/xarray_zarr.py,sha256=qI_fEKo28hu_B_qMPx67gigysw8qw5ePMU9P0wbDcIc,531
 anemoi/datasets/create/functions/sources/zenodo.py,sha256=n7_sfZHJVDjVnSwc07muO9kS7TwOM4S8DIlJf5pnDiU,1237
-anemoi/datasets/create/functions/sources/xarray/__init__.py,sha256=Ft3p7k8WEz3kBc26b1hWuk2y_13_SdPuizwKRb6cLi0,2954
-anemoi/datasets/create/functions/sources/xarray/coordinates.py,sha256=0iOu2qKTBfaegLYXQA1EEi2bRDL5m2H1hwYfuLjBQk4,6001
+anemoi/datasets/create/functions/sources/xarray/__init__.py,sha256=0HMGeuI2y-DlQJiikYvBLkZeLwdKCWggAadzXnRiMho,2962
+anemoi/datasets/create/functions/sources/xarray/coordinates.py,sha256=NgOlA7426pdhq20IKJv2-mrBnjFSFO4asPgcXQph05c,6198
 anemoi/datasets/create/functions/sources/xarray/field.py,sha256=3xy14ywRxYydvxucDlE8g6PeKfP5mcc5NL-2mGDv9nU,3658
 anemoi/datasets/create/functions/sources/xarray/fieldlist.py,sha256=GnYbierm_TuTPMG9WOn99QBi1DxaFo0hra69KGvSiEg,5589
-anemoi/datasets/create/functions/sources/xarray/flavour.py,sha256=8jkV-wUlze2IVQ2WLnfj9-5INYhat3Gw3ERFOrf_VqM,14667
-anemoi/datasets/create/functions/sources/xarray/grid.py,sha256=h3O20PsdpJc9V_oae_WEUw8qMnalV_4yjFNAZHvuxPI,3629
+anemoi/datasets/create/functions/sources/xarray/flavour.py,sha256=17bnp_VQ4lRIEMYz4JC6_XBbn56UoknX74SYVhgO_Yk,14841
+anemoi/datasets/create/functions/sources/xarray/grid.py,sha256=ND9TSNxg-_8gGhXLzCo_CFoTrZXLWLYjsF-d88AKr28,3630
 anemoi/datasets/create/functions/sources/xarray/metadata.py,sha256=TSFQ_2ijVtEiXdwnjYr9wPRsTBwqfo_pNNZDFPXFAts,4558
-anemoi/datasets/create/functions/sources/xarray/time.py,sha256=6IXwu9gI1e4LjZ6L6-Bgnq2LgRXvSmkujFWEFTxOymA,4491
+anemoi/datasets/create/functions/sources/xarray/time.py,sha256=494pqiQ2p9cryXobGsuf5AgvMBzionKTpnyVoI0HtiA,5973
 anemoi/datasets/create/functions/sources/xarray/variable.py,sha256=tJUiJCPJ6Ft-U9isaEv8vLzPk52Dnnss9zTJX1hXQvI,4522
 anemoi/datasets/create/input/__init__.py,sha256=v9hJbgkH5hl3Zp5papYHwyEJ3JF2u4ILiTfnjJ1oOpI,2135
 anemoi/datasets/create/input/action.py,sha256=qMKyQLTjAJDBey_1_iIbI6_bqdNVyXarJzDtCH4kiRM,4188
@@ -82,11 +83,11 @@ anemoi/datasets/create/input/join.py,sha256=uvsIio-KMP3aaZyHK2k_7sdsqYbgIoqKQQ3D
 anemoi/datasets/create/input/misc.py,sha256=HhCVxVeONqdV7WiiYiAeZb3_QeoR0gcVgAAanon6U4Y,2306
 anemoi/datasets/create/input/pipe.py,sha256=YBEAypMXaaWRzwZQCRG0oQwvwSmZhNCAa_1hPlYJuvc,1170
 anemoi/datasets/create/input/repeated_dates.py,sha256=NueAwQ8uwD54FQEOzHfPGEC8iw7HRiTmdSR2ytFdWrw,6536
-anemoi/datasets/create/input/result.py,sha256=p903jet3hY9g0_Um-COUUsdSIqmGsgD8BUlx16_ZAks,13754
+anemoi/datasets/create/input/result.py,sha256=35wCEKwmybe5uuTigqmESh_y_JB-KgTqLT_f2_lIqhQ,14804
 anemoi/datasets/create/input/step.py,sha256=Jvl3QpkNgSIkhdc2ekrCMlqSWmP9fPXD0CvPYcocAsM,3133
 anemoi/datasets/create/input/template.py,sha256=-prOGI7vbcqph4uS-Mf5-mQ1ws3LbdFzKZxx4d7Ojp8,1844
 anemoi/datasets/create/input/trace.py,sha256=J-8jDy28wNZa4aSV1KIQMwc1KolcoH3R2xjLl-_eLzM,2183
-anemoi/datasets/create/statistics/__init__.py,sha256=oMwWQ-LteeaVFqu6hzKEcsRz_5KMjzDbwIZ_1HEIvto,12699
+anemoi/datasets/create/statistics/__init__.py,sha256=HphcJfdD6mYxhH8hzDMAcSFH3wblHW_ZONT70qX-BYI,12771
 anemoi/datasets/create/statistics/summary.py,sha256=sgmhA24y3VRyjmDUgTnPIqcHSlWBbFA0qynx6gJ9Xw8,3370
 anemoi/datasets/data/__init__.py,sha256=usTH1SLkeveBbYkbBTc7rP4G3mCGNa8A74vPjXjlOSw,1067
 anemoi/datasets/data/concat.py,sha256=SYASNNngnAyz3rC4ENzrGAu91rnhd6uwP0kGX9aMZAQ,5269
@@ -94,28 +95,28 @@ anemoi/datasets/data/dataset.py,sha256=GS-V1rHze1qynIBz56PQOpQMVzWcSpbIppZ7-SLtY
 anemoi/datasets/data/debug.css,sha256=z2X_ZDSnZ9C3pyZPWnQiEyAxuMxUaxJxET4oaCImTAQ,211
 anemoi/datasets/data/debug.py,sha256=xNpFeJqdlN9Qhlhbp4pxXJP0dQTf0V4bm2cK4hsrSJM,6326
 anemoi/datasets/data/ensemble.py,sha256=AsP7Xx0ZHLoZs6a4EC0jtyGYIcOvZvvKXhgNsIvqIN8,1137
-anemoi/datasets/data/forwards.py,sha256=SfB3o8Y-nQEOkS5gI_6wtipvzB5peF_YkwW9cOMhLMg,8167
+anemoi/datasets/data/forwards.py,sha256=-5h18GgkSOH_sNRpwJ8ZGv_KuJkPqzmCDwljPStAQHI,8223
 anemoi/datasets/data/grids.py,sha256=KiHpN1Ne0DnQM42buBy0sIAZHNaC2kuEFwkk6ohIooU,8046
 anemoi/datasets/data/indexing.py,sha256=625m__JG5m_tDMrkz1hB6Vydenwt0oHuyAlc-o3Zwos,4799
 anemoi/datasets/data/interpolate.py,sha256=usnYgjoDPfnUNg_lZl3KsFTsJ7kOsf7YaHwarsBU7Ag,4163
-anemoi/datasets/data/join.py,sha256=dtCBbMTicqrRPxfBULi3RwEcQBLhQpIcvCjdN5A3XUU,4892
+anemoi/datasets/data/join.py,sha256=mmCSJVLsdY1qBLJ5pLQvC5n6-cJB-u40xoUP5ffaOd0,5244
 anemoi/datasets/data/masked.py,sha256=GKAKHwiiVrDX13V_mDiQ3G1Y7zXSksc0IkABPEvVw-M,3825
 anemoi/datasets/data/misc.py,sha256=8ZaLhgHmUmP77X0_6scp36BJJzz544nHVQRAamW6x-Y,9821
 anemoi/datasets/data/missing.py,sha256=OyW8cRow1v641Vgv-IY5BnUxH2Ob5P2QjgrVR1Szjl0,7157
 anemoi/datasets/data/rescale.py,sha256=c_onY5f-2XDGmLP5mawNFueQWNS-BnTpo6xaBJF4aWA,4138
-anemoi/datasets/data/select.py,sha256=tProUPIiRwSe_H9cwwn_4iMX1uZEIQ7q5KsegBgSmdc,3971
+anemoi/datasets/data/select.py,sha256=HiTFpZm5RVD2Vck4tjtFABUuIQYuYz_F09GrnIvnFDU,4319
 anemoi/datasets/data/statistics.py,sha256=lZCcKw9s7ttMBEp6ANyxtbXoZZvchhE7SClq-D4AUR8,1645
-anemoi/datasets/data/stores.py,sha256=7CRqiAzOq71pzRezC2f_elTRDmJQzfu8bi0vrnylatI,12194
+anemoi/datasets/data/stores.py,sha256=w8SgIGV7YlysazF6FLAVd81EQcAyWbj9RhR6svp1_iU,12301
 anemoi/datasets/data/subset.py,sha256=peyHlIvno0uUzqW0HS9zjqThmyXn6rpvVWh-925XHnE,4692
-anemoi/datasets/data/unchecked.py,sha256=xLOCU-O3OpfAi3bd-XZEpfDTnZfFGqdhh55d8D_u3wQ,4184
+anemoi/datasets/data/unchecked.py,sha256=4qqHuDJNQ89HDOQFsSVACRYpT34xgxYpiH4T6J16HXE,4304
 anemoi/datasets/data/xy.py,sha256=_k1z_2INvA0v7zsm5lxfVvf7ymBFu6Tr9PxvtdB9kFg,3002
 anemoi/datasets/dates/__init__.py,sha256=-RcoD33OQl4GOL92r7ksT8014KhvURjUCb1C9SNXjp0,7970
 anemoi/datasets/dates/groups.py,sha256=7zXEtMwFWFFQarKNyNDLxpTu7OFiZWSYIrvAIeWLEPg,5391
 anemoi/datasets/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 anemoi/datasets/utils/fields.py,sha256=VWf9PPmJ1WU1-oJ5tNkbC3_ik9gqnFxqwrc1EFysBYk,1397
-anemoi_datasets-0.5.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_datasets-0.5.5.dist-info/METADATA,sha256=IXuERodLJ1TIa4E3qplvDYy2ULOqGzZwvTaIntkLoo8,16324
-anemoi_datasets-0.5.5.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
-anemoi_datasets-0.5.5.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
-anemoi_datasets-0.5.5.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_datasets-0.5.5.dist-info/RECORD,,
+anemoi_datasets-0.5.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_datasets-0.5.7.dist-info/METADATA,sha256=UTGUJaqsj9oc2Z3-iLLc709HlXxBxp1KPp7CHVfrEhE,16324
+anemoi_datasets-0.5.7.dist-info/WHEEL,sha256=GV9aMThwP_4oNCtvEC2ec3qUYutgWeAzklro_0m4WJQ,91
+anemoi_datasets-0.5.7.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
+anemoi_datasets-0.5.7.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_datasets-0.5.7.dist-info/RECORD,,