anemoi-datasets 0.5.12__py3-none-any.whl → 0.5.13__py3-none-any.whl

anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.5.12'
-__version_tuple__ = version_tuple = (0, 5, 12)
+__version__ = version = '0.5.13'
+__version_tuple__ = version_tuple = (0, 5, 13)

anemoi/datasets/create/__init__.py

@@ -622,10 +622,14 @@ class Load(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixi
 
         check_shape(cube, dates, dates_in_data)
 
-        def check_dates_in_data(lst, lst2):
-            lst2 = [np.datetime64(_) for _ in lst2]
-            lst = [np.datetime64(_) for _ in lst]
-            assert lst == lst2, ("Dates in data are not the requested ones:", lst, lst2)
+        def check_dates_in_data(dates_in_data, requested_dates):
+            requested_dates = [np.datetime64(_) for _ in requested_dates]
+            dates_in_data = [np.datetime64(_) for _ in dates_in_data]
+            assert dates_in_data == requested_dates, (
+                "Dates in data are not the requested ones:",
+                dates_in_data,
+                requested_dates,
+            )
 
         check_dates_in_data(dates_in_data, dates)
 

anemoi/datasets/create/check.py

@@ -58,7 +58,7 @@ class DatasetName:
             raise ValueError(self.error_message)
 
     def _parse(self, name):
-        pattern = r"^(\w+)-([\w-]+)-(\w+)-(\w+)-(\d\d\d\d)-(\d\d\d\d)-(\d+h)-v(\d+)-?([a-zA-Z0-9-]+)?$"
+        pattern = r"^(\w+)-([\w-]+)-(\w+)-(\w+)-(\d\d\d\d)-(\d\d\d\d)-(\d+h|\d+m)-v(\d+)-?([a-zA-Z0-9-]+)?$"
         match = re.match(pattern, name)
 
         if not match:

anemoi/datasets/create/functions/__init__.py

@@ -22,6 +22,7 @@ def assert_is_fieldlist(obj):
 def import_function(name, kind):
 
     from anemoi.transform.filters import filter_registry
+    from anemoi.transform.sources import source_registry
 
     name = name.replace("-", "_")
 
@@ -45,7 +46,20 @@ def import_function(name, kind):
         if filter_registry.lookup(name, return_none=True):
 
             def proc(context, data, *args, **kwargs):
-                return filter_registry.create(name, *args, **kwargs)(data)
+                filter = filter_registry.create(name, *args, **kwargs)
+                filter.context = context
+                # filter = filter_registry.create(context, name, *args, **kwargs)
+                return filter.forward(data)
+
+            return proc
+
+    if kind == "sources":
+        if source_registry.lookup(name, return_none=True):
+
+            def proc(context, data, *args, **kwargs):
+                source = source_registry.create(name, *args, **kwargs)
+                # source = source_registry.create(context, name, *args, **kwargs)
+                return source.forward(data)
 
             return proc
 

anemoi/datasets/create/functions/filters/orog_to_z.py

@@ -0,0 +1,58 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+from collections import defaultdict
+
+from earthkit.data.indexing.fieldlist import FieldArray
+
+
+class NewDataField:
+    def __init__(self, field, data, new_name):
+        self.field = field
+        self.data = data
+        self.new_name = new_name
+
+    def to_numpy(self, *args, **kwargs):
+        return self.data
+
+    def metadata(self, key=None, **kwargs):
+        if key is None:
+            return self.field.metadata(**kwargs)
+
+        value = self.field.metadata(key, **kwargs)
+        if key == "param":
+            return self.new_name
+        return value
+
+    def __getattr__(self, name):
+        return getattr(self.field, name)
+
+
+def execute(context, input, orog, z="z"):
+    """Convert orography [m] to z (geopotential height)"""
+    result = FieldArray()
+
+    processed_fields = defaultdict(dict)
+
+    for f in input:
+        key = f.metadata(namespace="mars")
+        param = key.pop("param")
+        if param == orog:
+            key = tuple(key.items())
+
+            if param in processed_fields[key]:
+                raise ValueError(f"Duplicate field {param} for {key}")
+
+            output = f.to_numpy(flatten=True) * 9.80665
+            result.append(NewDataField(f, output, z))
+        else:
+            result.append(f)
+
+    return result

anemoi/datasets/create/functions/filters/sum.py

@@ -0,0 +1,71 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+from collections import defaultdict
+
+from earthkit.data.indexing.fieldlist import FieldArray
+
+
+class NewDataField:
+    def __init__(self, field, data, new_name):
+        self.field = field
+        self.data = data
+        self.new_name = new_name
+
+    def to_numpy(self, *args, **kwargs):
+        return self.data
+
+    def metadata(self, key=None, **kwargs):
+        if key is None:
+            return self.field.metadata(**kwargs)
+
+        value = self.field.metadata(key, **kwargs)
+        if key == "param":
+            return self.new_name
+        return value
+
+    def __getattr__(self, name):
+        return getattr(self.field, name)
+
+
+def execute(context, input, params, output):
+    """Computes the sum over a set of variables"""
+    result = FieldArray()
+
+    needed_fields = defaultdict(dict)
+
+    for f in input:
+        key = f.metadata(namespace="mars")
+        param = key.pop("param")
+        if param in params:
+            key = tuple(key.items())
+
+            if param in needed_fields[key]:
+                raise ValueError(f"Duplicate field {param} for {key}")
+
+            needed_fields[key][param] = f
+        else:
+            result.append(f)
+
+    for keys, values in needed_fields.items():
+
+        if len(values) != len(params):
+            raise ValueError("Missing fields")
+
+        s = None
+        for k, v in values.items():
+            c = v.to_numpy(flatten=True)
+            if s is None:
+                s = c
+            else:
+                s += c
+        result.append(NewDataField(values[list(values.keys())[0]], s, output))
+
+    return result

anemoi/datasets/create/functions/filters/wz_to_w.py

@@ -0,0 +1,79 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+from collections import defaultdict
+
+from earthkit.data.indexing.fieldlist import FieldArray
+
+
+class NewDataField:
+    def __init__(self, field, data, new_name):
+        self.field = field
+        self.data = data
+        self.new_name = new_name
+
+    def to_numpy(self, *args, **kwargs):
+        return self.data
+
+    def metadata(self, key=None, **kwargs):
+        if key is None:
+            return self.field.metadata(**kwargs)
+
+        value = self.field.metadata(key, **kwargs)
+        if key == "param":
+            return self.new_name
+        return value
+
+    def __getattr__(self, name):
+        return getattr(self.field, name)
+
+
+def execute(context, input, wz, t, w="w"):
+    """Convert geometric vertical velocity (m/s) to vertical velocity (Pa / s)"""
+    result = FieldArray()
+
+    params = (wz, t)
+    pairs = defaultdict(dict)
+
+    for f in input:
+        key = f.metadata(namespace="mars")
+        param = key.pop("param")
+        if param in params:
+            key = tuple(key.items())
+
+            if param in pairs[key]:
+                raise ValueError(f"Duplicate field {param} for {key}")
+
+            pairs[key][param] = f
+            if param == t:
+                result.append(f)
+        else:
+            result.append(f)
+
+    for keys, values in pairs.items():
+
+        if len(values) != 2:
+            raise ValueError("Missing fields")
+
+        wz_pl = values[wz].to_numpy(flatten=True)
+        t_pl = values[t].to_numpy(flatten=True)
+        pressure = keys[4][1] * 100  # TODO: REMOVE HARDCODED INDICES
+
+        w_pl = wz_to_w(wz_pl, t_pl, pressure)
+        result.append(NewDataField(values[wz], w_pl, w))
+
+    return result
+
+
+def wz_to_w(wz, t, pressure):
+    g = 9.81
+    Rd = 287.058
+
+    return -wz * g * pressure / (t * Rd)

anemoi/datasets/create/functions/sources/accumulations.py

@@ -379,6 +379,7 @@ def accumulations(context, dates, **request):
     KWARGS = {
         ("od", "oper"): dict(patch=_scda),
         ("od", "elda"): dict(base_times=(6, 18)),
+        ("od", "enfo"): dict(base_times=(0, 6, 12, 18)),
         ("ea", "oper"): dict(data_accumulation_period=1, base_times=(6, 18)),
         ("ea", "enda"): dict(data_accumulation_period=3, base_times=(6, 18)),
         ("rr", "oper"): dict(base_times=(0, 3, 6, 9, 12, 15, 18, 21)),

anemoi/datasets/create/functions/sources/xarray/__init__.py

@@ -29,7 +29,7 @@ def check(what, ds, paths, **kwargs):
         raise ValueError(f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, {what}s={paths})")
 
 
-def load_one(emoji, context, dates, dataset, options={}, flavour=None, **kwargs):
+def load_one(emoji, context, dates, dataset, *, options={}, flavour=None, patch=None, **kwargs):
     import xarray as xr
 
     """
@@ -54,10 +54,10 @@ def load_one(emoji, context, dates, dataset, options={}, flavour=None, **kwargs)
     else:
         data = xr.open_dataset(dataset, **options)
 
-    fs = XarrayFieldList.from_xarray(data, flavour)
+    fs = XarrayFieldList.from_xarray(data, flavour=flavour, patch=patch)
 
     if len(dates) == 0:
-        return fs.sel(**kwargs)
+        result = fs.sel(**kwargs)
     else:
         result = MultiFieldList([fs.sel(valid_datetime=date, **kwargs) for date in dates])
 

anemoi/datasets/create/functions/sources/xarray/field.py

@@ -92,6 +92,10 @@ class XArrayField(Field):
     def grid_points(self):
         return self.owner.grid_points()
 
+    def to_latlon(self, flatten=True):
+        assert flatten
+        return dict(lat=self.latitudes, lon=self.longitudes)
+
     @property
     def resolution(self):
         return None
@@ -120,6 +124,6 @@ class XArrayField(Field):
     def __repr__(self):
         return repr(self._metadata)
 
-    def _values(self):
+    def _values(self, dtype=None):
         # we don't use .values as this will download the data
         return self.selection

anemoi/datasets/create/functions/sources/xarray/fieldlist.py

@@ -16,6 +16,7 @@ from earthkit.data.core.fieldlist import FieldList
 
 from .field import EmptyFieldList
 from .flavour import CoordinateGuesser
+from .patch import patch_dataset
 from .time import Time
 from .variable import FilteredVariable
 from .variable import Variable
@@ -49,7 +50,11 @@ class XarrayFieldList(FieldList):
         raise IndexError(k)
 
     @classmethod
-    def from_xarray(cls, ds, flavour=None):
+    def from_xarray(cls, ds, *, flavour=None, patch=None):
+
+        if patch is not None:
+            ds = patch_dataset(ds, patch)
+
         variables = []
 
         if isinstance(flavour, str):
@@ -83,6 +88,8 @@ class XarrayFieldList(FieldList):
             _skip_attr(variable, "bounds")
             _skip_attr(variable, "grid_mapping")
 
+        LOG.debug("Xarray data_vars: %s", ds.data_vars)
+
         # Select only geographical variables
         for name in ds.data_vars:
 
@@ -97,6 +104,7 @@ class XarrayFieldList(FieldList):
                 c = guess.guess(ds[coord], coord)
                 assert c, f"Could not guess coordinate for {coord}"
                 if coord not in variable.dims:
+                    LOG.debug("%s: coord=%s (not a dimension): dims=%s", variable, coord, variable.dims)
                     c.is_dim = False
                 coordinates.append(c)
 
@@ -104,6 +112,7 @@ class XarrayFieldList(FieldList):
             assert grid_coords <= 2
 
             if grid_coords < 2:
+                LOG.debug("Skipping %s (not 2D): %s", variable, [(c, c.is_grid, c.is_dim) for c in coordinates])
                 continue
 
             v = Variable(

anemoi/datasets/create/functions/sources/xarray/metadata.py

@@ -24,6 +24,7 @@ class _MDMapping:
     def __init__(self, variable):
         self.variable = variable
         self.time = variable.time
+        # Aliases
         self.mapping = dict(param="variable")
         for c in variable.coordinates:
             for v in c.mars_names:
@@ -34,7 +35,6 @@ class _MDMapping:
         return self.mapping.get(key, key)
 
     def from_user(self, kwargs):
-        print("from_user", kwargs, self)
         return {self._from_user(k): v for k, v in kwargs.items()}
 
     def __repr__(self):
@@ -81,22 +81,16 @@ class XArrayMetadata(RawMetadata):
     def _valid_datetime(self):
         return self._get("valid_datetime")
 
-    def _get(self, key, **kwargs):
+    def get(self, key, astype=None, **kwargs):
 
         if key in self._d:
+            if astype is not None:
+                return astype(self._d[key])
             return self._d[key]
 
-        if key.startswith("mars."):
-            key = key[5:]
-            if key not in self.MARS_KEYS:
-                if kwargs.get("raise_on_missing", False):
-                    raise KeyError(f"Invalid key '{key}' in namespace='mars'")
-                else:
-                    return kwargs.get("default", None)
-
         key = self._mapping._from_user(key)
 
-        return super()._get(key, **kwargs)
+        return super().get(key, astype=astype, **kwargs)
 
 
 class XArrayFieldGeography(Geography):

anemoi/datasets/create/functions/sources/xarray/patch.py

@@ -0,0 +1,44 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import logging
+
+LOG = logging.getLogger(__name__)
+
+
+def patch_attributes(ds, attributes):
+    for name, value in attributes.items():
+        variable = ds[name]
+        variable.attrs.update(value)
+
+    return ds
+
+
+def patch_coordinates(ds, coordinates):
+    for name in coordinates:
+        ds = ds.assign_coords({name: ds[name]})
+
+    return ds
+
+
+PATCHES = {
+    "attributes": patch_attributes,
+    "coordinates": patch_coordinates,
+}
+
+
+def patch_dataset(ds, patch):
+    for what, values in patch.items():
+        if what not in PATCHES:
+            raise ValueError(f"Unknown patch type {what!r}")
+
+        ds = PATCHES[what](ds, values)
+
+    return ds

anemoi/datasets/create/functions/sources/xarray/time.py

@@ -62,12 +62,18 @@ class Time:
 
         raise NotImplementedError(f"{len(date_coordinate)=} {len(time_coordinate)=} {len(step_coordinate)=}")
 
+    def select_valid_datetime(self, variable):
+        raise NotImplementedError(f"{self.__class__.__name__}.select_valid_datetime()")
+
 
 class Constant(Time):
 
     def fill_time_metadata(self, coords_values, metadata):
         return None
 
+    def select_valid_datetime(self, variable):
+        return None
+
 
 class Analysis(Time):
 
@@ -83,6 +89,9 @@ class Analysis(Time):
 
         return valid_datetime
 
+    def select_valid_datetime(self, variable):
+        return self.time_coordinate_name
+
 
 class ForecastFromValidTimeAndStep(Time):
 
@@ -116,6 +125,9 @@ class ForecastFromValidTimeAndStep(Time):
 
         return valid_datetime
 
+    def select_valid_datetime(self, variable):
+        return self.time_coordinate_name
+
 
 class ForecastFromValidTimeAndBaseTime(Time):
 
@@ -138,6 +150,9 @@ class ForecastFromValidTimeAndBaseTime(Time):
 
         return valid_datetime
 
+    def select_valid_datetime(self, variable):
+        return self.time_coordinate_name
+
 
 class ForecastFromBaseTimeAndDate(Time):
 

anemoi/datasets/create/functions/sources/xarray/variable.py

@@ -37,7 +37,7 @@ class Variable:
         self.coordinates = coordinates
 
         self._metadata = metadata.copy()
-        self._metadata.update({"variable": variable.name})
+        self._metadata.update({"variable": variable.name, "param": variable.name})
 
         self.time = time
 
@@ -45,6 +45,9 @@ class Variable:
         self.names = {c.variable.name: c for c in coordinates if c.is_dim and not c.scalar and not c.is_grid}
         self.by_name = {c.variable.name: c for c in coordinates}
 
+        # We need that alias for the time dimension
+        self._aliases = dict(valid_datetime="time")
+
         self.length = math.prod(self.shape)
 
     @property
@@ -96,15 +99,28 @@ class Variable:
 
         k, v = kwargs.popitem()
 
+        user_provided_k = k
+
+        if k == "valid_datetime":
+            # Ask the Time object to select the valid datetime
+            k = self.time.select_valid_datetime(self)
+            if k is None:
+                return None
+
         c = self.by_name.get(k)
 
+        # assert c is not None, f"Could not find coordinate {k} in {self.variable.name} {self.coordinates} {list(self.by_name)}"
+
         if c is None:
             missing[k] = v
             return self.sel(missing, **kwargs)
 
         i = c.index(v)
         if i is None:
-            LOG.warning(f"Could not find {k}={v} in {c}")
+            if k != user_provided_k:
+                LOG.warning(f"Could not find {user_provided_k}={v} in {c} (alias of {k})")
+            else:
+                LOG.warning(f"Could not find {k}={v} in {c}")
             return None
 
         coordinates = [x.reduced(i) if c is x else x for x in self.coordinates]

anemoi/datasets/create/input/repeated_dates.py

@@ -72,6 +72,11 @@ class DateMapperClosest(DateMapper):
                 end += self.frequency
 
         to_try = sorted(to_try - self.tried)
+        info = {k: "no-data" for k in to_try}
+
+        if not to_try:
+            LOG.warning(f"No new dates to try for {group_of_dates} in {self.source}")
+            # return []
 
         if to_try:
             result = self.source.select(
@@ -82,19 +87,32 @@ class DateMapperClosest(DateMapper):
                 )
             )
 
+            cnt = 0
             for f in result.datasource:
+                cnt += 1
                 # We could keep the fields in a dictionary, but we don't want to keep the fields in memory
                 date = as_datetime(f.metadata("valid_datetime"))
 
                 if self.skip_all_nans:
                     if np.isnan(f.to_numpy()).all():
                         LOG.warning(f"Skipping {date} because all values are NaN")
+                        info[date] = "all-nans"
                         continue
 
+                info[date] = "ok"
                 self.found.add(date)
 
+            if cnt == 0:
+                raise ValueError(f"No data found for {group_of_dates} in {self.source}")
+
             self.tried.update(to_try)
 
+        if not self.found:
+            for k, v in info.items():
+                LOG.warning(f"{k}: {v}")
+
+            raise ValueError(f"No matching data found for {asked_dates} in {self.source}")
+
         new_dates = defaultdict(list)
 
         for date in asked_dates:

anemoi/datasets/create/statistics/__init__.py

@@ -98,7 +98,7 @@ def fix_variance(x, name, count, sums, squares):
 
     variances = squares / count - mean * mean
     assert variances.shape == squares.shape == mean.shape
-    if all(variances >= 0):
+    if np.all(variances >= 0):
         LOG.warning(f"All individual variances for {name} are positive, setting variance to 0.")
         return 0
 
@@ -108,7 +108,7 @@ def fix_variance(x, name, count, sums, squares):
     #     return 0
 
     LOG.warning(f"ERROR at least one individual variance is negative ({np.nanmin(variances)}).")
-    return x
+    return 0
 
 
 def check_variance(x, variables_names, minimum, maximum, mean, count, sums, squares):

anemoi/datasets/create/utils.py

@@ -54,6 +54,10 @@ def to_datetime(*args, **kwargs):
 
 
 def make_list_int(value):
+    # Convert a string like "1/2/3" or "1/to/3" or "1/to/10/by/2" to a list of integers.
+    # Moved to anemoi.utils.humanize
+    # replace with from anemoi.utils.humanize import make_list_int
+    # when anemoi-utils is released and pyproject.toml is updated
     if isinstance(value, str):
         if "/" not in value:
             return [value]

anemoi/datasets/data/complement.py

@@ -0,0 +1,164 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import logging
+from functools import cached_property
+
+from ..grids import nearest_grid_points
+from .debug import Node
+from .forwards import Combined
+from .indexing import apply_index_to_slices_changes
+from .indexing import index_to_slices
+from .indexing import update_tuple
+from .misc import _auto_adjust
+from .misc import _open
+
+LOG = logging.getLogger(__name__)
+
+
+class Complement(Combined):
+
+    def __init__(self, target, source, what="variables", interpolation="nearest"):
+        super().__init__([target, source])
+
+        # We had the variables of dataset[1] to dataset[0]
+        # interpoated on the grid of dataset[0]
+
+        self.target = target
+        self.source = source
+
+        self._variables = []
+
+        # Keep the same order as the original dataset
+        for v in self.source.variables:
+            if v not in self.target.variables:
+                self._variables.append(v)
+
+        if not self._variables:
+            raise ValueError("Augment: no missing variables")
+
+    @property
+    def variables(self):
+        return self._variables
+
+    @property
+    def name_to_index(self):
+        return {v: i for i, v in enumerate(self.variables)}
+
+    @property
+    def shape(self):
+        shape = self.target.shape
+        return (shape[0], len(self._variables)) + shape[2:]
+
+    @property
+    def variables_metadata(self):
+        return {k: v for k, v in self.source.variables_metadata.items() if k in self._variables}
+
+    def check_same_variables(self, d1, d2):
+        pass
+
+    @cached_property
+    def missing(self):
+        missing = self.source.missing.copy()
+        missing = missing | self.target.missing
+        return set(missing)
+
+    def tree(self):
+        """Generates a hierarchical tree structure for the `Cutout` instance and
+        its associated datasets.
+
+        Returns:
+            Node: A `Node` object representing the `Cutout` instance as the root
+            node, with each dataset in `self.datasets` represented as a child
+            node.
+        """
+        return Node(self, [d.tree() for d in (self.target, self.source)])
+
+    def __getitem__(self, index):
+        if isinstance(index, (int, slice)):
+            index = (index, slice(None), slice(None), slice(None))
+        return self._get_tuple(index)
+
+
+class ComplementNone(Complement):
+
+    def __init__(self, target, source):
+        super().__init__(target, source)
+
+    def _get_tuple(self, index):
+        index, changes = index_to_slices(index, self.shape)
+        result = self.source[index]
+        return apply_index_to_slices_changes(result, changes)
+
+
+class ComplementNearest(Complement):
+
+    def __init__(self, target, source):
+        super().__init__(target, source)
+
+        self._nearest_grid_points = nearest_grid_points(
+            self.source.latitudes,
+            self.source.longitudes,
+            self.target.latitudes,
+            self.target.longitudes,
+        )
+
+    def check_compatibility(self, d1, d2):
+        pass
+
+    def _get_tuple(self, index):
+        variable_index = 1
+        index, changes = index_to_slices(index, self.shape)
+        index, previous = update_tuple(index, variable_index, slice(None))
+        source_index = [self.source.name_to_index[x] for x in self.variables[previous]]
+        source_data = self.source[index[0], source_index, index[2], ...]
+        target_data = source_data[..., self._nearest_grid_points]
+
+        result = target_data[..., index[3]]
+
+        return apply_index_to_slices_changes(result, changes)
+
+
+def complement_factory(args, kwargs):
+    from .select import Select
+
+    assert len(args) == 0, args
+
+    source = kwargs.pop("source")
+    target = kwargs.pop("complement")
+    what = kwargs.pop("what", "variables")
+    interpolation = kwargs.pop("interpolation", "none")
+
+    if what != "variables":
+        raise NotImplementedError(f"Complement what={what} not implemented")
+
+    if interpolation not in ("none", "nearest"):
+        raise NotImplementedError(f"Complement method={interpolation} not implemented")
+
+    source = _open(source)
+    target = _open(target)
+    # `select` is the same as `variables`
+    (source, target), kwargs = _auto_adjust([source, target], kwargs, exclude=["select"])
+
+    Class = {
+        None: ComplementNone,
+        "none": ComplementNone,
+        "nearest": ComplementNearest,
+    }[interpolation]
+
+    complement = Class(target=target, source=source)._subset(**kwargs)
+
+    # Will join the datasets along the variables axis
+    reorder = source.variables
+    complemented = _open([target, complement])
+    ordered = (
+        Select(complemented, complemented._reorder_to_columns(reorder), {"reoder": reorder})._subset(**kwargs).mutate()
+    )
+    return ordered

anemoi/datasets/data/dataset.py

@@ -168,6 +168,16 @@ class Dataset:
             bbox = kwargs.pop("area")
             return Cropping(self, bbox)._subset(**kwargs).mutate()
 
+        if "number" in kwargs or "numbers" or "member" in kwargs or "members" in kwargs:
+            from .ensemble import Number
+
+            members = {}
+            for key in ["number", "numbers", "member", "members"]:
+                if key in kwargs:
+                    members[key] = kwargs.pop(key)
+
+            return Number(self, **members)._subset(**kwargs).mutate()
+
         if "set_missing_dates" in kwargs:
             from .missing import MissingDates
 
@@ -251,13 +261,19 @@ class Dataset:
         return sorted([v for k, v in self.name_to_index.items() if k not in vars])
 
     def _reorder_to_columns(self, vars):
+        if isinstance(vars, str) and vars == "sort":
+            # Sorting the variables alphabetically.
+            # This is cruical for pre-training then transfer learning in combination with
+            # cutout and adjust = 'all'
+
+            indices = [self.name_to_index[k] for k, v in sorted(self.name_to_index.items(), key=lambda x: x[0])]
+            assert set(indices) == set(range(len(self.name_to_index)))
+            return indices
+
         if isinstance(vars, (list, tuple)):
             vars = {k: i for i, k in enumerate(vars)}
 
-        indices = []
-
-        for k, v in sorted(vars.items(), key=lambda x: x[1]):
-            indices.append(self.name_to_index[k])
+        indices = [self.name_to_index[k] for k, v in sorted(vars.items(), key=lambda x: x[1])]
 
         # Make sure we don't forget any variables
         assert set(indices) == set(range(len(self.name_to_index)))
@@ -469,7 +485,7 @@ class Dataset:
         sample_count = min(4, len(indices))
         count = len(indices)
 
-        p = slice(0, count, count // (sample_count - 1))
+        p = slice(0, count, count // max(1, sample_count - 1))
         samples = list(range(*p.indices(count)))
 
         samples.append(count - 1)  # Add last
@@ -502,3 +518,50 @@ class Dataset:
                 result.append(v)
 
         return result
+
+    def plot(self, date, variable, member=0, **kwargs):
+        """For debugging purposes, plot a field.
+
+        Parameters
+        ----------
+        date : int or datetime.datetime or numpy.datetime64 or str
+            The date to plot.
+        variable : int or str
+            The variable to plot.
+        member : int, optional
+            The ensemble member to plot.
+
+        **kwargs:
+            Additional arguments to pass to matplotlib.pyplot.tricontourf
+
+
+        Returns
+        -------
+            matplotlib.pyplot.Axes
+        """
+
+        from anemoi.utils.devtools import plot_values
+        from earthkit.data.utils.dates import to_datetime
+
+        if not isinstance(date, int):
+            date = np.datetime64(to_datetime(date)).astype(self.dates[0].dtype)
+            index = np.where(self.dates == date)[0]
+            if len(index) == 0:
+                raise ValueError(
+                    f"Date {date} not found in the dataset {self.dates[0]} to {self.dates[-1]} by {self.frequency}"
+                )
+            date_index = index[0]
+        else:
+            date_index = date
+
+        if isinstance(variable, int):
+            variable_index = variable
+        else:
+            if variable not in self.variables:
+                raise ValueError(f"Unknown variable {variable} (available: {self.variables})")
+
+            variable_index = self.name_to_index[variable]
+
+        values = self[date_index, variable_index, member]
+
+        return plot_values(values, self.latitudes, self.longitudes, **kwargs)

anemoi/datasets/data/ensemble.py

@@ -10,13 +10,68 @@
 
 import logging
 
+import numpy as np
+
 from .debug import Node
+from .forwards import Forwards
 from .forwards import GivenAxis
+from .indexing import apply_index_to_slices_changes
+from .indexing import index_to_slices
+from .indexing import update_tuple
 from .misc import _auto_adjust
 from .misc import _open
 
 LOG = logging.getLogger(__name__)
 
+OFFSETS = dict(number=1, numbers=1, member=0, members=0)
+
+
+class Number(Forwards):
+    def __init__(self, forward, **kwargs):
+        super().__init__(forward)
+
+        self.members = []
+        for key, values in kwargs.items():
+            if not isinstance(values, (list, tuple)):
+                values = [values]
+            self.members.extend([int(v) - OFFSETS[key] for v in values])
+
+        self.members = sorted(set(self.members))
+        for n in self.members:
+            if not (0 <= n < forward.shape[2]):
+                raise ValueError(f"Member {n} is out of range. `number(s)` is one-based, `member(s)` is zero-based.")
+
+        self.mask = np.array([n in self.members for n in range(forward.shape[2])], dtype=bool)
+        self._shape, _ = update_tuple(forward.shape, 2, len(self.members))
+
+    @property
+    def shape(self):
+        return self._shape
+
+    def __getitem__(self, index):
+        if isinstance(index, int):
+            result = self.forward[index]
+            result = result[:, self.mask, :]
+            return result
+
+        if isinstance(index, slice):
+            result = self.forward[index]
+            result = result[:, :, self.mask, :]
+            return result
+
+        index, changes = index_to_slices(index, self.shape)
+        result = self.forward[index]
+        result = result[:, :, self.mask, :]
+        return apply_index_to_slices_changes(result, changes)
+
+    def tree(self):
+        return Node(self, [self.forward.tree()], numbers=[n + 1 for n in self.members])
+
+    def metadata_specific(self):
+        return {
+            "numbers": [n + 1 for n in self.members],
+        }
+
 
 class Ensemble(GivenAxis):
     def tree(self):

anemoi/datasets/data/join.py

@@ -118,6 +118,7 @@ class Join(Combined):
     def variables_metadata(self):
         result = {}
         variables = [v for v in self.variables if not (v.startswith("(") and v.endswith(")"))]
+
         for d in self.datasets:
             md = d.variables_metadata
             for v in variables:
@@ -130,8 +131,6 @@ class Join(Combined):
                 if v not in result:
                     LOG.error("Missing metadata for %r.", v)
 
-            raise ValueError("Some variables are missing metadata.")
-
         return result
 
     @cached_property

anemoi/datasets/data/merge.py

@@ -134,6 +134,9 @@ class Merge(Combined):
     def tree(self):
         return Node(self, [d.tree() for d in self.datasets], allow_gaps_in_dates=self.allow_gaps_in_dates)
 
+    def metadata_specific(self):
+        return {"allow_gaps_in_dates": self.allow_gaps_in_dates}
+
     @debug_indexing
     def __getitem__(self, n):
         if isinstance(n, tuple):

anemoi/datasets/data/misc.py

@@ -194,7 +194,7 @@ def _open(a):
     raise NotImplementedError(f"Unsupported argument: {type(a)}")
 
 
-def _auto_adjust(datasets, kwargs):
+def _auto_adjust(datasets, kwargs, exclude=None):
 
     if "adjust" not in kwargs:
         return datasets, kwargs
@@ -214,6 +214,9 @@ def _auto_adjust(datasets, kwargs):
     for a in adjust_list:
         adjust_set.update(ALIASES.get(a, [a]))
 
+    if exclude is not None:
+        adjust_set -= set(exclude)
+
     extra = set(adjust_set) - set(ALIASES["all"])
     if extra:
         raise ValueError(f"Invalid adjust keys: {extra}")
@@ -335,6 +338,12 @@ def _open_dataset(*args, **kwargs):
         assert not sets, sets
         return cutout_factory(args, kwargs).mutate()
 
+    if "complement" in kwargs:
+        from .complement import complement_factory
+
+        assert not sets, sets
+        return complement_factory(args, kwargs).mutate()
+
     for name in ("datasets", "dataset"):
         if name in kwargs:
             datasets = kwargs.pop(name)

anemoi/datasets/grids.py

@@ -152,7 +152,7 @@ def cutout_mask(
     plot=None,
 ):
     """Return a mask for the points in [global_lats, global_lons] that are inside of [lats, lons]"""
-    from scipy.spatial import KDTree
+    from scipy.spatial import cKDTree
 
     # TODO: transform min_distance from lat/lon to xyz
 
@@ -195,13 +195,13 @@ def cutout_mask(
         min_distance = min_distance_km / 6371.0
     else:
         points = {"lam": lam_points, "global": global_points, None: global_points}[min_distance_km]
-        distances, _ = KDTree(points).query(points, k=2)
+        distances, _ = cKDTree(points).query(points, k=2)
         min_distance = np.min(distances[:, 1])
 
         LOG.info(f"cutout_mask using min_distance = {min_distance * 6371.0} km")
 
-    # Use a KDTree to find the nearest points
-    distances, indices = KDTree(lam_points).query(global_points, k=neighbours)
+    # Use a cKDTree to find the nearest points
+    distances, indices = cKDTree(lam_points).query(global_points, k=neighbours)
 
     # Centre of the Earth
     zero = np.array([0.0, 0.0, 0.0])
@@ -255,7 +255,7 @@ def thinning_mask(
     cropping_distance=2.0,
 ):
     """Return the list of points in [lats, lons] closest to [global_lats, global_lons]"""
-    from scipy.spatial import KDTree
+    from scipy.spatial import cKDTree
 
     assert global_lats.ndim == 1
     assert global_lons.ndim == 1
@@ -291,20 +291,20 @@ def thinning_mask(
     xyx = latlon_to_xyz(lats, lons)
     points = np.array(xyx).transpose()
 
-    # Use a KDTree to find the nearest points
-    _, indices = KDTree(points).query(global_points, k=1)
+    # Use a cKDTree to find the nearest points
+    _, indices = cKDTree(points).query(global_points, k=1)
 
     return np.array([i for i in indices])
 
 
 def outline(lats, lons, neighbours=5):
-    from scipy.spatial import KDTree
+    from scipy.spatial import cKDTree
 
     xyx = latlon_to_xyz(lats, lons)
     grid_points = np.array(xyx).transpose()
 
-    # Use a KDTree to find the nearest points
-    _, indices = KDTree(grid_points).query(grid_points, k=neighbours)
+    # Use a cKDTree to find the nearest points
+    _, indices = cKDTree(grid_points).query(grid_points, k=neighbours)
 
     # Centre of the Earth
     zero = np.array([0.0, 0.0, 0.0])
@@ -379,6 +379,19 @@ def serialise_mask(mask):
     return result
 
 
+def nearest_grid_points(source_latitudes, source_longitudes, target_latitudes, target_longitudes):
+    from scipy.spatial import cKDTree
+
+    source_xyz = latlon_to_xyz(source_latitudes, source_longitudes)
+    source_points = np.array(source_xyz).transpose()
+
+    target_xyz = latlon_to_xyz(target_latitudes, target_longitudes)
+    target_points = np.array(target_xyz).transpose()
+
+    _, indices = cKDTree(source_points).query(target_points, k=1)
+    return indices
+
+
 if __name__ == "__main__":
     global_lats, global_lons = np.meshgrid(
         np.linspace(90, -90, 90),

{anemoi_datasets-0.5.12.dist-info → anemoi_datasets-0.5.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: anemoi-datasets
-Version: 0.5.12
+Version: 0.5.13
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_datasets-0.5.12.dist-info → anemoi_datasets-0.5.13.dist-info}/RECORD

@@ -1,7 +1,7 @@
 anemoi/datasets/__init__.py,sha256=0GOHATiKgkUqLRgAVQhNP1aPO7ULfSr8DqUf2ANPEv8,1010
 anemoi/datasets/__main__.py,sha256=5NW2A3OgTimB4ptwYThivIRSeCrvabMuvnr8mmnVx0E,715
-anemoi/datasets/_version.py,sha256=ZArwdbjC4yDqbe9dEwhZVS693DGl1K3jnRxJy_dutjo,413
-anemoi/datasets/grids.py,sha256=bq7pB_6uswILT3t8C8SeUpUrBww31dw5au_USrped6c,10919
+anemoi/datasets/_version.py,sha256=Ywt_J-kUrkp4bt0vX46Ot5kDjsAA9dtf8IfOdhjGcXA,413
+anemoi/datasets/grids.py,sha256=xHZwE3pJs0wP05x9qCgMcTuBnBGoCU51YKx5JQ7Ukts,11398
 anemoi/datasets/testing.py,sha256=7HGOz5_V9MbkHTDJ4KbklGRndBMrFfVrBBu6a9k0_qY,1825
 anemoi/datasets/commands/__init__.py,sha256=O5W3yHZywRoAqmRUioAr3zMCh0hGVV18wZYGvc00ioM,698
 anemoi/datasets/commands/cleanup.py,sha256=2rD34bHtfOCLwQh7yXa02IJmmOYMOma4YDj0PM-2-Jc,1456
@@ -20,20 +20,21 @@ anemoi/datasets/commands/publish.py,sha256=z1MV9_1BsEnw81Y_17fHkKGYe8_ZJo9eeQ1kG
 anemoi/datasets/commands/scan.py,sha256=mXzYEcYsncxC7ItyL_TlVRiWji6OFYfVxO5OMD9mbEI,3304
 anemoi/datasets/compute/__init__.py,sha256=hCW0QcLHJmE-C1r38P27_ZOvCLNewex5iQEtZqx2ckI,393
 anemoi/datasets/compute/recentre.py,sha256=tKs-YZLhqsMRBNEUF41hcuMmyvbRuFX07xJq-Cqg2_w,4954
-anemoi/datasets/create/__init__.py,sha256=q8JIS6mcfJhMD8-uJN-EMkaG4DwKNTIJy53iGKHhiXw,36259
-anemoi/datasets/create/check.py,sha256=1fT1au1LmSGKTQYhnmMK9ImSZnLLevkKX1a0jUtEuVc,6148
+anemoi/datasets/create/__init__.py,sha256=ImonAvvz-HqoZMF4vczyWEV48tGiZZ8RfM01SsxJ6ew,36427
+anemoi/datasets/create/check.py,sha256=q205XxzR7UtBRI5qOANav_NAVqAERs0aLJ8oBL3VNc4,6153
 anemoi/datasets/create/chunks.py,sha256=c7ufk-EamAGqxOI4ScTFlBzdEiH6V1R0f0SPavtZ2Xw,2457
 anemoi/datasets/create/config.py,sha256=Tq9kJ-bKhYSTWCXNa8lZIJpO3RteZfCr5hQVM12IgD0,8892
 anemoi/datasets/create/patch.py,sha256=YkDiFGV0DDg4WShVrIHhtRaAp9c82mANnR93YTVl36o,4171
 anemoi/datasets/create/persistent.py,sha256=V3agRYKbmYqAyhsznsgC3PLSoFbFCSCXqcQZvvJfVTw,4381
 anemoi/datasets/create/size.py,sha256=5fIF0yo63dTowojfeG9-MNkgypP5SA94WralxTZwxw4,1055
-anemoi/datasets/create/utils.py,sha256=jwbGQ_o-4U8n-lgrrs2Oia1WrGonEHfxaCg8dUlA6S0,3088
+anemoi/datasets/create/utils.py,sha256=3vRCPOrW0a7ZVx7gsflORwab74LV_BVVkgo_3SgyLpw,3347
 anemoi/datasets/create/writer.py,sha256=6XSIgNwKkjrkdnSvr69mXD8McjT3iYJ0d1rOnxaGuCQ,1394
 anemoi/datasets/create/zarr.py,sha256=0DkHwKaBpvTOrU1KmScLEfff_KfT1Rw5DXiRMD2d3Ws,5506
-anemoi/datasets/create/functions/__init__.py,sha256=e361XrjNV_4LCBmxVMluZ3KczFn9bwLSk5vcwy0zzkg,1382
+anemoi/datasets/create/functions/__init__.py,sha256=qeoEndq4f0gosfyfb-t0CZcLdngJ4GlZIgGs0bp9j0U,1965
 anemoi/datasets/create/functions/filters/__init__.py,sha256=SP6ReV1WYIf2Typf1FUaRHhphFGpU9kBoYtI-bYdu5U,395
 anemoi/datasets/create/functions/filters/empty.py,sha256=EGLufFcNFoqIXTZj7jQFjtFahMfgCVWj6W5j--u5Q-Q,636
 anemoi/datasets/create/functions/filters/noop.py,sha256=5YqumPxlGDOZOrYWayLr8PTycVWG2X_0PmoFi3Hj584,458
+anemoi/datasets/create/functions/filters/orog_to_z.py,sha256=PZwqiTVBLlwp2yuHCW_D8Epcb0fPNjzuYYtmL3Gu1XY,1658
 anemoi/datasets/create/functions/filters/pressure_level_relative_humidity_to_specific_humidity.py,sha256=jjmocA4WDKCAL49QUFk_3S0JRiPMmeVM7Wlxmfr1v6c,1857
 anemoi/datasets/create/functions/filters/pressure_level_specific_humidity_to_relative_humidity.py,sha256=e8LvXUq-qNKJrcjb1DSUXaPeFfxcWxFjGAkm47cOnE8,1855
 anemoi/datasets/create/functions/filters/rename.py,sha256=cDF3xmdhwzIZn_nwaO3hxG4fb2vpKtJtmy0ZdLGXyHI,2481
@@ -43,10 +44,12 @@ anemoi/datasets/create/functions/filters/single_level_relative_humidity_to_dewpo
 anemoi/datasets/create/functions/filters/single_level_relative_humidity_to_specific_humidity.py,sha256=BnuLrIFcOh_qJBmxwdJqjGqoH0ca5zyKdZgF6QPmJY8,4090
 anemoi/datasets/create/functions/filters/single_level_specific_humidity_to_relative_humidity.py,sha256=xqfklEwCqrQlhU6NV8vlVEZdY-hN3SpPpcNny2geVUI,12686
 anemoi/datasets/create/functions/filters/speeddir_to_uv.py,sha256=d5t78GToTTXCb1S3HyhTJ2tuwZDnk7UBsHPV4Wn4M_w,2249
+anemoi/datasets/create/functions/filters/sum.py,sha256=_f_xyIAbGdKoCXdd5zO7XOL2AHq_c04DFO8s6PhRR8I,1979
 anemoi/datasets/create/functions/filters/unrotate_winds.py,sha256=tDFXUSF2flD83W7GgwP1RoVXBUO0445DvQdImulzDzA,2429
 anemoi/datasets/create/functions/filters/uv_to_speeddir.py,sha256=niNuTSmyxLn4MGeNL1lowl5M0dH7har-flXy3ZtmKPM,1762
+anemoi/datasets/create/functions/filters/wz_to_w.py,sha256=SbTYE6rRjObR-sJEDYyc0-1Kw39zZOAheGMznD7Ic9A,2161
 anemoi/datasets/create/functions/sources/__init__.py,sha256=TMm8LerGY7--b0AMUqnz07ZGo-F7I9FF0DGlozcTtSg,1364
-anemoi/datasets/create/functions/sources/accumulations.py,sha256=tI5Aga1aLDp8DqfYbU_9AI1VIqDZzhMlZUAR4ik-CUI,12316
+anemoi/datasets/create/functions/sources/accumulations.py,sha256=xs3Ql3h_jfJDR0mzGIh54adVaHacHE3LQpBL5_8Rx8k,12373
 anemoi/datasets/create/functions/sources/constants.py,sha256=GaiUpJPYupiLWl8O9GEZ9KmlD88pH6dlBiUVrhNl_uA,918
 anemoi/datasets/create/functions/sources/empty.py,sha256=YTpOJ3rcb_eS9CbnpwPWBR9r1APIAaG6a_N803YFZFE,500
 anemoi/datasets/create/functions/sources/forcings.py,sha256=p442lCOXm8TJFRlP0mgwZujveo9gCtdAGLS4KSIqYfk,661
@@ -61,15 +64,16 @@ anemoi/datasets/create/functions/sources/tendencies.py,sha256=z8iDelu0vvDE8S-Rus
 anemoi/datasets/create/functions/sources/xarray_kerchunk.py,sha256=8evD6Sype3ffCbmQ0jMBpgR97UeNvkTB5rwchhy4YzY,1446
 anemoi/datasets/create/functions/sources/xarray_zarr.py,sha256=3JvoGfQZ4NCUcfxDAbNZOL7z2VRNJzr1H3r8dsWbrgk,545
 anemoi/datasets/create/functions/sources/zenodo.py,sha256=rPL9uNPeFTdI9XvVEahtHkxzE18MyrjNXZjpt_sNeH4,1251
-anemoi/datasets/create/functions/sources/xarray/__init__.py,sha256=fEJI2OGOtk-CJqCzaege0uwPKGpqEkeKxgt2bN-W934,3136
+anemoi/datasets/create/functions/sources/xarray/__init__.py,sha256=byu5zPP_4b7CjgSKvO3iL4xyZPmdoEVX93Tl7LBZc0c,3174
 anemoi/datasets/create/functions/sources/xarray/coordinates.py,sha256=-FkcAaio2KumOd20eb1hLv9rRhjnu-CyqtqzrMsZx18,6213
-anemoi/datasets/create/functions/sources/xarray/field.py,sha256=70_3wxBE-flftWqyHAyBP7FhwZOHgrSW6WDmA8QGszs,3680
-anemoi/datasets/create/functions/sources/xarray/fieldlist.py,sha256=wBHPQPm1oJnaVGO6p0opx9tEOIJkdINw_cCTG1y9GYk,5884
+anemoi/datasets/create/functions/sources/xarray/field.py,sha256=VfEuY-o1KZS1Bn4l7pR8FCx9hTtDbzKzPqJfwunwvRE,3816
+anemoi/datasets/create/functions/sources/xarray/fieldlist.py,sha256=3wCLbdqpPlBlzJHKp_ETxAochPA9iFDyF94JVn1DOB8,6281
 anemoi/datasets/create/functions/sources/xarray/flavour.py,sha256=6mqldGyx40Zgy4_VkuGWKgrSuPbWKe__nmEradQO5qg,14855
 anemoi/datasets/create/functions/sources/xarray/grid.py,sha256=OuLBVv_CdgtLgGACpqhjX8fwtYzM7tfJiwUOXbG_ifw,3644
-anemoi/datasets/create/functions/sources/xarray/metadata.py,sha256=GXHjPm8MsFU9fBfJCjllPWtKahBcKWtQwj0yA7zt4hU,4573
-anemoi/datasets/create/functions/sources/xarray/time.py,sha256=RKODUhgGtApvee3C8SdcBpz2wlQ4zJhyov1UPvblsr4,5987
-anemoi/datasets/create/functions/sources/xarray/variable.py,sha256=j2NN-zx-yjprIJLcidxMxFC4QQZ7I8Dd0nKMrgRpsdM,4385
+anemoi/datasets/create/functions/sources/xarray/metadata.py,sha256=zbbb0ssKhZJvogLJ1WPJMBVVHl40GjHWbmE6RzLwAz4,4336
+anemoi/datasets/create/functions/sources/xarray/patch.py,sha256=k1v7bUs-sO7-431T0bh5CSTE1FtgjhIlaPQ2-kSpc2E,1051
+anemoi/datasets/create/functions/sources/xarray/time.py,sha256=jGnaupnNQr9x4F7ijahzxtMQltC5fLbrEKajq5dIxR8,6458
+anemoi/datasets/create/functions/sources/xarray/variable.py,sha256=IdxZGOu1DMaUVlDGyVHuZiGUsN4buJoxexSFUD_NyFg,5029
 anemoi/datasets/create/input/__init__.py,sha256=cAwfW9AQiG2PfmZ2Irll7HX8HyiC0Nk1Q9OhoQ84ZAg,1625
 anemoi/datasets/create/input/action.py,sha256=SApZApq-_mlOwk1NTERgQlPdPL8lBlIk6rxYX3JBw_E,3857
 anemoi/datasets/create/input/concat.py,sha256=DwxgoTSTqNDsVcX5btUBAA7vXtX3G5m-zJ-jDrmAC-c,3279
@@ -81,28 +85,29 @@ anemoi/datasets/create/input/function.py,sha256=F5GQgbtFYmyqFAgNGoGDuWw-xqkcCLzu
 anemoi/datasets/create/input/join.py,sha256=wQP1-vVg4as-R5i3pstgK6HmTJAY7WyWYhCEF6FIU1c,1991
 anemoi/datasets/create/input/misc.py,sha256=r7NC_QRYA8iiJJbSFgQnNuixymATK0CPZknGxgYcLOk,1975
 anemoi/datasets/create/input/pipe.py,sha256=KfPCtiqyfqkXbmC-2LTqHkCQ7bJY46XMvNDnp9QeHTQ,1344
-anemoi/datasets/create/input/repeated_dates.py,sha256=biwnjgaRGzJiFk5fAY-YA4FotzdlwkY56GKUetzOulg,6840
+anemoi/datasets/create/input/repeated_dates.py,sha256=59EvJ_cQwA-p_42cmMFy3pBAAWV0xwPg4E3q2PIofcM,7461
 anemoi/datasets/create/input/result.py,sha256=-pcVcaaj3G_xcNKWWTgzVH5Ds5-ETWmErN0KeQGitAw,20013
 anemoi/datasets/create/input/step.py,sha256=CoowF9mc3kepT8XQ2ObxO750rnQEkYNTviIHQ1m-4UA,2886
 anemoi/datasets/create/input/template.py,sha256=Vgi4wQ1aeswLbji0fIzshYhISmzdrt7b0BmgeJJjYGc,1859
 anemoi/datasets/create/input/trace.py,sha256=DYXMSnwKqOIx0XWZTKNJojWz4EqaFLknTh6ysxsW9uY,2198
-anemoi/datasets/create/statistics/__init__.py,sha256=l6VE00sfcfqBg6cDFJTian-DLnvwt1QYEYq0lCHZ0PY,12786
+anemoi/datasets/create/statistics/__init__.py,sha256=iJ3mZ6eEI88wPXUKyOhNKqhakyHoceX9ICEKXVOriTo,12789
 anemoi/datasets/create/statistics/summary.py,sha256=wmnz4fZkr6fomXgI8JlMutU8gakfrXTc5ixf3Np7gZA,3385
 anemoi/datasets/data/__init__.py,sha256=AW1-Ycj77pWQsZcDGsp0pgTS5rFW6XC4CzuUEIUPAIk,1558
+anemoi/datasets/data/complement.py,sha256=hTvA_zTGIHAvZYPv1npVpRpxPg8nXbNTEBAU8r33rlc,5105
 anemoi/datasets/data/concat.py,sha256=udtYINuoLOEYYKhi_VpG2-emv80pwZbFAZKwNwXJk3s,5244
-anemoi/datasets/data/dataset.py,sha256=73NjQo7wUSWqfdpyVVZIpWgEmy1x9-beN-kVtgfhBJE,15587
+anemoi/datasets/data/dataset.py,sha256=x_ID6Ga_TbBfECqhOC4i3CTqo0UD34KJVqUWpvI9Si4,17926
 anemoi/datasets/data/debug.css,sha256=z2X_ZDSnZ9C3pyZPWnQiEyAxuMxUaxJxET4oaCImTAQ,211
 anemoi/datasets/data/debug.py,sha256=IjCMwtAvknF51PCl_YRYgMZB2iX_9DC5DKILNgl_UHQ,6300
-anemoi/datasets/data/ensemble.py,sha256=KNIXDfjYSIo6JVn1bD9X92yffd4Gg83wn_2sGxqAnWU,1111
+anemoi/datasets/data/ensemble.py,sha256=7ognsmoHDGw0cCs3hsARoV32J1qlQys6iUCJ7XSrARI,2923
 anemoi/datasets/data/fill_missing.py,sha256=4btLi-D-hFTsS_57_gIC1nK5AVifAO-V4M-fqMrtrxk,4636
 anemoi/datasets/data/forwards.py,sha256=P9DfSY5B9w9gtkKfV6TIzXel_LY83g-2nEreJy2rYkU,8916
 anemoi/datasets/data/grids.py,sha256=p7_nT7RLH6uKcxeAzQiGYk9lFxU_OOikDrwlb2rdEqI,15765
 anemoi/datasets/data/indexing.py,sha256=9lycQXSqUIbYj52JlFv0w_Gf6soVZnbVGswYMvGPpqs,4773
 anemoi/datasets/data/interpolate.py,sha256=D27lSH8yNhm0aoO0U3UoRbr3kni7OWXSu_X4jCbIrA0,4137
-anemoi/datasets/data/join.py,sha256=xl7SrjhggZdX_bdni5-cn8-BYGUYfEtogQeIqCgSL7U,5525
+anemoi/datasets/data/join.py,sha256=IG9Bj4o4Z25cl5YGMqtl75UuSZCWIJwGIUB0fsfnkE8,5456
 anemoi/datasets/data/masked.py,sha256=eAVGVmQR7tWsd3xXYGXGyq28uRLwL50vOXWTNNdHxl0,4530
-anemoi/datasets/data/merge.py,sha256=6vYRy3-P4J9GgTPkdoPFs9CbZ5F0m8FAukS-P66BR_M,5448
-anemoi/datasets/data/misc.py,sha256=IKqtvcU36nyB4z6dfu7W_gnCpaeX20fK2C5A2seWdCA,10061
+anemoi/datasets/data/merge.py,sha256=dr0sX2ufm-qOgOAMV5oh8qQwPvSdYbU-mhux6u-cmQw,5547
+anemoi/datasets/data/misc.py,sha256=J1v84jHpRgDK0DUrNmII5oqt3jft8rSTve2GtxqTKa8,10310
 anemoi/datasets/data/missing.py,sha256=SWEjiC1usBjZtlKMr73uKetnoQZoflVQVGqLP2gJR7A,7131
 anemoi/datasets/data/rescale.py,sha256=wMU7tFZebnOqJJxaIGOqNqhpNKGsPNZMC1YxuiHvri4,4112
 anemoi/datasets/data/select.py,sha256=XW_ohlhrF8FLe13pdM3DRZDxbHxntcsO0F56GRqZQY0,4293
@@ -115,9 +120,9 @@ anemoi/datasets/dates/__init__.py,sha256=wX2FvlmRfHV5HDmllIxwfrC1LuRlb7i6SguLLas
 anemoi/datasets/dates/groups.py,sha256=i7x8z0kv6E8qUfm1tMZR1aaOqNwQzEkV-VWpOvHjoX4,5390
 anemoi/datasets/utils/__init__.py,sha256=hCW0QcLHJmE-C1r38P27_ZOvCLNewex5iQEtZqx2ckI,393
 anemoi/datasets/utils/fields.py,sha256=l7xKOiRLgk9Eewykqu7xZP9xOajG2dx2CiDlGvBVejU,1411
-anemoi_datasets-0.5.12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_datasets-0.5.12.dist-info/METADATA,sha256=x548Cd_PFUCse1QR5dj-8rNfUx94ZU45g__IjPaGLgY,15598
-anemoi_datasets-0.5.12.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-anemoi_datasets-0.5.12.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
-anemoi_datasets-0.5.12.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_datasets-0.5.12.dist-info/RECORD,,
+anemoi_datasets-0.5.13.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_datasets-0.5.13.dist-info/METADATA,sha256=ArEcpcTEQ_Tt270GtB5GEeDX_1SJMKM8P4NWMdSve8M,15598
+anemoi_datasets-0.5.13.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+anemoi_datasets-0.5.13.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
+anemoi_datasets-0.5.13.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_datasets-0.5.13.dist-info/RECORD,,

{anemoi_datasets-0.5.12.dist-info → anemoi_datasets-0.5.13.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (75.8.0)
 Root-Is-Purelib: true
 Tag: py3-none-any