anemoi-datasets 0.3.10__py3-none-any.whl → 0.4.0__py3-none-any.whl

anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.10'
-__version_tuple__ = version_tuple = (0, 3, 10)
+__version__ = version = '0.4.0'
+__version_tuple__ = version_tuple = (0, 4, 0)

anemoi/datasets/commands/inspect.py

@@ -382,7 +382,7 @@ class NoVersion(Version):
     @property
     def last_date(self):
         monthly = find(self.metadata, "monthly")
-        time = max([int(t) for t in find(self.metadata["climetlab"], "time")])
+        time = max([int(t) for t in find(self.metadata["earthkit-data"], "time")])
         assert isinstance(time, int), (time, type(time))
         if time > 100:
             time = time // 100
@@ -390,7 +390,7 @@ class NoVersion(Version):
 
     @property
     def frequency(self):
-        time = find(self.metadata["climetlab"], "time")
+        time = find(self.metadata["earthkit-data"], "time")
         return 24 // len(time)
 
     @property
@@ -444,9 +444,9 @@ class Version0_4(Version):
         z = self.zarr
 
         # for backward compatibility
-        if "climetlab" in z.attrs:
-            climetlab_version = z.attrs["climetlab"].get("versions", {}).get("climetlab", "unkwown")
-            print(f"climetlab version used to create this zarr: {climetlab_version}. Not supported.")
+        if "earthkit-data" in z.attrs:
+            ekd_version = z.attrs["earthkit-data"].get("versions", {}).get("earthkit-data", "unkwown")
+            print(f"earthkit-data version used to create this zarr: {ekd_version}. Not supported.")
             return
 
         version = z.attrs.get("version")
@@ -455,7 +455,7 @@ class Version0_4(Version):
             print(" Cannot find metadata information about versions.")
         else:
             print(f"Zarr format (version {version})", end="")
-            print(f" created by climetlab={versions.pop('climetlab')}", end="")
+            print(f" created by earthkit-data={versions.pop('earthkit-data')}", end="")
             timestamp = z.attrs.get("creation_timestamp")
             timestamp = datetime.datetime.fromisoformat(timestamp)
             print(f" on {timestamp}", end="")

anemoi/datasets/commands/scan.py

@@ -3,7 +3,7 @@ import os
 import sys
 from collections import defaultdict
 
-import climetlab as cml
+import earthkit.data as ekd
 import tqdm
 import yaml
 
@@ -50,9 +50,9 @@ class Scan(Command):
         for path in tqdm.tqdm(paths, leave=False):
             if not match(path):
                 continue
-            for field in tqdm.tqdm(cml.load_source("file", path), leave=False):
-                dates.add(field.valid_datetime())
-                mars = field.as_mars()
+            for field in tqdm.tqdm(ekd.from_source("file", path), leave=False):
+                dates.add(field.datetime()["valid_time"])
+                mars = field.metadata(namespace="mars")
                 keys = tuple(mars.get(k) for k in KEYS)
                 gribs[keys].add(path)
                 for k, v in mars.items():

anemoi/datasets/compute/recentre.py

@@ -10,10 +10,10 @@
 import logging
 
 import numpy as np
-from climetlab.core.temporary import temp_file
-from climetlab.readers.grib.output import new_grib_output
+from earthkit.data.core.temporary import temp_file
+from earthkit.data.readers.grib.output import new_grib_output
 
-from anemoi.datasets.create.functions import assert_is_fieldset
+from anemoi.datasets.create.functions import assert_is_fieldlist
 
 LOG = logging.getLogger(__name__)
 
@@ -96,7 +96,7 @@ def recentre(
 
     for i, centre_field in enumerate(centre):
         param = centre_field.metadata("param")
-        centre_field_as_mars = centre_field.as_mars()
+        centre_field_as_mars = centre_field.metadata(namespace="mars")
 
         # load the centre field
         centre_np = centre_field.to_numpy()
@@ -106,8 +106,13 @@ def recentre(
 
         for j in range(n_numbers):
             ensemble_field = members[i * n_numbers + j]
-            ensemble_field_as_mars = ensemble_field.as_mars()
-            check_compatible(centre_field, ensemble_field, centre_field_as_mars, ensemble_field_as_mars)
+            ensemble_field_as_mars = ensemble_field.metadata(namespace="mars")
+            check_compatible(
+                centre_field,
+                ensemble_field,
+                centre_field_as_mars,
+                ensemble_field_as_mars,
+            )
             members_np[j] = ensemble_field.to_numpy()
 
             ensemble_field_as_mars = tuple(sorted(ensemble_field_as_mars.items()))
@@ -149,10 +154,10 @@ def recentre(
     if output is not None:
         return path
 
-    from climetlab import load_source
+    from earthkit.data import from_source
 
-    ds = load_source("file", path)
-    assert_is_fieldset(ds)
+    ds = from_source("file", path)
+    assert_is_fieldlist(ds)
     # save a reference to the tmp file so it is deleted
     # only when the dataset is not used anymore
     ds._tmp = tmp

anemoi/datasets/create/config.py

@@ -12,7 +12,7 @@ import os
 from copy import deepcopy
 
 import yaml
-from climetlab.core.order import normalize_order_by
+from earthkit.data.core.order import normalize_order_by
 
 from .utils import load_json_or_yaml
 

anemoi/datasets/create/functions/__init__.py

@@ -13,10 +13,10 @@ import importlib
 import entrypoints
 
 
-def assert_is_fieldset(obj):
-    from climetlab.readers.grib.index import FieldSet
+def assert_is_fieldlist(obj):
+    from earthkit.data.indexing.fieldlist import FieldList
 
-    assert isinstance(obj, FieldSet), type(obj)
+    assert isinstance(obj, FieldList), type(obj)
 
 
 def import_function(name, kind):

anemoi/datasets/create/functions/filters/empty.py

@@ -7,10 +7,10 @@
 # nor does it submit to any jurisdiction.
 #
 
-import climetlab as cml
+import earthkit.data as ekd
 
 
 def execute(context, input, **kwargs):
-    # Usefull to create a pipeline that returns an empty result
-    # So we can reference an earlier step in a function like 'contants'
-    return cml.load_source("empty")
+    # Useful to create a pipeline that returns an empty result
+    # So we can reference an earlier step in a function like 'constants'
+    return ekd.from_source("empty")

anemoi/datasets/create/functions/filters/rename.py

@@ -9,7 +9,7 @@
 
 import re
 
-from climetlab.indexing.fieldset import FieldArray
+from earthkit.data.indexing.fieldlist import FieldArray
 
 
 class RenamedFieldMapping:
@@ -26,8 +26,8 @@ class RenamedFieldMapping:
         self.what = what
         self.renaming = renaming
 
-    def metadata(self, key):
-        value = self.field.metadata(key)
+    def metadata(self, key, **kwargs):
+        value = self.field.metadata(key, **kwargs)
         if key == self.what:
             return self.renaming.get(value, value)
         return value
@@ -48,10 +48,10 @@ class RenamedFieldFormat:
         self.format = format
         self.bits = re.findall(r"{(\w+)}", format)
 
-    def metadata(self, key):
-        value = self.field.metadata(key)
+    def metadata(self, key, **kwargs):
+        value = self.field.metadata(key, **kwargs)
         if "{" + key + "}" in self.format:
-            bits = {b: self.field.metadata(b) for b in self.bits}
+            bits = {b: self.field.metadata(b, **kwargs) for b in self.bits}
             return self.format.format(**bits)
         return value
 

anemoi/datasets/create/functions/filters/rotate_winds.py

@@ -9,61 +9,8 @@
 
 from collections import defaultdict
 
-from climetlab.indexing.fieldset import FieldArray
-
-
-def rotate_winds(lats, lons, x_wind, y_wind, source_projection, target_projection):
-    """Code provided by MetNO"""
-    import numpy as np
-    import pyproj
-
-    if source_projection == target_projection:
-        return x_wind, x_wind
-
-    source_projection = pyproj.Proj(source_projection)
-    target_projection = pyproj.Proj(target_projection)
-
-    transformer = pyproj.transformer.Transformer.from_proj(source_projection, target_projection)
-
-    # To compute the new vector components:
-    # 1) perturb each position in the direction of the winds
-    # 2) convert the perturbed positions into the new coordinate system
-    # 3) measure the new x/y components.
-    #
-    # A complication occurs when using the longlat "projections", since this is not a cartesian grid
-    # (i.e. distances in each direction is not consistent), we need to deal with the fact that the
-    # width of a longitude varies with latitude
-    orig_speed = np.sqrt(x_wind**2 + y_wind**2)
-
-    x0, y0 = source_projection(lons, lats)
-
-    if source_projection.name != "longlat":
-        x1 = x0 + x_wind
-        y1 = y0 + y_wind
-    else:
-        # Reduce the perturbation, since x_wind and y_wind are in meters, which would create
-        # large perturbations in lat, lon. Also, deal with the fact that the width of longitude
-        # varies with latitude.
-        factor = 3600000.0
-        x1 = x0 + x_wind / factor / np.cos(np.deg2rad(lats))
-        y1 = y0 + y_wind / factor
-
-    X0, Y0 = transformer.transform(x0, y0)
-    X1, Y1 = transformer.transform(x1, y1)
-
-    new_x_wind = X1 - X0
-    new_y_wind = Y1 - Y0
-    if target_projection.name == "longlat":
-        new_x_wind *= np.cos(np.deg2rad(lats))
-
-    if target_projection.name == "longlat" or source_projection.name == "longlat":
-        # Ensure the wind speed is not changed (which might not the case since the units in longlat
-        # is degrees, not meters)
-        curr_speed = np.sqrt(new_x_wind**2 + new_y_wind**2)
-        new_x_wind *= orig_speed / curr_speed
-        new_y_wind *= orig_speed / curr_speed
-
-    return new_x_wind, new_y_wind
+from earthkit.data.indexing.fieldlist import FieldArray
+from earthkit.geo.rotate import rotate_vector
 
 
 class NewDataField:
@@ -94,7 +41,7 @@ def execute(
     wind_pairs = defaultdict(dict)
 
     for f in input:
-        key = f.as_mars()
+        key = f.metadata(namespace="mars")
         param = key.pop("param")
 
         if param not in wind_params:
@@ -118,11 +65,11 @@ def execute(
         assert x.grid_mapping == y.grid_mapping
 
         lats, lons = x.grid_points()
-        x_new, y_new = rotate_winds(
+        x_new, y_new = rotate_vector(
             lats,
             lons,
-            x.to_numpy(reshape=False),
-            y.to_numpy(reshape=False),
+            x.to_numpy(flatten=True),
+            y.to_numpy(flatten=True),
             (source_projection if source_projection is not None else CRS.from_cf(x.grid_mapping)),
             target_projection,
         )

anemoi/datasets/create/functions/filters/unrotate_winds.py

@@ -9,60 +9,9 @@
 
 from collections import defaultdict
 
-import numpy as np
-from climetlab.indexing.fieldset import FieldArray
-
-
-def normalise(x):
-    return max(min(x, 1.0), -1.0)
-
-
-def normalise_longitude(lon, minimum):
-    while lon < minimum:
-        lon += 360
-
-    while lon >= minimum + 360:
-        lon -= 360
-
-    return lon
-
-
-def rotate_winds(
-    lats,
-    lons,
-    raw_lats,
-    raw_lons,
-    x_wind,
-    y_wind,
-    south_pole_latitude,
-    south_pole_longitude,
-    south_pole_rotation_angle=0,
-):
-    # Code from MIR
-    assert south_pole_rotation_angle == 0
-    C = np.deg2rad(90 - south_pole_latitude)
-    cos_C = np.cos(C)
-    sin_C = np.sin(C)
-
-    new_x = np.zeros_like(x_wind)
-    new_y = np.zeros_like(y_wind)
-
-    for i, (vx, vy, lat, lon, raw_lat, raw_lon) in enumerate(zip(x_wind, y_wind, lats, lons, raw_lats, raw_lons)):
-        lonRotated = south_pole_longitude - lon
-        lon_rotated = normalise_longitude(lonRotated, -180)
-        lon_unrotated = raw_lon
-
-        a = np.deg2rad(lon_rotated)
-        b = np.deg2rad(lon_unrotated)
-        q = 1 if (sin_C * lon_rotated < 0.0) else -1.0  # correct quadrant
-
-        cos_c = normalise(np.cos(a) * np.cos(b) + np.sin(a) * np.sin(b) * cos_C)
-        sin_c = q * np.sqrt(1.0 - cos_c * cos_c)
-
-        new_x[i] = cos_c * vx + sin_c * vy
-        new_y[i] = -sin_c * vx + cos_c * vy
-
-    return new_x, new_y
+# import numpy as np
+from earthkit.data.indexing.fieldlist import FieldArray
+from earthkit.geo.rotate import unrotate_vector
 
 
 class NewDataField:
@@ -85,7 +34,7 @@ def execute(context, input, u, v):
     wind_pairs = defaultdict(dict)
 
     for f in input:
-        key = f.as_mars()
+        key = f.metadata(namespace="mars")
         param = key.pop("param")
 
         if param not in wind_params:
@@ -107,18 +56,19 @@ def execute(context, input, u, v):
         y = pairs[v]
 
         lats, lons = x.grid_points()
-        raw_lats, raw_longs = x.grid_points_raw()
+        raw_lats, raw_longs = x.grid_points_unrotated()
 
         assert x.rotation == y.rotation
 
-        u_new, v_new = rotate_winds(
+        u_new, v_new = unrotate_vector(
             lats,
             lons,
-            raw_lats,
-            raw_longs,
-            x.to_numpy(reshape=False),
-            y.to_numpy(reshape=False),
-            *x.rotation,
+            x.to_numpy(flatten=True),
+            y.to_numpy(flatten=True),
+            *x.rotation[:2],
+            south_pole_rotation_angle=x.rotation[2],
+            lat_unrotated=raw_lats,
+            lon_unrotated=raw_longs,
         )
 
         result.append(NewDataField(x, u_new))
@@ -128,9 +78,9 @@ def execute(context, input, u, v):
 
 
 if __name__ == "__main__":
-    from climetlab import load_source
+    from earthkit.data import from_source
 
-    source = load_source(
+    source = from_source(
         "mars",
         date=-1,
         param="10u/10v",

anemoi/datasets/create/functions/sources/accumulations.py

@@ -11,11 +11,11 @@ import logging
 import warnings
 from copy import deepcopy
 
-import climetlab as cml
+import earthkit.data as ekd
 import numpy as np
-from climetlab.core.temporary import temp_file
-from climetlab.readers.grib.output import new_grib_output
-from climetlab.utils.availability import Availability
+from earthkit.data.core.temporary import temp_file
+from earthkit.data.readers.grib.output import new_grib_output
+from earthkit.data.utils.availability import Availability
 
 from anemoi.datasets.create.utils import to_datetime_list
 
@@ -26,7 +26,7 @@ LOG = logging.getLogger(__name__)
 
 def member(field):
     # Bug in eccodes has number=0 randomly
-    number = field.metadata("number")
+    number = field.metadata("number", default=0)
     if number is None:
         number = 0
     return number
@@ -54,16 +54,25 @@ class Accumulation:
 
     def check(self, field):
         if self._check is None:
-            self._check = field.as_mars()
+            self._check = field.metadata(namespace="mars")
 
-            assert self.param == field.metadata("param"), (self.param, field.metadata("param"))
-            assert self.date == field.metadata("date"), (self.date, field.metadata("date"))
-            assert self.time == field.metadata("time"), (self.time, field.metadata("time"))
+            assert self.param == field.metadata("param"), (
+                self.param,
+                field.metadata("param"),
+            )
+            assert self.date == field.metadata("date"), (
+                self.date,
+                field.metadata("date"),
+            )
+            assert self.time == field.metadata("time"), (
+                self.time,
+                field.metadata("time"),
+            )
             assert self.number == member(field), (self.number, member(field))
 
             return
 
-        mars = field.as_mars()
+        mars = field.metadata(namespace="mars")
         keys1 = sorted(self._check.keys())
         keys2 = sorted(mars.keys())
 
@@ -196,7 +205,11 @@ class AccumulationFromLastStep(Accumulation):
 
     def compute(self, values, startStep, endStep):
 
-        assert endStep - startStep == self.frequency, (startStep, endStep, self.frequency)
+        assert endStep - startStep == self.frequency, (
+            startStep,
+            endStep,
+            self.frequency,
+        )
 
         if self.startStep is None:
             self.startStep = startStep
@@ -307,14 +320,13 @@ def compute_accumulations(
                 )
 
     compressed = Availability(requests)
-    ds = cml.load_source("empty")
+    ds = ekd.from_source("empty")
     for r in compressed.iterate():
         request.update(r)
         if context.use_grib_paramid and "param" in request:
             request = use_grib_paramid(request)
         print("🌧️", request)
-
-        ds = ds + cml.load_source("mars", **request)
+        ds = ds + ekd.from_source("mars", **request)
 
     accumulations = {}
     for a in [AccumulationClass(out, frequency=frequency, **r) for r in requests]:
@@ -341,7 +353,7 @@ def compute_accumulations(
 
     out.close()
 
-    ds = cml.load_source("file", path)
+    ds = ekd.from_source("file", path)
 
     assert len(ds) / len(param) / len(number) == len(dates), (
         len(ds),

anemoi/datasets/create/functions/sources/constants.py

@@ -6,15 +6,19 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 #
-from climetlab import load_source
+from earthkit.data import from_source
 
 
 def constants(context, dates, template, param):
     from warnings import warn
 
-    warn("The source `constants` is deprecated, use `forcings` instead.", DeprecationWarning, stacklevel=2)
-    context.trace("✅", f"load_source(constants, {template}, {param}")
-    return load_source("constants", source_or_dataset=template, date=dates, param=param)
+    warn(
+        "The source `constants` is deprecated, use `forcings` instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    context.trace("✅", f"from_source(constants, {template}, {param}")
+    return from_source("forcings", source_or_dataset=template, date=dates, param=param)
 
 
 execute = constants

anemoi/datasets/create/functions/sources/empty.py

@@ -7,8 +7,8 @@
 # nor does it submit to any jurisdiction.
 #
 
-import climetlab as cml
+import earthkit.data as ekd
 
 
 def execute(context, dates, **kwargs):
-    return cml.load_source("empty")
+    return ekd.from_source("empty")

anemoi/datasets/create/functions/sources/forcings.py

@@ -6,12 +6,12 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 #
-from climetlab import load_source
+from earthkit.data import from_source
 
 
 def forcings(context, dates, template, param):
-    context.trace("✅", f"load_source(forcings, {template}, {param}")
-    return load_source("constants", source_or_dataset=template, date=dates, param=param)
+    context.trace("✅", f"from_source(forcings, {template}, {param}")
+    return from_source("forcings", source_or_dataset=template, date=dates, param=param)
 
 
 execute = forcings

anemoi/datasets/create/functions/sources/grib.py

@@ -10,8 +10,8 @@
 
 import glob
 
-from climetlab import load_source
-from climetlab.utils.patterns import Pattern
+from earthkit.data import from_source
+from earthkit.data.utils.patterns import Pattern
 
 
 def check(ds, paths, **kwargs):
@@ -33,7 +33,7 @@ def _expand(paths):
 def execute(context, dates, path, *args, **kwargs):
     given_paths = path if isinstance(path, list) else [path]
 
-    ds = load_source("empty")
+    ds = from_source("empty")
     dates = [d.isoformat() for d in dates]
 
     for path in given_paths:
@@ -45,7 +45,7 @@ def execute(context, dates, path, *args, **kwargs):
 
         for path in _expand(paths):
             context.trace("📁", "PATH", path)
-            s = load_source("file", path)
+            s = from_source("file", path)
             s = s.sel(valid_datetime=dates, **kwargs)
             ds = ds + s
 

anemoi/datasets/create/functions/sources/hindcasts.py

@@ -10,11 +10,11 @@ import datetime
 import warnings
 from copy import deepcopy
 
-import climetlab as cml
+import earthkit.data as ekd
 import numpy as np
-from climetlab.core.temporary import temp_file
-from climetlab.readers.grib.output import new_grib_output
-from climetlab.utils.availability import Availability
+from earthkit.data.core.temporary import temp_file
+from earthkit.data.readers.grib.output import new_grib_output
+from earthkit.utils.availability import Availability
 
 from anemoi.datasets.create.functions.sources.mars import mars
 
@@ -53,9 +53,18 @@ class Accumulation:
         if self._check is None:
             self._check = field.as_mars()
 
-            assert self.param == field.metadata("param"), (self.param, field.metadata("param"))
-            assert self.date == field.metadata("date"), (self.date, field.metadata("date"))
-            assert self.time == field.metadata("time"), (self.time, field.metadata("time"))
+            assert self.param == field.metadata("param"), (
+                self.param,
+                field.metadata("param"),
+            )
+            assert self.date == field.metadata("date"), (
+                self.date,
+                field.metadata("date"),
+            )
+            assert self.time == field.metadata("time"), (
+                self.time,
+                field.metadata("time"),
+            )
             assert self.number == member(field), (self.number, member(field))
 
             return
@@ -189,7 +198,11 @@ class AccumulationFromLastStep(Accumulation):
 
     def compute(self, values, startStep, endStep):
 
-        assert endStep - startStep == self.frequency, (startStep, endStep, self.frequency)
+        assert endStep - startStep == self.frequency, (
+            startStep,
+            endStep,
+            self.frequency,
+        )
 
         if self.startStep is None:
             self.startStep = startStep
@@ -299,11 +312,11 @@ def compute_accumulations(
                 )
 
     compressed = Availability(requests)
-    ds = cml.load_source("empty")
+    ds = ekd.from_source("empty")
     for r in compressed.iterate():
         request.update(r)
         print("🌧️", request)
-        ds = ds + cml.load_source("mars", **request)
+        ds = ds + ekd.from_source("mars", **request)
 
     accumulations = {}
     for a in [AccumulationClass(out, frequency=frequency, **r) for r in requests]:
@@ -330,7 +343,7 @@ def compute_accumulations(
 
     out.close()
 
-    ds = cml.load_source("file", path)
+    ds = ekd.from_source("file", path)
 
     assert len(ds) / len(param) / len(number) == len(dates), (
         len(ds),

anemoi/datasets/create/functions/sources/mars.py

@@ -10,8 +10,8 @@ import datetime
 from copy import deepcopy
 
 from anemoi.utils.humanize import did_you_mean
-from climetlab import load_source
-from climetlab.utils.availability import Availability
+from earthkit.data import from_source
+from earthkit.data.utils.availability import Availability
 
 from anemoi.datasets.create.utils import to_datetime_list
 
@@ -176,7 +176,7 @@ def mars(context, dates, *requests, date_key="date", **kwargs):
         requests = [kwargs]
 
     requests = factorise_requests(dates, *requests, date_key=date_key)
-    ds = load_source("empty")
+    ds = from_source("empty")
     for r in requests:
         r = {k: v for k, v in r.items() if v != ("-",)}
 
@@ -184,14 +184,14 @@ def mars(context, dates, *requests, date_key="date", **kwargs):
             r = use_grib_paramid(r)
 
         if DEBUG:
-            context.trace("✅", f"load_source(mars, {r}")
+            context.trace("✅", f"from_source(mars, {r}")
 
         for k, v in r.items():
             if k not in MARS_KEYS:
                 raise ValueError(
                     f"⚠️ Unknown key {k}={v} in MARS request. Did you mean '{did_you_mean(k, MARS_KEYS)}' ?"
                 )
-        ds = ds + load_source("mars", **r)
+        ds = ds + from_source("mars", **r)
     return ds
 
 

anemoi/datasets/create/functions/sources/netcdf.py

@@ -9,8 +9,8 @@
 
 import glob
 
-from climetlab import load_source
-from climetlab.utils.patterns import Pattern
+from earthkit.data import from_source
+from earthkit.data.utils.patterns import Pattern
 
 
 def _expand(paths):
@@ -44,7 +44,7 @@ def load_netcdfs(emoji, what, context, dates, path, *args, **kwargs):
     given_paths = path if isinstance(path, list) else [path]
 
     dates = [d.isoformat() for d in dates]
-    ds = load_source("empty")
+    ds = from_source("empty")
 
     for path in given_paths:
         paths = Pattern(path, ignore_missing_keys=True).substitute(*args, date=dates, **kwargs)
@@ -53,7 +53,7 @@ def load_netcdfs(emoji, what, context, dates, path, *args, **kwargs):
 
         for path in _expand(paths):
             context.trace(emoji, what.upper(), path)
-            s = load_source("opendap", path)
+            s = from_source("opendap", path)
             s = s.sel(
                 valid_datetime=dates,
                 param=kwargs["param"],

anemoi/datasets/create/functions/sources/source.py

@@ -6,7 +6,7 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 #
-from climetlab import load_source
+from earthkit.data import from_source
 
 from anemoi.datasets.create.utils import to_datetime_list
 
@@ -15,12 +15,12 @@ DEBUG = True
 
 def source(context, dates, **kwargs):
     name = kwargs.pop("name")
-    context.trace("✅", f"load_source({name}, {dates}, {kwargs}")
+    context.trace("✅", f"from_source({name}, {dates}, {kwargs}")
     if kwargs["date"] == "$from_dates":
         kwargs["date"] = list({d.strftime("%Y%m%d") for d in dates})
     if kwargs["time"] == "$from_dates":
         kwargs["time"] = list({d.strftime("%H%M") for d in dates})
-    return load_source(name, **kwargs)
+    return from_source(name, **kwargs)
 
 
 execute = source

anemoi/datasets/create/functions/sources/tendencies.py

@@ -9,10 +9,10 @@
 import datetime
 from collections import defaultdict
 
-from climetlab.core.temporary import temp_file
-from climetlab.readers.grib.output import new_grib_output
+from earthkit.data.core.temporary import temp_file
+from earthkit.data.readers.grib.output import new_grib_output
 
-from anemoi.datasets.create.functions import assert_is_fieldset
+from anemoi.datasets.create.functions import assert_is_fieldlist
 from anemoi.datasets.create.utils import to_datetime_list
 
 
@@ -36,7 +36,7 @@ def normalise_time_delta(t):
 def group_by_field(ds):
     d = defaultdict(list)
     for field in ds.order_by("valid_datetime"):
-        m = field.as_mars()
+        m = field.metadata(namespace="mars")
         for k in ("date", "time", "step"):
             m.pop(k, None)
         keys = tuple(m.items())
@@ -103,10 +103,10 @@ def tendencies(dates, time_increment, **kwargs):
 
     out.close()
 
-    from climetlab import load_source
+    from earthkit.data import from_source
 
-    ds = load_source("file", path)
-    assert_is_fieldset(ds)
+    ds = from_source("file", path)
+    assert_is_fieldlist(ds)
     # save a reference to the tmp file so it is deleted
     # only when the dataset is not used anymore
     ds._tmp = tmp

anemoi/datasets/create/input.py

@@ -15,8 +15,8 @@ from functools import cached_property
 from functools import wraps
 
 import numpy as np
-from climetlab.core.order import build_remapping
-from climetlab.indexing.fieldset import FieldSet
+from earthkit.data.core.fieldlist import FieldList
+from earthkit.data.core.order import build_remapping
 
 from anemoi.datasets.dates import Dates
 
@@ -75,18 +75,18 @@ def is_function(name, kind):
         return False
 
 
-def assert_fieldset(method):
+def assert_fieldlist(method):
     @wraps(method)
     def wrapper(self, *args, **kwargs):
         result = method(self, *args, **kwargs)
-        assert isinstance(result, FieldSet), type(result)
+        assert isinstance(result, FieldList), type(result)
         return result
 
     return wrapper
 
 
-def assert_is_fieldset(obj):
-    assert isinstance(obj, FieldSet), type(obj)
+def assert_is_fieldlist(obj):
+    assert isinstance(obj, FieldList), type(obj)
 
 
 def _data_request(data):
@@ -101,12 +101,12 @@ def _data_request(data):
             continue
 
         if date is None:
-            date = field.valid_datetime()
+            date = field.datetime()["valid_time"]
 
-        if field.valid_datetime() != date:
+        if field.datetime()["valid_time"] != date:
             continue
 
-        as_mars = field.as_mars()
+        as_mars = field.metadata(namespace="mars")
         step = as_mars.get("step")
         levtype = as_mars.get("levtype", "sfc")
         param = as_mars["param"]
@@ -399,12 +399,12 @@ class EmptyResult(Result):
         super().__init__(context, action_path + ["empty"], dates)
 
     @cached_property
-    @assert_fieldset
+    @assert_fieldlist
     @trace_datasource
     def datasource(self):
-        from climetlab import load_source
+        from earthkit.data import from_source
 
-        return load_source("empty")
+        return from_source("empty")
 
     @property
     def variables(self):
@@ -423,7 +423,7 @@ class FunctionResult(Result):
         return f"{self.action.name}({shorten(self.dates)})"
 
     @cached_property
-    @assert_fieldset
+    @assert_fieldlist
     @notify_result
     @trace_datasource
     def datasource(self):
@@ -452,7 +452,7 @@ class JoinResult(Result):
         self.results = [r for r in results if not r.empty]
 
     @cached_property
-    @assert_fieldset
+    @assert_fieldlist
     @notify_result
     @trace_datasource
     def datasource(self):
@@ -504,11 +504,11 @@ class UnShiftResult(Result):
         return f"{self.action.delta}({shorten(self.dates)})"
 
     @cached_property
-    @assert_fieldset
+    @assert_fieldlist
     @notify_result
     @trace_datasource
     def datasource(self):
-        from climetlab.indexing.fieldset import FieldArray
+        from earthkit.data.indexing.fieldlist import FieldArray
 
         class DateShiftedField:
             def __init__(self, field, delta):
@@ -615,7 +615,7 @@ class StepAction(Action):
 
 class StepFunctionResult(StepResult):
     @cached_property
-    @assert_fieldset
+    @assert_fieldlist
     @notify_result
     @trace_datasource
     def datasource(self):
@@ -638,7 +638,7 @@ class StepFunctionResult(StepResult):
 class FilterStepResult(StepResult):
     @property
     @notify_result
-    @assert_fieldset
+    @assert_fieldlist
     @trace_datasource
     def datasource(self):
         ds = self.upstream_result.datasource
@@ -665,7 +665,7 @@ class ConcatResult(Result):
         self.results = [r for r in results if not r.empty]
 
     @cached_property
-    @assert_fieldset
+    @assert_fieldlist
     @notify_result
     @trace_datasource
     def datasource(self):

anemoi/datasets/create/loaders.py

@@ -176,7 +176,7 @@ class DatasetHandlerWithStatistics(GenericDatasetHandler):
 
 class Loader(DatasetHandlerWithStatistics):
     def build_input(self):
-        from climetlab.core.order import build_remapping
+        from earthkit.data.core.order import build_remapping
 
         builder = build_input(
             self.main_config.input,
@@ -581,7 +581,16 @@ class StatisticsAdder(DatasetHandlerWithStatistics):
         if not all(self.registry.get_flags(sync=False)):
             raise Exception(f"❗Zarr {self.path} is not fully built, not writting statistics into dataset.")
 
-        for k in ["mean", "stdev", "minimum", "maximum", "sums", "squares", "count", "has_nans"]:
+        for k in [
+            "mean",
+            "stdev",
+            "minimum",
+            "maximum",
+            "sums",
+            "squares",
+            "count",
+            "has_nans",
+        ]:
             self._add_dataset(name=k, array=stats[k])
 
         self.registry.add_to_history("compute_statistics_end")
@@ -643,8 +652,16 @@ class GenericAdditions(GenericDatasetHandler):
             for k in ["minimum", "maximum", "sums", "squares", "count", "has_nans"]:
                 agg[k][i, ...] = stats[k]
 
-        assert len(found) + len(missing) == len(self.dates), (len(found), len(missing), len(self.dates))
-        assert found.union(missing) == set(self.dates), (found, missing, set(self.dates))
+        assert len(found) + len(missing) == len(self.dates), (
+            len(found),
+            len(missing),
+            len(self.dates),
+        )
+        assert found.union(missing) == set(self.dates), (
+            found,
+            missing,
+            set(self.dates),
+        )
 
         if len(ifound) < 2:
             LOG.warn(f"Not enough data found in {self.path} to compute {self.__class__.__name__}. Skipped.")
@@ -656,7 +673,10 @@ class GenericAdditions(GenericDatasetHandler):
             agg[k] = agg[k][mask, ...]
 
         for k in ["minimum", "maximum", "sums", "squares", "count", "has_nans"]:
-            assert agg[k].shape == agg["count"].shape, (agg[k].shape, agg["count"].shape)
+            assert agg[k].shape == agg["count"].shape, (
+                agg[k].shape,
+                agg["count"].shape,
+            )
 
         minimum = np.nanmin(agg["minimum"], axis=0)
         maximum = np.nanmax(agg["maximum"], axis=0)
@@ -699,7 +719,16 @@ class GenericAdditions(GenericDatasetHandler):
         self.tmp_storage.delete()
 
     def _write(self, summary):
-        for k in ["mean", "stdev", "minimum", "maximum", "sums", "squares", "count", "has_nans"]:
+        for k in [
+            "mean",
+            "stdev",
+            "minimum",
+            "maximum",
+            "sums",
+            "squares",
+            "count",
+            "has_nans",
+        ]:
             name = self.final_storage_name(k)
             self._add_dataset(name=name, array=summary[k])
         self.registry.add_to_history(f"compute_statistics_{self.__class__.__name__.lower()}_end")

anemoi/datasets/create/statistics/__init__.py

@@ -71,7 +71,7 @@ def to_datetime(date):
     if isinstance(date, str):
         return np.datetime64(date)
     if isinstance(date, datetime.datetime):
-        return np.datetime64(date)
+        return np.datetime64(date, "s")
     return date
 
 
@@ -242,6 +242,7 @@ class StatAggregator:
 
         found = set()
         offset = 0
+
         for _, _dates, stats in self.owner._gather_data():
             assert isinstance(stats, dict), stats
             assert stats["minimum"].shape[0] == len(_dates), (stats["minimum"].shape, len(_dates))

anemoi/datasets/create/utils.py

@@ -13,8 +13,8 @@ from contextlib import contextmanager
 
 import numpy as np
 import yaml
-from climetlab import settings
-from climetlab.utils.humanize import seconds  # noqa: F401
+from earthkit.data import settings
+from earthkit.data.utils.humanize import seconds  # noqa: F401
 from tqdm.auto import tqdm
 
 
@@ -27,7 +27,8 @@ def cache_context(dirname):
         return no_cache_context()
 
     os.makedirs(dirname, exist_ok=True)
-    return settings.temporary("cache-directory", dirname)
+    # return settings.temporary("cache-directory", dirname)
+    return settings.temporary({"cache-policy": "user", "user-cache-directory": dirname})
 
 
 def bytes(n):
@@ -51,13 +52,13 @@ def bytes(n):
 
 
 def to_datetime_list(*args, **kwargs):
-    from climetlab.utils.dates import to_datetime_list as to_datetime_list_
+    from earthkit.data.utils.dates import to_datetime_list as to_datetime_list_
 
     return to_datetime_list_(*args, **kwargs)
 
 
 def to_datetime(*args, **kwargs):
-    from climetlab.utils.dates import to_datetime as to_datetime_
+    from earthkit.data.utils.dates import to_datetime as to_datetime_
 
     return to_datetime_(*args, **kwargs)
 

anemoi/datasets/data/dataset.py

@@ -187,8 +187,8 @@ class Dataset:
                 specific=self.metadata_specific(),
                 frequency=self.frequency,
                 variables=self.variables,
-                start_date=self.dates[0],
-                end_date=self.dates[-1],
+                start_date=self.dates[0].astype(str),
+                end_date=self.dates[-1].astype(str),
             )
         )
 
@@ -200,8 +200,8 @@ class Dataset:
             variables=self.variables,
             shape=self.shape,
             frequency=self.frequency,
-            start_date=self.dates[0],
-            end_date=self.dates[-1],
+            start_date=self.dates[0].astype(str),
+            end_date=self.dates[-1].astype(str),
             **kwargs,
         )
 

anemoi/datasets/data/stores.py

@@ -58,15 +58,17 @@ class HTTPStore(ReadOnlyStore):
 
 
 class S3Store(ReadOnlyStore):
-    """We write our own S3Store because the one used by zarr (fsspec) does not play well with fork() and multiprocessing."""
+    """We write our own S3Store because the one used by zarr (fsspec)
+    does not play well with fork() and multiprocessing. Also, we get
+    to control the s3 client.
+    """
 
     def __init__(self, url):
-        import boto3
+        from anemoi.utils.s3 import s3_client
 
-        self.bucket, self.key = url[5:].split("/", 1)
+        _, _, self.bucket, self.key = url.split("/", 3)
 
-        # TODO: get the profile name from the url
-        self.s3 = boto3.Session(profile_name=None).client("s3")
+        self.s3 = s3_client(self.bucket)
 
     def __getitem__(self, key):
         try:
@@ -111,6 +113,12 @@ def open_zarr(path, dont_fail=False, cache=None):
 
         if DEBUG_ZARR_LOADING:
             if isinstance(store, str):
+                import os
+
+                if not os.path.isdir(store):
+                    raise NotImplementedError(
+                        "DEBUG_ZARR_LOADING is only implemented for DirectoryStore. Please disable it for other backends."
+                    )
                 store = zarr.storage.DirectoryStore(store)
             store = DebugStore(store)
 

{anemoi_datasets-0.3.10.dist-info → anemoi_datasets-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.3.10
+Version: 0.4.0
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License: Apache License
@@ -223,7 +223,7 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: anemoi-utils[provenance] >=0.3.4
+Requires-Dist: anemoi-utils[provenance] >=0.3.5
 Requires-Dist: numpy
 Requires-Dist: pyyaml
 Requires-Dist: semantic-version
@@ -231,7 +231,8 @@ Requires-Dist: tqdm
 Requires-Dist: zarr <=2.17
 Provides-Extra: all
 Requires-Dist: boto3 ; extra == 'all'
-Requires-Dist: climetlab >=0.23.2 ; extra == 'all'
+Requires-Dist: earthkit-data[mars] >=0.9 ; extra == 'all'
+Requires-Dist: earthkit-geo >=0.2 ; extra == 'all'
 Requires-Dist: earthkit-meteo ; extra == 'all'
 Requires-Dist: ecmwflibs >=0.6.3 ; extra == 'all'
 Requires-Dist: entrypoints ; extra == 'all'
@@ -239,14 +240,16 @@ Requires-Dist: pyproj ; extra == 'all'
 Requires-Dist: requests ; extra == 'all'
 Requires-Dist: s3fs ; extra == 'all'
 Provides-Extra: create
-Requires-Dist: climetlab >=0.23.2 ; extra == 'create'
+Requires-Dist: earthkit-data[mars] >=0.9 ; extra == 'create'
+Requires-Dist: earthkit-geo >=0.2 ; extra == 'create'
 Requires-Dist: earthkit-meteo ; extra == 'create'
 Requires-Dist: ecmwflibs >=0.6.3 ; extra == 'create'
 Requires-Dist: entrypoints ; extra == 'create'
 Requires-Dist: pyproj ; extra == 'create'
 Provides-Extra: dev
 Requires-Dist: boto3 ; extra == 'dev'
-Requires-Dist: climetlab >=0.23.2 ; extra == 'dev'
+Requires-Dist: earthkit-data[mars] >=0.9 ; extra == 'dev'
+Requires-Dist: earthkit-geo >=0.2 ; extra == 'dev'
 Requires-Dist: earthkit-meteo ; extra == 'dev'
 Requires-Dist: ecmwflibs >=0.6.3 ; extra == 'dev'
 Requires-Dist: entrypoints ; extra == 'dev'
@@ -257,13 +260,11 @@ Requires-Dist: pytest ; extra == 'dev'
 Requires-Dist: requests ; extra == 'dev'
 Requires-Dist: s3fs ; extra == 'dev'
 Requires-Dist: sphinx ; extra == 'dev'
-Requires-Dist: sphinx-argparse ; extra == 'dev'
 Requires-Dist: sphinx-rtd-theme ; extra == 'dev'
 Provides-Extra: docs
 Requires-Dist: nbsphinx ; extra == 'docs'
 Requires-Dist: pandoc ; extra == 'docs'
 Requires-Dist: sphinx ; extra == 'docs'
-Requires-Dist: sphinx-argparse ; extra == 'docs'
 Requires-Dist: sphinx-rtd-theme ; extra == 'docs'
 Provides-Extra: remote
 Requires-Dist: boto3 ; extra == 'remote'

{anemoi_datasets-0.3.10.dist-info → anemoi_datasets-0.4.0.dist-info}/RECORD

@@ -1,53 +1,53 @@
 anemoi/datasets/__init__.py,sha256=Z1gqZWhecLcT0RZQqYBLlz01MUlUZd0kWEj_RavbITM,782
 anemoi/datasets/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
-anemoi/datasets/_version.py,sha256=QTWoiVZ8kBQVv_3SZGJVwOxMqBDiTg3_qUjTLFx1eco,413
+anemoi/datasets/_version.py,sha256=j90u3VVU4UrJf1fgMUhaZarHK_Do2XGYXr-vZvOFzVo,411
 anemoi/datasets/grids.py,sha256=3YBMMJodgYhavarXPAlMZHaMtDT9v2IbTmAXZTqf8Qo,8481
 anemoi/datasets/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
 anemoi/datasets/commands/compare.py,sha256=p2jQOAC3JhScCLF0GjTCO8goYLWLN8p7vzy_gf5fFcI,1473
 anemoi/datasets/commands/copy.py,sha256=SxAeN51owyN5gwtwpt30xhJSIJRlJb9YOUt_4K4m-D8,11780
 anemoi/datasets/commands/create.py,sha256=POdOsVDlvRrHFFkI3SNXNgNIbSxkVUUPMoo660x7Ma0,987
-anemoi/datasets/commands/inspect.py,sha256=G3fzcgiLaU8jln7GKvgamN7Y06-qC_JnFw2SbNn1_E4,18646
-anemoi/datasets/commands/scan.py,sha256=HxsLdCgBMSdEXjlJfPq5M_9LxXHHQIoZ1ZEHO_AoPgA,2881
+anemoi/datasets/commands/inspect.py,sha256=NipFE2SDxuc6uzjZJT1wDJSFO24yVh95YJpiHz0WBmw,18666
+anemoi/datasets/commands/scan.py,sha256=MaTdne4JrtlqO3LhOUr43DZhZ6O-RZwC7uQ7C6PG7Os,2910
 anemoi/datasets/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-anemoi/datasets/compute/recentre.py,sha256=GRxI6rY_KyXJnZGPxU_UO9YDb-rY_raK70Fiwv1mjhs,4792
+anemoi/datasets/compute/recentre.py,sha256=oIqhENljB9ad-wnMJCJ3P0Xf1v76zQjqYCu1TuySgSI,4919
 anemoi/datasets/create/__init__.py,sha256=Q8uXUdbE-SRYYaZd5cPQ2RVbSoHnGX7-eKdOJHYVhDk,5951
 anemoi/datasets/create/check.py,sha256=DLjw-eyaCNxPhoKFsP4Yn_l3SIr57YHdyPR-tE5vx80,5791
 anemoi/datasets/create/chunks.py,sha256=YEDcr0K2KiiceSTiBuZzj0TbRbzZ9J546XO7rrrTFQw,2441
-anemoi/datasets/create/config.py,sha256=uLIp1WHg3hbqwwMV9EepMwJQsXJAGImkbo0okBeEVd4,7683
-anemoi/datasets/create/input.py,sha256=3G7sqdn7R1pLBeeswXwwi8VRAHrBnjq1PdRYHJBe594,27741
-anemoi/datasets/create/loaders.py,sha256=-fJ9qKjsCd8Wvnobn34WsQpE9uAjon5M4REgCpW5q_w,30594
+anemoi/datasets/create/config.py,sha256=Dek5SlifOie8U3TNnWgLKmyOATomO3SQisEJ663lKPI,7687
+anemoi/datasets/create/input.py,sha256=s7agDYIB8XzmEOVm6Kq1B5lpv749u_xvzt6XHHM2kwg,27800
+anemoi/datasets/create/loaders.py,sha256=mT3dSUHEfHrhlM-AiYRKs_zQxGyMeQ2zCkuLJW5gB4s,30953
 anemoi/datasets/create/patch.py,sha256=xjCLhvIQKRqmypsKInRU1CvFh1uoaB3YGSQP1UVZZik,3682
 anemoi/datasets/create/persistent.py,sha256=nT8gvhVPdI1H3zW_F7uViGKIlQQ94jCDrMSWTmhQ2_A,4290
 anemoi/datasets/create/size.py,sha256=A1w6RkaL0L9IlwIdmYsCTJTecmY_QtvbkGf__jvQle0,1068
 anemoi/datasets/create/template.py,sha256=2roItOYJzjGB0bKS28f6EjfpomP0ppT4v6T9fYzjRxQ,4263
-anemoi/datasets/create/utils.py,sha256=H1-auNSZUSDW0Aog8CHnIfZlzgKE1XPoi1I40CqquA4,3676
+anemoi/datasets/create/utils.py,sha256=Z1d8A_q0hg0yvbdnR7hw-Ia-zj4wE6jEc9YUI3C06yg,3783
 anemoi/datasets/create/writer.py,sha256=G1qAPvdn8anGnpWYhvSSP4u3Km_tHKPdMXm0G4skKSk,1379
 anemoi/datasets/create/zarr.py,sha256=hwM_PaYTa_IgFY1VC7qdYTWQ5MXCWWlMrzXsV_eAY0Q,4776
-anemoi/datasets/create/functions/__init__.py,sha256=v5RRSAK8LiNdPbxy0c9WxqMx3AMHi42FK-DtFh2h_EU,938
+anemoi/datasets/create/functions/__init__.py,sha256=5HmelLkXDjFOhNhX0Z78aV3ZlW2txiJliJwT4jfLEN4,945
 anemoi/datasets/create/functions/filters/__init__.py,sha256=Xe9G54CKvCI3ji-7k0R5l0WZZdhlydRgawsXuBcX_hg,379
-anemoi/datasets/create/functions/filters/empty.py,sha256=X7gAZS_jl9X0ztGs3Arn8DkOB6iP4Draq4EAKa4I8hc,617
+anemoi/datasets/create/functions/filters/empty.py,sha256=QGj7YEfbo3gwlmwHi1lPATjST0332TH2-uc6_wKENjI,621
 anemoi/datasets/create/functions/filters/noop.py,sha256=ZP434Z1rFlqdgXse_1ZzqC2XAqRQlYlXlVfGLx7rK8g,444
-anemoi/datasets/create/functions/filters/rename.py,sha256=cGoHr-IS-PhYEtZvXDpH03gtfZ9uT7uEZqu9EDxAxN0,2105
-anemoi/datasets/create/functions/filters/rotate_winds.py,sha256=fUdh8ILcMzMzckGlvwzdgG-c7w5R9NnWfaijp28Bf5M,4092
-anemoi/datasets/create/functions/filters/unrotate_winds.py,sha256=nsa3EHly8ppWd2WH4ROoMczM8WFu5qKaIhO_UFcL9TY,3502
+anemoi/datasets/create/functions/filters/rename.py,sha256=U0jbZe7GYqbvw85NpJrsNkVEm3SO5RR661yCnMLS3jE,2160
+anemoi/datasets/create/functions/filters/rotate_winds.py,sha256=Li3Xgf92yRkIm_2XDPOhStcSLMQei-bjh63DtDhPNDA,2065
+anemoi/datasets/create/functions/filters/unrotate_winds.py,sha256=hiIwgWi_2lk_ntxsPFMyZ6Ku8_5p91ht36VN_2kHYDA,2414
 anemoi/datasets/create/functions/sources/__init__.py,sha256=Xe9G54CKvCI3ji-7k0R5l0WZZdhlydRgawsXuBcX_hg,379
-anemoi/datasets/create/functions/sources/accumulations.py,sha256=klbp-akoZlOk9jByDFsgPfHRCdfLvpatTLMxDPZaNZc,12943
-anemoi/datasets/create/functions/sources/constants.py,sha256=aqquu6HDc8t-zsF9KRFLaj0eV4S0UPZ59BVna8E3bU8,785
-anemoi/datasets/create/functions/sources/empty.py,sha256=SBuAfC33imbfcRnFnnOR44y8Q3KSQcqx3juIcXfCa3c,481
-anemoi/datasets/create/functions/sources/forcings.py,sha256=EVcdu8puMSW451qj3LKCWWXaSf2LlmF8YXVs8hSMxkU,643
-anemoi/datasets/create/functions/sources/grib.py,sha256=YQNuGnlh2EYb2NIHYpzlipwUTmOhrmyQtP3zgk8MAUU,1661
-anemoi/datasets/create/functions/sources/hindcasts.py,sha256=0Psnsx2J0cRLMpJuNN-gESm1xJFC1gmQzI8sdnXCoYE,13042
-anemoi/datasets/create/functions/sources/mars.py,sha256=JWsbzyoXF95HPk2VWzmX53f_SJwXhKkaJvXtXJMGLig,5285
-anemoi/datasets/create/functions/sources/netcdf.py,sha256=kic6PH7SAK3gseXChD38IDXw6Zcg2zhF4SeDXB2LQ8Q,2084
+anemoi/datasets/create/functions/sources/accumulations.py,sha256=aFDgVQQjSPRbI5K_HLo3v1IBZzCp31gkCPqfRSETJ9E,13191
+anemoi/datasets/create/functions/sources/constants.py,sha256=QscxBGAKYeSRhhzLcsK5Yy3kM4s3rltS2eDgDE6OmQA,819
+anemoi/datasets/create/functions/sources/empty.py,sha256=ZrXGs8Y3VrLSV8C8YlJTJcHV7Bmi7xPiUlrq8R0JZQY,485
+anemoi/datasets/create/functions/sources/forcings.py,sha256=tF3EyIs5AGF1Ppvp6dIExONM-kGF-wcnMO1sZc_wDuo,646
+anemoi/datasets/create/functions/sources/grib.py,sha256=XOq1Efn-r9rX5bJj0ARhmKtMgKP6rraq_YRMXALxP54,1669
+anemoi/datasets/create/functions/sources/hindcasts.py,sha256=_O94pLbn-ShD9KBU5sBlppVm4wojkkz8iwsckndIXyw,13241
+anemoi/datasets/create/functions/sources/mars.py,sha256=Am9jHddLEHxsnKQ-0Ug0kgQknkEJ2UwU_v-Yy9yv_8c,5293
+anemoi/datasets/create/functions/sources/netcdf.py,sha256=hsRqNIakef5AQCk4aPHxCW6U5qLJmW3DLfN3WSXTWzY,2092
 anemoi/datasets/create/functions/sources/opendap.py,sha256=T0CPinscfafrVLaye5ue-PbiCNbcNqf_3m6pphN9rCU,543
 anemoi/datasets/create/functions/sources/recentre.py,sha256=t07LIXG3Hp9gmPkPriILVt86TxubsHyS1EL1lzwgtXY,1810
-anemoi/datasets/create/functions/sources/source.py,sha256=hPQnV_6UIxFw97uRKcTA8TplcgG1kC8NlFHoEaaLet4,1418
-anemoi/datasets/create/functions/sources/tendencies.py,sha256=kwS_GZt8R9kpfs5RrvxPb0Gj-5nDP0sgJgfSRCAwwww,4057
-anemoi/datasets/create/statistics/__init__.py,sha256=eXyOdlgXBt6QdVWM7ZVyUWdFMv6iNsFefkjvOVvZAlQ,11010
+anemoi/datasets/create/functions/sources/source.py,sha256=J3O4M0nB1a-67IJuY_aWqDDqyNGXB_uzxVbicFldO4U,1422
+anemoi/datasets/create/functions/sources/tendencies.py,sha256=OIYKdRTabDvxnBXCcffWWBcorVlhZVNj3VkcVuQjIYA,4088
+anemoi/datasets/create/statistics/__init__.py,sha256=0_cUKLeYRD19wpSdx-N5SBl4KvdFnKEo9kT2_Lg_Pxo,11016
 anemoi/datasets/create/statistics/summary.py,sha256=sgmhA24y3VRyjmDUgTnPIqcHSlWBbFA0qynx6gJ9Xw8,3370
 anemoi/datasets/data/__init__.py,sha256=to9L_RZVQ4OgyHUpX6lcvt4GqJdZjBa5HCTaWx1aGKo,1046
 anemoi/datasets/data/concat.py,sha256=AkpyOs16OjW7X0cdyYFQfWSCV6dteXBp-x9WlokO-DI,3550
-anemoi/datasets/data/dataset.py,sha256=efs1R84G8ayhaWpfP_loi2BJkd1EAEykUPVCjcNfs_Y,7623
+anemoi/datasets/data/dataset.py,sha256=hcspK-Fjp-rYdOYZo4qlnL_GUeGpXk4NDINWjEpeSNc,7671
 anemoi/datasets/data/debug.css,sha256=z2X_ZDSnZ9C3pyZPWnQiEyAxuMxUaxJxET4oaCImTAQ,211
 anemoi/datasets/data/debug.py,sha256=PcyrjgxaLzeb_vf12pvUtPPVvBRHNm1SimythZvqsP4,6303
 anemoi/datasets/data/ensemble.py,sha256=AsP7Xx0ZHLoZs6a4EC0jtyGYIcOvZvvKXhgNsIvqIN8,1137
@@ -59,15 +59,15 @@ anemoi/datasets/data/masked.py,sha256=czAv1ZfZ9q6Wr4RqI2Xj8SEm7yoCgJrwMl-CPDs_wS
 anemoi/datasets/data/misc.py,sha256=tuNsUY06nWh3Raf_RCi8bzCXsMB4t2hOuIkNGV4epj8,10501
 anemoi/datasets/data/select.py,sha256=Oje3KG1shRawjuBy2-GM8s_Nk_68l-uujvx5SGW0tUM,3781
 anemoi/datasets/data/statistics.py,sha256=lZCcKw9s7ttMBEp6ANyxtbXoZZvchhE7SClq-D4AUR8,1645
-anemoi/datasets/data/stores.py,sha256=yy914zMHIYKm5q6mHOqGeK0dC_26VFeqKLXyb7x9NXE,11190
+anemoi/datasets/data/stores.py,sha256=MBl81LUUWic2xJS8l8nbdm3aHvSZ1zCEC8HNEzGv-q4,11460
 anemoi/datasets/data/subset.py,sha256=9urVTXdnwCgqn0_BRYquMi8oiXn4ubAf0n4586hWfKw,3814
 anemoi/datasets/data/unchecked.py,sha256=xhdMg-ToI1UfBWHNsWyn1y2meZWngZtHx-33L0KqKp8,4037
 anemoi/datasets/dates/__init__.py,sha256=4ItowfLLh90T8L_JOjtv98lE6M7gAaWt7dV3niUrFvk,4473
 anemoi/datasets/dates/groups.py,sha256=iq310Pi7ullglOhcNblv14MmcT8FPgYCD5s45qAfV_s,3383
 anemoi/datasets/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-anemoi_datasets-0.3.10.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_datasets-0.3.10.dist-info/METADATA,sha256=z0W6UcinpdFzA4kRKjgMqyTQn4vTZu81robfOyVJ_nI,16020
-anemoi_datasets-0.3.10.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
-anemoi_datasets-0.3.10.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
-anemoi_datasets-0.3.10.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_datasets-0.3.10.dist-info/RECORD,,
+anemoi_datasets-0.4.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_datasets-0.4.0.dist-info/METADATA,sha256=IZd5nok8kl49TlacMsgs7VcucD3YVF6Otf6u5AKscOw,16099
+anemoi_datasets-0.4.0.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
+anemoi_datasets-0.4.0.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
+anemoi_datasets-0.4.0.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_datasets-0.4.0.dist-info/RECORD,,