anemoi-datasets 0.5.24__py3-none-any.whl → 0.5.26__py3-none-any.whl

anemoi/datasets/data/records/backends/__init__.py

@@ -0,0 +1,157 @@
+# (C) Copyright 2025 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import json
+import os
+
+import numpy as np
+
+
+class Backend:
+    def __init__(self, path, **kwargs):
+        self.path = path
+        self.kwargs = kwargs
+
+    def read(self, i, **kwargs):
+        raise NotImplementedError("Must be implemented in subclass")
+
+    def read_metadata(self):
+        raise NotImplementedError("Must be implemented in subclass")
+
+    def read_statistics(self):
+        raise NotImplementedError("Must be implemented in subclass")
+
+
+class Npz1Backend(Backend):
+    def read(self, i, **kwargs):
+        path = os.path.join(self.path, "data", str(int(i / 10)), f"{i}.npz")
+        with open(path, "rb") as f:
+            return dict(np.load(f))
+
+    def read_metadata(self):
+        with open(os.path.join(self.path, "metadata.json"), "r") as f:
+            return json.load(f)
+
+    def read_statistics(self):
+        path = os.path.join(self.path, "statistics.npz")
+        dic = {}
+        for k, v in dict(np.load(path)).items():
+            key, group = k.split(":")
+            if group not in dic:
+                dic[group] = {}
+            dic[group][key] = v
+        return dic
+
+
+class Npz2Backend(Backend):
+    def read(self, i, **kwargs):
+        path = os.path.join(self.path, "data_", str(int(i / 10)), f"{i}_.npz")
+        with open(path, "rb") as f:
+            return dict(np.load(f))
+
+    def read_metadata(self):
+        with open(os.path.join(self.path, "metadata.json"), "r") as f:
+            return json.load(f)
+
+    def read_statistics(self):
+        path = os.path.join(self.path, "statistics_.npz")
+        dic = {}
+        for k, v in dict(np.load(path)).items():
+            key, group = k.split(":")
+            if group not in dic:
+                dic[group] = {}
+            dic[group][key] = v
+        return dic
+
+
+def backend_factory(backend, *args, **kwargs):
+    BACKENDS = dict(
+        npz1=Npz1Backend,
+        npz2=Npz2Backend,
+    )
+    return BACKENDS[backend](*args, **kwargs)
+
+
+class WriteBackend(Backend):
+    def __init__(self, path, **kwargs):
+        super().__init__(path, **kwargs)
+
+    def write(self, i, data, **kwargs):
+        raise NotImplementedError("Must be implemented in subclass")
+
+    def write_metadata(self, metadata):
+        raise NotImplementedError("Must be implemented in subclass")
+
+    def write_statistics(self, statistics):
+        raise NotImplementedError("Must be implemented in subclass")
+
+
+class Npz1WriteBackend(WriteBackend):
+    def write(self, i, data, **kwargs):
+        path = os.path.join(self.path, "data", str(int(i / 10)))
+        os.makedirs(path, exist_ok=True)
+        out_path = os.path.join(path, f"{i}.npz")
+        np.savez(out_path, **data)
+
+    def write_metadata(self, metadata):
+        from anemoi.datasets.create import json_tidy
+
+        os.makedirs(self.path, exist_ok=True)
+        with open(os.path.join(self.path, "metadata.json"), "w") as f:
+            json.dump(metadata, f, indent=2, default=json_tidy)
+
+    def write_statistics(self, statistics):
+        flatten = {}
+        for name, d in statistics.items():
+            assert isinstance(d, dict), f"Statistics for {name} must be a dict, got {type(d)}"
+            for k, v in d.items():
+                assert isinstance(
+                    v, (int, float, np.ndarray)
+                ), f"Statistics value for {k} in {name} must be int, float or ndarray, got {type(v)}"
+                flatten[k + ":" + name] = v
+
+        path = os.path.join(self.path, "statistics.npz")
+        np.savez(path, **flatten)
+
+
+class Npz2WriteBackend(WriteBackend):
+    def write(self, i, data, **kwargs):
+        path = os.path.join(self.path, "data_", str(int(i / 10)))
+        os.makedirs(path, exist_ok=True)
+        out_path = os.path.join(path, f"{i}_.npz")
+        np.savez(out_path, **data)
+
+    def write_metadata(self, metadata):
+        from anemoi.datasets.create import json_tidy
+
+        os.makedirs(self.path, exist_ok=True)
+        with open(os.path.join(self.path, "metadata.json"), "w") as f:
+            json.dump(metadata, f, indent=2, default=json_tidy)
+
+    def write_statistics(self, statistics):
+        flatten = {}
+        for name, d in statistics.items():
+            assert isinstance(d, dict), f"Statistics for {name} must be a dict, got {type(d)}"
+            for k, v in d.items():
+                assert isinstance(
+                    v, (int, float, np.ndarray)
+                ), f"Statistics value for {k} in {name} must be int, float or ndarray, got {type(v)}"
+                flatten[k + ":" + name] = v
+
+        os.makedirs(self.path, exist_ok=True)
+        path = os.path.join(self.path, "statistics_.npz")
+        np.savez(path, **flatten)
+
+
+def writer_backend_factory(backend, *args, **kwargs):
+    WRITE_BACKENDS = dict(
+        npz1=Npz1WriteBackend,
+        npz2=Npz2WriteBackend,
+    )
+    return WRITE_BACKENDS[backend](*args, **kwargs)

anemoi/datasets/data/stores.py

@@ -107,51 +107,6 @@ class S3Store(ReadOnlyStore):
         return response["Body"].read()
 
 
-class PlanetaryComputerStore(ReadOnlyStore):
-    """We write our own Store to access catalogs on Planetary Computer,
-    as it requires some extra arguments to use xr.open_zarr.
-    """
-
-    def __init__(self, data_catalog_id: str) -> None:
-        """Initialize the PlanetaryComputerStore with a data catalog ID.
-
-        Parameters
-        ----------
-        data_catalog_id : str
-            The data catalog ID.
-        """
-        self.data_catalog_id = data_catalog_id
-
-        import planetary_computer
-        import pystac_client
-
-        catalog = pystac_client.Client.open(
-            "https://planetarycomputer.microsoft.com/api/stac/v1/",
-            modifier=planetary_computer.sign_inplace,
-        )
-        collection = catalog.get_collection(self.data_catalog_id)
-
-        asset = collection.assets["zarr-abfs"]
-
-        if "xarray:storage_options" in asset.extra_fields:
-            store = {
-                "store": asset.href,
-                "storage_options": asset.extra_fields["xarray:storage_options"],
-                **asset.extra_fields["xarray:open_kwargs"],
-            }
-        else:
-            store = {
-                "filename_or_obj": asset.href,
-                **asset.extra_fields["xarray:open_kwargs"],
-            }
-
-        self.store = store
-
-    def __getitem__(self, key: str) -> bytes:
-        """Retrieve an item from the store."""
-        raise NotImplementedError()
-
-
 class DebugStore(ReadOnlyStore):
     """A store to debug the zarr loading."""
 
@@ -190,11 +145,11 @@ def name_to_zarr_store(path_or_url: str) -> ReadOnlyStore:
 
     if store.startswith("http://") or store.startswith("https://"):
 
-        parsed = urlparse(store)
-
         if store.endswith(".zip"):
             import multiurl
 
+            parsed = urlparse(store)
+
             # Zarr cannot handle zip files over HTTP
             tmpdir = tempfile.gettempdir()
             name = os.path.basename(parsed.path)
@@ -210,15 +165,7 @@ def name_to_zarr_store(path_or_url: str) -> ReadOnlyStore:
             os.rename(path + ".tmp", path)
             return name_to_zarr_store(path)
 
-        bits = parsed.netloc.split(".")
-        if len(bits) == 5 and (bits[1], bits[3], bits[4]) == ("s3", "amazonaws", "com"):
-            s3_url = f"s3://{bits[0]}{parsed.path}"
-            store = S3Store(s3_url, region=bits[2])
-        elif store.startswith("https://planetarycomputer.microsoft.com/"):
-            data_catalog_id = store.rsplit("/", 1)[-1]
-            store = PlanetaryComputerStore(data_catalog_id).store
-        else:
-            store = HTTPStore(store)
+        return HTTPStore(store)
 
     return store
 
@@ -565,6 +512,10 @@ def zarr_lookup(name: str, fail: bool = True) -> Optional[str]:
     config = load_config()["datasets"]
     use_search_path_not_found = config.get("use_search_path_not_found", False)
 
+    if name.endswith(".zarr/"):
+        LOG.warning("Removing trailing slash from path: %s", name)
+        name = name[:-1]
+
     if name.endswith(".zarr") or name.endswith(".zip"):
 
         if os.path.exists(name):

anemoi/datasets/data/subset.py

@@ -185,6 +185,11 @@ class Subset(Forwards):
         n = self.indices[n]
         return self.dataset[n]
 
+    def get_aux(self, n: FullIndex) -> NDArray[Any]:
+        assert n >= 0, n
+        n = self.indices[n]
+        return self.dataset.get_aux(n)
+
     @debug_indexing
     def _get_slice(self, s: slice) -> NDArray[Any]:
         """Get slice of data.

anemoi/datasets/grids.py

@@ -605,6 +605,7 @@ def nearest_grid_points(
     target_latitudes: NDArray[Any],
     target_longitudes: NDArray[Any],
     max_distance: float = None,
+    k: int = 1,
 ) -> NDArray[Any]:
     """Find the nearest grid points from source to target coordinates.
 
@@ -621,6 +622,8 @@ def nearest_grid_points(
     max_distance: float, optional
         Maximum distance between nearest point and point to interpolate. Defaults to None.
         For example, 1e-3 is 1 km.
+    k : int, optional
+        The number of k closest neighbors to consider for interpolation
 
     Returns
     -------
@@ -637,10 +640,10 @@ def nearest_grid_points(
     target_xyz = latlon_to_xyz(target_latitudes, target_longitudes)
     target_points = np.array(target_xyz).transpose()
     if max_distance is None:
-        _, indices = cKDTree(source_points).query(target_points, k=1)
+        distances, indices = cKDTree(source_points).query(target_points, k=k)
     else:
-        _, indices = cKDTree(source_points).query(target_points, k=1, distance_upper_bound=max_distance)
-    return indices
+        distances, indices = cKDTree(source_points).query(target_points, k=k, distance_upper_bound=max_distance)
+    return distances, indices
 
 
 if __name__ == "__main__":

{anemoi_datasets-0.5.24.dist-info → anemoi_datasets-0.5.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: anemoi-datasets
-Version: 0.5.24
+Version: 0.5.26
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License
@@ -226,7 +226,7 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
 Requires-Dist: anemoi-transform>=0.1.10
-Requires-Dist: anemoi-utils[provenance]>=0.4.21
+Requires-Dist: anemoi-utils[provenance]>=0.4.26
 Requires-Dist: cfunits
 Requires-Dist: numcodecs<0.16
 Requires-Dist: numpy
@@ -262,6 +262,7 @@ Requires-Dist: requests; extra == "remote"
 Provides-Extra: tests
 Requires-Dist: anemoi-datasets[xarray]; extra == "tests"
 Requires-Dist: pytest; extra == "tests"
+Requires-Dist: pytest-xdist; extra == "tests"
 Provides-Extra: xarray
 Requires-Dist: adlfs; extra == "xarray"
 Requires-Dist: gcsfs; extra == "xarray"

{anemoi_datasets-0.5.24.dist-info → anemoi_datasets-0.5.26.dist-info}/RECORD

@@ -1,8 +1,8 @@
 anemoi/datasets/__init__.py,sha256=i_wsAT3ezEYF7o5dpqGrpoG4wmLS-QIBug18uJbSYMs,1065
 anemoi/datasets/__main__.py,sha256=ErwAqE3rBc7OaNO2JRsEOhWpB8ldjAt7BFSuRhbnlqQ,936
-anemoi/datasets/_version.py,sha256=lcZoRfXOiuB3TPgHkv3cgIWkhKIQEcb5AqCY1rvinxg,513
+anemoi/datasets/_version.py,sha256=Yk8LV8Lu80AtsqyYb-tECJmugSSgoanvPUOHGqIfkwA,513
 anemoi/datasets/check.py,sha256=hbEMUurl2IjZbp56dBgOfAEsAmmgymgRM5ySaMJSTdk,2755
-anemoi/datasets/grids.py,sha256=Hhj1aOXHvDjmI46M_UlLSjCs1qYqxH-uqd_kapDSdbU,18134
+anemoi/datasets/grids.py,sha256=_i5hgtFvT8Un7etkSTPYnrudLf2otz5iNqSRSl3_oDI,18271
 anemoi/datasets/testing.py,sha256=fy_JzavUwLlK_2rtXAT-UGUyo5gjyQW2y826zf334Wg,2645
 anemoi/datasets/commands/__init__.py,sha256=O5W3yHZywRoAqmRUioAr3zMCh0hGVV18wZYGvc00ioM,698
 anemoi/datasets/commands/check.py,sha256=zEJqE5XkNLsebemg5XIvvZTv2bA6R49zVaka9HRySKU,2655
@@ -11,53 +11,33 @@ anemoi/datasets/commands/compare-lam.py,sha256=F5GYRsKOtdhDePhifgf1TCj5L2T8EVIA2
 anemoi/datasets/commands/compare.py,sha256=jzhjbbt1U-YANTVRBhrwSh2CcYgk4qX2IiTMJtcn82s,3678
 anemoi/datasets/commands/copy.py,sha256=hP7BSqkGzK8-n3BA2vlpFcbO3INHmShPZ75Aw_K_g4Y,17302
 anemoi/datasets/commands/create.py,sha256=3myohTCLsM6oUuZHIfLaTlDaq-DOcJZRM4Vks007fZg,6543
-anemoi/datasets/commands/finalise-additions.py,sha256=2LqU7ke3i-yRQbjkgldX6e2QlyE-tKqp0b6QOhJF19g,1985
-anemoi/datasets/commands/finalise.py,sha256=-YtN9wFFDrM_i_V9YHoXZsajF3eAax-73Zsi4uHAFCI,1709
-anemoi/datasets/commands/grib-index.py,sha256=H9snsk1w2tL6ObBhlHVxFXxCm5sB-eUGrX772G8Nb2s,3057
-anemoi/datasets/commands/init-additions.py,sha256=2vMom5L38UvLLopzP2z-R_Fq31fU2uMvKXoDq5d8oI4,1931
+anemoi/datasets/commands/finalise-additions.py,sha256=GXjGAJILFORXXkE_wfgnk5w4jYug18Q2TpwVah5Ctto,1982
+anemoi/datasets/commands/finalise.py,sha256=cZIiqpJsaN1rqBKOStOA6pJh5n1tisrMFcGGT2UGpKY,1706
+anemoi/datasets/commands/grib-index.py,sha256=0_cISIRw0Iwf6NX5VasxlGbPWX1p7xDMciLglS9G1Yg,3075
+anemoi/datasets/commands/init-additions.py,sha256=wIsintXpf3aG2VhuBJJYI8ZZGXLrum7sM05IX8ImRXk,1928
 anemoi/datasets/commands/init.py,sha256=5IKyJ_hJA4lLIbpT88XtcGzXccHLSGwSoqVSvVJGxPg,2852
 anemoi/datasets/commands/inspect.py,sha256=kaDHXP8Cv8PsGqEXUF5Yruf5OQHwOIkjCS0SNxMs6eg,26578
-anemoi/datasets/commands/load-additions.py,sha256=zqmRherIHXb5WIB4cnAuCBEsxFJmUpGjafvm6RtQ7Co,2004
-anemoi/datasets/commands/load.py,sha256=U1Y8qByjreu7H9WeX4G8-tyKsz48va10fkW1L4U4wWg,2034
+anemoi/datasets/commands/load-additions.py,sha256=bg0JYaPPXhGXKL-p0nngAeI2rPhSTFpZ7bY04mKuSKk,2001
+anemoi/datasets/commands/load.py,sha256=FuKdRYuNlnv_p5ScwPKCiVDEVc9cA1Khvbd-T7N4SvU,2031
 anemoi/datasets/commands/patch.py,sha256=Q9FDabWxlvK1QaeH4D9zhNpoSGB4h7EliWgcV76iFBs,1599
 anemoi/datasets/commands/publish.py,sha256=7YusLCWYdVLuexZzvyh8ztYoBOBzVmve3uJs-XKeMAE,1469
 anemoi/datasets/commands/scan.py,sha256=6Uoyd7WkM4ypoqmZargXIG50uRKzHE3AlvkAr7sCBy4,4262
 anemoi/datasets/compute/__init__.py,sha256=hCW0QcLHJmE-C1r38P27_ZOvCLNewex5iQEtZqx2ckI,393
 anemoi/datasets/compute/recentre.py,sha256=kwxDB8qpgOCFZSQJvjAmVcpH5zWsfk5FSoIureqNHd4,5915
-anemoi/datasets/create/__init__.py,sha256=8TJwN_t7Epg6yBSbbRW2L039Xh_aqUsUgZBNR3U2yl0,51165
+anemoi/datasets/create/__init__.py,sha256=LAa-e6TnH5eHmHEpkMk9_a24wlpTFEKWllPgSl-mcW8,50855
 anemoi/datasets/create/check.py,sha256=xqobSfh3655ZoKs-CjHWBiEpIfrHU_vkqwiIsAOrqvs,10795
 anemoi/datasets/create/chunks.py,sha256=kZV3dWoCuv3Bttc0wysJB7OPbXsD99exKyrrj4HGFwQ,4025
 anemoi/datasets/create/config.py,sha256=xrSlaY2p5zssfLIt8A1CP9WwJReSXVWBMQM7bT1aFbU,13448
-anemoi/datasets/create/filter.py,sha256=Hu4o3Z2omIdcu5ycJqmBkY_ZSKTG5JkjbIuxXM8ADfs,1254
+anemoi/datasets/create/filter.py,sha256=LadfjHmDUc01eadLKAX8aeFXXZVfZ7BCHbxhRzEsYn4,1380
 anemoi/datasets/create/patch.py,sha256=u4CeIuo3Ncrbhu9CTyaUbcmaJfBfMrrFVpgEikM9pE4,5398
 anemoi/datasets/create/persistent.py,sha256=XkEBjymXrR-y9KPVLtz9xdd0IB14wSEhcANUhUUzGVw,7832
 anemoi/datasets/create/size.py,sha256=nHjX1manYhQIrcQWDCgBBBiAVDbfoMAG6ybVjytSaKI,1454
 anemoi/datasets/create/source.py,sha256=xoV8uH_y6aBSE4_PWuy5w7Q7cX-tGm8e-2xC9flSAT4,1336
-anemoi/datasets/create/testing.py,sha256=FzTSsbv_JBGViGrD1jT6z_T2yaA0KCrbJ3SCgp-rFPQ,2406
 anemoi/datasets/create/typing.py,sha256=Hs2uDG8ZVtQ-Q-5I9-W0Pik2p1hZH5-JPVjpJXRXP7M,484
 anemoi/datasets/create/utils.py,sha256=94JKPYcNSurA62yFAytW5dUFVz-r-fdwiPOkmu121pM,5572
 anemoi/datasets/create/writer.py,sha256=nZBJvYZ63g_c9FfL65bAeG10Y6bX2R7CgtZvY0kW3fI,2203
 anemoi/datasets/create/zarr.py,sha256=N9PGGD-dYvcc97BZjLVWm5XQeYTiK9gwvhtreRiQzBI,9437
-anemoi/datasets/create/filters/__init__.py,sha256=rhetUFPZKe-vwDIfMY33SbYrJMq909dUJRgBzuX6Q6E,901
-anemoi/datasets/create/filters/empty.py,sha256=Dw1kUnAlFt6b5ds0kmrw9Gak09XjSqF8m1_MpHZNV9I,1013
-anemoi/datasets/create/filters/legacy.py,sha256=6JY6uX7m-8NZjoZ1sqs0EAqT-uorvnZ-eFOwMU3LmRU,2536
-anemoi/datasets/create/filters/noop.py,sha256=WHl-k3NojGJMX4iNYxQ6Ln21pM8ERP4z8pQ5zLRDvXs,1019
-anemoi/datasets/create/filters/orog_to_z.py,sha256=vnZ1hD9LXoOfHCIbzkurMuBl_NSfXSiiHS2yZt8ndeQ,1784
-anemoi/datasets/create/filters/pressure_level_relative_humidity_to_specific_humidity.py,sha256=dBAQFNAc3GEZ_HwyDrcctFaKZQYddh2ldnDA2XSfSRg,2646
-anemoi/datasets/create/filters/pressure_level_specific_humidity_to_relative_humidity.py,sha256=51t6kbNZsXK87H0HVR9j0a54siokID47ve9r1a8rOLE,2663
-anemoi/datasets/create/filters/rename.py,sha256=pKi3CU6fvox2sPH7szXdA79NjdIcSz59IB7HsiS_9Co,5779
-anemoi/datasets/create/filters/rotate_winds.py,sha256=fVyAbypO_EsENHjQCujbEXp2gUEb97sMoG0s4YiPXfc,3102
-anemoi/datasets/create/filters/single_level_dewpoint_to_relative_humidity.py,sha256=hCS3yiN9nZf-P6shQmBm5Or9rMOwU1fTwHw_qFIjT9s,2378
-anemoi/datasets/create/filters/single_level_relative_humidity_to_dewpoint.py,sha256=pGwu6YprJ6PwJ8ZRD5k4Mz_wqSXz52jvXP13WDIGOTw,2642
-anemoi/datasets/create/filters/single_level_relative_humidity_to_specific_humidity.py,sha256=a9QVOG8hTVqkDtX4MJ5NB0y-huOKsOm_K4ujakQ36fg,5160
-anemoi/datasets/create/filters/single_level_specific_humidity_to_relative_humidity.py,sha256=bXgm5nKgBZaP1E4tcjSLqJsEl6BlJaNLr3MsR8V9sJ4,14682
-anemoi/datasets/create/filters/speeddir_to_uv.py,sha256=8NXsus1LaYOzAAr7XCHKCh8HAz8BI0A1ZZz_RNDB0-w,2762
-anemoi/datasets/create/filters/sum.py,sha256=aGT6JkdHJ3i2SKzklqiyJ4ZFV3bVMYhHOSoxkdYuzp8,2151
-anemoi/datasets/create/filters/transform.py,sha256=gIDLvaJlnn3Nc6P29aPOvNYM6yBWcIGrR2e_1bM6_Nw,1418
-anemoi/datasets/create/filters/unrotate_winds.py,sha256=3AJf0crnVVySLlXLIdfEUxRRlQeKgheUuD-UCrSrgo8,2798
-anemoi/datasets/create/filters/uv_to_speeddir.py,sha256=Zdc34AG5Bsz-Z7JGuznyRJr6F-BnWKXPiI3mjmOpbek,2883
-anemoi/datasets/create/filters/wz_to_w.py,sha256=slOiX5RibG48Zrkss8Qjpb-8ZTnvSvmKlk1Hy45_wzU,2812
-anemoi/datasets/create/input/__init__.py,sha256=XeURpmbReQvpELltGFKzg3oZFXWRdUxW9SK3K662SBQ,3364
+anemoi/datasets/create/input/__init__.py,sha256=vsB_whJG87IWnjaGgIMCCg8v9pfuC_vQk8BB3u5j33o,2886
 anemoi/datasets/create/input/action.py,sha256=pc_2RPbs3laF8vBhNkbFdib40kTcF5QW6QL0p8VLNzA,7778
 anemoi/datasets/create/input/concat.py,sha256=bU8SWfBVfK8bRAmmN4UO9zpIGxwQvRUk9_vwrKPOTE4,5355
 anemoi/datasets/create/input/context.py,sha256=qrLccxMe9UkyQxsNuR6JSK7oLzZq21dt38AxZ9kYzsY,2714
@@ -70,11 +50,11 @@ anemoi/datasets/create/input/misc.py,sha256=FVaH_ym52RZI_fnLSMM_dKTQmWTrInucP780
 anemoi/datasets/create/input/pipe.py,sha256=-tCz161IwXoI8pl1hilA9T_j5eHSr-sgbijFLp9HHNc,2083
 anemoi/datasets/create/input/repeated_dates.py,sha256=HaPzDCNHQBY1VVp6gvd3drwjWjYpSBh-GLgHqBRJTz0,12012
 anemoi/datasets/create/input/result.py,sha256=BmeZVN63ZnUkiOwT0mkE4DdB06OmVwdRZkiV4ACPNrI,24309
-anemoi/datasets/create/input/step.py,sha256=WcR9NgRvUKF60Fo5veLvRCAQMrOd55x1gOEAmd2t2r4,5948
+anemoi/datasets/create/input/step.py,sha256=NkmJ4cD9sURy_hwaeQN8kkdOfVtIW6xcxIClEWWdSvY,5376
 anemoi/datasets/create/input/template.py,sha256=Iycw9VmfA0WEIDP_Of8bp-8HsV0EUfwbnm0WjxiO4GA,4092
 anemoi/datasets/create/input/trace.py,sha256=dakPYMmwKq6s17Scww1CN-xYBD3btJTGeDknOhAcnEM,3320
 anemoi/datasets/create/sources/__init__.py,sha256=XNiiGaC6NbxnGfl6glPw-gTJASi3vsGKwVlfkMqYGk4,950
-anemoi/datasets/create/sources/accumulations.py,sha256=9I8_AQtD62Ji-7SMyyYBTdIZBG2rlPY9V2YjqLeWYl8,32538
+anemoi/datasets/create/sources/accumulations.py,sha256=vo2f7cejjkKneZ-9UVRJ_jwNiJJ1XNabxP57UHJZAZs,32572
 anemoi/datasets/create/sources/accumulations2.py,sha256=UwKJOtfbJGNTceVpLtHC8dJhROJbxzjF3V_HR7wTwrk,20661
 anemoi/datasets/create/sources/anemoi_dataset.py,sha256=2xJJTmKlv87F_2ECMKeehaeW7_oWLlDcLt8C_Prp1RI,2017
 anemoi/datasets/create/sources/constants.py,sha256=5O6d9tEuAmVjl5vNkNfmkaAjKXFlw1UjeueTsF1GZCI,1528
@@ -89,6 +69,7 @@ anemoi/datasets/create/sources/mars.py,sha256=tesQz7Ne6SLBChE_cNJU6Sxr6e0LXFlUKQ
 anemoi/datasets/create/sources/netcdf.py,sha256=UnehMwEMJquqaOeU33zNyFUYfzqQx4Rg-GRmUcgMcbE,1222
 anemoi/datasets/create/sources/opendap.py,sha256=sTm0wXE_BHk9q8vaNNE_Y6BhTOmhxPweS8RTjP4HYjU,1254
 anemoi/datasets/create/sources/patterns.py,sha256=siTExxLY5HWdIgOsufyQuG7Qvkt38oElZOFwz9h0JFg,2283
+anemoi/datasets/create/sources/planetary_computer.py,sha256=Erk6fKJt63gj_pgbklBWhAKjzjtAfq_DRizMfWdqPPU,1578
 anemoi/datasets/create/sources/recentre.py,sha256=OtobkmaWzGD3UacjXfK_Oerjf7EnQi85LIs9xBYJK7A,4044
 anemoi/datasets/create/sources/source.py,sha256=x8k---A2_3AglYqNsXLlv1ti4f9n_gVKmmqtyQGLPTs,2117
 anemoi/datasets/create/sources/tendencies.py,sha256=saHGYl-MnvBEeZX-n1zgT8lehA7LC2G5dMNnxklI9-U,5590
@@ -97,48 +78,54 @@ anemoi/datasets/create/sources/xarray_kerchunk.py,sha256=vdFaFzze8VLjYUgIX8Lc39E
 anemoi/datasets/create/sources/xarray_zarr.py,sha256=5eQOpB3sBD49RarTME81s0ynIVkha2pP0ymA4TNnLYY,1201
 anemoi/datasets/create/sources/zenodo.py,sha256=KEetFEk5GzGFpoos8rbBQBTa2XElWG7oTYjfZXgbu0Q,2065
 anemoi/datasets/create/sources/xarray_support/README.md,sha256=56olM9Jh0vI0_bU9GI-IqbBcz4DZXWONqvdzN_VeAFE,78
-anemoi/datasets/create/sources/xarray_support/__init__.py,sha256=8Dv7KQW8O3VvHOsSbqYdjaaomYIhXIKgSGatnNEweNU,5564
-anemoi/datasets/create/sources/xarray_support/coordinates.py,sha256=rPEuijS77mQ9V9tpN7wjg-w9rBxj7bZf_c30lLgSscE,11029
-anemoi/datasets/create/sources/xarray_support/field.py,sha256=YRxx6kh1qO2qQ6I_VyR51h3dwNiiFM7CNwQNfpp-p-E,6375
+anemoi/datasets/create/sources/xarray_support/__init__.py,sha256=X-Th4__2fVciyLqCUQU2mwYt4dph3l1_VH837iDeuqI,4714
+anemoi/datasets/create/sources/xarray_support/coordinates.py,sha256=dAwkIChEwhy7XY64BsXeRQOpcShQRufhsJROoUk68gY,11179
+anemoi/datasets/create/sources/xarray_support/field.py,sha256=1iLAMPpsYxzGuB4pqXA0TwEJQCCZDPFiiojfzFjnkzU,6280
 anemoi/datasets/create/sources/xarray_support/fieldlist.py,sha256=UyUljq2Ax-PpQ-bvG4Dsi_lkZucuPgCy120EadDeUMU,8271
-anemoi/datasets/create/sources/xarray_support/flavour.py,sha256=UyfzBjYMNfugMCq-r5Ie3qDuorLwaalPi_0oZHckZcg,32073
+anemoi/datasets/create/sources/xarray_support/flavour.py,sha256=oA5pYwe9HUHwgH09PgTYhJFSMmsOfRVsIEQvlVfCfM4,33366
 anemoi/datasets/create/sources/xarray_support/grid.py,sha256=lsE8bQwBH9pflzvsJ89Z6ExYPdHJd54xorMNzL2gTd0,6181
 anemoi/datasets/create/sources/xarray_support/metadata.py,sha256=OJ35Y4m9BpPmnrabD9qiuHUEfejc6YfTIWPm8prHokk,10876
-anemoi/datasets/create/sources/xarray_support/patch.py,sha256=Snk8bz7gp0HrG0MrY5hrXu7VC0tKgtoiWXByi2sBYJc,2037
+anemoi/datasets/create/sources/xarray_support/patch.py,sha256=8NHSDO2lLcRPBvmuPyQ5foAkYHFVFNWJhMVeXNs6x7o,3027
 anemoi/datasets/create/sources/xarray_support/time.py,sha256=Y_lZTUOXWJH4jcSgyL4WTDwrtPXi7MUiumaXfRoqqAY,12486
-anemoi/datasets/create/sources/xarray_support/variable.py,sha256=fcazws9vuizmx55JCXwbkwffg4WxJllPrEg2US1VysE,9163
+anemoi/datasets/create/sources/xarray_support/variable.py,sha256=zNXTNh58BLg9atnZJauto9z0e_gNRjreo4eUntqlJP8,9245
 anemoi/datasets/create/statistics/__init__.py,sha256=_BuPcuUrwQAEcMQVds93EV9M5ys2ao8jCWKV4OVoSSA,18291
 anemoi/datasets/create/statistics/summary.py,sha256=JdtChTmsr1Y958_nka36HltTbeZkawuGbprbfZD7Ux8,4790
 anemoi/datasets/data/__init__.py,sha256=wzhk_7VQImge12Xkg99xuiFOC7DAjBW1mu446y0Iq60,3057
-anemoi/datasets/data/complement.py,sha256=N1vJAO2bijrWAxXQi9AFAPVEBe4vikSIKEXcX1EqQHI,10590
+anemoi/datasets/data/complement.py,sha256=Sbn6ajVLcLOW_1xqPNCCteV7Et_1p2CLSZkQFT_oyYI,11991
 anemoi/datasets/data/concat.py,sha256=eY5rujcdal00BJCv00mKSlxp0FKVvPQd7uqrBnL9fj4,8996
-anemoi/datasets/data/dataset.py,sha256=d-LZPCczG2-ZAua29FGP3_QWzwdnWPHhKpG1dHFQKio,31290
+anemoi/datasets/data/dataset.py,sha256=it02CVYdzU9QXSkU9jVR9BTWv-JDMlnH0ujyIgf40pM,32326
 anemoi/datasets/data/debug.css,sha256=z2X_ZDSnZ9C3pyZPWnQiEyAxuMxUaxJxET4oaCImTAQ,211
 anemoi/datasets/data/debug.py,sha256=hVa1jAQ-TK7CoKJNyyUC0eZPobFG-FpkVXEaO_3B-MA,10796
 anemoi/datasets/data/ensemble.py,sha256=-36kMjuT2y5jUeSnjCRTCyE4um6DLAADBVSKSTkHZZg,5352
 anemoi/datasets/data/fill_missing.py,sha256=ceONpzD-PWLMTtG4WOw6USw-Cd1O55VYzfpAiEsROK8,8797
-anemoi/datasets/data/forwards.py,sha256=3DHmjed5lDG-4tUIafvbAE5z7bJ-odVT6yG68EdQzoY,19904
+anemoi/datasets/data/forwards.py,sha256=pFGTBgUfaUvOtcBQVweDVNpmONYc-apk-Rq9U2lpVvc,20117
 anemoi/datasets/data/grids.py,sha256=8fZSitKTStBT-fsQWwTXiTgyrmYjh_jQ5dJi1U3lRz0,22056
 anemoi/datasets/data/indexing.py,sha256=DasVd1j0FB0iTw6eqvhiLka4ztf2zJcI5NgWxmtxzCw,7526
 anemoi/datasets/data/interpolate.py,sha256=-kSYwdjKH7zJtfITdbqdH6KyOFGVZDyHg4TaFk9shEI,9279
 anemoi/datasets/data/join.py,sha256=ZEHOsCecKBkKKH-vki404Sm7r7cV368ECO7PXPpay3s,9212
 anemoi/datasets/data/masked.py,sha256=giOvHLcGbLf6mZPqZjAxQd1kvydmkepDFh2EqchXLTQ,10213
 anemoi/datasets/data/merge.py,sha256=SvQhJHf-C-Kn7hEjFqomienk-epPPjMtoccRNCJpMtw,8733
-anemoi/datasets/data/misc.py,sha256=6JOag2VJBJhz-eEEWL9F5nriEJyKz9aZL2P3FYcOSJo,20951
+anemoi/datasets/data/misc.py,sha256=x9E8qDqd7Z7blkva6Giu7RtzF0PtKeL8YSzpxvhPw1k,22899
 anemoi/datasets/data/missing.py,sha256=ogfVDponbs0bGHMxps32Fj_fq4gT26R70yEMco5gdK8,12593
+anemoi/datasets/data/padded.py,sha256=BYTLDNRatjEB2lri9IlLcMsFgxnQT2F5rZ0XxExjE7c,7881
 anemoi/datasets/data/rescale.py,sha256=nGfJ5tWCncMJ7NMXkLbmt6z0ELrD6FxpbjJreQ3W91g,7004
 anemoi/datasets/data/select.py,sha256=Xs6uOzJL0CoOGeWA_E5_ukr8Jav2kXbZ41vhk7Vr8PE,8277
 anemoi/datasets/data/statistics.py,sha256=Hi9tPtNPBFaD0jcBa5vxoZp1radEMS-1RXwA3RbWrK8,3173
-anemoi/datasets/data/stores.py,sha256=9RuNKbBv3SeEKRME63GZKzlJonMlvvyQk6F4xhSheL4,20023
-anemoi/datasets/data/subset.py,sha256=3h3Y7Mcw-fxs9-S0z8WNk6TcV5Hk5u9ZWfpJCTAZt4g,8804
+anemoi/datasets/data/stores.py,sha256=gQicl-VcvmxgWf8qtSpnCSwvatF8xv1lV_WFQRMpPWc,18282
+anemoi/datasets/data/subset.py,sha256=hEYRSc1QOdtJJpIWy85ZuZ9a6UNk-Eo9vAYsgRg2UPs,8950
 anemoi/datasets/data/unchecked.py,sha256=c7YIa9gFxOOjqyyOqrhGaFWQ1pN7_0W1Q8ABUTkI8e8,7311
 anemoi/datasets/data/xy.py,sha256=-jWzYismrK3eI3YCKIBpU1BCmraRncmVn0_2IUY--lk,7579
+anemoi/datasets/data/observations/__init__.py,sha256=zpYL-WEws9FZ6QwAfj8kcZXL0d7uZuSrMTxZ6wY3npg,11260
+anemoi/datasets/data/observations/legacy_obs_dataset.py,sha256=6Aj32XWmNmCsd0azhuB6aDcpDaJ7lRdgJ9KWlU9fXxs,7342
+anemoi/datasets/data/observations/multi.py,sha256=nDeN99LRNVyjUCNTFCL3R7iAQNsf1tSen-fSw9-VsiY,2184
+anemoi/datasets/data/records/__init__.py,sha256=lWJBukNArbBBdcfG8bh5MzJoSOjk5Fi8zcwHWo5a0I8,12405
+anemoi/datasets/data/records/backends/__init__.py,sha256=KAbzMflpo9ZFAComabAdYTNgbtr1mrrSFBQaUlQoLfs,5247
 anemoi/datasets/dates/__init__.py,sha256=pEArHDQ7w5E0WC8Vvf9ypyKSdm6gnhoN9TmooITB7C4,13617
 anemoi/datasets/dates/groups.py,sha256=IOveL6IyTXZwEdXZEnRAnpu9pINY95VN7LzcpLfJ09E,10105
 anemoi/datasets/utils/__init__.py,sha256=hCW0QcLHJmE-C1r38P27_ZOvCLNewex5iQEtZqx2ckI,393
-anemoi_datasets-0.5.24.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
-anemoi_datasets-0.5.24.dist-info/METADATA,sha256=VEJ0AfuyX7vg00NEIQUDecbv3eOHiTTtT1VE_zlAlYI,16107
-anemoi_datasets-0.5.24.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-anemoi_datasets-0.5.24.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
-anemoi_datasets-0.5.24.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_datasets-0.5.24.dist-info/RECORD,,
+anemoi_datasets-0.5.26.dist-info/licenses/LICENSE,sha256=8HznKF1Vi2IvfLsKNE5A2iVyiri3pRjRPvPC9kxs6qk,11354
+anemoi_datasets-0.5.26.dist-info/METADATA,sha256=o_5TIgYbXJxVZraRVkbPKqF6hRsI2Kkgmb4yL8I9lsk,16153
+anemoi_datasets-0.5.26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+anemoi_datasets-0.5.26.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
+anemoi_datasets-0.5.26.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_datasets-0.5.26.dist-info/RECORD,,

{anemoi_datasets-0.5.24.dist-info → anemoi_datasets-0.5.26.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

anemoi/datasets/create/filters/__init__.py

@@ -1,33 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-#
-
-from typing import Any
-
-from anemoi.utils.registry import Registry
-
-filter_registry = Registry(__name__)
-
-
-def create_filter(context: Any, config: Any) -> Any:
-    """Create a filter based on the provided configuration.
-
-    Parameters
-    ----------
-    context : Any
-        The context in which the filter is created.
-    config : Any
-        The configuration for the filter.
-
-    Returns
-    -------
-    Any
-        The created filter.
-    """
-    return filter_registry.from_config(config, context)

anemoi/datasets/create/filters/empty.py

@@ -1,37 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-from typing import Any
-
-import earthkit.data as ekd
-from anemoi.transform.fields import new_empty_fieldlist
-
-from .legacy import legacy_filter
-
-
-@legacy_filter(__file__)
-def execute(context: Any, input: ekd.FieldList, **kwargs: Any) -> ekd.FieldList:
-    """Create a pipeline that returns an empty result.
-
-    Parameters
-    ----------
-    context : Any
-        The context in which the function is executed.
-    input : List[Any]
-        List of input fields.
-    **kwargs : Any
-        Additional keyword arguments.
-
-    Returns
-    -------
-    Any
-        An empty result.
-    """
-    return new_empty_fieldlist()