anemoi-datasets 0.3.3__py3-none-any.whl → 0.3.6__py3-none-any.whl

anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.3'
-__version_tuple__ = version_tuple = (0, 3, 3)
+__version__ = version = '0.3.6'
+__version_tuple__ = version_tuple = (0, 3, 6)

anemoi/datasets/commands/compare.py

@@ -14,6 +14,8 @@ from . import Command
 
 
 class Compare(Command):
+    """Compare two datasets. This command compares the variables in two datasets and prints the mean of the common variables. It does not compare the data itself (yet)."""
+
     def add_arguments(self, command_parser):
         command_parser.add_argument("dataset1")
         command_parser.add_argument("dataset2")

anemoi/datasets/commands/{inspect/zarr.py → inspect.py}

@@ -24,6 +24,9 @@ from anemoi.utils.text import table
 
 from anemoi.datasets import open_dataset
 from anemoi.datasets.data.stores import open_zarr
+from anemoi.datasets.data.stores import zarr_lookup
+
+from . import Command
 
 LOG = logging.getLogger(__name__)
 
@@ -78,46 +81,12 @@ class Version:
         self.metadata = metadata
         self.version = version
         self.dataset = None
-        # try:
         self.dataset = open_dataset(self.path)
-        # except Exception as e:
-        #     LOG.error("Error opening dataset '%s': %s", self.path, e)
 
     def describe(self):
         print(f"📦 Path          : {self.path}")
         print(f"🔢 Format version: {self.version}")
 
-    def probe(self):
-        if "cos_local_time" not in self.name_to_index:
-            print("⚠️ probe: no cos_local_time")
-            return
-
-        try:
-            lon = self.longitudes
-        except AttributeError:
-            print("⚠️ probe: no longitudes")
-            return
-        # print(json.dumps(self.metadata, indent=4))
-        cos_local_time = self.name_to_index["cos_local_time"]
-        data = self.data
-        start, end, frequency = self.first_date, self.last_date, self.frequency
-        date = start
-        same = 0
-        for i in range(10):
-            field = data[i, cos_local_time]
-            buggy = cos_local_time_bug(lon, date).reshape(field.hape)
-            diff = np.abs(field - buggy)
-            if np.max(diff) < 1e-5:
-                same += 1
-            date += datetime.timedelta(hours=frequency)
-            if date > end:
-                break
-        if same > 1:
-            print("❌ probe: cos_local_time is buggy")
-            return
-
-        print("✅ probe: cos_local_time is fixed")
-
     @property
     def name_to_index(self):
         return find(self.metadata, "name_to_index")
@@ -586,29 +555,35 @@ VERSIONS = {
 }
 
 
-class InspectZarr:
-    """Inspect a checkpoint or zarr file."""
+class InspectZarr(Command):
+    """Inspect a zarr dataset."""
 
-    def inspect_zarr(self, path, **kwargs):
-        version = self._info(path)
+    def add_arguments(self, command_parser):
+        command_parser.add_argument("path", metavar="DATASET")
+        command_parser.add_argument("--detailed", action="store_true")
+
+        command_parser.add_argument("--progress", action="store_true")
+        command_parser.add_argument("--statistics", action="store_true")
+        command_parser.add_argument("--size", action="store_true", help="Print size")
 
-        # try:
-        #     with open("/tmp/probe.json", "w") as f:
-        #         json.dump(version.metadata, f, indent=4, sort_keys=True)
-        # except Exception:
-        #     pass
+    def run(self, args):
+        self.inspect_zarr(**vars(args))
+
+    def inspect_zarr(self, path, progress=False, statistics=False, detailed=False, size=False, **kwargs):
+        version = self._info(path)
 
         dotted_line()
         version.describe()
 
         try:
-            if kwargs.get("probe"):
-                return version.probe()
-            if kwargs.get("progress"):
+            if progress:
                 return version.progress()
-            if kwargs.get("statistics"):
+
+            if statistics:
                 return version.brute_force_statistics()
-            version.info(kwargs.get("detailed"), kwargs.get("size"))
+
+            version.info(detailed, size)
+
         except Exception as e:
             LOG.error("Error inspecting zarr file '%s': %s", path, e)
 
@@ -616,14 +591,7 @@ class InspectZarr:
             raise
 
     def _info(self, path):
-        if path.endswith("/"):
-            path = path[:-1]
-
-        try:
-            z = open_zarr(path)
-        except Exception as e:
-            LOG.error("Error opening zarr file '%s': %s", path, e)
-            raise
+        z = open_zarr(zarr_lookup(path))
 
         metadata = dict(z.attrs)
         version = metadata.get("version", "0.0.0")
@@ -640,3 +608,6 @@ class InspectZarr:
                 candidate = klass
 
         return candidate(path, z, metadata, version)
+
+
+command = InspectZarr

anemoi/datasets/compute/recentre.py

@@ -29,7 +29,7 @@ CLIP_VARIABLES = (
     "swl1",
 )
 
-SKIP = ("class", "stream", "type", "number", "expver", "_leg_number", "anoffset")
+SKIP = ("class", "stream", "type", "number", "expver", "_leg_number", "anoffset", "time", "date", "step")
 
 
 def check_compatible(f1, f2, centre_field_as_mars, ensemble_field_as_mars):

anemoi/datasets/create/functions/filters/rename.py

@@ -13,6 +13,14 @@ from climetlab.indexing.fieldset import FieldArray
 
 
 class RenamedFieldMapping:
+    """Rename a field based on the value of another field.
+
+    Args:
+        field (Field): The field to be renamed.
+        what (str): The name of the field that will be used to rename the field.
+        renaming (dict): A dictionary mapping the values of 'what' to the new names.
+    """
+
     def __init__(self, field, what, renaming):
         self.field = field
         self.what = what
@@ -29,6 +37,12 @@ class RenamedFieldMapping:
 
 
 class RenamedFieldFormat:
+    """Rename a field based on a format string.
+
+    Args:
+        format (str): A string that defines the new name of the field.
+    """
+
     def __init__(self, field, format):
         self.field = field
         self.format = format

anemoi/datasets/create/functions/sources/grib.py

@@ -8,6 +8,8 @@
 #
 
 
+import glob
+
 from climetlab import load_source
 from climetlab.utils.patterns import Pattern
 
@@ -22,6 +24,12 @@ def check(ds, paths, **kwargs):
         raise ValueError(f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, paths={paths})")
 
 
+def _expand(paths):
+    for path in paths:
+        for p in glob.glob(path):
+            yield p
+
+
 def execute(context, dates, path, *args, **kwargs):
     given_paths = path if isinstance(path, list) else [path]
 
@@ -35,7 +43,7 @@ def execute(context, dates, path, *args, **kwargs):
             if name in kwargs:
                 raise ValueError(f"MARS interpolation parameter '{name}' not supported")
 
-        for path in paths:
+        for path in _expand(paths):
             context.trace("📁", "PATH", path)
             s = load_source("file", path)
             s = s.sel(valid_datetime=dates, **kwargs)

anemoi/datasets/create/functions/sources/netcdf.py

@@ -7,10 +7,29 @@
 # nor does it submit to any jurisdiction.
 #
 
+import glob
+
 from climetlab import load_source
 from climetlab.utils.patterns import Pattern
 
 
+def _expand(paths):
+    for path in paths:
+        if path.startswith("file://"):
+            path = path[7:]
+
+        if path.startswith("http://"):
+            yield path
+            continue
+
+        if path.startswith("https://"):
+            yield path
+            continue
+
+        for p in glob.glob(path):
+            yield p
+
+
 def check(what, ds, paths, **kwargs):
     count = 1
     for k, v in kwargs.items():
@@ -32,7 +51,7 @@ def load_netcdfs(emoji, what, context, dates, path, *args, **kwargs):
 
         levels = kwargs.get("level", kwargs.get("levelist"))
 
-        for path in paths:
+        for path in _expand(paths):
             context.trace(emoji, what.upper(), path)
             s = load_source("opendap", path)
             s = s.sel(

anemoi/datasets/create/loaders.py

@@ -177,11 +177,14 @@ class InitialiserLoader(Loader):
                     return
                 if isinstance(obj, (dict, DictObj)):
                     if "grid" in obj:
+                        previous = obj["grid"]
                         obj["grid"] = "20./20."
-                        LOG.warn(f"Running in test mode. Setting grid to {obj['grid']}")
+                        LOG.warn(f"Running in test mode. Setting grid to {obj['grid']} instead of {previous}")
                     if "number" in obj:
-                        obj["number"] = obj["number"][0:3]
-                        LOG.warn(f"Running in test mode. Setting number to {obj['number']}")
+                        if isinstance(obj["number"], (list, tuple)):
+                            previous = obj["number"]
+                            obj["number"] = previous[0:3]
+                            LOG.warn(f"Running in test mode. Setting number to {obj['number']} instead of {previous}")
                     for k, v in obj.items():
                         set_to_test_mode(v)
 

anemoi/datasets/data/concat.py

@@ -12,7 +12,7 @@ import numpy as np
 
 from .debug import Node
 from .debug import debug_indexing
-from .forewards import Combined
+from .forwards import Combined
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices

anemoi/datasets/data/ensemble.py

@@ -8,7 +8,7 @@
 import logging
 
 from .debug import Node
-from .forewards import GivenAxis
+from .forwards import GivenAxis
 from .misc import _auto_adjust
 from .misc import _open
 

anemoi/datasets/data/{forewards.py → forwards.py}

@@ -91,12 +91,18 @@ class Forwards(Dataset):
     def metadata_specific(self, **kwargs):
         return super().metadata_specific(
             forward=self.forward.metadata_specific(),
+            **self.subclass_metadata_specific(),
             **kwargs,
         )
 
     def source(self, index):
         return self.forward.source(index)
 
+    def subclass_metadata_specific(self):
+        raise NotImplementedError(
+            f"subclass_metadata_specific() must be implemented in derived class {self.__class__.__name__}"
+        )
+
 
 class Combined(Forwards):
     def __init__(self, datasets):

anemoi/datasets/data/grids.py

@@ -12,8 +12,8 @@ import numpy as np
 
 from .debug import Node
 from .debug import debug_indexing
-from .forewards import Combined
-from .forewards import GivenAxis
+from .forwards import Combined
+from .forwards import GivenAxis
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -91,7 +91,7 @@ class Concat(Combined):
         return Node(self, [d.tree() for d in self.datasets])
 
 
-class Grids(GivenAxis):
+class GridsBase(GivenAxis):
     def __init__(self, datasets, axis):
         super().__init__(datasets, axis)
         # Shape: (dates, variables, ensemble, 1d-values)
@@ -106,7 +106,7 @@ class Grids(GivenAxis):
         pass
 
 
-class ConcatGrids(Grids):
+class Grids(GridsBase):
     # TODO: select the statistics of the most global grid?
     @property
     def latitudes(self):
@@ -127,7 +127,7 @@ class ConcatGrids(Grids):
         return Node(self, [d.tree() for d in self.datasets], mode="concat")
 
 
-class Cutout(Grids):
+class Cutout(GridsBase):
     def __init__(self, datasets, axis):
         from anemoi.datasets.grids import cutout_mask
 
@@ -220,7 +220,7 @@ def grids_factory(args, kwargs):
     datasets = [_open(e) for e in grids]
     datasets, kwargs = _auto_adjust(datasets, kwargs)
 
-    return ConcatGrids(datasets, axis=axis)._subset(**kwargs)
+    return Grids(datasets, axis=axis)._subset(**kwargs)
 
 
 def cutout_factory(args, kwargs):

anemoi/datasets/data/join.py

@@ -13,7 +13,7 @@ import numpy as np
 from .debug import Node
 from .debug import Source
 from .debug import debug_indexing
-from .forewards import Combined
+from .forwards import Combined
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices

anemoi/datasets/data/masked.py

@@ -14,7 +14,7 @@ from ..grids import cropping_mask
 from .dataset import Dataset
 from .debug import Node
 from .debug import debug_indexing
-from .forewards import Forwards
+from .forwards import Forwards
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -83,6 +83,9 @@ class Thinning(Masked):
     def tree(self):
         return Node(self, [self.forward.tree()], thinning=self.thinning, method=self.method)
 
+    def subclass_metadata_specific(self):
+        return dict(thinning=self.thinning, method=self.method)
+
 
 class Cropping(Masked):
     def __init__(self, forward, area):
@@ -104,3 +107,6 @@ class Cropping(Masked):
 
     def tree(self):
         return Node(self, [self.forward.tree()], area=self.area)
+
+    def metadata_specific(self, **kwargs):
+        return super().metadata_specific(area=self.area, **kwargs)

anemoi/datasets/data/misc.py

@@ -133,9 +133,9 @@ def _as_date(d, dates, last):
 
         if "-" in d and ":" in d:
             date, time = d.replace(" ", "T").split("T")
-            year, month, day = date.split("-")
-            hour, minute, second = time.split(":")
-            return np.datetime64(f"{year:04}-{month:02}-{day:02}T{hour}:{minute}:{second}")
+            year, month, day = [int(_) for _ in date.split("-")]
+            hour, minute, second = [int(_) for _ in time.split(":")]
+            return np.datetime64(f"{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}")
 
         if "-" in d:
             assert ":" not in d

anemoi/datasets/data/select.py

@@ -11,7 +11,7 @@ from functools import cached_property
 from .debug import Node
 from .debug import Source
 from .debug import debug_indexing
-from .forewards import Forwards
+from .forwards import Forwards
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -88,6 +88,10 @@ class Select(Forwards):
     def tree(self):
         return Node(self, [self.dataset.tree()], **self.title)
 
+    def subclass_metadata_specific(self):
+        # return dict(indices=self.indices)
+        return {}
+
 
 class Rename(Forwards):
     def __init__(self, dataset, rename):
@@ -105,8 +109,8 @@ class Rename(Forwards):
     def name_to_index(self):
         return {k: i for i, k in enumerate(self.variables)}
 
-    def metadata_specific(self, **kwargs):
-        return super().metadata_specific(rename=self.rename, **kwargs)
-
     def tree(self):
         return Node(self, [self.forward.tree()], rename=self.rename)
+
+    def subclass_metadata_specific(self):
+        return dict(rename=self.rename)

anemoi/datasets/data/statistics.py

@@ -10,7 +10,7 @@ from functools import cached_property
 
 from . import open_dataset
 from .debug import Node
-from .forewards import Forwards
+from .forwards import Forwards
 
 LOG = logging.getLogger(__name__)
 
@@ -34,11 +34,8 @@ class Statistics(Forwards):
             delta = self.frequency
         return self._statistic.statistics_tendencies(delta)
 
-    def metadata_specific(self, **kwargs):
-        return super().metadata_specific(
-            statistics=self._statistic.metadata_specific(),
-            **kwargs,
-        )
+    def subclass_metadata_specific(self):
+        return dict(statistics=self._statistic.metadata_specific())
 
     def tree(self):
         return Node(self, [self.forward.tree()])

anemoi/datasets/data/stores.py

@@ -347,7 +347,7 @@ class ZarrWithMissingDates(Zarr):
         return "zarr*"
 
 
-def zarr_lookup(name):
+def zarr_lookup(name, fail=True):
 
     if name.endswith(".zarr") or name.endswith(".zip"):
         return name
@@ -372,4 +372,7 @@ def zarr_lookup(name):
         except zarr.errors.PathNotFoundError:
             pass
 
-    raise ValueError(f"Cannot find a dataset that matched '{name}'. Tried: {tried}")
+    if fail:
+        raise ValueError(f"Cannot find a dataset that matched '{name}'. Tried: {tried}")
+
+    return None

anemoi/datasets/data/subset.py

@@ -13,7 +13,7 @@ import numpy as np
 from .debug import Node
 from .debug import Source
 from .debug import debug_indexing
-from .forewards import Forwards
+from .forwards import Forwards
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -109,3 +109,9 @@ class Subset(Forwards):
 
     def tree(self):
         return Node(self, [self.dataset.tree()], **self.reason)
+
+    def subclass_metadata_specific(self):
+        return {
+            # "indices": self.indices,
+            "reason": self.reason,
+        }

anemoi/datasets/data/unchecked.py

@@ -11,7 +11,7 @@ from functools import wraps
 
 from .concat import ConcatMixin
 from .debug import Node
-from .forewards import Combined
+from .forwards import Combined
 from .misc import _auto_adjust
 from .misc import _open
 

anemoi/datasets/dates/__init__.py

@@ -116,8 +116,8 @@ class StartEndDates(Dates):
         start = no_time_zone(start)
         end = no_time_zone(end)
 
-        if end <= start:
-            raise ValueError(f"End date {end} must be after start date {start}")
+        # if end <= start:
+        #     raise ValueError(f"End date {end} must be after start date {start}")
 
         increment = datetime.timedelta(hours=frequency)
 

{anemoi_datasets-0.3.3.dist-info → anemoi_datasets-0.3.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.3.3
+Version: 0.3.6
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License: Apache License
@@ -230,20 +230,14 @@ Requires-Dist: semantic-version
 Requires-Dist: tqdm
 Requires-Dist: zarr <=2.17
 Provides-Extra: all
-Requires-Dist: anemoi-utils[provenance] >=0.3 ; extra == 'all'
 Requires-Dist: boto3 ; extra == 'all'
 Requires-Dist: climetlab >=0.23.2 ; extra == 'all'
 Requires-Dist: earthkit-meteo ; extra == 'all'
 Requires-Dist: ecmwflibs >=0.6.3 ; extra == 'all'
 Requires-Dist: entrypoints ; extra == 'all'
-Requires-Dist: numpy ; extra == 'all'
 Requires-Dist: pyproj ; extra == 'all'
-Requires-Dist: pyyaml ; extra == 'all'
 Requires-Dist: requests ; extra == 'all'
 Requires-Dist: s3fs ; extra == 'all'
-Requires-Dist: semantic-version ; extra == 'all'
-Requires-Dist: tqdm ; extra == 'all'
-Requires-Dist: zarr <=2.17 ; extra == 'all'
 Provides-Extra: create
 Requires-Dist: climetlab >=0.23.2 ; extra == 'create'
 Requires-Dist: earthkit-meteo ; extra == 'create'
@@ -252,11 +246,14 @@ Requires-Dist: entrypoints ; extra == 'create'
 Requires-Dist: pyproj ; extra == 'create'
 Provides-Extra: dev
 Requires-Dist: boto3 ; extra == 'dev'
+Requires-Dist: climetlab >=0.23.2 ; extra == 'dev'
 Requires-Dist: earthkit-meteo ; extra == 'dev'
 Requires-Dist: ecmwflibs >=0.6.3 ; extra == 'dev'
+Requires-Dist: entrypoints ; extra == 'dev'
 Requires-Dist: nbsphinx ; extra == 'dev'
 Requires-Dist: pandoc ; extra == 'dev'
 Requires-Dist: pyproj ; extra == 'dev'
+Requires-Dist: pytest ; extra == 'dev'
 Requires-Dist: requests ; extra == 'dev'
 Requires-Dist: s3fs ; extra == 'dev'
 Requires-Dist: sphinx ; extra == 'dev'
@@ -272,4 +269,6 @@ Provides-Extra: remote
 Requires-Dist: boto3 ; extra == 'remote'
 Requires-Dist: requests ; extra == 'remote'
 Requires-Dist: s3fs ; extra == 'remote'
+Provides-Extra: tests
+Requires-Dist: pytest ; extra == 'tests'
 

{anemoi_datasets-0.3.3.dist-info → anemoi_datasets-0.3.6.dist-info}/RECORD

@@ -1,22 +1,21 @@
 anemoi/datasets/__init__.py,sha256=DC7ttKT--pmhBQALX_Cn7P28dngsJucKi5y-Ydm28QM,700
 anemoi/datasets/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
-anemoi/datasets/_version.py,sha256=FKnJIExgNrZG2xJ0y_dGNBpxGbGBYylvfat-jHhLUuM,411
+anemoi/datasets/_version.py,sha256=IKAQ4gPrCQ2FWMXOFRqouULC2EQI1zCb4iXHsnfbmTQ,411
 anemoi/datasets/grids.py,sha256=3YBMMJodgYhavarXPAlMZHaMtDT9v2IbTmAXZTqf8Qo,8481
 anemoi/datasets/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
-anemoi/datasets/commands/compare.py,sha256=tN3eqihvnZ0rFc0OUzrfI34PHDlYfc2l90ZIQBE1TDQ,1300
+anemoi/datasets/commands/compare.py,sha256=p2jQOAC3JhScCLF0GjTCO8goYLWLN8p7vzy_gf5fFcI,1473
 anemoi/datasets/commands/copy.py,sha256=fba-zjD0iTHHXHhPEcm8VhDzsXQXDUxlbtTA1TovyT0,9991
 anemoi/datasets/commands/create.py,sha256=POdOsVDlvRrHFFkI3SNXNgNIbSxkVUUPMoo660x7Ma0,987
+anemoi/datasets/commands/inspect.py,sha256=G3fzcgiLaU8jln7GKvgamN7Y06-qC_JnFw2SbNn1_E4,18646
 anemoi/datasets/commands/scan.py,sha256=HxsLdCgBMSdEXjlJfPq5M_9LxXHHQIoZ1ZEHO_AoPgA,2881
-anemoi/datasets/commands/inspect/__init__.py,sha256=v6fPUTdMRdmUiEUUs0F74QlzPr-x5XEEOql3mkFme7E,1500
-anemoi/datasets/commands/inspect/zarr.py,sha256=Q1waDTgdJZwJXNST4jkO4DCIbqbf2T_2Us2k6yKGToo,19684
 anemoi/datasets/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-anemoi/datasets/compute/recentre.py,sha256=j8LdC8kq1t4PW7WFTXf93hSxok10un8ENIPwCehzbP8,4768
+anemoi/datasets/compute/recentre.py,sha256=GRxI6rY_KyXJnZGPxU_UO9YDb-rY_raK70Fiwv1mjhs,4792
 anemoi/datasets/create/__init__.py,sha256=jji65Zni5aPTvS269fAMix4pN9ukmSoK0z5SVsbpr5E,5807
 anemoi/datasets/create/check.py,sha256=DLjw-eyaCNxPhoKFsP4Yn_l3SIr57YHdyPR-tE5vx80,5791
 anemoi/datasets/create/chunks.py,sha256=YEDcr0K2KiiceSTiBuZzj0TbRbzZ9J546XO7rrrTFQw,2441
 anemoi/datasets/create/config.py,sha256=uLIp1WHg3hbqwwMV9EepMwJQsXJAGImkbo0okBeEVd4,7683
 anemoi/datasets/create/input.py,sha256=3G7sqdn7R1pLBeeswXwwi8VRAHrBnjq1PdRYHJBe594,27741
-anemoi/datasets/create/loaders.py,sha256=BDeb2CI_oWqIGcBnt39nRGIt3r3dK4rIidNO3pBunTk,29865
+anemoi/datasets/create/loaders.py,sha256=Y9HwH_yPDvnrOTpXVwZxB6VKqGYrHC4xMKhizUhR4lU,30081
 anemoi/datasets/create/patch.py,sha256=xjCLhvIQKRqmypsKInRU1CvFh1uoaB3YGSQP1UVZZik,3682
 anemoi/datasets/create/persistent.py,sha256=nT8gvhVPdI1H3zW_F7uViGKIlQQ94jCDrMSWTmhQ2_A,4290
 anemoi/datasets/create/size.py,sha256=A1w6RkaL0L9IlwIdmYsCTJTecmY_QtvbkGf__jvQle0,1068
@@ -28,7 +27,7 @@ anemoi/datasets/create/functions/__init__.py,sha256=v5RRSAK8LiNdPbxy0c9WxqMx3AMH
 anemoi/datasets/create/functions/filters/__init__.py,sha256=Xe9G54CKvCI3ji-7k0R5l0WZZdhlydRgawsXuBcX_hg,379
 anemoi/datasets/create/functions/filters/empty.py,sha256=X7gAZS_jl9X0ztGs3Arn8DkOB6iP4Draq4EAKa4I8hc,617
 anemoi/datasets/create/functions/filters/noop.py,sha256=ZP434Z1rFlqdgXse_1ZzqC2XAqRQlYlXlVfGLx7rK8g,444
-anemoi/datasets/create/functions/filters/rename.py,sha256=Nvmwor6S5E-mhp60hF58KkFthxbYodGcpG-sJPqbkJ0,1673
+anemoi/datasets/create/functions/filters/rename.py,sha256=cGoHr-IS-PhYEtZvXDpH03gtfZ9uT7uEZqu9EDxAxN0,2105
 anemoi/datasets/create/functions/filters/rotate_winds.py,sha256=fUdh8ILcMzMzckGlvwzdgG-c7w5R9NnWfaijp28Bf5M,4092
 anemoi/datasets/create/functions/filters/unrotate_winds.py,sha256=nsa3EHly8ppWd2WH4ROoMczM8WFu5qKaIhO_UFcL9TY,3502
 anemoi/datasets/create/functions/sources/__init__.py,sha256=Xe9G54CKvCI3ji-7k0R5l0WZZdhlydRgawsXuBcX_hg,379
@@ -36,10 +35,10 @@ anemoi/datasets/create/functions/sources/accumulations.py,sha256=li1tpEew1XUv4sJ
 anemoi/datasets/create/functions/sources/constants.py,sha256=aqquu6HDc8t-zsF9KRFLaj0eV4S0UPZ59BVna8E3bU8,785
 anemoi/datasets/create/functions/sources/empty.py,sha256=SBuAfC33imbfcRnFnnOR44y8Q3KSQcqx3juIcXfCa3c,481
 anemoi/datasets/create/functions/sources/forcings.py,sha256=EVcdu8puMSW451qj3LKCWWXaSf2LlmF8YXVs8hSMxkU,643
-anemoi/datasets/create/functions/sources/grib.py,sha256=p1vYlRfV0irlkVBTYhNfQ7OrJjHaMfe2KkSphyNtyU0,1540
+anemoi/datasets/create/functions/sources/grib.py,sha256=YQNuGnlh2EYb2NIHYpzlipwUTmOhrmyQtP3zgk8MAUU,1661
 anemoi/datasets/create/functions/sources/hindcasts.py,sha256=0Psnsx2J0cRLMpJuNN-gESm1xJFC1gmQzI8sdnXCoYE,13042
 anemoi/datasets/create/functions/sources/mars.py,sha256=Jau-ceN_cI3Z2-uql92iS4-Emh9Pie7omdRkFB5oe1I,4025
-anemoi/datasets/create/functions/sources/netcdf.py,sha256=XeV3ZlQWPZD2Yxxi5HzvhKKS-04WpLnVSORv-cBa-nM,1726
+anemoi/datasets/create/functions/sources/netcdf.py,sha256=kic6PH7SAK3gseXChD38IDXw6Zcg2zhF4SeDXB2LQ8Q,2084
 anemoi/datasets/create/functions/sources/opendap.py,sha256=T0CPinscfafrVLaye5ue-PbiCNbcNqf_3m6pphN9rCU,543
 anemoi/datasets/create/functions/sources/recentre.py,sha256=t07LIXG3Hp9gmPkPriILVt86TxubsHyS1EL1lzwgtXY,1810
 anemoi/datasets/create/functions/sources/source.py,sha256=hPQnV_6UIxFw97uRKcTA8TplcgG1kC8NlFHoEaaLet4,1418
@@ -47,28 +46,28 @@ anemoi/datasets/create/functions/sources/tendencies.py,sha256=kwS_GZt8R9kpfs5Rrv
 anemoi/datasets/create/statistics/__init__.py,sha256=X50drgE-ltuNe7bSIyvyeC4GeTqGTQGbglh2-2aVWKE,15445
 anemoi/datasets/create/statistics/summary.py,sha256=sgmhA24y3VRyjmDUgTnPIqcHSlWBbFA0qynx6gJ9Xw8,3370
 anemoi/datasets/data/__init__.py,sha256=tacn6K_VZ-pYhLmGePG5sze8kmqGpqscYb-bMyQnWtk,888
-anemoi/datasets/data/concat.py,sha256=U6IZi6NkI6yccrDamgasENBqwyJ1m0ZesuDtHXoqEh8,3551
+anemoi/datasets/data/concat.py,sha256=AkpyOs16OjW7X0cdyYFQfWSCV6dteXBp-x9WlokO-DI,3550
 anemoi/datasets/data/dataset.py,sha256=UDnidq2amyCT2COH05pGfDCJcmkdMj1ubtHk9cl-qcE,7384
 anemoi/datasets/data/debug.css,sha256=z2X_ZDSnZ9C3pyZPWnQiEyAxuMxUaxJxET4oaCImTAQ,211
 anemoi/datasets/data/debug.py,sha256=PcyrjgxaLzeb_vf12pvUtPPVvBRHNm1SimythZvqsP4,6303
-anemoi/datasets/data/ensemble.py,sha256=PcrdNL4DhAuWYSXgNxC6igDXpDndXC_QrbLrL4Lvj-Y,1138
-anemoi/datasets/data/forewards.py,sha256=4IsaNDhYlLiCbawUvTynm2vdpGPqdXcrSoAENwsJoqI,7456
-anemoi/datasets/data/grids.py,sha256=vgZMIQbv5SnIcnPu2ujsrAQ8VyBz5o2a1SnxsjXkDuw,7495
+anemoi/datasets/data/ensemble.py,sha256=AsP7Xx0ZHLoZs6a4EC0jtyGYIcOvZvvKXhgNsIvqIN8,1137
+anemoi/datasets/data/forwards.py,sha256=t9YQCN7j75VMInt0uP9JUJoh1klF1Z1xnwy5_kDMDQs,7700
+anemoi/datasets/data/grids.py,sha256=rooOeR6rvjl4U8B4LO3N23fcgxvGE7ZUmhVryk1QS4M,7493
 anemoi/datasets/data/indexing.py,sha256=625m__JG5m_tDMrkz1hB6Vydenwt0oHuyAlc-o3Zwos,4799
-anemoi/datasets/data/join.py,sha256=m_lpxWPy8-xYOjPbVoBV3V92VGtBFIriiDWvQM6KqXc,4893
-anemoi/datasets/data/masked.py,sha256=KZZ-3nq9saj_W8PTN9V4YdZ24BayHgECj12i4yjyKpc,3525
-anemoi/datasets/data/misc.py,sha256=m_28VIhX546RIoVfGpimPOThl5EwOhkun2UgWMAUxqw,10355
-anemoi/datasets/data/select.py,sha256=JoEepq8iRSSX6L75hzhLrBFhy0RJInuBM3C_Eu2Ryv0,3608
-anemoi/datasets/data/statistics.py,sha256=rWuG5qlfQoo9shOXR6TleJbJONwYggxxLy_HRet8azM,1582
-anemoi/datasets/data/stores.py,sha256=damJzNScaGenARAv8xpNa7d32f03MpGk5adRoRi34yw,10851
-anemoi/datasets/data/subset.py,sha256=RjfOMu7p69DZXRxQpvTfDOjVAURhgUO2pWyuZpXlJGY,3671
-anemoi/datasets/data/unchecked.py,sha256=qeUKthbvVVSPH-P366q1DEofvPzZSSXCXA49x-RkBOc,4038
-anemoi/datasets/dates/__init__.py,sha256=zOph2N_mXYbjSvqEWYF1mmm-UZpljb61WLrdFJmi0qQ,4469
+anemoi/datasets/data/join.py,sha256=dtCBbMTicqrRPxfBULi3RwEcQBLhQpIcvCjdN5A3XUU,4892
+anemoi/datasets/data/masked.py,sha256=Fzkehyka70CiS0LYSy_uyVYu2gKLwDSxlbm8GiC_pYs,3742
+anemoi/datasets/data/misc.py,sha256=cu2rMNtq8M8yzcR1CGHHQksE0p9C2SLVigOTMH6ilMs,10400
+anemoi/datasets/data/select.py,sha256=U3AEid80mrJKu0SF4lLc-bRWMVcAZwHNUHUHRehvuHU,3680
+anemoi/datasets/data/statistics.py,sha256=PKRgcCiZEb1HjkIveVGhE3TzUy9Qe3AYWGFD72Urah8,1514
+anemoi/datasets/data/stores.py,sha256=oEjUra0zzIysiUvh-RBQRzcbviggejEQiMO5RfpjPyM,10896
+anemoi/datasets/data/subset.py,sha256=9urVTXdnwCgqn0_BRYquMi8oiXn4ubAf0n4586hWfKw,3814
+anemoi/datasets/data/unchecked.py,sha256=xhdMg-ToI1UfBWHNsWyn1y2meZWngZtHx-33L0KqKp8,4037
+anemoi/datasets/dates/__init__.py,sha256=4ItowfLLh90T8L_JOjtv98lE6M7gAaWt7dV3niUrFvk,4473
 anemoi/datasets/dates/groups.py,sha256=iq310Pi7ullglOhcNblv14MmcT8FPgYCD5s45qAfV_s,3383
 anemoi/datasets/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-anemoi_datasets-0.3.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_datasets-0.3.3.dist-info/METADATA,sha256=Tf99e7CzYgXE32yyPro7S7xKUzRfmGQYn5jj03j5mdM,16090
-anemoi_datasets-0.3.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-anemoi_datasets-0.3.3.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
-anemoi_datasets-0.3.3.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_datasets-0.3.3.dist-info/RECORD,,
+anemoi_datasets-0.3.6.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_datasets-0.3.6.dist-info/METADATA,sha256=sEnXtS3eh-ix9Z1aR8RFDqBlnmK-XE6A42ga2uE_Utc,16017
+anemoi_datasets-0.3.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+anemoi_datasets-0.3.6.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
+anemoi_datasets-0.3.6.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_datasets-0.3.6.dist-info/RECORD,,

anemoi/datasets/commands/inspect/__init__.py

@@ -1,37 +0,0 @@
-# (C) Copyright 2023 European Centre for Medium-Range Weather Forecasts.
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-import os
-
-from .. import Command
-from .zarr import InspectZarr
-
-
-class Inspect(Command, InspectZarr):
-    """Inspect a zarr dataset."""
-
-    def add_arguments(self, command_parser):
-        # g = command_parser.add_mutually_exclusive_group()
-        command_parser.add_argument("path", metavar="PATH", nargs="+")
-        command_parser.add_argument("--detailed", action="store_true")
-        # command_parser.add_argument("--probe", action="store_true")
-        command_parser.add_argument("--progress", action="store_true")
-        command_parser.add_argument("--statistics", action="store_true")
-        command_parser.add_argument("--size", action="store_true", help="Print size")
-
-    def run(self, args):
-        dic = vars(args)
-        for path in dic.pop("path"):
-            if os.path.isdir(path) or path.endswith(".zarr.zip") or path.endswith(".zarr"):
-                self.inspect_zarr(path=path, **dic)
-            else:
-                raise ValueError(f"Unknown file type: {path}")
-                # self.inspect_checkpoint(path=path, **dic)
-
-
-command = Inspect