anemoi-datasets 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.3.3'
-__version_tuple__ = version_tuple = (0, 3, 3)
+__version__ = version = '0.3.5'
+__version_tuple__ = version_tuple = (0, 3, 5)

anemoi/datasets/commands/{inspect/zarr.py → inspect.py}

@@ -24,6 +24,9 @@ from anemoi.utils.text import table
 
 from anemoi.datasets import open_dataset
 from anemoi.datasets.data.stores import open_zarr
+from anemoi.datasets.data.stores import zarr_lookup
+
+from . import Command
 
 LOG = logging.getLogger(__name__)
 
@@ -78,46 +81,12 @@ class Version:
         self.metadata = metadata
         self.version = version
         self.dataset = None
-        # try:
         self.dataset = open_dataset(self.path)
-        # except Exception as e:
-        #     LOG.error("Error opening dataset '%s': %s", self.path, e)
 
     def describe(self):
         print(f"📦 Path          : {self.path}")
         print(f"🔢 Format version: {self.version}")
 
-    def probe(self):
-        if "cos_local_time" not in self.name_to_index:
-            print("⚠️ probe: no cos_local_time")
-            return
-
-        try:
-            lon = self.longitudes
-        except AttributeError:
-            print("⚠️ probe: no longitudes")
-            return
-        # print(json.dumps(self.metadata, indent=4))
-        cos_local_time = self.name_to_index["cos_local_time"]
-        data = self.data
-        start, end, frequency = self.first_date, self.last_date, self.frequency
-        date = start
-        same = 0
-        for i in range(10):
-            field = data[i, cos_local_time]
-            buggy = cos_local_time_bug(lon, date).reshape(field.hape)
-            diff = np.abs(field - buggy)
-            if np.max(diff) < 1e-5:
-                same += 1
-            date += datetime.timedelta(hours=frequency)
-            if date > end:
-                break
-        if same > 1:
-            print("❌ probe: cos_local_time is buggy")
-            return
-
-        print("✅ probe: cos_local_time is fixed")
-
     @property
     def name_to_index(self):
         return find(self.metadata, "name_to_index")
@@ -586,29 +555,35 @@ VERSIONS = {
 }
 
 
-class InspectZarr:
-    """Inspect a checkpoint or zarr file."""
+class InspectZarr(Command):
+    """Inspect a zarr dataset."""
 
-    def inspect_zarr(self, path, **kwargs):
-        version = self._info(path)
+    def add_arguments(self, command_parser):
+        command_parser.add_argument("path", metavar="DATASET")
+        command_parser.add_argument("--detailed", action="store_true")
+
+        command_parser.add_argument("--progress", action="store_true")
+        command_parser.add_argument("--statistics", action="store_true")
+        command_parser.add_argument("--size", action="store_true", help="Print size")
 
-        # try:
-        #     with open("/tmp/probe.json", "w") as f:
-        #         json.dump(version.metadata, f, indent=4, sort_keys=True)
-        # except Exception:
-        #     pass
+    def run(self, args):
+        self.inspect_zarr(**vars(args))
+
+    def inspect_zarr(self, path, progress=False, statistics=False, detailed=False, size=False, **kwargs):
+        version = self._info(path)
 
         dotted_line()
         version.describe()
 
         try:
-            if kwargs.get("probe"):
-                return version.probe()
-            if kwargs.get("progress"):
+            if progress:
                 return version.progress()
-            if kwargs.get("statistics"):
+
+            if statistics:
                 return version.brute_force_statistics()
-            version.info(kwargs.get("detailed"), kwargs.get("size"))
+
+            version.info(detailed, size)
+
         except Exception as e:
             LOG.error("Error inspecting zarr file '%s': %s", path, e)
 
@@ -616,14 +591,7 @@ class InspectZarr:
             raise
 
     def _info(self, path):
-        if path.endswith("/"):
-            path = path[:-1]
-
-        try:
-            z = open_zarr(path)
-        except Exception as e:
-            LOG.error("Error opening zarr file '%s': %s", path, e)
-            raise
+        z = open_zarr(zarr_lookup(path))
 
         metadata = dict(z.attrs)
         version = metadata.get("version", "0.0.0")
@@ -640,3 +608,6 @@ class InspectZarr:
                 candidate = klass
 
         return candidate(path, z, metadata, version)
+
+
+command = InspectZarr

anemoi/datasets/create/functions/sources/grib.py

@@ -8,6 +8,8 @@
 #
 
 
+import glob
+
 from climetlab import load_source
 from climetlab.utils.patterns import Pattern
 
@@ -22,6 +24,12 @@ def check(ds, paths, **kwargs):
         raise ValueError(f"Expected {count} fields, got {len(ds)} (kwargs={kwargs}, paths={paths})")
 
 
+def _expand(paths):
+    for path in paths:
+        for p in glob.glob(path):
+            yield p
+
+
 def execute(context, dates, path, *args, **kwargs):
     given_paths = path if isinstance(path, list) else [path]
 
@@ -35,7 +43,7 @@ def execute(context, dates, path, *args, **kwargs):
             if name in kwargs:
                 raise ValueError(f"MARS interpolation parameter '{name}' not supported")
 
-        for path in paths:
+        for path in _expand(paths):
             context.trace("📁", "PATH", path)
             s = load_source("file", path)
             s = s.sel(valid_datetime=dates, **kwargs)

anemoi/datasets/create/functions/sources/netcdf.py

@@ -7,10 +7,29 @@
 # nor does it submit to any jurisdiction.
 #
 
+import glob
+
 from climetlab import load_source
 from climetlab.utils.patterns import Pattern
 
 
+def _expand(paths):
+    for path in paths:
+        if path.startswith("file://"):
+            path = path[7:]
+
+        if path.startswith("http://"):
+            yield path
+            continue
+
+        if path.startswith("https://"):
+            yield path
+            continue
+
+        for p in glob.glob(path):
+            yield p
+
+
 def check(what, ds, paths, **kwargs):
     count = 1
     for k, v in kwargs.items():
@@ -32,7 +51,7 @@ def load_netcdfs(emoji, what, context, dates, path, *args, **kwargs):
 
         levels = kwargs.get("level", kwargs.get("levelist"))
 
-        for path in paths:
+        for path in _expand(paths):
             context.trace(emoji, what.upper(), path)
             s = load_source("opendap", path)
             s = s.sel(

anemoi/datasets/data/concat.py

@@ -12,7 +12,7 @@ import numpy as np
 
 from .debug import Node
 from .debug import debug_indexing
-from .forewards import Combined
+from .forwards import Combined
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices

anemoi/datasets/data/ensemble.py

@@ -8,7 +8,7 @@
 import logging
 
 from .debug import Node
-from .forewards import GivenAxis
+from .forwards import GivenAxis
 from .misc import _auto_adjust
 from .misc import _open
 

anemoi/datasets/data/{forewards.py → forwards.py}

@@ -91,12 +91,18 @@ class Forwards(Dataset):
     def metadata_specific(self, **kwargs):
         return super().metadata_specific(
             forward=self.forward.metadata_specific(),
+            **self.subclass_metadata_specific(),
             **kwargs,
         )
 
     def source(self, index):
         return self.forward.source(index)
 
+    def subclass_metadata_specific(self):
+        raise NotImplementedError(
+            f"subclass_metadata_specific() must be implemented in derived class {self.__class__.__name__}"
+        )
+
 
 class Combined(Forwards):
     def __init__(self, datasets):

anemoi/datasets/data/grids.py

@@ -12,8 +12,8 @@ import numpy as np
 
 from .debug import Node
 from .debug import debug_indexing
-from .forewards import Combined
-from .forewards import GivenAxis
+from .forwards import Combined
+from .forwards import GivenAxis
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -91,7 +91,7 @@ class Concat(Combined):
         return Node(self, [d.tree() for d in self.datasets])
 
 
-class Grids(GivenAxis):
+class GridsBase(GivenAxis):
     def __init__(self, datasets, axis):
         super().__init__(datasets, axis)
         # Shape: (dates, variables, ensemble, 1d-values)
@@ -106,7 +106,7 @@ class Grids(GivenAxis):
         pass
 
 
-class ConcatGrids(Grids):
+class Grids(GridsBase):
     # TODO: select the statistics of the most global grid?
     @property
     def latitudes(self):
@@ -127,7 +127,7 @@ class ConcatGrids(Grids):
         return Node(self, [d.tree() for d in self.datasets], mode="concat")
 
 
-class Cutout(Grids):
+class Cutout(GridsBase):
     def __init__(self, datasets, axis):
         from anemoi.datasets.grids import cutout_mask
 
@@ -220,7 +220,7 @@ def grids_factory(args, kwargs):
     datasets = [_open(e) for e in grids]
     datasets, kwargs = _auto_adjust(datasets, kwargs)
 
-    return ConcatGrids(datasets, axis=axis)._subset(**kwargs)
+    return Grids(datasets, axis=axis)._subset(**kwargs)
 
 
 def cutout_factory(args, kwargs):

anemoi/datasets/data/join.py

@@ -13,7 +13,7 @@ import numpy as np
 from .debug import Node
 from .debug import Source
 from .debug import debug_indexing
-from .forewards import Combined
+from .forwards import Combined
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices

anemoi/datasets/data/masked.py

@@ -14,7 +14,7 @@ from ..grids import cropping_mask
 from .dataset import Dataset
 from .debug import Node
 from .debug import debug_indexing
-from .forewards import Forwards
+from .forwards import Forwards
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -83,6 +83,9 @@ class Thinning(Masked):
     def tree(self):
         return Node(self, [self.forward.tree()], thinning=self.thinning, method=self.method)
 
+    def subclass_metadata_specific(self):
+        return dict(thinning=self.thinning, method=self.method)
+
 
 class Cropping(Masked):
     def __init__(self, forward, area):
@@ -104,3 +107,6 @@ class Cropping(Masked):
 
     def tree(self):
         return Node(self, [self.forward.tree()], area=self.area)
+
+    def metadata_specific(self, **kwargs):
+        return super().metadata_specific(area=self.area, **kwargs)

anemoi/datasets/data/select.py

@@ -11,7 +11,7 @@ from functools import cached_property
 from .debug import Node
 from .debug import Source
 from .debug import debug_indexing
-from .forewards import Forwards
+from .forwards import Forwards
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -88,6 +88,10 @@ class Select(Forwards):
     def tree(self):
         return Node(self, [self.dataset.tree()], **self.title)
 
+    def subclass_metadata_specific(self):
+        # return dict(indices=self.indices)
+        return {}
+
 
 class Rename(Forwards):
     def __init__(self, dataset, rename):
@@ -105,8 +109,8 @@ class Rename(Forwards):
     def name_to_index(self):
         return {k: i for i, k in enumerate(self.variables)}
 
-    def metadata_specific(self, **kwargs):
-        return super().metadata_specific(rename=self.rename, **kwargs)
-
     def tree(self):
         return Node(self, [self.forward.tree()], rename=self.rename)
+
+    def subclass_metadata_specific(self):
+        return dict(rename=self.rename)

anemoi/datasets/data/statistics.py

@@ -10,7 +10,7 @@ from functools import cached_property
 
 from . import open_dataset
 from .debug import Node
-from .forewards import Forwards
+from .forwards import Forwards
 
 LOG = logging.getLogger(__name__)
 
@@ -34,11 +34,8 @@ class Statistics(Forwards):
             delta = self.frequency
         return self._statistic.statistics_tendencies(delta)
 
-    def metadata_specific(self, **kwargs):
-        return super().metadata_specific(
-            statistics=self._statistic.metadata_specific(),
-            **kwargs,
-        )
+    def subclass_metadata_specific(self):
+        return dict(statistics=self._statistic.metadata_specific())
 
     def tree(self):
         return Node(self, [self.forward.tree()])

anemoi/datasets/data/stores.py

@@ -347,7 +347,7 @@ class ZarrWithMissingDates(Zarr):
         return "zarr*"
 
 
-def zarr_lookup(name):
+def zarr_lookup(name, fail=True):
 
     if name.endswith(".zarr") or name.endswith(".zip"):
         return name
@@ -372,4 +372,7 @@ def zarr_lookup(name):
         except zarr.errors.PathNotFoundError:
             pass
 
-    raise ValueError(f"Cannot find a dataset that matched '{name}'. Tried: {tried}")
+    if fail:
+        raise ValueError(f"Cannot find a dataset that matched '{name}'. Tried: {tried}")
+
+    return None

anemoi/datasets/data/subset.py

@@ -13,7 +13,7 @@ import numpy as np
 from .debug import Node
 from .debug import Source
 from .debug import debug_indexing
-from .forewards import Forwards
+from .forwards import Forwards
 from .indexing import apply_index_to_slices_changes
 from .indexing import expand_list_indexing
 from .indexing import index_to_slices
@@ -109,3 +109,9 @@ class Subset(Forwards):
 
     def tree(self):
         return Node(self, [self.dataset.tree()], **self.reason)
+
+    def subclass_metadata_specific(self):
+        return {
+            # "indices": self.indices,
+            "reason": self.reason,
+        }

anemoi/datasets/data/unchecked.py

@@ -11,7 +11,7 @@ from functools import wraps
 
 from .concat import ConcatMixin
 from .debug import Node
-from .forewards import Combined
+from .forwards import Combined
 from .misc import _auto_adjust
 from .misc import _open
 

anemoi/datasets/dates/__init__.py

@@ -116,8 +116,8 @@ class StartEndDates(Dates):
         start = no_time_zone(start)
         end = no_time_zone(end)
 
-        if end <= start:
-            raise ValueError(f"End date {end} must be after start date {start}")
+        # if end <= start:
+        #     raise ValueError(f"End date {end} must be after start date {start}")
 
         increment = datetime.timedelta(hours=frequency)
 

{anemoi_datasets-0.3.3.dist-info → anemoi_datasets-0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.3.3
+Version: 0.3.5
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License: Apache License
@@ -238,6 +238,7 @@ Requires-Dist: ecmwflibs >=0.6.3 ; extra == 'all'
 Requires-Dist: entrypoints ; extra == 'all'
 Requires-Dist: numpy ; extra == 'all'
 Requires-Dist: pyproj ; extra == 'all'
+Requires-Dist: pytest ; extra == 'all'
 Requires-Dist: pyyaml ; extra == 'all'
 Requires-Dist: requests ; extra == 'all'
 Requires-Dist: s3fs ; extra == 'all'
@@ -272,4 +273,6 @@ Provides-Extra: remote
 Requires-Dist: boto3 ; extra == 'remote'
 Requires-Dist: requests ; extra == 'remote'
 Requires-Dist: s3fs ; extra == 'remote'
+Provides-Extra: test
+Requires-Dist: pytest ; extra == 'test'
 

{anemoi_datasets-0.3.3.dist-info → anemoi_datasets-0.3.5.dist-info}/RECORD

@@ -1,14 +1,13 @@
 anemoi/datasets/__init__.py,sha256=DC7ttKT--pmhBQALX_Cn7P28dngsJucKi5y-Ydm28QM,700
 anemoi/datasets/__main__.py,sha256=cLA2PidDTOUHaDGzd0_E5iioKYNe-PSTv567Y2fuwQk,723
-anemoi/datasets/_version.py,sha256=FKnJIExgNrZG2xJ0y_dGNBpxGbGBYylvfat-jHhLUuM,411
+anemoi/datasets/_version.py,sha256=3eLsZPTwWh0zKBhR6n3c4iAL9geCCdJGMojIL6dF0IA,411
 anemoi/datasets/grids.py,sha256=3YBMMJodgYhavarXPAlMZHaMtDT9v2IbTmAXZTqf8Qo,8481
 anemoi/datasets/commands/__init__.py,sha256=qAybFZPBBQs0dyx7dZ3X5JsLpE90pwrqt1vSV7cqEIw,706
 anemoi/datasets/commands/compare.py,sha256=tN3eqihvnZ0rFc0OUzrfI34PHDlYfc2l90ZIQBE1TDQ,1300
 anemoi/datasets/commands/copy.py,sha256=fba-zjD0iTHHXHhPEcm8VhDzsXQXDUxlbtTA1TovyT0,9991
 anemoi/datasets/commands/create.py,sha256=POdOsVDlvRrHFFkI3SNXNgNIbSxkVUUPMoo660x7Ma0,987
+anemoi/datasets/commands/inspect.py,sha256=G3fzcgiLaU8jln7GKvgamN7Y06-qC_JnFw2SbNn1_E4,18646
 anemoi/datasets/commands/scan.py,sha256=HxsLdCgBMSdEXjlJfPq5M_9LxXHHQIoZ1ZEHO_AoPgA,2881
-anemoi/datasets/commands/inspect/__init__.py,sha256=v6fPUTdMRdmUiEUUs0F74QlzPr-x5XEEOql3mkFme7E,1500
-anemoi/datasets/commands/inspect/zarr.py,sha256=Q1waDTgdJZwJXNST4jkO4DCIbqbf2T_2Us2k6yKGToo,19684
 anemoi/datasets/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 anemoi/datasets/compute/recentre.py,sha256=j8LdC8kq1t4PW7WFTXf93hSxok10un8ENIPwCehzbP8,4768
 anemoi/datasets/create/__init__.py,sha256=jji65Zni5aPTvS269fAMix4pN9ukmSoK0z5SVsbpr5E,5807
@@ -36,10 +35,10 @@ anemoi/datasets/create/functions/sources/accumulations.py,sha256=li1tpEew1XUv4sJ
 anemoi/datasets/create/functions/sources/constants.py,sha256=aqquu6HDc8t-zsF9KRFLaj0eV4S0UPZ59BVna8E3bU8,785
 anemoi/datasets/create/functions/sources/empty.py,sha256=SBuAfC33imbfcRnFnnOR44y8Q3KSQcqx3juIcXfCa3c,481
 anemoi/datasets/create/functions/sources/forcings.py,sha256=EVcdu8puMSW451qj3LKCWWXaSf2LlmF8YXVs8hSMxkU,643
-anemoi/datasets/create/functions/sources/grib.py,sha256=p1vYlRfV0irlkVBTYhNfQ7OrJjHaMfe2KkSphyNtyU0,1540
+anemoi/datasets/create/functions/sources/grib.py,sha256=YQNuGnlh2EYb2NIHYpzlipwUTmOhrmyQtP3zgk8MAUU,1661
 anemoi/datasets/create/functions/sources/hindcasts.py,sha256=0Psnsx2J0cRLMpJuNN-gESm1xJFC1gmQzI8sdnXCoYE,13042
 anemoi/datasets/create/functions/sources/mars.py,sha256=Jau-ceN_cI3Z2-uql92iS4-Emh9Pie7omdRkFB5oe1I,4025
-anemoi/datasets/create/functions/sources/netcdf.py,sha256=XeV3ZlQWPZD2Yxxi5HzvhKKS-04WpLnVSORv-cBa-nM,1726
+anemoi/datasets/create/functions/sources/netcdf.py,sha256=kic6PH7SAK3gseXChD38IDXw6Zcg2zhF4SeDXB2LQ8Q,2084
 anemoi/datasets/create/functions/sources/opendap.py,sha256=T0CPinscfafrVLaye5ue-PbiCNbcNqf_3m6pphN9rCU,543
 anemoi/datasets/create/functions/sources/recentre.py,sha256=t07LIXG3Hp9gmPkPriILVt86TxubsHyS1EL1lzwgtXY,1810
 anemoi/datasets/create/functions/sources/source.py,sha256=hPQnV_6UIxFw97uRKcTA8TplcgG1kC8NlFHoEaaLet4,1418
@@ -47,28 +46,28 @@ anemoi/datasets/create/functions/sources/tendencies.py,sha256=kwS_GZt8R9kpfs5Rrv
 anemoi/datasets/create/statistics/__init__.py,sha256=X50drgE-ltuNe7bSIyvyeC4GeTqGTQGbglh2-2aVWKE,15445
 anemoi/datasets/create/statistics/summary.py,sha256=sgmhA24y3VRyjmDUgTnPIqcHSlWBbFA0qynx6gJ9Xw8,3370
 anemoi/datasets/data/__init__.py,sha256=tacn6K_VZ-pYhLmGePG5sze8kmqGpqscYb-bMyQnWtk,888
-anemoi/datasets/data/concat.py,sha256=U6IZi6NkI6yccrDamgasENBqwyJ1m0ZesuDtHXoqEh8,3551
+anemoi/datasets/data/concat.py,sha256=AkpyOs16OjW7X0cdyYFQfWSCV6dteXBp-x9WlokO-DI,3550
 anemoi/datasets/data/dataset.py,sha256=UDnidq2amyCT2COH05pGfDCJcmkdMj1ubtHk9cl-qcE,7384
 anemoi/datasets/data/debug.css,sha256=z2X_ZDSnZ9C3pyZPWnQiEyAxuMxUaxJxET4oaCImTAQ,211
 anemoi/datasets/data/debug.py,sha256=PcyrjgxaLzeb_vf12pvUtPPVvBRHNm1SimythZvqsP4,6303
-anemoi/datasets/data/ensemble.py,sha256=PcrdNL4DhAuWYSXgNxC6igDXpDndXC_QrbLrL4Lvj-Y,1138
-anemoi/datasets/data/forewards.py,sha256=4IsaNDhYlLiCbawUvTynm2vdpGPqdXcrSoAENwsJoqI,7456
-anemoi/datasets/data/grids.py,sha256=vgZMIQbv5SnIcnPu2ujsrAQ8VyBz5o2a1SnxsjXkDuw,7495
+anemoi/datasets/data/ensemble.py,sha256=AsP7Xx0ZHLoZs6a4EC0jtyGYIcOvZvvKXhgNsIvqIN8,1137
+anemoi/datasets/data/forwards.py,sha256=t9YQCN7j75VMInt0uP9JUJoh1klF1Z1xnwy5_kDMDQs,7700
+anemoi/datasets/data/grids.py,sha256=rooOeR6rvjl4U8B4LO3N23fcgxvGE7ZUmhVryk1QS4M,7493
 anemoi/datasets/data/indexing.py,sha256=625m__JG5m_tDMrkz1hB6Vydenwt0oHuyAlc-o3Zwos,4799
-anemoi/datasets/data/join.py,sha256=m_lpxWPy8-xYOjPbVoBV3V92VGtBFIriiDWvQM6KqXc,4893
-anemoi/datasets/data/masked.py,sha256=KZZ-3nq9saj_W8PTN9V4YdZ24BayHgECj12i4yjyKpc,3525
+anemoi/datasets/data/join.py,sha256=dtCBbMTicqrRPxfBULi3RwEcQBLhQpIcvCjdN5A3XUU,4892
+anemoi/datasets/data/masked.py,sha256=Fzkehyka70CiS0LYSy_uyVYu2gKLwDSxlbm8GiC_pYs,3742
 anemoi/datasets/data/misc.py,sha256=m_28VIhX546RIoVfGpimPOThl5EwOhkun2UgWMAUxqw,10355
-anemoi/datasets/data/select.py,sha256=JoEepq8iRSSX6L75hzhLrBFhy0RJInuBM3C_Eu2Ryv0,3608
-anemoi/datasets/data/statistics.py,sha256=rWuG5qlfQoo9shOXR6TleJbJONwYggxxLy_HRet8azM,1582
-anemoi/datasets/data/stores.py,sha256=damJzNScaGenARAv8xpNa7d32f03MpGk5adRoRi34yw,10851
-anemoi/datasets/data/subset.py,sha256=RjfOMu7p69DZXRxQpvTfDOjVAURhgUO2pWyuZpXlJGY,3671
-anemoi/datasets/data/unchecked.py,sha256=qeUKthbvVVSPH-P366q1DEofvPzZSSXCXA49x-RkBOc,4038
-anemoi/datasets/dates/__init__.py,sha256=zOph2N_mXYbjSvqEWYF1mmm-UZpljb61WLrdFJmi0qQ,4469
+anemoi/datasets/data/select.py,sha256=U3AEid80mrJKu0SF4lLc-bRWMVcAZwHNUHUHRehvuHU,3680
+anemoi/datasets/data/statistics.py,sha256=PKRgcCiZEb1HjkIveVGhE3TzUy9Qe3AYWGFD72Urah8,1514
+anemoi/datasets/data/stores.py,sha256=oEjUra0zzIysiUvh-RBQRzcbviggejEQiMO5RfpjPyM,10896
+anemoi/datasets/data/subset.py,sha256=9urVTXdnwCgqn0_BRYquMi8oiXn4ubAf0n4586hWfKw,3814
+anemoi/datasets/data/unchecked.py,sha256=xhdMg-ToI1UfBWHNsWyn1y2meZWngZtHx-33L0KqKp8,4037
+anemoi/datasets/dates/__init__.py,sha256=4ItowfLLh90T8L_JOjtv98lE6M7gAaWt7dV3niUrFvk,4473
 anemoi/datasets/dates/groups.py,sha256=iq310Pi7ullglOhcNblv14MmcT8FPgYCD5s45qAfV_s,3383
 anemoi/datasets/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-anemoi_datasets-0.3.3.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_datasets-0.3.3.dist-info/METADATA,sha256=Tf99e7CzYgXE32yyPro7S7xKUzRfmGQYn5jj03j5mdM,16090
-anemoi_datasets-0.3.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-anemoi_datasets-0.3.3.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
-anemoi_datasets-0.3.3.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_datasets-0.3.3.dist-info/RECORD,,
+anemoi_datasets-0.3.5.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_datasets-0.3.5.dist-info/METADATA,sha256=rrQ3_Oz3ToYC03NGG2unQIwQ9kCVW6tQOvGwnZzTJ-0,16190
+anemoi_datasets-0.3.5.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+anemoi_datasets-0.3.5.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
+anemoi_datasets-0.3.5.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_datasets-0.3.5.dist-info/RECORD,,

anemoi/datasets/commands/inspect/__init__.py

@@ -1,37 +0,0 @@
-# (C) Copyright 2023 European Centre for Medium-Range Weather Forecasts.
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-import os
-
-from .. import Command
-from .zarr import InspectZarr
-
-
-class Inspect(Command, InspectZarr):
-    """Inspect a zarr dataset."""
-
-    def add_arguments(self, command_parser):
-        # g = command_parser.add_mutually_exclusive_group()
-        command_parser.add_argument("path", metavar="PATH", nargs="+")
-        command_parser.add_argument("--detailed", action="store_true")
-        # command_parser.add_argument("--probe", action="store_true")
-        command_parser.add_argument("--progress", action="store_true")
-        command_parser.add_argument("--statistics", action="store_true")
-        command_parser.add_argument("--size", action="store_true", help="Print size")
-
-    def run(self, args):
-        dic = vars(args)
-        for path in dic.pop("path"):
-            if os.path.isdir(path) or path.endswith(".zarr.zip") or path.endswith(".zarr"):
-                self.inspect_zarr(path=path, **dic)
-            else:
-                raise ValueError(f"Unknown file type: {path}")
-                # self.inspect_checkpoint(path=path, **dic)
-
-
-command = Inspect