anemoi-datasets 0.5.11__py3-none-any.whl → 0.5.12__py3-none-any.whl

anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.5.11'
-__version_tuple__ = version_tuple = (0, 5, 11)
+__version__ = version = '0.5.12'
+__version_tuple__ = version_tuple = (0, 5, 12)

anemoi/datasets/data/dataset.py

@@ -15,6 +15,7 @@ import pprint
 import warnings
 from functools import cached_property
 
+import numpy as np
 from anemoi.utils.dates import frequency_to_seconds
 from anemoi.utils.dates import frequency_to_string
 from anemoi.utils.dates import frequency_to_timedelta
@@ -42,6 +43,9 @@ def _tidy(v):
     if isinstance(v, slice):
         return (v.start, v.stop, v.step)
 
+    if isinstance(v, np.integer):
+        return int(v)
+
     return v
 
 
@@ -241,7 +245,8 @@ class Dataset:
         if not isinstance(vars, (list, tuple, set)):
             vars = [vars]
 
-        assert set(vars) <= set(self.name_to_index)
+        if not set(vars) <= set(self.name_to_index):
+            raise ValueError(f"drop: unknown variables: {set(vars) - set(self.name_to_index)}")
 
         return sorted([v for k, v in self.name_to_index.items() if k not in vars])
 

anemoi/datasets/data/grids.py

@@ -289,14 +289,15 @@ class Cutout(GridsBase):
         """
         index, changes = index_to_slices(index, self.shape)
         # Select data from each LAM
-        lam_data = [lam[index] for lam in self.lams]
+        lam_data = [lam[index[:3]] for lam in self.lams]
 
         # First apply spatial indexing on `self.globe` and then apply the mask
         globe_data_sliced = self.globe[index[:3]]
         globe_data = globe_data_sliced[..., self.global_mask]
 
-        # Concatenate LAM data with global data
-        result = np.concatenate(lam_data + [globe_data], axis=self.axis)
+        # Concatenate LAM data with global data, apply the grid slicing
+        result = np.concatenate(lam_data + [globe_data], axis=self.axis)[..., index[3]]
+
         return apply_index_to_slices_changes(result, changes)
 
     def collect_supporting_arrays(self, collected, *path):
@@ -324,7 +325,8 @@ class Cutout(GridsBase):
         """
         shapes = [np.sum(mask) for mask in self.masks]
         global_shape = np.sum(self.global_mask)
-        return tuple(self.lams[0].shape[:-1] + (sum(shapes) + global_shape,))
+        total_shape = sum(shapes) + global_shape
+        return tuple(self.lams[0].shape[:-1] + (int(total_shape),))
 
     def check_same_resolution(self, d1, d2):
         # Turned off because we are combining different resolutions

anemoi/datasets/data/join.py

@@ -124,7 +124,14 @@ class Join(Combined):
                 if v in md:
                     result[v] = md[v]
 
-        assert len(result) == len(variables), (result, variables)
+        if len(result) != len(variables):
+            LOG.error("Some variables are missing metadata.")
+            for v in variables:
+                if v not in result:
+                    LOG.error("Missing metadata for %r.", v)
+
+            raise ValueError("Some variables are missing metadata.")
+
         return result
 
     @cached_property

{anemoi_datasets-0.5.11.dist-info → anemoi_datasets-0.5.12.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: anemoi-datasets
-Version: 0.5.11
+Version: 0.5.12
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
-License: Apache License
+License:                                  Apache License
                                    Version 2.0, January 2004
                                 http://www.apache.org/licenses/
         
@@ -224,40 +224,39 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.9
 License-File: LICENSE
-Requires-Dist: anemoi-transform >=0.1
-Requires-Dist: anemoi-utils[provenance] >=0.4.9
+Requires-Dist: anemoi-transform>=0.1
+Requires-Dist: anemoi-utils[provenance]>=0.4.9
 Requires-Dist: cfunits
 Requires-Dist: numpy
 Requires-Dist: pyyaml
 Requires-Dist: semantic-version
 Requires-Dist: tqdm
-Requires-Dist: zarr <=2.17
+Requires-Dist: zarr<=2.17
 Provides-Extra: all
-Requires-Dist: anemoi-datasets[create,remote,xarray] ; extra == 'all'
+Requires-Dist: anemoi-datasets[create,remote,xarray]; extra == "all"
 Provides-Extra: create
-Requires-Dist: earthkit-data[mars] >=0.10.7 ; extra == 'create'
-Requires-Dist: earthkit-geo >=0.2 ; extra == 'create'
-Requires-Dist: earthkit-meteo ; extra == 'create'
-Requires-Dist: eccodes >=2.38.1 ; extra == 'create'
-Requires-Dist: entrypoints ; extra == 'create'
-Requires-Dist: pyproj ; extra == 'create'
+Requires-Dist: earthkit-data[mars]>=0.10.7; extra == "create"
+Requires-Dist: earthkit-geo>=0.2; extra == "create"
+Requires-Dist: earthkit-meteo; extra == "create"
+Requires-Dist: eccodes>=2.38.1; extra == "create"
+Requires-Dist: entrypoints; extra == "create"
+Requires-Dist: pyproj; extra == "create"
 Provides-Extra: dev
-Requires-Dist: anemoi-datasets[all,docs,tests] ; extra == 'dev'
+Requires-Dist: anemoi-datasets[all,docs,tests]; extra == "dev"
 Provides-Extra: docs
-Requires-Dist: nbsphinx ; extra == 'docs'
-Requires-Dist: pandoc ; extra == 'docs'
-Requires-Dist: sphinx ; extra == 'docs'
-Requires-Dist: sphinx-argparse ; extra == 'docs'
-Requires-Dist: sphinx-rtd-theme ; extra == 'docs'
+Requires-Dist: nbsphinx; extra == "docs"
+Requires-Dist: pandoc; extra == "docs"
+Requires-Dist: sphinx; extra == "docs"
+Requires-Dist: sphinx-argparse; extra == "docs"
+Requires-Dist: sphinx-rtd-theme; extra == "docs"
 Provides-Extra: remote
-Requires-Dist: boto3 ; extra == 'remote'
-Requires-Dist: requests ; extra == 'remote'
+Requires-Dist: boto3; extra == "remote"
+Requires-Dist: requests; extra == "remote"
 Provides-Extra: tests
-Requires-Dist: pytest ; extra == 'tests'
+Requires-Dist: pytest; extra == "tests"
 Provides-Extra: xarray
-Requires-Dist: gcsfs ; extra == 'xarray'
-Requires-Dist: kerchunk ; extra == 'xarray'
-Requires-Dist: pandas ; extra == 'xarray'
-Requires-Dist: planetary-computer ; extra == 'xarray'
-Requires-Dist: pystac-client ; extra == 'xarray'
-
+Requires-Dist: gcsfs; extra == "xarray"
+Requires-Dist: kerchunk; extra == "xarray"
+Requires-Dist: pandas; extra == "xarray"
+Requires-Dist: planetary-computer; extra == "xarray"
+Requires-Dist: pystac-client; extra == "xarray"

{anemoi_datasets-0.5.11.dist-info → anemoi_datasets-0.5.12.dist-info}/RECORD

@@ -1,6 +1,6 @@
 anemoi/datasets/__init__.py,sha256=0GOHATiKgkUqLRgAVQhNP1aPO7ULfSr8DqUf2ANPEv8,1010
 anemoi/datasets/__main__.py,sha256=5NW2A3OgTimB4ptwYThivIRSeCrvabMuvnr8mmnVx0E,715
-anemoi/datasets/_version.py,sha256=9QMo8Qn8JXkug8VyV8nVyp5woiC_5GOBRXHVqZu_cf4,413
+anemoi/datasets/_version.py,sha256=ZArwdbjC4yDqbe9dEwhZVS693DGl1K3jnRxJy_dutjo,413
 anemoi/datasets/grids.py,sha256=bq7pB_6uswILT3t8C8SeUpUrBww31dw5au_USrped6c,10919
 anemoi/datasets/testing.py,sha256=7HGOz5_V9MbkHTDJ4KbklGRndBMrFfVrBBu6a9k0_qY,1825
 anemoi/datasets/commands/__init__.py,sha256=O5W3yHZywRoAqmRUioAr3zMCh0hGVV18wZYGvc00ioM,698
@@ -90,16 +90,16 @@ anemoi/datasets/create/statistics/__init__.py,sha256=l6VE00sfcfqBg6cDFJTian-DLnv
 anemoi/datasets/create/statistics/summary.py,sha256=wmnz4fZkr6fomXgI8JlMutU8gakfrXTc5ixf3Np7gZA,3385
 anemoi/datasets/data/__init__.py,sha256=AW1-Ycj77pWQsZcDGsp0pgTS5rFW6XC4CzuUEIUPAIk,1558
 anemoi/datasets/data/concat.py,sha256=udtYINuoLOEYYKhi_VpG2-emv80pwZbFAZKwNwXJk3s,5244
-anemoi/datasets/data/dataset.py,sha256=rknyaODW9uNSd456139ZUaqTrd59RHUab5nCFWKdSaQ,15414
+anemoi/datasets/data/dataset.py,sha256=73NjQo7wUSWqfdpyVVZIpWgEmy1x9-beN-kVtgfhBJE,15587
 anemoi/datasets/data/debug.css,sha256=z2X_ZDSnZ9C3pyZPWnQiEyAxuMxUaxJxET4oaCImTAQ,211
 anemoi/datasets/data/debug.py,sha256=IjCMwtAvknF51PCl_YRYgMZB2iX_9DC5DKILNgl_UHQ,6300
 anemoi/datasets/data/ensemble.py,sha256=KNIXDfjYSIo6JVn1bD9X92yffd4Gg83wn_2sGxqAnWU,1111
 anemoi/datasets/data/fill_missing.py,sha256=4btLi-D-hFTsS_57_gIC1nK5AVifAO-V4M-fqMrtrxk,4636
 anemoi/datasets/data/forwards.py,sha256=P9DfSY5B9w9gtkKfV6TIzXel_LY83g-2nEreJy2rYkU,8916
-anemoi/datasets/data/grids.py,sha256=IjR0Ay3AyPm3FSjT4S4-qhcbMpNzdK7jXC8yYhH6Jss,15682
+anemoi/datasets/data/grids.py,sha256=p7_nT7RLH6uKcxeAzQiGYk9lFxU_OOikDrwlb2rdEqI,15765
 anemoi/datasets/data/indexing.py,sha256=9lycQXSqUIbYj52JlFv0w_Gf6soVZnbVGswYMvGPpqs,4773
 anemoi/datasets/data/interpolate.py,sha256=D27lSH8yNhm0aoO0U3UoRbr3kni7OWXSu_X4jCbIrA0,4137
-anemoi/datasets/data/join.py,sha256=z6mVXJM4On4V2mtniHZGTWtEH4JNGY5czxjH62c_Q7s,5287
+anemoi/datasets/data/join.py,sha256=xl7SrjhggZdX_bdni5-cn8-BYGUYfEtogQeIqCgSL7U,5525
 anemoi/datasets/data/masked.py,sha256=eAVGVmQR7tWsd3xXYGXGyq28uRLwL50vOXWTNNdHxl0,4530
 anemoi/datasets/data/merge.py,sha256=6vYRy3-P4J9GgTPkdoPFs9CbZ5F0m8FAukS-P66BR_M,5448
 anemoi/datasets/data/misc.py,sha256=IKqtvcU36nyB4z6dfu7W_gnCpaeX20fK2C5A2seWdCA,10061
@@ -115,9 +115,9 @@ anemoi/datasets/dates/__init__.py,sha256=wX2FvlmRfHV5HDmllIxwfrC1LuRlb7i6SguLLas
 anemoi/datasets/dates/groups.py,sha256=i7x8z0kv6E8qUfm1tMZR1aaOqNwQzEkV-VWpOvHjoX4,5390
 anemoi/datasets/utils/__init__.py,sha256=hCW0QcLHJmE-C1r38P27_ZOvCLNewex5iQEtZqx2ckI,393
 anemoi/datasets/utils/fields.py,sha256=l7xKOiRLgk9Eewykqu7xZP9xOajG2dx2CiDlGvBVejU,1411
-anemoi_datasets-0.5.11.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_datasets-0.5.11.dist-info/METADATA,sha256=L4oe7KIUvfomyWOms-zfinIGzjJ41RfwC61wifz_C0E,15593
-anemoi_datasets-0.5.11.dist-info/WHEEL,sha256=R06PA3UVYHThwHvxuRWMqaGcr-PuniXahwjmQRFMEkY,91
-anemoi_datasets-0.5.11.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
-anemoi_datasets-0.5.11.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_datasets-0.5.11.dist-info/RECORD,,
+anemoi_datasets-0.5.12.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_datasets-0.5.12.dist-info/METADATA,sha256=x548Cd_PFUCse1QR5dj-8rNfUx94ZU45g__IjPaGLgY,15598
+anemoi_datasets-0.5.12.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+anemoi_datasets-0.5.12.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
+anemoi_datasets-0.5.12.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_datasets-0.5.12.dist-info/RECORD,,

{anemoi_datasets-0.5.11.dist-info → anemoi_datasets-0.5.12.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.5.0)
+Generator: setuptools (75.6.0)
 Root-Is-Purelib: true
 Tag: py3-none-any