anemoi-datasets 0.5.13__py3-none-any.whl → 0.5.14__py3-none-any.whl

anemoi/datasets/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.5.13'
-__version_tuple__ = version_tuple = (0, 5, 13)
+__version__ = version = '0.5.14'
+__version_tuple__ = version_tuple = (0, 5, 14)

anemoi/datasets/commands/create.py

@@ -83,11 +83,12 @@ class Create(Command):
         task("load", options)
         task("finalise", options)
 
-        task("patch", options)
-
         task("init_additions", options)
         task("run_additions", options)
         task("finalise_additions", options)
+
+        task("patch", options)
+
         task("cleanup", options)
         task("verify", options)
 
@@ -153,6 +154,7 @@ class Create(Command):
 
         with ExecutorClass(max_workers=1) as executor:
             executor.submit(task, "finalise-additions", options).result()
+            executor.submit(task, "patch", options).result()
             executor.submit(task, "cleanup", options).result()
             executor.submit(task, "verify", options).result()
 

anemoi/datasets/create/__init__.py

@@ -79,7 +79,10 @@ def json_tidy(o):
         )
         return o.isoformat()
 
-    raise TypeError(repr(o) + " is not JSON serializable")
+    if isinstance(o, (np.float32, np.float64)):
+        return float(o)
+
+    raise TypeError(f"{repr(o)} is not JSON serializable {type(o)}")
 
 
 def build_statistics_dates(dates, start, end):
@@ -596,6 +599,8 @@ class Load(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixi
         # There is one cube to load for each result.
         dates = list(result.group_of_dates)
 
+        LOG.debug(f"Loading cube for {len(dates)} dates")
+
         cube = result.get_cube()
         shape = cube.extended_user_shape
         dates_in_data = cube.user_coords["valid_datetime"]
@@ -642,12 +647,14 @@ class Load(Actor, HasRegistryMixin, HasStatisticTempMixin, HasElementForDataMixi
         indexes = dates_to_indexes(self.dates, dates_in_data)
 
         array = ViewCacheArray(self.data_array, shape=shape, indexes=indexes)
+        LOG.info(f"Loading array shape={shape}, indexes={len(indexes)}")
         self.load_cube(cube, array)
 
         stats = compute_statistics(array.cache, self.variables_names, allow_nans=self._get_allow_nans())
         self.tmp_statistics.write(indexes, stats, dates=dates_in_data)
-
+        LOG.info("Flush data array")
         array.flush()
+        LOG.info("Flushed data array")
 
     def _get_allow_nans(self):
         config = self.main_config
@@ -740,6 +747,11 @@ class AdditionsMixin:
         if not self.delta.total_seconds() % frequency.total_seconds() == 0:
             LOG.debug(f"Delta {self.delta} is not a multiple of frequency {frequency}. Skipping.")
             return True
+
+        if self.dataset.zarr_metadata.get("build", {}).get("additions", None) is False:
+            LOG.warning(f"Additions are disabled for {self.path} in the recipe.")
+            return True
+
         return False
 
     @cached_property

anemoi/datasets/create/functions/sources/accumulations.py

@@ -253,6 +253,7 @@ def _compute_accumulations(
     data_accumulation_period=None,
     patch=_identity,
     base_times=None,
+    use_cdsapi_dataset=None,
 ):
     adjust_step = isinstance(user_accumulation_period, int)
 
@@ -311,7 +312,9 @@ def _compute_accumulations(
 
                 requests.append(patch(r))
 
-    ds = mars(context, dates, *requests, request_already_using_valid_datetime=True)
+    ds = mars(
+        context, dates, *requests, request_already_using_valid_datetime=True, use_cdsapi_dataset=use_cdsapi_dataset
+    )
 
     accumulations = {}
     for a in [AccumulationClass(out, frequency=frequency, **r) for r in requests]:
@@ -366,7 +369,7 @@ def _scda(request):
     return request
 
 
-def accumulations(context, dates, **request):
+def accumulations(context, dates, use_cdsapi_dataset=None, **request):
     _to_list(request["param"])
     class_ = request.get("class", "od")
     stream = request.get("stream", "oper")
@@ -395,6 +398,7 @@ def accumulations(context, dates, **request):
         dates,
         request,
         user_accumulation_period=user_accumulation_period,
+        use_cdsapi_dataset=use_cdsapi_dataset,
         **kwargs,
     )
 

anemoi/datasets/create/functions/sources/eccc_fstd.py

@@ -0,0 +1,16 @@
+# (C) Copyright 2025 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+from .xarray import load_many
+
+
+def execute(context, dates, path, *args, **kwargs):
+    options = {"engine": "fstd"}
+    return load_many("🍁", context, dates, path, *args, options=options, **kwargs)

anemoi/datasets/create/functions/sources/mars.py

@@ -246,6 +246,7 @@ def mars(
     *requests,
     request_already_using_valid_datetime=False,
     date_key="date",
+    use_cdsapi_dataset=None,
     **kwargs,
 ):
 
@@ -305,7 +306,10 @@ def mars(
                     f"⚠️ Unknown key {k}={v} in MARS request. Did you mean '{did_you_mean(k, MARS_KEYS)}' ?"
                 )
         try:
-            ds = ds + from_source("mars", **r)
+            if use_cdsapi_dataset:
+                ds = ds + from_source("cds", use_cdsapi_dataset, r)
+            else:
+                ds = ds + from_source("mars", **r)
         except Exception as e:
             if "File is empty:" not in str(e):
                 raise

anemoi/datasets/create/input/result.py

@@ -459,7 +459,7 @@ class Result:
         if self.group_of_dates is not None:
             dates = f" {len(self.group_of_dates)} dates"
             dates += " ("
-            dates += "/".join(d.strftime("%Y-%m-%d:%H") for d in self.group_of_dates)
+            dates += "/".join(d.strftime("%Y-%m-%dT%H:%M") for d in self.group_of_dates)
             if len(dates) > 100:
                 dates = dates[:100] + "..."
             dates += ")"

anemoi/datasets/create/statistics/__init__.py

@@ -18,6 +18,7 @@ import shutil
 import socket
 
 import numpy as np
+import tqdm
 from anemoi.utils.provenance import gather_provenance_info
 
 from ..check import check_data_values
@@ -134,7 +135,7 @@ def check_variance(x, variables_names, minimum, maximum, mean, count, sums, squa
 
 def compute_statistics(array, check_variables_names=None, allow_nans=False):
     """Compute statistics for a given array, provides minimum, maximum, sum, squares, count and has_nans as a dictionary."""
-
+    LOG.info(f"Computing statistics for {array.shape} array")
     nvars = array.shape[1]
 
     LOG.debug(f"Stats {nvars}, {array.shape}, {check_variables_names}")
@@ -149,7 +150,7 @@ def compute_statistics(array, check_variables_names=None, allow_nans=False):
     maximum = np.zeros(stats_shape, dtype=np.float64)
     has_nans = np.zeros(stats_shape, dtype=np.bool_)
 
-    for i, chunk in enumerate(array):
+    for i, chunk in tqdm.tqdm(enumerate(array), delay=1, total=array.shape[0], desc="Computing statistics"):
         values = chunk.reshape((nvars, -1))
 
         for j, name in enumerate(check_variables_names):
@@ -166,6 +167,8 @@ def compute_statistics(array, check_variables_names=None, allow_nans=False):
         count[i] = np.sum(~np.isnan(values), axis=1)
         has_nans[i] = np.isnan(values).any()
 
+    LOG.info(f"Statistics computed for {nvars} variables.")
+
     return {
         "minimum": minimum,
         "maximum": maximum,

anemoi/datasets/data/misc.py

@@ -103,6 +103,30 @@ def _as_date(d, dates, last):
 
     if isinstance(d, str):
 
+        def isfloat(s):
+            try:
+                float(s)
+                return True
+            except ValueError:
+                return False
+
+        if d.endswith("%") and isfloat(d[:-1]):
+            x = float(d[:-1])
+            if not 0 <= x <= 100:
+                raise ValueError(f"Invalid date: {d}")
+            i_float = x * len(dates) / 100
+
+            epsilon = 2 ** (-30)
+            if len(dates) > 1 / epsilon:
+                LOG.warning("Too many dates to use percentage, one date may be lost in rounding")
+
+            if last:
+                index = int(i_float + epsilon) - 1
+            else:
+                index = int(i_float - epsilon)
+            index = max(0, min(len(dates) - 1, index))
+            return dates[index]
+
         if "-" in d and ":" in d:
             date, time = d.replace(" ", "T").split("T")
             year, month, day = [int(_) for _ in date.split("-")]

anemoi/datasets/grids.py

@@ -62,6 +62,8 @@ def plot_mask(path, mask, lats, lons, global_lats, global_lons):
         plt.savefig(path + "-global-zoomed.png")
 
 
+# TODO: Use the one from anemoi.utils.grids instead
+# from anemoi.utils.grids import ...
 def xyz_to_latlon(x, y, z):
     return (
         np.rad2deg(np.arcsin(np.minimum(1.0, np.maximum(-1.0, z)))),
@@ -69,6 +71,8 @@ def xyz_to_latlon(x, y, z):
     )
 
 
+# TODO: Use the one from anemoi.utils.grids instead
+# from anemoi.utils.grids import ...
 def latlon_to_xyz(lat, lon, radius=1.0):
     # https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#From_geodetic_to_ECEF_coordinates
     # We assume that the Earth is a sphere of radius 1 so N(phi) = 1
@@ -380,6 +384,8 @@ def serialise_mask(mask):
 
 
 def nearest_grid_points(source_latitudes, source_longitudes, target_latitudes, target_longitudes):
+    # TODO: Use the one from anemoi.utils.grids instead
+    # from anemoi.utils.grids import ...
     from scipy.spatial import cKDTree
 
     source_xyz = latlon_to_xyz(source_latitudes, source_longitudes)

{anemoi_datasets-0.5.13.dist-info → anemoi_datasets-0.5.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: anemoi-datasets
-Version: 0.5.13
+Version: 0.5.14
 Summary: A package to hold various functions to support training of ML models on ECMWF data.
 Author-email: "European Centre for Medium-Range Weather Forecasts (ECMWF)" <software.support@ecmwf.int>
 License:                                  Apache License

{anemoi_datasets-0.5.13.dist-info → anemoi_datasets-0.5.14.dist-info}/RECORD

@@ -1,13 +1,13 @@
 anemoi/datasets/__init__.py,sha256=0GOHATiKgkUqLRgAVQhNP1aPO7ULfSr8DqUf2ANPEv8,1010
 anemoi/datasets/__main__.py,sha256=5NW2A3OgTimB4ptwYThivIRSeCrvabMuvnr8mmnVx0E,715
-anemoi/datasets/_version.py,sha256=Ywt_J-kUrkp4bt0vX46Ot5kDjsAA9dtf8IfOdhjGcXA,413
-anemoi/datasets/grids.py,sha256=xHZwE3pJs0wP05x9qCgMcTuBnBGoCU51YKx5JQ7Ukts,11398
+anemoi/datasets/_version.py,sha256=y_fPpU6VK_LLaj-dweUMkuluSUpLZldJCnJxFDfcohM,413
+anemoi/datasets/grids.py,sha256=qG4TDESkbS48AJr7Q5PKgAagkwrz4KpSktnNmn_zTsQ,11673
 anemoi/datasets/testing.py,sha256=7HGOz5_V9MbkHTDJ4KbklGRndBMrFfVrBBu6a9k0_qY,1825
 anemoi/datasets/commands/__init__.py,sha256=O5W3yHZywRoAqmRUioAr3zMCh0hGVV18wZYGvc00ioM,698
 anemoi/datasets/commands/cleanup.py,sha256=2rD34bHtfOCLwQh7yXa02IJmmOYMOma4YDj0PM-2-Jc,1456
 anemoi/datasets/commands/compare.py,sha256=8YOlFi9Ydw81qNx4kmI35gvoldUAjYae1enBhxqLrvQ,3256
 anemoi/datasets/commands/copy.py,sha256=2owwwcih6mzrmgxrKNmm1B9ngjxZ6q3ZVo3S7grrGWQ,11305
-anemoi/datasets/commands/create.py,sha256=SRHQdMPwBmbjfi2JDYmF3uC7R6dIrcrOOBs0aaVSQmQ,5505
+anemoi/datasets/commands/create.py,sha256=Es8BV2d9iazAjiGWaXf5EUmjkGk6-whGHaOEYG34CBU,5567
 anemoi/datasets/commands/finalise-additions.py,sha256=ZCUa5CbScR_tEImcJe9AhhGryyrsq1bYhM-dtjaIw7M,1552
 anemoi/datasets/commands/finalise.py,sha256=glFj3lhPwf5niybx8u053zMItNjIxYebt3r6E6W2D9Y,1307
 anemoi/datasets/commands/init-additions.py,sha256=JFTPu5cGNsmojbkb5DRhKF0rSku0S_2O6wRf7EvjEnM,1540
@@ -20,7 +20,7 @@ anemoi/datasets/commands/publish.py,sha256=z1MV9_1BsEnw81Y_17fHkKGYe8_ZJo9eeQ1kG
 anemoi/datasets/commands/scan.py,sha256=mXzYEcYsncxC7ItyL_TlVRiWji6OFYfVxO5OMD9mbEI,3304
 anemoi/datasets/compute/__init__.py,sha256=hCW0QcLHJmE-C1r38P27_ZOvCLNewex5iQEtZqx2ckI,393
 anemoi/datasets/compute/recentre.py,sha256=tKs-YZLhqsMRBNEUF41hcuMmyvbRuFX07xJq-Cqg2_w,4954
-anemoi/datasets/create/__init__.py,sha256=ImonAvvz-HqoZMF4vczyWEV48tGiZZ8RfM01SsxJ6ew,36427
+anemoi/datasets/create/__init__.py,sha256=0kGaWi3WA3rD-Jfps6HBa6v8J8CNTvXtoaXeT9DpfgU,36913
 anemoi/datasets/create/check.py,sha256=q205XxzR7UtBRI5qOANav_NAVqAERs0aLJ8oBL3VNc4,6153
 anemoi/datasets/create/chunks.py,sha256=c7ufk-EamAGqxOI4ScTFlBzdEiH6V1R0f0SPavtZ2Xw,2457
 anemoi/datasets/create/config.py,sha256=Tq9kJ-bKhYSTWCXNa8lZIJpO3RteZfCr5hQVM12IgD0,8892
@@ -49,13 +49,14 @@ anemoi/datasets/create/functions/filters/unrotate_winds.py,sha256=tDFXUSF2flD83W
 anemoi/datasets/create/functions/filters/uv_to_speeddir.py,sha256=niNuTSmyxLn4MGeNL1lowl5M0dH7har-flXy3ZtmKPM,1762
 anemoi/datasets/create/functions/filters/wz_to_w.py,sha256=SbTYE6rRjObR-sJEDYyc0-1Kw39zZOAheGMznD7Ic9A,2161
 anemoi/datasets/create/functions/sources/__init__.py,sha256=TMm8LerGY7--b0AMUqnz07ZGo-F7I9FF0DGlozcTtSg,1364
-anemoi/datasets/create/functions/sources/accumulations.py,sha256=xs3Ql3h_jfJDR0mzGIh54adVaHacHE3LQpBL5_8Rx8k,12373
+anemoi/datasets/create/functions/sources/accumulations.py,sha256=lOICpg8T1sKn7n0X0EkMrNDFBYIqmHYSF_-_LYIEPaw,12527
 anemoi/datasets/create/functions/sources/constants.py,sha256=GaiUpJPYupiLWl8O9GEZ9KmlD88pH6dlBiUVrhNl_uA,918
+anemoi/datasets/create/functions/sources/eccc_fstd.py,sha256=pcF-N9IarrXuoZ85lmKRbESr8QsSc2VWxUGKT9aNw_M,597
 anemoi/datasets/create/functions/sources/empty.py,sha256=YTpOJ3rcb_eS9CbnpwPWBR9r1APIAaG6a_N803YFZFE,500
 anemoi/datasets/create/functions/sources/forcings.py,sha256=p442lCOXm8TJFRlP0mgwZujveo9gCtdAGLS4KSIqYfk,661
 anemoi/datasets/create/functions/sources/grib.py,sha256=BSieYEnoG92-8lPlrK7CIaGX7qozpkixqHarucDZtLk,4543
 anemoi/datasets/create/functions/sources/hindcasts.py,sha256=AWtSW_VoeWIjwjojyAL8FskKMq9fTUfnQ1OI30_J2qA,1677
-anemoi/datasets/create/functions/sources/mars.py,sha256=_QDAhb1onZRBcH21jMdlhw8gp1taiQyt3bpkbQpGWRY,8736
+anemoi/datasets/create/functions/sources/mars.py,sha256=AXz2q1XhvIEVtj0gJNWqcZNkyvXSL-8r6i93GhnmGn4,8890
 anemoi/datasets/create/functions/sources/netcdf.py,sha256=GZvd8-d9lQFQ6sJL1usJPtnVqRz2UWqJOX3Fup1CAcI,547
 anemoi/datasets/create/functions/sources/opendap.py,sha256=vkbk14-1PFwN9o1wlGROLoF3P-t9WtUVSum6aW_2JoM,545
 anemoi/datasets/create/functions/sources/recentre.py,sha256=cCA4TIdRp5r7PaGe7skOuwB1Np9-FcuRX1JXIpUSuyc,1825
@@ -86,11 +87,11 @@ anemoi/datasets/create/input/join.py,sha256=wQP1-vVg4as-R5i3pstgK6HmTJAY7WyWYhCE
 anemoi/datasets/create/input/misc.py,sha256=r7NC_QRYA8iiJJbSFgQnNuixymATK0CPZknGxgYcLOk,1975
 anemoi/datasets/create/input/pipe.py,sha256=KfPCtiqyfqkXbmC-2LTqHkCQ7bJY46XMvNDnp9QeHTQ,1344
 anemoi/datasets/create/input/repeated_dates.py,sha256=59EvJ_cQwA-p_42cmMFy3pBAAWV0xwPg4E3q2PIofcM,7461
-anemoi/datasets/create/input/result.py,sha256=-pcVcaaj3G_xcNKWWTgzVH5Ds5-ETWmErN0KeQGitAw,20013
+anemoi/datasets/create/input/result.py,sha256=AW6BGDQkrL1XQLXt-h-3lzFOCrVgXjAm3SYk2ZlDfrE,20016
 anemoi/datasets/create/input/step.py,sha256=CoowF9mc3kepT8XQ2ObxO750rnQEkYNTviIHQ1m-4UA,2886
 anemoi/datasets/create/input/template.py,sha256=Vgi4wQ1aeswLbji0fIzshYhISmzdrt7b0BmgeJJjYGc,1859
 anemoi/datasets/create/input/trace.py,sha256=DYXMSnwKqOIx0XWZTKNJojWz4EqaFLknTh6ysxsW9uY,2198
-anemoi/datasets/create/statistics/__init__.py,sha256=iJ3mZ6eEI88wPXUKyOhNKqhakyHoceX9ICEKXVOriTo,12789
+anemoi/datasets/create/statistics/__init__.py,sha256=Z3UXauDVYyfNjsBvfJGjK1unnxEkqNIfpjLCQCiLS2k,12994
 anemoi/datasets/create/statistics/summary.py,sha256=wmnz4fZkr6fomXgI8JlMutU8gakfrXTc5ixf3Np7gZA,3385
 anemoi/datasets/data/__init__.py,sha256=AW1-Ycj77pWQsZcDGsp0pgTS5rFW6XC4CzuUEIUPAIk,1558
 anemoi/datasets/data/complement.py,sha256=hTvA_zTGIHAvZYPv1npVpRpxPg8nXbNTEBAU8r33rlc,5105
@@ -107,7 +108,7 @@ anemoi/datasets/data/interpolate.py,sha256=D27lSH8yNhm0aoO0U3UoRbr3kni7OWXSu_X4j
 anemoi/datasets/data/join.py,sha256=IG9Bj4o4Z25cl5YGMqtl75UuSZCWIJwGIUB0fsfnkE8,5456
 anemoi/datasets/data/masked.py,sha256=eAVGVmQR7tWsd3xXYGXGyq28uRLwL50vOXWTNNdHxl0,4530
 anemoi/datasets/data/merge.py,sha256=dr0sX2ufm-qOgOAMV5oh8qQwPvSdYbU-mhux6u-cmQw,5547
-anemoi/datasets/data/misc.py,sha256=J1v84jHpRgDK0DUrNmII5oqt3jft8rSTve2GtxqTKa8,10310
+anemoi/datasets/data/misc.py,sha256=lYRE-vfRIra9bhyG3-kdruoQvxyYWaE9EN9qTFoM7bA,11074
 anemoi/datasets/data/missing.py,sha256=SWEjiC1usBjZtlKMr73uKetnoQZoflVQVGqLP2gJR7A,7131
 anemoi/datasets/data/rescale.py,sha256=wMU7tFZebnOqJJxaIGOqNqhpNKGsPNZMC1YxuiHvri4,4112
 anemoi/datasets/data/select.py,sha256=XW_ohlhrF8FLe13pdM3DRZDxbHxntcsO0F56GRqZQY0,4293
@@ -120,9 +121,9 @@ anemoi/datasets/dates/__init__.py,sha256=wX2FvlmRfHV5HDmllIxwfrC1LuRlb7i6SguLLas
 anemoi/datasets/dates/groups.py,sha256=i7x8z0kv6E8qUfm1tMZR1aaOqNwQzEkV-VWpOvHjoX4,5390
 anemoi/datasets/utils/__init__.py,sha256=hCW0QcLHJmE-C1r38P27_ZOvCLNewex5iQEtZqx2ckI,393
 anemoi/datasets/utils/fields.py,sha256=l7xKOiRLgk9Eewykqu7xZP9xOajG2dx2CiDlGvBVejU,1411
-anemoi_datasets-0.5.13.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-anemoi_datasets-0.5.13.dist-info/METADATA,sha256=ArEcpcTEQ_Tt270GtB5GEeDX_1SJMKM8P4NWMdSve8M,15598
-anemoi_datasets-0.5.13.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-anemoi_datasets-0.5.13.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
-anemoi_datasets-0.5.13.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
-anemoi_datasets-0.5.13.dist-info/RECORD,,
+anemoi_datasets-0.5.14.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+anemoi_datasets-0.5.14.dist-info/METADATA,sha256=FCUiHOFlg1S8dOZasmQkDBLKpeoEAZrXP4L8t9oFEuA,15598
+anemoi_datasets-0.5.14.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+anemoi_datasets-0.5.14.dist-info/entry_points.txt,sha256=yR-o-4uiPEA_GLBL81SkMYnUoxq3CAV3hHulQiRtGG0,66
+anemoi_datasets-0.5.14.dist-info/top_level.txt,sha256=DYn8VPs-fNwr7fNH9XIBqeXIwiYYd2E2k5-dUFFqUz0,7
+anemoi_datasets-0.5.14.dist-info/RECORD,,