anemoi-datasets 0.5.26__py3-none-any.whl → 0.5.28__py3-none-any.whl

anemoi/datasets/__init__.py

@@ -7,7 +7,6 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-from typing import List
 
 from .data import MissingDateError
 from .data import add_dataset_path
@@ -23,7 +22,7 @@ except ImportError:  # pragma: no cover
     # Local copy or not installed with setuptools
     __version__ = "999"
 
-__all__: List[str] = [
+__all__: list[str] = [
     "add_dataset_path",
     "add_named_dataset",
     "list_dataset_names",

anemoi/datasets/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.5.26'
-__version_tuple__ = version_tuple = (0, 5, 26)
+__version__ = version = '0.5.28'
+__version_tuple__ = version_tuple = (0, 5, 28)
+
+__commit_id__ = commit_id = None

anemoi/datasets/commands/check.py

@@ -77,7 +77,7 @@ class Check(Command):
 
         recipe_filename = os.path.basename(recipe)
         recipe_name = os.path.splitext(recipe_filename)[0]
-        in_recipe_name = yaml.safe_load(open(recipe, "r", encoding="utf-8"))["name"]
+        in_recipe_name = yaml.safe_load(open(recipe, encoding="utf-8"))["name"]
         if recipe_name != in_recipe_name:
             print(f"Recipe name {recipe_name} does not match the name in the recipe file {in_recipe_name}")
 

anemoi/datasets/commands/copy.py

@@ -14,7 +14,6 @@ import sys
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures import as_completed
 from typing import Any
-from typing import Optional
 
 import tqdm
 from anemoi.utils.remote import Transfer
@@ -136,7 +135,7 @@ class ZarrCopier:
             return zarr.storage.NestedDirectoryStore(path)
         return path
 
-    def copy_chunk(self, n: int, m: int, source: Any, target: Any, _copy: Any, verbosity: int) -> Optional[slice]:
+    def copy_chunk(self, n: int, m: int, source: Any, target: Any, _copy: Any, verbosity: int) -> slice | None:
         """Copy a chunk of data from source to target.
 
         Parameters

anemoi/datasets/commands/create.py

@@ -120,7 +120,7 @@ class Create(Command):
         task("finalise", options)
 
         task("init_additions", options)
-        task("run_additions", options)
+        task("load_additions", options)
         task("finalise_additions", options)
 
         task("patch", options)

anemoi/datasets/commands/inspect.py

@@ -14,10 +14,6 @@ import os
 from copy import deepcopy
 from functools import cached_property
 from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
-from typing import Union
 
 import numpy as np
 import semantic_version
@@ -39,7 +35,7 @@ from . import Command
 LOG = logging.getLogger(__name__)
 
 
-def compute_directory_size(path: str) -> Union[tuple[int, int], tuple[None, None]]:
+def compute_directory_size(path: str) -> tuple[int, int] | tuple[None, None]:
     """Compute the total size and number of files in a directory.
 
     Parameters
@@ -104,7 +100,7 @@ def cos_local_time_bug(lon: float, date: datetime.datetime) -> float:
     return np.cos(radians)
 
 
-def find(config: Union[dict, list], name: str) -> Any:
+def find(config: dict | list, name: str) -> Any:
     """Recursively search for a key in a nested dictionary or list.
 
     Parameters
@@ -167,7 +163,7 @@ class Version:
         print(f"🔢 Format version: {self.version}")
 
     @property
-    def name_to_index(self) -> Dict[str, int]:
+    def name_to_index(self) -> dict[str, int]:
         """Get a mapping of variable names to their indices."""
         return find(self.metadata, "name_to_index")
 
@@ -208,30 +204,30 @@ class Version:
         return self.metadata["resolution"]
 
     @property
-    def field_shape(self) -> Optional[tuple]:
+    def field_shape(self) -> tuple | None:
         """Get the field shape of the dataset."""
         return self.metadata.get("field_shape")
 
     @property
-    def proj_string(self) -> Optional[str]:
+    def proj_string(self) -> str | None:
         """Get the projection string of the dataset."""
         return self.metadata.get("proj_string")
 
     @property
-    def shape(self) -> Optional[tuple]:
+    def shape(self) -> tuple | None:
         """Get the shape of the dataset."""
         if self.data and hasattr(self.data, "shape"):
             return self.data.shape
 
     @property
-    def n_missing_dates(self) -> Optional[int]:
+    def n_missing_dates(self) -> int | None:
         """Get the number of missing dates in the dataset."""
         if "missing_dates" in self.metadata:
             return len(self.metadata["missing_dates"])
         return None
 
     @property
-    def uncompressed_data_size(self) -> Optional[int]:
+    def uncompressed_data_size(self) -> int | None:
         """Get the uncompressed data size of the dataset."""
         if self.data and hasattr(self.data, "dtype") and hasattr(self.data, "size"):
             return self.data.dtype.itemsize * self.data.size
@@ -258,7 +254,7 @@ class Version:
         print()
         shape_str = "📐 Shape      : "
         if self.shape:
-            shape_str += " × ".join(["{:,}".format(s) for s in self.shape])
+            shape_str += " × ".join([f"{s:,}" for s in self.shape])
         if self.uncompressed_data_size:
             shape_str += f" ({bytes(self.uncompressed_data_size)})"
         print(shape_str)
@@ -293,17 +289,17 @@ class Version:
         print()
 
     @property
-    def variables(self) -> List[str]:
+    def variables(self) -> list[str]:
         """Get the list of variables in the dataset."""
         return [v[0] for v in sorted(self.name_to_index.items(), key=lambda x: x[1])]
 
     @property
-    def total_size(self) -> Optional[int]:
+    def total_size(self) -> int | None:
         """Get the total size of the dataset."""
         return self.zarr.attrs.get("total_size")
 
     @property
-    def total_number_of_files(self) -> Optional[int]:
+    def total_number_of_files(self) -> int | None:
         """Get the total number of files in the dataset."""
         return self.zarr.attrs.get("total_number_of_files")
 
@@ -348,7 +344,7 @@ class Version:
         return False
 
     @property
-    def statistics_started(self) -> Optional[datetime.datetime]:
+    def statistics_started(self) -> datetime.datetime | None:
         """Get the timestamp when statistics computation started."""
         for d in reversed(self.metadata.get("history", [])):
             if d["action"] == "compute_statistics_start":
@@ -356,12 +352,12 @@ class Version:
         return None
 
     @property
-    def build_flags(self) -> Optional[NDArray[Any]]:
+    def build_flags(self) -> NDArray[Any] | None:
         """Get the build flags of the dataset."""
         return self.zarr.get("_build_flags")
 
     @cached_property
-    def copy_flags(self) -> Optional[NDArray[Any]]:
+    def copy_flags(self) -> NDArray[Any] | None:
         """Get the copy flags of the dataset."""
         if "_copy" not in self.zarr:
             return None
@@ -381,7 +377,7 @@ class Version:
         return not all(self.copy_flags)
 
     @property
-    def build_lengths(self) -> Optional[NDArray]:
+    def build_lengths(self) -> NDArray | None:
         """Get the build lengths of the dataset."""
         return self.zarr.get("_build_lengths")
 
@@ -396,17 +392,13 @@ class Version:
             print(
                 "📈 Progress:",
                 progress(built, total, width=50),
-                "{:.0f}%".format(built / total * 100),
+                f"{built / total * 100:.0f}%",
             )
             return
 
-        if self.build_flags is None:
-            print("🪫 Dataset not initialised")
-            return
-
-        build_flags = self.build_flags
+        build_flags = self.build_flags or np.array([], dtype=bool)
 
-        build_lengths = self.build_lengths
+        build_lengths = self.build_lengths or np.array([], dtype=bool)
         assert build_flags.size == build_lengths.size
 
         latest_write_timestamp = self.zarr.attrs.get("latest_write_timestamp")
@@ -422,7 +414,7 @@ class Version:
             print(
                 "📈 Progress:",
                 progress(built, total, width=50),
-                "{:.0f}%".format(built / total * 100),
+                f"{built / total * 100:.0f}%",
             )
             start = self.initialised
             if self.initialised:
@@ -623,7 +615,7 @@ class Version0_6(Version):
     """Represents version 0.6 of a dataset."""
 
     @property
-    def initialised(self) -> Optional[datetime.datetime]:
+    def initialised(self) -> datetime.datetime | None:
         """Get the initialization timestamp of the dataset."""
         for record in self.metadata.get("history", []):
             if record["action"] == "initialised":
@@ -659,12 +651,12 @@ class Version0_6(Version):
         return all(build_flags)
 
     @property
-    def name_to_index(self) -> Dict[str, int]:
+    def name_to_index(self) -> dict[str, int]:
         """Get a mapping of variable names to their indices."""
         return {n: i for i, n in enumerate(self.metadata["variables"])}
 
     @property
-    def variables(self) -> List[str]:
+    def variables(self) -> list[str]:
         """Get the list of variables in the dataset."""
         return self.metadata["variables"]
 
@@ -706,7 +698,7 @@ class Version0_13(Version0_12):
     """Represents version 0.13 of a dataset."""
 
     @property
-    def build_flags(self) -> Optional[NDArray]:
+    def build_flags(self) -> NDArray | None:
         """Get the build flags for the dataset."""
         if "_build" not in self.zarr:
             return None
@@ -714,7 +706,7 @@ class Version0_13(Version0_12):
         return build.get("flags")
 
     @property
-    def build_lengths(self) -> Optional[NDArray]:
+    def build_lengths(self) -> NDArray | None:
         """Get the build lengths for the dataset."""
         if "_build" not in self.zarr:
             return None
@@ -792,10 +784,10 @@ class InspectZarr(Command):
 
         try:
             if progress:
-                return version.progress()
+                version.progress()
 
             if statistics:
-                return version.brute_force_statistics()
+                version.brute_force_statistics()
 
             version.info(detailed, size)
 

anemoi/datasets/commands/recipe/__init__.py

@@ -0,0 +1,93 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import argparse
+import logging
+import sys
+from typing import Any
+
+import yaml
+
+from anemoi.datasets.create import validate_config
+
+from .. import Command
+from .format import format_recipe
+from .migrate import migrate_recipe
+
+LOG = logging.getLogger(__name__)
+
+
+class Recipe(Command):
+    def add_arguments(self, command_parser: Any) -> None:
+        """Add arguments to the command parser.
+
+        Parameters
+        ----------
+        command_parser : Any
+            Command parser object.
+        """
+
+        command_parser.add_argument("--validate", action="store_true", help="Validate recipe.")
+        command_parser.add_argument("--format", action="store_true", help="Format the recipe.")
+        command_parser.add_argument("--migrate", action="store_true", help="Migrate the recipe to the latest version.")
+
+        group = command_parser.add_mutually_exclusive_group()
+        group.add_argument("--inplace", action="store_true", help="Overwrite the recipe file in place.")
+        group.add_argument("--output", type=str, help="Output file path for the converted recipe.")
+
+        command_parser.add_argument(
+            "path",
+            help="Path to recipe.",
+        )
+
+    def run(self, args: Any) -> None:
+
+        if not args.validate and not args.format and not args.migrate:
+            args.validate = True
+
+        with open(args.path) as file:
+            config = yaml.safe_load(file)
+
+        assert isinstance(config, dict)
+
+        if args.validate:
+            if args.inplace and (not args.format and not args.migrate):
+                argparse.ArgumentError(None, "--inplace is not supported with --validate.")
+
+            if args.output and (not args.format and not args.migrate):
+                argparse.ArgumentError(None, "--output is not supported with --validate.")
+
+            validate_config(config)
+            LOG.info(f"{args.path}: Recipe is valid.")
+            return
+
+        if args.migrate:
+            config = migrate_recipe(args, config)
+            if config is None:
+                LOG.info(f"{args.path}: No changes needed.")
+                return
+
+            args.format = True
+
+        if args.format:
+            formatted = format_recipe(args, config)
+            assert "dates" in formatted
+            f = sys.stdout
+            if args.output:
+                f = open(args.output, "w")
+
+            if args.inplace:
+                f = open(args.path, "w")
+
+            print(formatted, file=f)
+            f.close()
+
+
+command = Recipe

anemoi/datasets/commands/recipe/format.py

@@ -0,0 +1,55 @@
+# (C) Copyright 2025 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import datetime
+import logging
+
+from ...dumper import yaml_dump
+
+LOG = logging.getLogger(__name__)
+
+
+def make_dates(config):
+    if isinstance(config, dict):
+        return {k: make_dates(v) for k, v in config.items()}
+    if isinstance(config, list):
+        return [make_dates(v) for v in config]
+    if isinstance(config, str):
+        try:
+            return datetime.datetime.fromisoformat(config)
+        except ValueError:
+            return config
+    return config
+
+
+ORDER = (
+    "name",
+    "description",
+    "dataset_status",
+    "licence",
+    "attribution",
+    "env",
+    "dates",
+    "common",
+    "data_sources",
+    "input",
+    "output",
+    "statistics",
+    "build",
+    "platform",
+)
+
+
+def format_recipe(args, config: dict) -> str:
+
+    config = make_dates(config)
+    assert config
+
+    return yaml_dump(config, order=ORDER)