anemoi-datasets 0.5.26__py3-none-any.whl → 0.5.27__py3-none-any.whl

anemoi/datasets/create/config.py

@@ -12,8 +12,6 @@ import logging
 import os
 from copy import deepcopy
 from typing import Any
-from typing import Optional
-from typing import Union
 
 import yaml
 from anemoi.utils.config import DotDict
@@ -25,7 +23,7 @@ from anemoi.datasets.dates.groups import Groups
 LOG = logging.getLogger(__name__)
 
 
-def _get_first_key_if_dict(x: Union[str, dict]) -> str:
+def _get_first_key_if_dict(x: str | dict) -> str:
     """Returns the first key if the input is a dictionary, otherwise returns the input string.
 
     Parameters
@@ -97,7 +95,7 @@ def check_dict_value_and_set(dic: dict, key: str, value: Any) -> None:
     dic[key] = value
 
 
-def resolve_includes(config: Union[dict, list]) -> Union[dict, list]:
+def resolve_includes(config: dict | list) -> dict | list:
     """Resolves '<<' includes in a configuration dictionary or list.
 
     Parameters
@@ -123,7 +121,7 @@ def resolve_includes(config: Union[dict, list]) -> Union[dict, list]:
 class Config(DotDict):
     """Configuration class that extends DotDict to handle configuration loading and processing."""
 
-    def __init__(self, config: Optional[Union[str, dict]] = None, **kwargs):
+    def __init__(self, config: str | dict | None = None, **kwargs):
         """Initializes the Config object.
 
         Parameters
@@ -134,7 +132,6 @@ class Config(DotDict):
             Additional keyword arguments to update the configuration.
         """
         if isinstance(config, str):
-            self.config_path = os.path.realpath(config)
             config = load_any_dict_format(config)
         else:
             config = deepcopy(config if config is not None else {})

anemoi/datasets/create/filter.py

@@ -8,7 +8,6 @@
 # nor does it submit to any jurisdiction.
 
 from typing import Any
-from typing import Dict
 
 import earthkit.data as ekd
 
@@ -26,7 +25,7 @@ class TransformFilter:
         The configuration for the filter.
     """
 
-    def __init__(self, context: Any, name: str, config: Dict[str, Any]) -> None:
+    def __init__(self, context: Any, name: str, config: dict[str, Any]) -> None:
         from anemoi.transform.filters import create_filter
 
         self.name = name

anemoi/datasets/create/input/__init__.py

@@ -10,7 +10,6 @@
 import logging
 from copy import deepcopy
 from typing import Any
-from typing import Union
 
 from anemoi.datasets.dates.groups import GroupOfDates
 
@@ -28,7 +27,7 @@ class Context:
 class InputBuilder:
     """Builder class for creating input data from configuration and data sources."""
 
-    def __init__(self, config: dict, data_sources: Union[dict, list], **kwargs: Any) -> None:
+    def __init__(self, config: dict, data_sources: dict | list, **kwargs: Any) -> None:
         """Initialize the InputBuilder.
 
         Parameters

anemoi/datasets/create/input/action.py

@@ -11,8 +11,6 @@ import json
 import logging
 from copy import deepcopy
 from typing import Any
-from typing import Dict
-from typing import List
 
 from earthkit.data.core.order import build_remapping
 
@@ -39,7 +37,7 @@ class Action:
     """
 
     def __init__(
-        self, context: "ActionContext", action_path: List[str], /, *args: Any, **kwargs: Dict[str, Any]
+        self, context: "ActionContext", action_path: list[str], /, *args: Any, **kwargs: dict[str, Any]
     ) -> None:
         """Initialize an Action instance.
 
@@ -175,7 +173,7 @@ class ActionContext(Context):
         Whether to use GRIB parameter ID.
     """
 
-    def __init__(self, /, order_by: str, flatten_grid: bool, remapping: Dict[str, Any], use_grib_paramid: bool) -> None:
+    def __init__(self, /, order_by: str, flatten_grid: bool, remapping: dict[str, Any], use_grib_paramid: bool) -> None:
         """Initialize an ActionContext instance.
 
         Parameters
@@ -196,7 +194,7 @@ class ActionContext(Context):
         self.use_grib_paramid = use_grib_paramid
 
 
-def action_factory(config: Dict[str, Any], context: ActionContext, action_path: List[str]) -> Action:
+def action_factory(config: dict[str, Any], context: ActionContext, action_path: list[str]) -> Action:
     """Factory function to create an Action instance based on the configuration.
 
     Parameters

anemoi/datasets/create/input/concat.py

@@ -11,9 +11,6 @@ import logging
 from copy import deepcopy
 from functools import cached_property
 from typing import Any
-from typing import Dict
-from typing import List
-from typing import Union
 
 from earthkit.data import FieldList
 
@@ -39,9 +36,9 @@ class ConcatResult(Result):
     def __init__(
         self,
         context: object,
-        action_path: List[str],
+        action_path: list[str],
         group_of_dates: GroupOfDates,
-        results: List[Result],
+        results: list[Result],
         **kwargs: Any,
     ) -> None:
         """Initializes a ConcatResult instance.
@@ -74,7 +71,7 @@ class ConcatResult(Result):
         return _tidy(ds)
 
     @property
-    def variables(self) -> List[str]:
+    def variables(self) -> list[str]:
         """Returns the list of variables, ensuring all results have the same variables."""
         variables = None
         for f in self.results:
@@ -101,7 +98,7 @@ class ConcatResult(Result):
 class ConcatAction(Action):
     """Represents an action that concatenates multiple actions based on their dates."""
 
-    def __init__(self, context: object, action_path: List[str], *configs: Dict[str, Any]) -> None:
+    def __init__(self, context: object, action_path: list[str], *configs: dict[str, Any]) -> None:
         """Initializes a ConcatAction instance.
 
         Parameters
@@ -138,7 +135,7 @@ class ConcatAction(Action):
         return self._repr(content)
 
     @trace_select
-    def select(self, group_of_dates: GroupOfDates) -> Union[ConcatResult, EmptyResult]:
+    def select(self, group_of_dates: GroupOfDates) -> ConcatResult | EmptyResult:
         """Selects the concatenated result for the given group of dates.
 
         Parameters

anemoi/datasets/create/input/context.py

@@ -10,9 +10,6 @@
 import logging
 import textwrap
 from typing import Any
-from typing import List
-from typing import Tuple
-from typing import Union
 
 from anemoi.utils.humanize import plural
 
@@ -32,7 +29,7 @@ class Context:
         # results is a dictionary of reference path -> obj
         self.results = {}
 
-    def will_need_reference(self, key: Union[List, Tuple]) -> None:
+    def will_need_reference(self, key: list | tuple) -> None:
         """Marks a reference as needed.
 
         Parameters
@@ -44,7 +41,7 @@ class Context:
         key = tuple(key)
         self.used_references.add(key)
 
-    def notify_result(self, key: Union[List, Tuple], result: Any) -> None:
+    def notify_result(self, key: list | tuple, result: Any) -> None:
         """Notifies that a result is available for a reference.
 
         Parameters
@@ -68,7 +65,7 @@ class Context:
                 raise ValueError(f"Duplicate result {key}")
             self.results[key] = result
 
-    def get_result(self, key: Union[List, Tuple]) -> Any:
+    def get_result(self, key: list | tuple) -> Any:
         """Retrieves the result for a given reference.
 
         Parameters

anemoi/datasets/create/input/data_sources.py

@@ -10,9 +10,6 @@
 import logging
 from functools import cached_property
 from typing import Any
-from typing import Dict
-from typing import List
-from typing import Union
 
 from earthkit.data import FieldList
 
@@ -31,9 +28,9 @@ class DataSourcesAction(Action):
     def __init__(
         self,
         context: object,
-        action_path: List[str],
-        sources: Union[Dict[str, Any], List[Dict[str, Any]]],
-        input: Dict[str, Any],
+        action_path: list[str],
+        sources: dict[str, Any] | list[dict[str, Any]],
+        input: dict[str, Any],
     ) -> None:
         """Initializes a DataSourcesAction instance.
 
@@ -93,10 +90,10 @@ class DataSourcesResult(Result):
     def __init__(
         self,
         context: object,
-        action_path: List[str],
+        action_path: list[str],
         dates: object,
         input_result: Result,
-        sources_results: List[Result],
+        sources_results: list[Result],
     ) -> None:
         """Initializes a DataSourcesResult instance.
 

anemoi/datasets/create/input/empty.py

@@ -9,7 +9,6 @@
 
 import logging
 from functools import cached_property
-from typing import List
 
 from earthkit.data import FieldList
 
@@ -49,6 +48,6 @@ class EmptyResult(Result):
         return from_source("empty")
 
     @property
-    def variables(self) -> List[str]:
+    def variables(self) -> list[str]:
         """Returns an empty list of variables."""
         return []

anemoi/datasets/create/input/filter.py

@@ -10,7 +10,6 @@
 import logging
 from functools import cached_property
 from typing import Any
-from typing import Type
 
 from earthkit.data import FieldList
 
@@ -40,7 +39,7 @@ class FilterStepResult(StepResult):
 class FilterStepAction(StepAction):
     """Represents an action to filter a step result."""
 
-    result_class: Type[FilterStepResult] = FilterStepResult
+    result_class: type[FilterStepResult] = FilterStepResult
 
 
 class StepFunctionResult(StepResult):
@@ -86,7 +85,7 @@ class StepFunctionResult(StepResult):
 class FunctionStepAction(StepAction):
     """Represents an action to apply a function to a step result."""
 
-    result_class: Type[StepFunctionResult] = StepFunctionResult
+    result_class: type[StepFunctionResult] = StepFunctionResult
 
     def __init__(
         self,

anemoi/datasets/create/input/function.py

@@ -10,7 +10,6 @@
 import logging
 from functools import cached_property
 from typing import Any
-from typing import Dict
 
 from earthkit.data import FieldList
 
@@ -91,7 +90,7 @@ class FunctionAction(Action):
         The name of the function.
     """
 
-    def __init__(self, context: object, action_path: list, _name: str, source, **kwargs: Dict[str, Any]) -> None:
+    def __init__(self, context: object, action_path: list, _name: str, source, **kwargs: dict[str, Any]) -> None:
         """Initializes a FunctionAction instance.
 
         Parameters

anemoi/datasets/create/input/join.py

@@ -10,7 +10,6 @@
 import logging
 from functools import cached_property
 from typing import Any
-from typing import List
 
 from earthkit.data import FieldList
 
@@ -44,7 +43,7 @@ class JoinResult(Result):
     """
 
     def __init__(
-        self, context: object, action_path: list, group_of_dates: GroupOfDates, results: List[Result], **kwargs: Any
+        self, context: object, action_path: list, group_of_dates: GroupOfDates, results: list[Result], **kwargs: Any
     ) -> None:
         """Initializes a JoinResult instance.
 
@@ -60,7 +59,7 @@ class JoinResult(Result):
             The list of results.
         """
         super().__init__(context, action_path, group_of_dates)
-        self.results: List[Result] = [r for r in results if not r.empty]
+        self.results: list[Result] = [r for r in results if not r.empty]
 
     @cached_property
     @assert_fieldlist
@@ -105,7 +104,7 @@ class JoinAction(Action):
             The configuration dictionaries.
         """
         super().__init__(context, action_path, *configs)
-        self.actions: List[Action] = [action_factory(c, context, action_path + [str(i)]) for i, c in enumerate(configs)]
+        self.actions: list[Action] = [action_factory(c, context, action_path + [str(i)]) for i, c in enumerate(configs)]
 
     def __repr__(self) -> str:
         """Returns a string representation of the JoinAction instance."""
@@ -126,5 +125,5 @@ class JoinAction(Action):
         JoinResult
             The combined result for the given group of dates.
         """
-        results: List[Result] = [a.select(group_of_dates) for a in self.actions]
+        results: list[Result] = [a.select(group_of_dates) for a in self.actions]
         return JoinResult(self.context, self.action_path, group_of_dates, results)

anemoi/datasets/create/input/misc.py

@@ -8,11 +8,9 @@
 # nor does it submit to any jurisdiction.
 
 import logging
+from collections.abc import Callable
 from functools import wraps
 from typing import Any
-from typing import Callable
-from typing import Tuple
-from typing import Union
 
 from earthkit.data import FieldList
 from earthkit.data.core.fieldlist import MultiFieldList
@@ -20,7 +18,7 @@ from earthkit.data.core.fieldlist import MultiFieldList
 LOG = logging.getLogger(__name__)
 
 
-def parse_function_name(name: str) -> Tuple[str, Union[int, None]]:
+def parse_function_name(name: str) -> tuple[str, int | None]:
     """Parses a function name to extract the base name and an optional time delta.
 
     Parameters
@@ -88,7 +86,7 @@ def assert_is_fieldlist(obj: object) -> None:
     assert isinstance(obj, FieldList), type(obj)
 
 
-def _flatten(ds: Union[MultiFieldList, FieldList]) -> list:
+def _flatten(ds: MultiFieldList | FieldList) -> list:
     """Flattens a MultiFieldList or FieldList into a list of FieldList objects.
 
     Parameters
@@ -106,7 +104,7 @@ def _flatten(ds: Union[MultiFieldList, FieldList]) -> list:
     return [ds]
 
 
-def _tidy(ds: Union[MultiFieldList, FieldList], indent: int = 0) -> Union[MultiFieldList, FieldList]:
+def _tidy(ds: MultiFieldList | FieldList, indent: int = 0) -> MultiFieldList | FieldList:
     """Tidies up a MultiFieldList or FieldList by removing empty sources.
 
     Parameters

anemoi/datasets/create/input/repeated_dates.py

@@ -10,13 +10,8 @@
 
 import logging
 from collections import defaultdict
+from collections.abc import Generator
 from typing import Any
-from typing import Dict
-from typing import Generator
-from typing import List
-from typing import Optional
-from typing import Set
-from typing import Tuple
 
 import numpy as np
 from anemoi.transform.fields import new_field_with_valid_datetime
@@ -37,7 +32,7 @@ class DateMapper:
     """A factory class to create DateMapper instances based on the given mode."""
 
     @staticmethod
-    def from_mode(mode: str, source: Any, config: Dict[str, Any]) -> "DateMapper":
+    def from_mode(mode: str, source: Any, config: dict[str, Any]) -> "DateMapper":
         """Create a DateMapper instance based on the given mode.
 
         Parameters
@@ -87,10 +82,10 @@ class DateMapperClosest(DateMapper):
         self.maximum: Any = frequency_to_timedelta(maximum)
         self.frequency: Any = frequency_to_timedelta(frequency)
         self.skip_all_nans: bool = skip_all_nans
-        self.tried: Set[Any] = set()
-        self.found: Set[Any] = set()
+        self.tried: set[Any] = set()
+        self.found: set[Any] = set()
 
-    def transform(self, group_of_dates: Any) -> Generator[Tuple[Any, Any], None, None]:
+    def transform(self, group_of_dates: Any) -> Generator[tuple[Any, Any], None, None]:
         """Transform the group of dates to the closest available dates.
 
         Parameters
@@ -185,7 +180,7 @@ class DateMapperClosest(DateMapper):
 class DateMapperClimatology(DateMapper):
     """A DateMapper implementation that maps dates to specified climatology dates."""
 
-    def __init__(self, source: Any, year: int, day: int, hour: Optional[int] = None) -> None:
+    def __init__(self, source: Any, year: int, day: int, hour: int | None = None) -> None:
         """Initialize DateMapperClimatology.
 
         Parameters
@@ -201,9 +196,9 @@ class DateMapperClimatology(DateMapper):
         """
         self.year: int = year
         self.day: int = day
-        self.hour: Optional[int] = hour
+        self.hour: int | None = hour
 
-    def transform(self, group_of_dates: Any) -> Generator[Tuple[Any, Any], None, None]:
+    def transform(self, group_of_dates: Any) -> Generator[tuple[Any, Any], None, None]:
         """Transform the group of dates to the specified climatology dates.
 
         Parameters
@@ -239,7 +234,7 @@ class DateMapperClimatology(DateMapper):
 class DateMapperConstant(DateMapper):
     """A DateMapper implementation that maps dates to a constant date."""
 
-    def __init__(self, source: Any, date: Optional[Any] = None) -> None:
+    def __init__(self, source: Any, date: Any | None = None) -> None:
         """Initialize DateMapperConstant.
 
         Parameters
@@ -250,9 +245,9 @@ class DateMapperConstant(DateMapper):
             The constant date to map to.
         """
         self.source: Any = source
-        self.date: Optional[Any] = date
+        self.date: Any | None = date
 
-    def transform(self, group_of_dates: Any) -> Tuple[Any, Any]:
+    def transform(self, group_of_dates: Any) -> tuple[Any, Any]:
         """Transform the group of dates to a constant date.
 
         Parameters
@@ -289,7 +284,7 @@ class DateMapperResult(Result):
     def __init__(
         self,
         context: Any,
-        action_path: List[str],
+        action_path: list[str],
         group_of_dates: Any,
         source_result: Any,
         mapper: DateMapper,
@@ -336,7 +331,7 @@ class DateMapperResult(Result):
 class RepeatedDatesAction(Action):
     """An Action implementation that selects and transforms a group of dates."""
 
-    def __init__(self, context: Any, action_path: List[str], source: Any, mode: str, **kwargs: Any) -> None:
+    def __init__(self, context: Any, action_path: list[str], source: Any, mode: str, **kwargs: Any) -> None:
         """Initialize RepeatedDatesAction.
 
         Args: