anemoi-datasets 0.5.27__py3-none-any.whl → 0.5.29__py3-none-any.whl

anemoi/datasets/create/input/concat.py

@@ -1,161 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import logging
-from copy import deepcopy
-from functools import cached_property
-from typing import Any
-
-from earthkit.data import FieldList
-
-from anemoi.datasets.dates import DatesProvider
-
-from ...dates.groups import GroupOfDates
-from .action import Action
-from .action import action_factory
-from .empty import EmptyResult
-from .misc import _tidy
-from .misc import assert_fieldlist
-from .result import Result
-from .template import notify_result
-from .trace import trace_datasource
-from .trace import trace_select
-
-LOG = logging.getLogger(__name__)
-
-
-class ConcatResult(Result):
-    """Represents the result of concatenating multiple results."""
-
-    def __init__(
-        self,
-        context: object,
-        action_path: list[str],
-        group_of_dates: GroupOfDates,
-        results: list[Result],
-        **kwargs: Any,
-    ) -> None:
-        """Initializes a ConcatResult instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : List[str]
-            The action path.
-        group_of_dates : GroupOfDates
-            The group of dates.
-        results : List[Result]
-            The list of results.
-        kwargs : Any
-            Additional keyword arguments.
-        """
-        super().__init__(context, action_path, group_of_dates)
-        self.results = [r for r in results if not r.empty]
-
-    @cached_property
-    @assert_fieldlist
-    @notify_result
-    @trace_datasource
-    def datasource(self) -> FieldList:
-        """Returns the concatenated datasource from all results."""
-        ds = EmptyResult(self.context, self.action_path, self.group_of_dates).datasource
-        for i in self.results:
-            ds += i.datasource
-        return _tidy(ds)
-
-    @property
-    def variables(self) -> list[str]:
-        """Returns the list of variables, ensuring all results have the same variables."""
-        variables = None
-        for f in self.results:
-            if f.empty:
-                continue
-            if variables is None:
-                variables = f.variables
-            assert variables == f.variables, (variables, f.variables)
-        assert variables is not None, self.results
-        return variables
-
-    def __repr__(self) -> str:
-        """Returns a string representation of the ConcatResult instance.
-
-        Returns
-        -------
-        str
-            A string representation of the ConcatResult instance.
-        """
-        content = "\n".join([str(i) for i in self.results])
-        return self._repr(content)
-
-
-class ConcatAction(Action):
-    """Represents an action that concatenates multiple actions based on their dates."""
-
-    def __init__(self, context: object, action_path: list[str], *configs: dict[str, Any]) -> None:
-        """Initializes a ConcatAction instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : List[str]
-            The action path.
-        configs : Dict[str, Any]
-            The configuration dictionaries.
-        """
-        super().__init__(context, action_path, *configs)
-        parts = []
-        for i, cfg in enumerate(configs):
-            if "dates" not in cfg:
-                raise ValueError(f"Missing 'dates' in {cfg}")
-            cfg = deepcopy(cfg)
-            dates_cfg = cfg.pop("dates")
-            assert isinstance(dates_cfg, dict), dates_cfg
-            filtering_dates = DatesProvider.from_config(**dates_cfg)
-            action = action_factory(cfg, context, action_path + [str(i)])
-            parts.append((filtering_dates, action))
-        self.parts = parts
-
-    def __repr__(self) -> str:
-        """Returns a string representation of the ConcatAction instance.
-
-        Returns
-        -------
-        str
-            A string representation of the ConcatAction instance.
-        """
-        content = "\n".join([str(i) for i in self.parts])
-        return self._repr(content)
-
-    @trace_select
-    def select(self, group_of_dates: GroupOfDates) -> ConcatResult | EmptyResult:
-        """Selects the concatenated result for the given group of dates.
-
-        Parameters
-        ----------
-        group_of_dates : GroupOfDates
-            The group of dates.
-
-        Returns
-        -------
-        Union[ConcatResult, EmptyResult]
-            The concatenated result or an empty result.
-        """
-        from anemoi.datasets.dates.groups import GroupOfDates
-
-        results = []
-        for filtering_dates, action in self.parts:
-            newdates = GroupOfDates(sorted(set(group_of_dates) & set(filtering_dates)), group_of_dates.provider)
-            if newdates:
-                results.append(action.select(newdates))
-        if not results:
-            return EmptyResult(self.context, self.action_path, group_of_dates)
-
-        return ConcatResult(self.context, self.action_path, group_of_dates, results)

anemoi/datasets/create/input/context.py

@@ -1,86 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import logging
-import textwrap
-from typing import Any
-
-from anemoi.utils.humanize import plural
-
-from .trace import step
-from .trace import trace
-
-LOG = logging.getLogger(__name__)
-
-
-class Context:
-    """Class to handle the build context in the dataset creation process."""
-
-    def __init__(self) -> None:
-        """Initializes a Context instance."""
-        # used_references is a set of reference paths that will be needed
-        self.used_references = set()
-        # results is a dictionary of reference path -> obj
-        self.results = {}
-
-    def will_need_reference(self, key: list | tuple) -> None:
-        """Marks a reference as needed.
-
-        Parameters
-        ----------
-        key : Union[List, Tuple]
-            The reference key.
-        """
-        assert isinstance(key, (list, tuple)), key
-        key = tuple(key)
-        self.used_references.add(key)
-
-    def notify_result(self, key: list | tuple, result: Any) -> None:
-        """Notifies that a result is available for a reference.
-
-        Parameters
-        ----------
-        key : Union[List, Tuple]
-            The reference key.
-        result : Any
-            The result object.
-        """
-        trace(
-            "🎯",
-            step(key),
-            "notify result",
-            textwrap.shorten(repr(result).replace(",", ", "), width=40),
-            plural(len(result), "field"),
-        )
-        assert isinstance(key, (list, tuple)), key
-        key = tuple(key)
-        if key in self.used_references:
-            if key in self.results:
-                raise ValueError(f"Duplicate result {key}")
-            self.results[key] = result
-
-    def get_result(self, key: list | tuple) -> Any:
-        """Retrieves the result for a given reference.
-
-        Parameters
-        ----------
-        key : Union[List, Tuple]
-            The reference key.
-
-        Returns
-        -------
-        Any
-            The result for the given reference.
-        """
-        assert isinstance(key, (list, tuple)), key
-        key = tuple(key)
-        if key in self.results:
-            return self.results[key]
-        all_keys = sorted(list(self.results.keys()))
-        raise ValueError(f"Cannot find result {key} in {all_keys}")

anemoi/datasets/create/input/empty.py

@@ -1,53 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import logging
-from functools import cached_property
-
-from earthkit.data import FieldList
-
-from .misc import assert_fieldlist
-from .result import Result
-from .trace import trace_datasource
-
-LOG = logging.getLogger(__name__)
-
-
-class EmptyResult(Result):
-    """Class to represent an empty result in the dataset creation process."""
-
-    empty = True
-
-    def __init__(self, context: object, action_path: list, dates: object) -> None:
-        """Initializes an EmptyResult instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : list
-            The action path.
-        dates : object
-            The dates object.
-        """
-        super().__init__(context, action_path + ["empty"], dates)
-
-    @cached_property
-    @assert_fieldlist
-    @trace_datasource
-    def datasource(self) -> FieldList:
-        """Returns an empty datasource."""
-        from earthkit.data import from_source
-
-        return from_source("empty")
-
-    @property
-    def variables(self) -> list[str]:
-        """Returns an empty list of variables."""
-        return []

anemoi/datasets/create/input/filter.py

@@ -1,117 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import logging
-from functools import cached_property
-from typing import Any
-
-from earthkit.data import FieldList
-
-from .function import FunctionContext
-from .misc import _tidy
-from .misc import assert_fieldlist
-from .step import StepAction
-from .step import StepResult
-from .template import notify_result
-from .trace import trace_datasource
-
-LOG = logging.getLogger(__name__)
-
-
-class FilterStepResult(StepResult):
-    @property
-    @notify_result
-    @assert_fieldlist
-    @trace_datasource
-    def datasource(self) -> FieldList:
-        """Returns the filtered datasource."""
-        ds: FieldList = self.upstream_result.datasource
-        ds = ds.sel(**self.action.kwargs)
-        return _tidy(ds)
-
-
-class FilterStepAction(StepAction):
-    """Represents an action to filter a step result."""
-
-    result_class: type[FilterStepResult] = FilterStepResult
-
-
-class StepFunctionResult(StepResult):
-    @cached_property
-    @assert_fieldlist
-    @notify_result
-    @trace_datasource
-    def datasource(self) -> FieldList:
-        """Returns the datasource after applying the function."""
-
-        self.action.filter.context = FunctionContext(self)
-        try:
-            return _tidy(
-                self.action.filter.execute(
-                    self.upstream_result.datasource,
-                    *self.action.args[1:],
-                    **self.action.kwargs,
-                )
-            )
-
-        except Exception:
-            LOG.error(f"Error in {self.action.name}", exc_info=True)
-            raise
-
-    def _trace_datasource(self, *args: Any, **kwargs: Any) -> str:
-        """Traces the datasource for the given arguments.
-
-        Parameters
-        ----------
-        *args : Any
-            The arguments.
-        **kwargs : Any
-            The keyword arguments.
-
-        Returns
-        -------
-        str
-            A string representation of the traced datasource.
-        """
-        return f"{self.action.name}({self.group_of_dates})"
-
-
-class FunctionStepAction(StepAction):
-    """Represents an action to apply a function to a step result."""
-
-    result_class: type[StepFunctionResult] = StepFunctionResult
-
-    def __init__(
-        self,
-        context: object,
-        action_path: list,
-        previous_step: StepAction,
-        name: str,
-        filter: Any,
-        *args: Any,
-        **kwargs: Any,
-    ) -> None:
-        """Initializes a FunctionStepAction instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : list
-            The action path.
-        previous_step : StepAction
-            The previous step action.
-        *args : Any
-            Additional arguments.
-        **kwargs : Any
-            Additional keyword arguments.
-        """
-        super().__init__(context, action_path, previous_step, *args, **kwargs)
-        self.name = name
-        self.filter = filter

anemoi/datasets/create/input/function.py

@@ -1,232 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import logging
-from functools import cached_property
-from typing import Any
-
-from earthkit.data import FieldList
-
-from ...dates.groups import GroupOfDates
-from .action import Action
-from .misc import _tidy
-from .misc import assert_fieldlist
-from .result import Result
-from .template import notify_result
-from .template import substitute
-from .trace import trace
-from .trace import trace_datasource
-from .trace import trace_select
-
-LOG = logging.getLogger(__name__)
-
-
-class FunctionContext:
-    """A FunctionContext is passed to all functions, it will be used to pass information
-    to the functions from the other actions and filters and results.
-    """
-
-    def __init__(self, owner: Result) -> None:
-        """Initializes a FunctionContext instance.
-
-        Parameters
-        ----------
-        owner : object
-            The owner object.
-        """
-        self.owner = owner
-        self.use_grib_paramid: bool = owner.context.use_grib_paramid
-
-    def trace(self, emoji: str, *args: Any) -> None:
-        """Traces the given arguments with an emoji.
-
-        Parameters
-        ----------
-        emoji : str
-            The emoji to use.
-        *args : Any
-            The arguments to trace.
-        """
-        trace(emoji, *args)
-
-    def info(self, *args: Any, **kwargs: Any) -> None:
-        """Logs an info message.
-
-        Parameters
-        ----------
-        *args : Any
-            The arguments for the log message.
-        **kwargs : Any
-            The keyword arguments for the log message.
-        """
-        LOG.info(*args, **kwargs)
-
-    @property
-    def dates_provider(self) -> object:
-        """Returns the dates provider."""
-        return self.owner.group_of_dates.provider
-
-    @property
-    def partial_ok(self) -> bool:
-        """Returns whether partial results are acceptable."""
-        return self.owner.group_of_dates.partial_ok
-
-    def get_result(self, *args, **kwargs) -> Any:
-        return self.owner.context.get_result(*args, **kwargs)
-
-
-class FunctionAction(Action):
-    """Represents an action that executes a function.
-
-    Attributes
-    ----------
-    name : str
-        The name of the function.
-    """
-
-    def __init__(self, context: object, action_path: list, _name: str, source, **kwargs: dict[str, Any]) -> None:
-        """Initializes a FunctionAction instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : list
-            The action path.
-        _name : str
-            The name of the function.
-        **kwargs : Dict[str, Any]
-            Additional keyword arguments.
-        """
-        super().__init__(context, action_path, **kwargs)
-        self.name: str = _name
-        self.source = source
-
-    @trace_select
-    def select(self, group_of_dates: GroupOfDates) -> "FunctionResult":
-        """Selects the function result for the given group of dates.
-
-        Parameters
-        ----------
-        group_of_dates : GroupOfDates
-            The group of dates.
-
-        Returns
-        -------
-        FunctionResult
-            The function result instance.
-        """
-        return FunctionResult(self.context, self.action_path, group_of_dates, action=self)
-
-    def __repr__(self) -> str:
-        """Returns a string representation of the FunctionAction instance."""
-        content: str = ""
-        content += ",".join([self._short_str(a) for a in self.args])
-        content += " ".join([self._short_str(f"{k}={v}") for k, v in self.kwargs.items()])
-        content = self._short_str(content)
-        return self._repr(_inline_=content, _indent_=" ")
-
-    def _trace_select(self, group_of_dates: GroupOfDates) -> str:
-        """Traces the selection of the function for the given group of dates.
-
-        Parameters
-        ----------
-        group_of_dates : GroupOfDates
-            The group of dates.
-
-        Returns
-        -------
-        str
-            The trace string.
-        """
-        return f"{self.name}({group_of_dates})"
-
-
-class FunctionResult(Result):
-    """Represents the result of executing a function.
-
-    Attributes
-    ----------
-    action : Action
-        The action instance.
-    args : tuple
-        The positional arguments for the function.
-    kwargs : dict
-        The keyword arguments for the function.
-    """
-
-    def __init__(self, context: object, action_path: list, group_of_dates: GroupOfDates, action: Action) -> None:
-        """Initializes a FunctionResult instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : list
-            The action path.
-        group_of_dates : GroupOfDates
-            The group of dates.
-        action : Action
-            The action instance.
-        """
-        super().__init__(context, action_path, group_of_dates)
-        assert isinstance(action, Action), type(action)
-        self.action: Action = action
-
-        self.args, self.kwargs = substitute(context, (self.action.args, self.action.kwargs))
-
-    def _trace_datasource(self, *args: Any, **kwargs: Any) -> str:
-        """Traces the datasource for the given arguments.
-
-        Parameters
-        ----------
-        *args : Any
-            The arguments.
-        **kwargs : Any
-            The keyword arguments.
-
-        Returns
-        -------
-        str
-            The trace string.
-        """
-        return f"{self.action.name}({self.group_of_dates})"
-
-    @cached_property
-    @assert_fieldlist
-    @notify_result
-    @trace_datasource
-    def datasource(self) -> FieldList:
-        """Returns the datasource for the function result."""
-        # args, kwargs = resolve(self.context, (self.args, self.kwargs))
-        self.action.source.context = FunctionContext(self)
-
-        return _tidy(
-            self.action.source.execute(
-                list(self.group_of_dates),  # Will provide a list of datetime objects
-            )
-        )
-
-    def __repr__(self) -> str:
-        """Returns a string representation of the FunctionResult instance."""
-        try:
-            return f"{self.action.name}({self.group_of_dates})"
-        except Exception:
-            return f"{self.__class__.__name__}(unitialised)"
-
-    @property
-    def function(self) -> None:
-        """Raises NotImplementedError as this property is not implemented.
-
-        Raises
-        ------
-        NotImplementedError
-            Always raised.
-        """
-        raise NotImplementedError(f"Not implemented in {self.__class__.__name__}")