anemoi-datasets 0.5.26__py3-none-any.whl → 0.5.28__py3-none-any.whl

anemoi/datasets/create/input/function.py

@@ -1,233 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import logging
-from functools import cached_property
-from typing import Any
-from typing import Dict
-
-from earthkit.data import FieldList
-
-from ...dates.groups import GroupOfDates
-from .action import Action
-from .misc import _tidy
-from .misc import assert_fieldlist
-from .result import Result
-from .template import notify_result
-from .template import substitute
-from .trace import trace
-from .trace import trace_datasource
-from .trace import trace_select
-
-LOG = logging.getLogger(__name__)
-
-
-class FunctionContext:
-    """A FunctionContext is passed to all functions, it will be used to pass information
-    to the functions from the other actions and filters and results.
-    """
-
-    def __init__(self, owner: Result) -> None:
-        """Initializes a FunctionContext instance.
-
-        Parameters
-        ----------
-        owner : object
-            The owner object.
-        """
-        self.owner = owner
-        self.use_grib_paramid: bool = owner.context.use_grib_paramid
-
-    def trace(self, emoji: str, *args: Any) -> None:
-        """Traces the given arguments with an emoji.
-
-        Parameters
-        ----------
-        emoji : str
-            The emoji to use.
-        *args : Any
-            The arguments to trace.
-        """
-        trace(emoji, *args)
-
-    def info(self, *args: Any, **kwargs: Any) -> None:
-        """Logs an info message.
-
-        Parameters
-        ----------
-        *args : Any
-            The arguments for the log message.
-        **kwargs : Any
-            The keyword arguments for the log message.
-        """
-        LOG.info(*args, **kwargs)
-
-    @property
-    def dates_provider(self) -> object:
-        """Returns the dates provider."""
-        return self.owner.group_of_dates.provider
-
-    @property
-    def partial_ok(self) -> bool:
-        """Returns whether partial results are acceptable."""
-        return self.owner.group_of_dates.partial_ok
-
-    def get_result(self, *args, **kwargs) -> Any:
-        return self.owner.context.get_result(*args, **kwargs)
-
-
-class FunctionAction(Action):
-    """Represents an action that executes a function.
-
-    Attributes
-    ----------
-    name : str
-        The name of the function.
-    """
-
-    def __init__(self, context: object, action_path: list, _name: str, source, **kwargs: Dict[str, Any]) -> None:
-        """Initializes a FunctionAction instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : list
-            The action path.
-        _name : str
-            The name of the function.
-        **kwargs : Dict[str, Any]
-            Additional keyword arguments.
-        """
-        super().__init__(context, action_path, **kwargs)
-        self.name: str = _name
-        self.source = source
-
-    @trace_select
-    def select(self, group_of_dates: GroupOfDates) -> "FunctionResult":
-        """Selects the function result for the given group of dates.
-
-        Parameters
-        ----------
-        group_of_dates : GroupOfDates
-            The group of dates.
-
-        Returns
-        -------
-        FunctionResult
-            The function result instance.
-        """
-        return FunctionResult(self.context, self.action_path, group_of_dates, action=self)
-
-    def __repr__(self) -> str:
-        """Returns a string representation of the FunctionAction instance."""
-        content: str = ""
-        content += ",".join([self._short_str(a) for a in self.args])
-        content += " ".join([self._short_str(f"{k}={v}") for k, v in self.kwargs.items()])
-        content = self._short_str(content)
-        return self._repr(_inline_=content, _indent_=" ")
-
-    def _trace_select(self, group_of_dates: GroupOfDates) -> str:
-        """Traces the selection of the function for the given group of dates.
-
-        Parameters
-        ----------
-        group_of_dates : GroupOfDates
-            The group of dates.
-
-        Returns
-        -------
-        str
-            The trace string.
-        """
-        return f"{self.name}({group_of_dates})"
-
-
-class FunctionResult(Result):
-    """Represents the result of executing a function.
-
-    Attributes
-    ----------
-    action : Action
-        The action instance.
-    args : tuple
-        The positional arguments for the function.
-    kwargs : dict
-        The keyword arguments for the function.
-    """
-
-    def __init__(self, context: object, action_path: list, group_of_dates: GroupOfDates, action: Action) -> None:
-        """Initializes a FunctionResult instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : list
-            The action path.
-        group_of_dates : GroupOfDates
-            The group of dates.
-        action : Action
-            The action instance.
-        """
-        super().__init__(context, action_path, group_of_dates)
-        assert isinstance(action, Action), type(action)
-        self.action: Action = action
-
-        self.args, self.kwargs = substitute(context, (self.action.args, self.action.kwargs))
-
-    def _trace_datasource(self, *args: Any, **kwargs: Any) -> str:
-        """Traces the datasource for the given arguments.
-
-        Parameters
-        ----------
-        *args : Any
-            The arguments.
-        **kwargs : Any
-            The keyword arguments.
-
-        Returns
-        -------
-        str
-            The trace string.
-        """
-        return f"{self.action.name}({self.group_of_dates})"
-
-    @cached_property
-    @assert_fieldlist
-    @notify_result
-    @trace_datasource
-    def datasource(self) -> FieldList:
-        """Returns the datasource for the function result."""
-        # args, kwargs = resolve(self.context, (self.args, self.kwargs))
-        self.action.source.context = FunctionContext(self)
-
-        return _tidy(
-            self.action.source.execute(
-                list(self.group_of_dates),  # Will provide a list of datetime objects
-            )
-        )
-
-    def __repr__(self) -> str:
-        """Returns a string representation of the FunctionResult instance."""
-        try:
-            return f"{self.action.name}({self.group_of_dates})"
-        except Exception:
-            return f"{self.__class__.__name__}(unitialised)"
-
-    @property
-    def function(self) -> None:
-        """Raises NotImplementedError as this property is not implemented.
-
-        Raises
-        ------
-        NotImplementedError
-            Always raised.
-        """
-        raise NotImplementedError(f"Not implemented in {self.__class__.__name__}")

anemoi/datasets/create/input/join.py

@@ -1,130 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import logging
-from functools import cached_property
-from typing import Any
-from typing import List
-
-from earthkit.data import FieldList
-
-from ...dates.groups import GroupOfDates
-from .action import Action
-from .action import action_factory
-from .empty import EmptyResult
-from .misc import _tidy
-from .misc import assert_fieldlist
-from .result import Result
-from .template import notify_result
-from .trace import trace_datasource
-from .trace import trace_select
-
-LOG = logging.getLogger(__name__)
-
-
-class JoinResult(Result):
-    """Represents a result that combines multiple results.
-
-    Attributes
-    ----------
-    context : object
-        The context object.
-    action_path : list
-        The action path.
-    group_of_dates : GroupOfDates
-        The group of dates.
-    results : List[Result]
-        The list of results.
-    """
-
-    def __init__(
-        self, context: object, action_path: list, group_of_dates: GroupOfDates, results: List[Result], **kwargs: Any
-    ) -> None:
-        """Initializes a JoinResult instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : list
-            The action path.
-        group_of_dates : GroupOfDates
-            The group of dates.
-        results : List[Result]
-            The list of results.
-        """
-        super().__init__(context, action_path, group_of_dates)
-        self.results: List[Result] = [r for r in results if not r.empty]
-
-    @cached_property
-    @assert_fieldlist
-    @notify_result
-    @trace_datasource
-    def datasource(self) -> FieldList:
-        """Returns the combined datasource from all results."""
-        ds: FieldList = EmptyResult(self.context, self.action_path, self.group_of_dates).datasource
-        for i in self.results:
-            ds += i.datasource
-        return _tidy(ds)
-
-    def __repr__(self) -> str:
-        """Returns a string representation of the JoinResult instance."""
-        content: str = "\n".join([str(i) for i in self.results])
-        return self._repr(content)
-
-
-class JoinAction(Action):
-    """Represents an action that combines multiple actions.
-
-    Attributes
-    ----------
-    context : object
-        The context object.
-    action_path : list
-        The action path.
-    actions : List[Action]
-        The list of actions.
-    """
-
-    def __init__(self, context: object, action_path: list, *configs: dict) -> None:
-        """Initializes a JoinAction instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : list
-            The action path.
-        *configs : dict
-            The configuration dictionaries.
-        """
-        super().__init__(context, action_path, *configs)
-        self.actions: List[Action] = [action_factory(c, context, action_path + [str(i)]) for i, c in enumerate(configs)]
-
-    def __repr__(self) -> str:
-        """Returns a string representation of the JoinAction instance."""
-        content: str = "\n".join([str(i) for i in self.actions])
-        return self._repr(content)
-
-    @trace_select
-    def select(self, group_of_dates: GroupOfDates) -> JoinResult:
-        """Selects the results for the given group of dates.
-
-        Parameters
-        ----------
-        group_of_dates : GroupOfDates
-            The group of dates.
-
-        Returns
-        -------
-        JoinResult
-            The combined result for the given group of dates.
-        """
-        results: List[Result] = [a.select(group_of_dates) for a in self.actions]
-        return JoinResult(self.context, self.action_path, group_of_dates, results)

anemoi/datasets/create/input/pipe.py

@@ -1,66 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import json
-import logging
-from typing import Any
-
-from .action import Action
-from .action import action_factory
-from .step import step_factory
-from .trace import trace_select
-
-LOG = logging.getLogger(__name__)
-
-
-class PipeAction(Action):
-    """A class to represent a pipeline of actions."""
-
-    def __init__(self, context: Any, action_path: list, *configs: dict) -> None:
-        """Initialize the PipeAction.
-
-        Parameters
-        ----------
-        context : Any
-            The context for the action.
-        action_path : list
-            The path of the action.
-        configs : dict
-            The configurations for the actions.
-        """
-        super().__init__(context, action_path, *configs)
-        if len(configs) <= 1:
-            raise ValueError(
-                f"PipeAction requires at least two actions, got {len(configs)}\n{json.dumps(configs, indent=2)}"
-            )
-
-        current: Any = action_factory(configs[0], context, action_path + ["0"])
-        for i, c in enumerate(configs[1:]):
-            current = step_factory(c, context, action_path + [str(i + 1)], previous_step=current)
-        self.last_step: Any = current
-
-    @trace_select
-    def select(self, group_of_dates: Any) -> Any:
-        """Select data based on the group of dates.
-
-        Parameters
-        ----------
-        group_of_dates : Any
-            The group of dates to select data for.
-
-        Returns
-        -------
-        Any
-            The selected data.
-        """
-        return self.last_step.select(group_of_dates)
-
-    def __repr__(self) -> str:
-        """Return a string representation of the PipeAction."""
-        return f"PipeAction({self.last_step})"

anemoi/datasets/create/input/step.py

@@ -1,177 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import logging
-from copy import deepcopy
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
-from typing import Type
-
-from .action import Action
-from .action import ActionContext
-from .context import Context
-from .result import Result
-from .template import notify_result
-from .trace import trace_datasource
-from .trace import trace_select
-
-LOG = logging.getLogger(__name__)
-
-
-class StepResult(Result):
-    """Represents the result of a step in the data processing pipeline."""
-
-    def __init__(
-        self, context: Context, action_path: List[str], group_of_dates: Any, action: Action, upstream_result: Result
-    ) -> None:
-        """Initialize a StepResult instance.
-
-        Parameters
-        ----------
-        context
-            The context in which the step is executed.
-        action_path
-            The path of actions leading to this step.
-        group_of_dates
-            The group of dates associated with this step.
-        action
-            The action associated with this step.
-        upstream_result
-            The result of the upstream step.
-        """
-        super().__init__(context, action_path, group_of_dates)
-        assert isinstance(upstream_result, Result), type(upstream_result)
-        self.upstream_result: Result = upstream_result
-        self.action: Action = action
-
-    @property
-    @notify_result
-    @trace_datasource
-    def datasource(self) -> Any:
-        """Retrieve the datasource associated with this step result."""
-        raise NotImplementedError(f"Not implemented in {self.__class__.__name__}")
-
-
-class StepAction(Action):
-    """Represents an action that is part of a step in the data processing pipeline."""
-
-    result_class: Optional[Type[StepResult]] = None
-
-    def __init__(
-        self, context: ActionContext, action_path: List[str], previous_step: Any, *args: Any, **kwargs: Any
-    ) -> None:
-        """Initialize a StepAction instance.
-
-        Parameters
-        ----------
-        context
-            The context in which the action is executed.
-        action_path
-            The path of actions leading to this step.
-        previous_step
-            The previous step in the pipeline.
-        """
-        super().__init__(context, action_path, *args, **kwargs)
-        self.previous_step: Any = previous_step
-
-    @trace_select
-    def select(self, group_of_dates: Any) -> StepResult:
-        """Select the result for a given group of dates.
-
-        Parameters
-        ----------
-        group_of_dates
-            The group of dates to select the result for.
-
-        Returns
-        -------
-        unknown
-            The result of the step.
-        """
-        return self.result_class(
-            self.context,
-            self.action_path,
-            group_of_dates,
-            self,
-            self.previous_step.select(group_of_dates),
-        )
-
-    def __repr__(self) -> str:
-        """Return a string representation of the StepAction instance.
-
-        Returns
-        -------
-        unknown
-            String representation of the instance.
-        """
-        return self._repr(self.previous_step, _inline_=str(self.kwargs))
-
-
-def step_factory(config: Dict[str, Any], context: ActionContext, action_path: List[str], previous_step: Any) -> Any:
-    """Factory function to create a step action based on the given configuration.
-
-    Parameters
-    ----------
-    config
-        The configuration dictionary for the step.
-    context
-        The context in which the step is executed.
-    action_path
-        The path of actions leading to this step.
-    previous_step
-        The previous step in the pipeline.
-
-    Returns
-    -------
-    unknown
-        An instance of a step action.
-    """
-
-    from .filter import FilterStepAction
-    from .filter import FunctionStepAction
-
-    assert isinstance(context, Context), (type, context)
-    if not isinstance(config, dict):
-        raise ValueError(f"Invalid input config {config}")
-
-    config = deepcopy(config)
-    assert len(config) == 1, config
-
-    key = list(config.keys())[0]
-    cls = dict(
-        filter=FilterStepAction,
-        # rename=RenameAction,
-        # remapping=RemappingAction,
-    ).get(key)
-
-    if isinstance(config[key], list):
-        args, kwargs = config[key], {}
-
-    if isinstance(config[key], dict):
-        args, kwargs = [], config[key]
-
-    if isinstance(config[key], str):
-        args, kwargs = [config[key]], {}
-
-    if cls is not None:
-        return cls(context, action_path, previous_step, *args, **kwargs)
-
-    # Try filters from transform filter registry
-    from anemoi.transform.filters import filter_registry as transform_filter_registry
-
-    if transform_filter_registry.is_registered(key):
-        from ..filter import TransformFilter
-
-        return FunctionStepAction(
-            context, action_path + [key], previous_step, key, TransformFilter(context, key, config)
-        )
-
-    raise ValueError(f"Unknown step action `{key}`")