anemoi-datasets 0.5.27__py3-none-any.whl → 0.5.29__py3-none-any.whl

anemoi/datasets/create/input/join.py

@@ -1,129 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import logging
-from functools import cached_property
-from typing import Any
-
-from earthkit.data import FieldList
-
-from ...dates.groups import GroupOfDates
-from .action import Action
-from .action import action_factory
-from .empty import EmptyResult
-from .misc import _tidy
-from .misc import assert_fieldlist
-from .result import Result
-from .template import notify_result
-from .trace import trace_datasource
-from .trace import trace_select
-
-LOG = logging.getLogger(__name__)
-
-
-class JoinResult(Result):
-    """Represents a result that combines multiple results.
-
-    Attributes
-    ----------
-    context : object
-        The context object.
-    action_path : list
-        The action path.
-    group_of_dates : GroupOfDates
-        The group of dates.
-    results : List[Result]
-        The list of results.
-    """
-
-    def __init__(
-        self, context: object, action_path: list, group_of_dates: GroupOfDates, results: list[Result], **kwargs: Any
-    ) -> None:
-        """Initializes a JoinResult instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : list
-            The action path.
-        group_of_dates : GroupOfDates
-            The group of dates.
-        results : List[Result]
-            The list of results.
-        """
-        super().__init__(context, action_path, group_of_dates)
-        self.results: list[Result] = [r for r in results if not r.empty]
-
-    @cached_property
-    @assert_fieldlist
-    @notify_result
-    @trace_datasource
-    def datasource(self) -> FieldList:
-        """Returns the combined datasource from all results."""
-        ds: FieldList = EmptyResult(self.context, self.action_path, self.group_of_dates).datasource
-        for i in self.results:
-            ds += i.datasource
-        return _tidy(ds)
-
-    def __repr__(self) -> str:
-        """Returns a string representation of the JoinResult instance."""
-        content: str = "\n".join([str(i) for i in self.results])
-        return self._repr(content)
-
-
-class JoinAction(Action):
-    """Represents an action that combines multiple actions.
-
-    Attributes
-    ----------
-    context : object
-        The context object.
-    action_path : list
-        The action path.
-    actions : List[Action]
-        The list of actions.
-    """
-
-    def __init__(self, context: object, action_path: list, *configs: dict) -> None:
-        """Initializes a JoinAction instance.
-
-        Parameters
-        ----------
-        context : object
-            The context object.
-        action_path : list
-            The action path.
-        *configs : dict
-            The configuration dictionaries.
-        """
-        super().__init__(context, action_path, *configs)
-        self.actions: list[Action] = [action_factory(c, context, action_path + [str(i)]) for i, c in enumerate(configs)]
-
-    def __repr__(self) -> str:
-        """Returns a string representation of the JoinAction instance."""
-        content: str = "\n".join([str(i) for i in self.actions])
-        return self._repr(content)
-
-    @trace_select
-    def select(self, group_of_dates: GroupOfDates) -> JoinResult:
-        """Selects the results for the given group of dates.
-
-        Parameters
-        ----------
-        group_of_dates : GroupOfDates
-            The group of dates.
-
-        Returns
-        -------
-        JoinResult
-            The combined result for the given group of dates.
-        """
-        results: list[Result] = [a.select(group_of_dates) for a in self.actions]
-        return JoinResult(self.context, self.action_path, group_of_dates, results)

anemoi/datasets/create/input/pipe.py

@@ -1,66 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import json
-import logging
-from typing import Any
-
-from .action import Action
-from .action import action_factory
-from .step import step_factory
-from .trace import trace_select
-
-LOG = logging.getLogger(__name__)
-
-
-class PipeAction(Action):
-    """A class to represent a pipeline of actions."""
-
-    def __init__(self, context: Any, action_path: list, *configs: dict) -> None:
-        """Initialize the PipeAction.
-
-        Parameters
-        ----------
-        context : Any
-            The context for the action.
-        action_path : list
-            The path of the action.
-        configs : dict
-            The configurations for the actions.
-        """
-        super().__init__(context, action_path, *configs)
-        if len(configs) <= 1:
-            raise ValueError(
-                f"PipeAction requires at least two actions, got {len(configs)}\n{json.dumps(configs, indent=2)}"
-            )
-
-        current: Any = action_factory(configs[0], context, action_path + ["0"])
-        for i, c in enumerate(configs[1:]):
-            current = step_factory(c, context, action_path + [str(i + 1)], previous_step=current)
-        self.last_step: Any = current
-
-    @trace_select
-    def select(self, group_of_dates: Any) -> Any:
-        """Select data based on the group of dates.
-
-        Parameters
-        ----------
-        group_of_dates : Any
-            The group of dates to select data for.
-
-        Returns
-        -------
-        Any
-            The selected data.
-        """
-        return self.last_step.select(group_of_dates)
-
-    def __repr__(self) -> str:
-        """Return a string representation of the PipeAction."""
-        return f"PipeAction({self.last_step})"

anemoi/datasets/create/input/step.py

@@ -1,173 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-import logging
-from copy import deepcopy
-from typing import Any
-
-from .action import Action
-from .action import ActionContext
-from .context import Context
-from .result import Result
-from .template import notify_result
-from .trace import trace_datasource
-from .trace import trace_select
-
-LOG = logging.getLogger(__name__)
-
-
-class StepResult(Result):
-    """Represents the result of a step in the data processing pipeline."""
-
-    def __init__(
-        self, context: Context, action_path: list[str], group_of_dates: Any, action: Action, upstream_result: Result
-    ) -> None:
-        """Initialize a StepResult instance.
-
-        Parameters
-        ----------
-        context
-            The context in which the step is executed.
-        action_path
-            The path of actions leading to this step.
-        group_of_dates
-            The group of dates associated with this step.
-        action
-            The action associated with this step.
-        upstream_result
-            The result of the upstream step.
-        """
-        super().__init__(context, action_path, group_of_dates)
-        assert isinstance(upstream_result, Result), type(upstream_result)
-        self.upstream_result: Result = upstream_result
-        self.action: Action = action
-
-    @property
-    @notify_result
-    @trace_datasource
-    def datasource(self) -> Any:
-        """Retrieve the datasource associated with this step result."""
-        raise NotImplementedError(f"Not implemented in {self.__class__.__name__}")
-
-
-class StepAction(Action):
-    """Represents an action that is part of a step in the data processing pipeline."""
-
-    result_class: type[StepResult] | None = None
-
-    def __init__(
-        self, context: ActionContext, action_path: list[str], previous_step: Any, *args: Any, **kwargs: Any
-    ) -> None:
-        """Initialize a StepAction instance.
-
-        Parameters
-        ----------
-        context
-            The context in which the action is executed.
-        action_path
-            The path of actions leading to this step.
-        previous_step
-            The previous step in the pipeline.
-        """
-        super().__init__(context, action_path, *args, **kwargs)
-        self.previous_step: Any = previous_step
-
-    @trace_select
-    def select(self, group_of_dates: Any) -> StepResult:
-        """Select the result for a given group of dates.
-
-        Parameters
-        ----------
-        group_of_dates
-            The group of dates to select the result for.
-
-        Returns
-        -------
-        unknown
-            The result of the step.
-        """
-        return self.result_class(
-            self.context,
-            self.action_path,
-            group_of_dates,
-            self,
-            self.previous_step.select(group_of_dates),
-        )
-
-    def __repr__(self) -> str:
-        """Return a string representation of the StepAction instance.
-
-        Returns
-        -------
-        unknown
-            String representation of the instance.
-        """
-        return self._repr(self.previous_step, _inline_=str(self.kwargs))
-
-
-def step_factory(config: dict[str, Any], context: ActionContext, action_path: list[str], previous_step: Any) -> Any:
-    """Factory function to create a step action based on the given configuration.
-
-    Parameters
-    ----------
-    config
-        The configuration dictionary for the step.
-    context
-        The context in which the step is executed.
-    action_path
-        The path of actions leading to this step.
-    previous_step
-        The previous step in the pipeline.
-
-    Returns
-    -------
-    unknown
-        An instance of a step action.
-    """
-
-    from .filter import FilterStepAction
-    from .filter import FunctionStepAction
-
-    assert isinstance(context, Context), (type, context)
-    if not isinstance(config, dict):
-        raise ValueError(f"Invalid input config {config}")
-
-    config = deepcopy(config)
-    assert len(config) == 1, config
-
-    key = list(config.keys())[0]
-    cls = dict(
-        filter=FilterStepAction,
-        # rename=RenameAction,
-        # remapping=RemappingAction,
-    ).get(key)
-
-    if isinstance(config[key], list):
-        args, kwargs = config[key], {}
-
-    if isinstance(config[key], dict):
-        args, kwargs = [], config[key]
-
-    if isinstance(config[key], str):
-        args, kwargs = [config[key]], {}
-
-    if cls is not None:
-        return cls(context, action_path, previous_step, *args, **kwargs)
-
-    # Try filters from transform filter registry
-    from anemoi.transform.filters import filter_registry as transform_filter_registry
-
-    if transform_filter_registry.is_registered(key):
-        from ..filter import TransformFilter
-
-        return FunctionStepAction(
-            context, action_path + [key], previous_step, key, TransformFilter(context, key, config)
-        )
-
-    raise ValueError(f"Unknown step action `{key}`")

anemoi/datasets/create/input/template.py

@@ -1,161 +0,0 @@
-# (C) Copyright 2024 Anemoi contributors.
-#
-# This software is licensed under the terms of the Apache Licence Version 2.0
-# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
-#
-# In applying this licence, ECMWF does not waive the privileges and immunities
-# granted to it by virtue of its status as an intergovernmental organisation
-# nor does it submit to any jurisdiction.
-
-
-import logging
-import re
-from abc import ABC
-from abc import abstractmethod
-from collections.abc import Callable
-from functools import wraps
-from typing import Any
-
-from .context import Context
-
-LOG = logging.getLogger(__name__)
-
-
-def notify_result(method: Callable[..., Any]) -> Callable[..., Any]:
-    """Decorator to notify the context of the result of the method call.
-
-    Parameters
-    ----------
-    method : Callable[..., Any]
-        The method to wrap.
-
-    Returns
-    -------
-    Callable[..., Any]
-        The wrapped method.
-    """
-
-    @wraps(method)
-    def wrapper(self: Any, *args: Any, **kwargs: Any) -> Any:
-        result: Any = method(self, *args, **kwargs)
-        self.context.notify_result(self.action_path, result)
-        return result
-
-    return wrapper
-
-
-class Substitution(ABC):
-    """Abstract base class for substitutions in templates."""
-
-    @abstractmethod
-    def resolve(self, context: Context) -> Any:
-        """Resolve the substitution using the given context.
-
-        Parameters
-        ----------
-        context : Context
-            The context to use for resolution.
-
-        Returns
-        -------
-        Any
-            The resolved value.
-        """
-        pass
-
-
-class Reference(Substitution):
-    """A class to represent a reference to another value in the context."""
-
-    def __init__(self, context: Any, action_path: list[str]) -> None:
-        """Initialize a Reference instance.
-
-        Parameters
-        ----------
-        context : Any
-            The context in which the reference exists.
-        action_path : list of str
-            The action path to resolve.
-        """
-        self.context: Any = context
-        self.action_path: list[str] = action_path
-
-    def resolve(self, context: Context) -> Any:
-        """Resolve the reference using the given context.
-
-        Parameters
-        ----------
-        context : Context
-            The context to use for resolution.
-
-        Returns
-        -------
-        Any
-            The resolved value.
-        """
-        return context.get_result(self.action_path)
-
-
-def resolve(context: Context, x: Any) -> Any:
-    """Recursively resolve substitutions in the given structure using the context.
-
-    Parameters
-    ----------
-    context : Context
-        The context to use for resolution.
-    x : Union[tuple, list, dict, Substitution, Any]
-        The structure to resolve.
-
-    Returns
-    -------
-    Any
-        The resolved structure.
-    """
-    if isinstance(x, tuple):
-        return tuple([resolve(context, y) for y in x])
-
-    if isinstance(x, list):
-        return [resolve(context, y) for y in x]
-
-    if isinstance(x, dict):
-        return {k: resolve(context, v) for k, v in x.items()}
-
-    if isinstance(x, Substitution):
-        return x.resolve(context)
-
-    return x
-
-
-def substitute(context: Context, x: Any) -> Any:
-    """Recursively substitute references in the given structure using the context.
-
-    Parameters
-    ----------
-    context : Context
-        The context to use for substitution.
-    x : Union[tuple, list, dict, str, Any]
-        The structure to substitute.
-
-    Returns
-    -------
-    Any
-        The substituted structure.
-    """
-    if isinstance(x, tuple):
-        return tuple([substitute(context, y) for y in x])
-
-    if isinstance(x, list):
-        return [substitute(context, y) for y in x]
-
-    if isinstance(x, dict):
-        return {k: substitute(context, v) for k, v in x.items()}
-
-    if not isinstance(x, str):
-        return x
-
-    if re.match(r"^\${[\.\w\-]+}$", x):
-        path = x[2:-1].split(".")
-        context.will_need_reference(path)
-        return Reference(context, path)
-
-    return x