ewokscore 4.0.0__py3-none-any.whl

ewokscore/__init__.py

@@ -0,0 +1,3 @@
+from .bindings import *  # noqa: F403,F401
+from .task import Task  # noqa: F401
+from .taskwithprogress import TaskWithProgress  # noqa: F401

ewokscore/bindings.py

@@ -0,0 +1,98 @@
+from pathlib import Path
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+
+from .events import job_decorator as execute_graph_decorator
+from .graph import TaskGraph
+from .graph import load_graph as _load_graph
+from .graph.execute import sequential
+from .graph.graph_io import update_default_inputs
+from .graph.serialize import GraphRepresentation
+from .node import NodeIdType
+from .task import Task
+
+__all__ = [
+    "execute_graph",
+    "load_graph",
+    "save_graph",
+    "convert_graph",
+    "graph_is_supported",
+    "execute_graph_decorator",
+]
+
+
+def load_graph(
+    graph: Any,
+    inputs: Optional[List[dict]] = None,
+    representation: Optional[Union[GraphRepresentation, str]] = None,
+    root_dir: Optional[Union[str, Path]] = None,
+    root_module: Optional[str] = None,
+) -> TaskGraph:
+    taskgraph = _load_graph(
+        source=graph,
+        representation=representation,
+        root_dir=root_dir,
+        root_module=root_module,
+    )
+    if inputs:
+        update_default_inputs(taskgraph.graph, inputs)
+    return taskgraph
+
+
+def save_graph(
+    graph: TaskGraph,
+    destination,
+    representation: Optional[Union[GraphRepresentation, str]] = None,
+    **save_options,
+) -> Union[str, dict]:
+    return graph.dump(destination, representation=representation, **save_options)
+
+
+def convert_graph(
+    source,
+    destination,
+    inputs: Optional[List[dict]] = None,
+    load_options: Optional[dict] = None,
+    save_options: Optional[dict] = None,
+) -> Union[str, dict]:
+    if load_options is None:
+        load_options = dict()
+    if save_options is None:
+        save_options = dict()
+    graph = load_graph(source, inputs=inputs, **load_options)
+    return save_graph(graph, destination, **save_options)
+
+
+@execute_graph_decorator(engine="core")
+def execute_graph(
+    graph,
+    inputs: Optional[List[dict]] = None,
+    load_options: Optional[dict] = None,
+    varinfo: Optional[dict] = None,
+    execinfo: Optional[dict] = None,
+    task_options: Optional[dict] = None,
+    raise_on_error: Optional[bool] = True,
+    outputs: Optional[List[dict]] = None,
+    merge_outputs: Optional[bool] = True,
+    output_tasks: Optional[bool] = False,
+) -> Union[Dict[NodeIdType, Task], Dict[str, Any]]:
+    if load_options is None:
+        load_options = dict()
+    taskgraph = load_graph(graph, inputs=inputs, **load_options)
+    return sequential.execute_graph(
+        taskgraph.graph,
+        varinfo=varinfo,
+        execinfo=execinfo,
+        task_options=task_options,
+        raise_on_error=raise_on_error,
+        outputs=outputs,
+        merge_outputs=merge_outputs,
+        output_tasks=output_tasks,
+    )
+
+
+def graph_is_supported(graph: TaskGraph) -> bool:
+    return not graph.is_cyclic and not graph.has_conditional_links

ewokscore/dynamictask.py

@@ -0,0 +1,25 @@
+from typing import Callable
+from typing import Optional
+
+from ewoksutils.import_utils import import_method
+
+from .task import Task
+
+_GENERATORS = dict()
+
+
+def get_task_class_generator(generator_qualname: str) -> Callable[[str], Task]:
+    if not generator_qualname:
+        return Task.get_subclass
+    task_class_generator = _GENERATORS.get(generator_qualname, None)
+    if task_class_generator is not None:
+        return task_class_generator
+    task_class_generator = import_method(generator_qualname)
+    _GENERATORS[generator_qualname] = task_class_generator
+    return task_class_generator
+
+
+def get_dynamically_task_class(
+    generator_qualname: Optional[str], registry_name: str
+) -> Task:
+    return get_task_class_generator(generator_qualname)(registry_name)

ewokscore/engine.py

@@ -0,0 +1,92 @@
+from collections.abc import Mapping
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+
+from . import bindings
+from .engine_interface import Path
+from .engine_interface import RawExecInfoType
+from .engine_interface import TaskGraph
+from .engine_interface import WorkflowEngineWithSerialization
+from .node import NodeIdType
+from .task import Task
+
+
+class CoreWorkflowEngine(WorkflowEngineWithSerialization):
+
+    def execute_graph(
+        self,
+        graph: Any,
+        *,
+        inputs: Optional[List[dict]] = None,
+        load_options: Optional[dict] = None,
+        varinfo: Optional[dict] = None,
+        execinfo: RawExecInfoType = None,
+        task_options: Optional[dict] = None,
+        outputs: Optional[List[dict]] = None,
+        merge_outputs: Optional[bool] = True,
+        # Engine specific:
+        output_tasks: Optional[bool] = False,
+        raise_on_error: Optional[bool] = True,
+    ) -> Union[Dict[NodeIdType, Task], Dict[str, Any]]:
+        return bindings.execute_graph(
+            graph,
+            inputs=inputs,
+            load_options=load_options,
+            varinfo=varinfo,
+            execinfo=execinfo,
+            task_options=task_options,
+            raise_on_error=raise_on_error,
+            outputs=outputs,
+            merge_outputs=merge_outputs,
+            output_tasks=output_tasks,
+        )
+
+    def deserialize_graph(
+        self,
+        graph: Any,
+        *,
+        inputs: Optional[List[dict]] = None,
+        representation: Optional[str] = None,
+        root_dir: Optional[Union[str, Path]] = None,
+        root_module: Optional[str] = None,
+        # Serializer specific:
+        **load_options,
+    ) -> TaskGraph:
+        return bindings.load_graph(
+            graph,
+            inputs=inputs,
+            representation=representation,
+            root_dir=root_dir,
+            root_module=root_module,
+            **load_options,
+        )
+
+    def serialize_graph(
+        self,
+        graph: TaskGraph,
+        destination,
+        *,
+        representation: Optional[str] = None,
+        # Serializer specific:
+        **save_options,
+    ) -> Union[str, dict]:
+        return bindings.save_graph(
+            graph, destination, representation=representation, **save_options
+        )
+
+    def get_graph_representation(self, graph: Any) -> Optional[str]:
+        if isinstance(graph, Mapping):
+            return "json_dict"
+        if isinstance(graph, (str, Path)):
+            if graph == "test_core":
+                return "test_core"
+            if isinstance(graph, str) and "{" in graph and "}" in graph:
+                return "json_string"
+            filename = str(graph).lower()
+            if filename.endswith(".json"):
+                return "json"
+            elif filename.endswith((".yml", ".yaml")):
+                return "yaml"

ewokscore/engine_interface.py

@@ -0,0 +1,79 @@
+from abc import ABC
+from abc import abstractmethod
+from pathlib import Path
+from typing import Any
+from typing import List
+from typing import Optional
+from typing import Union
+
+from .events.contexts import RawExecInfoType
+from .graph import TaskGraph
+
+
+class WorkflowEngine(ABC):
+    """Python projects that provide Ewoks engines for deserializing, serializing and executing
+    computational Ewoks graphs can implement this interface.
+
+    To make it discoverable it can be added as an entry-point the the project. For
+    example in a `pyproject.toml` file:
+
+    .. code-block: toml
+
+        [project.entry-points."ewoks.engines"]
+        "<engine-name>" = "<project-name>.engine:MyWorkflowEngine"
+    """
+
+    @abstractmethod
+    def execute_graph(
+        self,
+        graph: TaskGraph,
+        *,
+        inputs: Optional[List[dict]] = None,
+        load_options: Optional[dict] = None,
+        varinfo: Optional[dict] = None,
+        execinfo: Optional[RawExecInfoType] = None,
+        task_options: Optional[dict] = None,
+        outputs: Optional[List[dict]] = None,
+        merge_outputs: Optional[bool] = True,
+        # Engine specific:
+        **execute_options,
+    ) -> Optional[dict]:
+        """Execute a computional Ewoks graph."""
+        pass
+
+
+class WorkflowEngineWithSerialization(WorkflowEngine):
+    """Ewoks engines with graph serialization capabilities."""
+
+    @abstractmethod
+    def deserialize_graph(
+        self,
+        graph: Any,
+        *,
+        inputs: Optional[List[dict]] = None,
+        representation: Optional[str] = None,
+        root_dir: Optional[Union[str, Path]] = None,
+        root_module: Optional[str] = None,
+        # Serializer specific:
+        **deserialize_options,
+    ) -> TaskGraph:
+        """Convert a computational graph representation to the canonical in-memory representation `TaskGraph`."""
+        pass
+
+    @abstractmethod
+    def serialize_graph(
+        self,
+        graph: TaskGraph,
+        destination: Any,
+        *,
+        representation: Optional[str] = None,
+        # Serializer specific:
+        **serialize_options,
+    ) -> Any:
+        """Convert the canonical computational graph representation `TaskGraph` to another representation."""
+        pass
+
+    @abstractmethod
+    def get_graph_representation(self, graph: Any) -> Optional[str]:
+        """Return the representation if the engine recognizes the graph object."""
+        pass

ewokscore/entry_points.py

@@ -0,0 +1,24 @@
+import sys
+from typing import Tuple
+
+if sys.version_info < (3, 9):
+    from importlib.metadata import EntryPoint
+
+    from importlib_metadata import entry_points as _entry_points
+
+    def entry_points(group: str) -> Tuple[EntryPoint]:
+        return tuple(_entry_points(group=group))
+
+elif sys.version_info < (3, 10):
+    from importlib.metadata import EntryPoint
+    from importlib.metadata import entry_points as _entry_points
+
+    def entry_points(group: str) -> Tuple[EntryPoint]:
+        return _entry_points().get(group, ())
+
+else:
+    from importlib.metadata import EntryPoint
+    from importlib.metadata import entry_points as _entry_points
+
+    def entry_points(group: str) -> Tuple[EntryPoint]:
+        return tuple(_entry_points(group=group))

ewokscore/events/__init__.py

@@ -0,0 +1,12 @@
+"""Ewoks sends job, workflow and node events during workflow execution."""
+
+from .contexts import job_context  # noqa F401
+from .contexts import job_decorator  # noqa F401
+from .contexts import node_context  # noqa F401
+from .contexts import workflow_context  # noqa F401
+from .global_state import add_handler  # noqa F401
+from .global_state import cleanup  # noqa F401
+from .global_state import remove_handler  # noqa F401
+from .send_events import send_job_event  # noqa F401
+from .send_events import send_task_event  # noqa F401
+from .send_events import send_workflow_event  # noqa F401

ewokscore/events/contexts.py

@@ -0,0 +1,97 @@
+"""Contexts for job, workflow or node events. Allows for initializing
+event fields and sending start/end events.
+"""
+
+from contextlib import contextmanager
+from functools import wraps
+from typing import Mapping
+from typing import Union
+
+from . import global_state
+from .initialize_events import init_job
+from .initialize_events import init_node
+from .initialize_events import init_workflow
+from .send_events import ExecInfoType
+from .send_events import send_job_event
+from .send_events import send_workflow_event
+
+RawExecInfoType = Union[Mapping, bool, str, None]
+
+
+def job_decorator(**static_job_info):
+    def _job_decorator(method):
+        @wraps(method)
+        def wrapper(*args, execinfo: RawExecInfoType = None, **kw):
+            with job_context(execinfo, **static_job_info) as execinfo:
+                return method(*args, execinfo=execinfo, **kw)
+
+        return wrapper
+
+    return _job_decorator
+
+
+@contextmanager
+def job_context(execinfo: RawExecInfoType, **static_job_info) -> ExecInfoType:
+    if execinfo is None:
+        execinfo = global_state.ENABLE_EWOKS_EVENTS_BY_DEFAULT
+
+    if isinstance(execinfo, str):
+        execinfo = {"job_id": execinfo}
+    elif isinstance(execinfo, bool):
+        if execinfo:
+            execinfo = dict()
+        else:
+            execinfo = None
+    elif execinfo is None:
+        pass
+    elif not isinstance(execinfo, Mapping):
+        raise TypeError
+
+    execinfo = init_job(execinfo, **static_job_info)
+    if execinfo is None:
+        yield None
+    else:
+        with _context(execinfo, "job", send_job_event, execinfo["job_id"]) as execinfo:
+            yield execinfo
+
+
+@contextmanager
+def workflow_context(execinfo: ExecInfoType, **kw) -> ExecInfoType:
+    execinfo = init_workflow(execinfo, **kw)
+    if execinfo is None:
+        yield None
+    else:
+        with _context(
+            execinfo, "workflow", send_workflow_event, execinfo["workflow_id"]
+        ) as execinfo:
+            yield execinfo
+
+
+@contextmanager
+def node_context(execinfo: ExecInfoType, **kw) -> ExecInfoType:
+    yield init_node(execinfo, **kw)
+
+
+@contextmanager
+def _context(
+    execinfo: ExecInfoType, context, send_context_event, obj_id
+) -> ExecInfoType:
+    contexts = execinfo.get("contexts")
+    if contexts is None:
+        contexts = {"job": list(), "workflow": list()}
+        execinfo["contexts"] = contexts
+    obj_ids = contexts[context]
+    first_context = obj_id not in obj_ids
+    if first_context:
+        send_context_event(execinfo=execinfo, event="start")
+        obj_ids.append(obj_id)
+    try:
+        yield execinfo
+    except BaseException as e:
+        if not execinfo.get("exception"):
+            execinfo["exception"] = e
+        raise
+    finally:
+        if first_context:
+            obj_ids.remove(obj_id)
+            send_context_event(execinfo=execinfo, event="end")

ewokscore/events/global_state.py

@@ -0,0 +1,181 @@
+"""Manage the EWOKS event logger which is a global object"""
+
+import logging
+import os
+from contextlib import contextmanager
+from typing import Any
+from typing import Dict
+from typing import Iterator
+from typing import List
+from typing import Optional
+from typing import Tuple
+
+from ewoksutils.logging_utils.asyncwrapper import AsyncHandlerWrapper
+from ewoksutils.logging_utils.cleanup import cleanup_handler
+from ewoksutils.logging_utils.cleanup import cleanup_logger
+from ewoksutils.logging_utils.cleanup import protect_logging_state
+
+from .handlers import instantiate_handler
+from .handlers import is_ewoks_event_handler
+
+_app_logger = logging.getLogger(__name__)
+EWOKS_EVENT_LOGGER_NAME = f"__{__name__}__"
+ENABLE_EWOKS_EVENTS_BY_DEFAULT = True
+
+
+def send(
+    *args,
+    handlers: Optional[List[Dict[str, Any]]] = None,
+    asynchronous: Optional[bool] = None,
+    **kw,
+) -> None:
+    """Log an EWOKS event with the EWOKS event handlers and the application handlers."""
+    with _ewoks_event_logger(
+        handlers=handlers, asynchronous=asynchronous, cleanup_on_different_handlers=True
+    ) as logger:
+        # Send to the EWOKS event handlers
+        logger.info(*args, **kw)
+        # Send to the application log handlers
+        _app_logger.info(*args, **kw)
+
+
+def add_handler(
+    handler: logging.Handler,
+    asynchronous: Optional[bool] = None,
+) -> None:
+    """Add a handler to the global EWOKS event logger."""
+    with _ewoks_event_logger() as logger:
+        if _has_handler_instance(logger, handler):
+            return
+        if asynchronous is None and is_ewoks_event_handler(handler):
+            asynchronous = handler.BLOCKING
+        if asynchronous:
+            handler = AsyncHandlerWrapper(handler)
+        logger.addHandler(handler)
+
+
+def remove_handler(handler: logging.Handler) -> None:
+    """Remove a handler from all loggers that receive EWOKS events."""
+    with _ewoks_event_logger() as logger:
+        for linstance, hinstance in _iter_handler_owners(logger, handler):
+            linstance.removeHandler(hinstance)
+            cleanup_handler(hinstance)
+
+
+def cleanup():
+    """Pending events will be dropped"""
+    with protect_logging_state():
+        _cleanup_ewoks_event_logger()
+
+
+def _after_fork_in_child():
+    cleanup()
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(after_in_child=_after_fork_in_child)
+
+
+@contextmanager
+def _ewoks_event_logger(
+    handlers: Optional[List[Dict[str, Any]]] = None,
+    asynchronous: Optional[bool] = None,
+    cleanup_on_different_handlers: bool = False,
+) -> Iterator[logging.Logger]:
+    """Initialize and yield the EWOKS event logger"""
+    # Issue with logging and forking:
+    # https://pythonspeed.com/articles/python-multiprocessing/
+
+    with protect_logging_state():
+        if _ewoks_event_logger_requires_cleanup(
+            handlers=handlers,
+            cleanup_on_different_handlers=cleanup_on_different_handlers,
+        ):
+            _cleanup_ewoks_event_logger()
+        if _ewoks_event_logger_requires_init():
+            _init_ewoks_event_logger(handlers, asynchronous)
+        yield logging.getLogger(EWOKS_EVENT_LOGGER_NAME)
+
+
+def _cleanup_ewoks_event_logger():
+    """Cleanup and delete the global EWOKS event logger"""
+    cleanup_logger(EWOKS_EVENT_LOGGER_NAME)
+
+
+def _ewoks_event_logger_requires_init() -> bool:
+    logger = logging.getLogger(EWOKS_EVENT_LOGGER_NAME)
+    return not hasattr(logger, "ewoks_pid")
+
+
+def _ewoks_event_logger_requires_cleanup(
+    handlers: Optional[List[Dict[str, Any]]] = None,
+    cleanup_on_different_handlers: bool = False,
+) -> bool:
+    logger = logging.getLogger(EWOKS_EVENT_LOGGER_NAME)
+
+    ewoks_pid = getattr(logger, "ewoks_pid", None)
+    if ewoks_pid is not None and ewoks_pid != os.getpid():
+        # Process forked
+        return True
+
+    if cleanup_on_different_handlers:
+        ewoks_handlers = getattr(logger, "ewoks_handlers", None)
+        if ewoks_handlers != handlers:
+            return True
+
+    return False
+
+
+def _init_ewoks_event_logger(
+    handlers: Optional[List[Dict[str, Any]]], asynchronous: Optional[bool]
+):
+    logger = logging.getLogger(EWOKS_EVENT_LOGGER_NAME)
+    logger.setLevel(logging.DEBUG)
+    logger.ewoks_pid = os.getpid()
+    logger.ewoks_handlers = handlers
+    logger.propagate = False
+    if not handlers:
+        return
+    for desc in handlers:
+        try:
+            kwargs = {
+                arg["name"]: arg["value"] for arg in desc.get("arguments", list())
+            }
+            handler = instantiate_handler(desc["class"], **kwargs)
+        except Exception as e:
+            raise RuntimeError(
+                "cannot create an EWOKS event handler from the description"
+            ) from e
+        asynchronous_handler = desc.get("asynchronous", asynchronous)
+        add_handler(handler, asynchronous=asynchronous_handler)
+
+
+def _iter_loggers(logger: logging.Logger) -> Iterator[logging.Logger]:
+    """Yield all loggers which will receive EWOKS events."""
+    _logger = logger
+    while _logger is not None:
+        yield _logger
+        if not _logger.propagate:
+            return
+        _logger = _logger.parent
+
+
+def _iter_handler_owners(
+    logger: logging.Logger, instance: logging.Handler
+) -> Iterator[Tuple[logging.Logger, logging.Handler]]:
+    """Yield all loggers which have a specific handler (or a handler that wraps the specific event handler)."""
+    for _logger in _iter_loggers(logger):
+        for handler in _logger.handlers:
+            if handler is instance:
+                yield _logger, instance
+            elif isinstance(handler, AsyncHandlerWrapper):
+                instance2 = handler.wrapped_handler
+                if instance2 is instance:
+                    yield _logger, instance2
+
+
+def _has_handler_instance(logger: logging.Logger, instance: logging.Handler) -> bool:
+    """Is this handler registered with a logger or it's parents?"""
+    for _ in _iter_handler_owners(logger, instance):
+        return True
+    return False

ewokscore/events/handlers/__init__.py

@@ -0,0 +1,7 @@
+# fmt: off
+from ewoksutils.import_utils import instantiate_class as instantiate_handler  # noqa F401
+
+from .base import *  # noqa F401
+from .sqlite3 import Sqlite3EwoksEventHandler  # noqa F401
+
+# fmt: on

ewokscore/events/handlers/base.py

@@ -0,0 +1,20 @@
+import logging
+
+from ewoksutils.event_utils import FIELD_TYPES
+
+__all__ = ["is_ewoks_event_handler", "EwoksEventHandlerMixIn", "EwoksEventHandler"]
+
+
+def is_ewoks_event_handler(handler):
+    return isinstance(handler, EwoksEventHandlerMixIn)
+
+
+class EwoksEventHandlerMixIn:
+    BLOCKING = False
+    FIELD_TYPES = FIELD_TYPES
+
+
+class EwoksEventHandler(EwoksEventHandlerMixIn, logging.Handler):
+    """Base class for handling ewoks events on the publishing side (implement the `emit` method)."""
+
+    pass

ewokscore/events/handlers/sqlite3.py

@@ -0,0 +1,8 @@
+from ewoksutils.logging_utils.sqlite3 import Sqlite3Handler
+
+from .base import EwoksEventHandlerMixIn
+
+
+class Sqlite3EwoksEventHandler(EwoksEventHandlerMixIn, Sqlite3Handler):
+    def __init__(self, uri: str):
+        super().__init__(uri=uri, table="ewoks_events", field_types=self.FIELD_TYPES)