pyworkflowy 0.1.0__py3-none-any.whl

pyworkflowy/__init__.py

@@ -0,0 +1,56 @@
+"""pyWorkflowy — a full workflow engine for async/parallelized Python tasks."""
+
+from __future__ import annotations
+
+from pyworkflowy._classbased import TaskBase
+from pyworkflowy._core import (
+    Backend,
+    Backoff,
+    DepFailurePolicy,
+    Task,
+    TaskContext,
+    TaskHandle,
+    TaskResult,
+    TaskStatus,
+    current_task,
+    task,
+)
+from pyworkflowy._persistence import Checkpointer, JSONCheckpointer, PickleCheckpointer
+from pyworkflowy._runner import TaskRunner, get_current_runner
+from pyworkflowy.exceptions import (
+    CheckpointError,
+    CycleError,
+    DependencyFailedError,
+    RetryExhaustedError,
+    TaskCancelledError,
+    TaskError,
+    TaskTimeoutError,
+)
+
+__all__ = [
+    "Backend",
+    "Backoff",
+    "CheckpointError",
+    "Checkpointer",
+    "CycleError",
+    "DepFailurePolicy",
+    "DependencyFailedError",
+    "JSONCheckpointer",
+    "PickleCheckpointer",
+    "RetryExhaustedError",
+    "Task",
+    "TaskBase",
+    "TaskCancelledError",
+    "TaskContext",
+    "TaskError",
+    "TaskHandle",
+    "TaskResult",
+    "TaskRunner",
+    "TaskStatus",
+    "TaskTimeoutError",
+    "current_task",
+    "get_current_runner",
+    "task",
+]
+
+__version__ = "0.1.0"

pyworkflowy/_backends.py

@@ -0,0 +1,204 @@
+"""Execution backends: asyncio, thread pool, process pool.
+
+Each backend implements :meth:`Backend.execute` which takes a task body and
+returns its value, honoring the runner-level concerns the caller has already
+arranged (cancellation flag, contextvars). Retries and timeouts are layered
+*on top* of these backends by :mod:`pyworkflowy._runner` — backends only run a
+single attempt.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+from collections.abc import Awaitable, Callable
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+from inspect import iscoroutinefunction
+from typing import TYPE_CHECKING, Any
+
+from pyworkflowy.exceptions import TaskCancelledError, TaskTimeoutError
+
+if TYPE_CHECKING:
+    from pyworkflowy._core import TaskContext
+
+__all__ = [
+    "BackendExecutor",
+    "ProcessBackend",
+    "ThreadBackend",
+    "asyncio_execute",
+    "build_backend",
+]
+
+
+# ---------- thread / process backend ----------
+
+
+class BackendExecutor:
+    """Protocol-shaped base for the thread/process backends.
+
+    Both real backends own a ``concurrent.futures`` executor and translate
+    submit/cancel into futures calls. The asyncio backend is *not* a
+    BackendExecutor — it runs cooperatively inside the runner's loop and
+    needs no pool.
+    """
+
+    name: str
+    _executor: ThreadPoolExecutor | ProcessPoolExecutor
+
+    def execute(
+        self,
+        fn: Callable[..., Any],
+        args: tuple[Any, ...],
+        kwargs: dict[str, Any],
+        *,
+        timeout: float | None,
+        cancel_event: threading.Event,
+        task_name: str,
+    ) -> Any:
+        raise NotImplementedError
+
+    def shutdown(self, *, wait: bool = True) -> None:
+        self._executor.shutdown(wait=wait)
+
+
+class ThreadBackend(BackendExecutor):
+    """Runs each attempt on a :class:`concurrent.futures.ThreadPoolExecutor`.
+
+    Cancellation is cooperative — the task body must check
+    ``current_task().cancel_event`` (or accept the timeout) to actually
+    stop. Threads cannot be force-killed, so a runaway task will hang
+    shutdown if it ignores cooperative signals.
+    """
+
+    name = "thread"
+
+    def __init__(self, max_workers: int) -> None:
+        self._executor = ThreadPoolExecutor(
+            max_workers=max_workers, thread_name_prefix="pyworkflowy"
+        )
+
+    def execute(
+        self,
+        fn: Callable[..., Any],
+        args: tuple[Any, ...],
+        kwargs: dict[str, Any],
+        *,
+        timeout: float | None,
+        cancel_event: threading.Event,
+        task_name: str,
+    ) -> Any:
+        future = self._executor.submit(fn, *args, **kwargs)
+        try:
+            return future.result(timeout=timeout)
+        except TimeoutError as exc:
+            cancel_event.set()
+            raise TaskTimeoutError(
+                f"Task {task_name!r} exceeded its timeout of {timeout}s on the thread backend"
+            ) from exc
+
+
+class ProcessBackend(BackendExecutor):
+    """Runs each attempt on a :class:`concurrent.futures.ProcessPoolExecutor`.
+
+    Task functions must be importable (top-level or class-level — not nested
+    closures or lambdas), because the multiprocessing pickler serializes the
+    function reference to ship it to the worker. On timeout, the runner
+    cancels the future and the worker is left to finish or be reaped; a
+    *force* terminate is intentionally avoided to keep pool stability.
+    """
+
+    name = "process"
+
+    def __init__(self, max_workers: int) -> None:
+        self._executor = ProcessPoolExecutor(max_workers=max_workers)
+
+    def execute(
+        self,
+        fn: Callable[..., Any],
+        args: tuple[Any, ...],
+        kwargs: dict[str, Any],
+        *,
+        timeout: float | None,
+        cancel_event: threading.Event,
+        task_name: str,
+    ) -> Any:
+        future = self._executor.submit(fn, *args, **kwargs)
+        try:
+            return future.result(timeout=timeout)
+        except TimeoutError as exc:
+            future.cancel()
+            raise TaskTimeoutError(
+                f"Task {task_name!r} exceeded its timeout of {timeout}s on the process backend"
+            ) from exc
+
+
+def build_backend(name: str, max_workers: int) -> BackendExecutor:
+    """Build a :class:`BackendExecutor` for ``name``.
+
+    The asyncio backend is special-cased upstream and does not produce a
+    :class:`BackendExecutor` — only ``thread`` and ``process`` return one.
+    """
+    if name == "thread":
+        return ThreadBackend(max_workers)
+    if name == "process":
+        return ProcessBackend(max_workers)
+    raise ValueError(f"Unknown backend {name!r}; expected 'thread' or 'process'.")
+
+
+# ---------- asyncio backend ----------
+
+
+async def asyncio_execute(
+    fn: Callable[..., Any],
+    args: tuple[Any, ...],
+    kwargs: dict[str, Any],
+    *,
+    timeout: float | None,
+    cancel_event: threading.Event,
+    task_name: str,
+    ctx: TaskContext,
+    setup: Callable[[], Any],
+    teardown: Callable[[Any], None],
+) -> Any:
+    """Run one attempt on the asyncio event loop.
+
+    Supports both sync and async ``fn``. Sync ``fn`` runs inline on the loop
+    (we deliberately don't shove it onto a thread — if the user wanted that,
+    they'd pick the ``thread`` backend). Cancellation works cooperatively
+    via asyncio's normal task cancellation machinery; the runner also sets
+    ``cancel_event`` for parity with the other backends.
+    """
+    token = setup()
+    try:
+        if iscoroutinefunction(fn):
+            coro: Awaitable[Any] = fn(*args, **kwargs)
+            try:
+                if timeout is not None:
+                    return await asyncio.wait_for(coro, timeout=timeout)
+                return await coro
+            except TimeoutError as exc:
+                cancel_event.set()
+                raise TaskTimeoutError(
+                    f"Task {task_name!r} exceeded its timeout of {timeout}s on the asyncio backend"
+                ) from exc
+            except asyncio.CancelledError as exc:
+                cancel_event.set()
+                raise TaskCancelledError(f"Task {task_name!r} was cancelled") from exc
+        # Sync fn on asyncio: run inline. Timeout is checked *after*; for
+        # CPU-bound sync code, prefer the thread backend.
+        if timeout is not None:
+            loop = asyncio.get_running_loop()
+            start = loop.time()
+            result = fn(*args, **kwargs)
+            elapsed = loop.time() - start
+            if elapsed > timeout:
+                raise TaskTimeoutError(
+                    f"Task {task_name!r} took {elapsed:.3f}s, exceeding timeout of {timeout}s "
+                    "(sync function on asyncio backend; consider backend='thread' for "
+                    "true timeout cancellation)",
+                    elapsed=elapsed,
+                )
+            return result
+        return fn(*args, **kwargs)
+    finally:
+        teardown(token)

pyworkflowy/_classbased.py

@@ -0,0 +1,108 @@
+"""Class-based task definition.
+
+Subclass :class:`TaskBase`, override ``run``, set class-level config attributes,
+and instantiate to get a :class:`Task`. The :class:`Task` instance forwards
+``.submit()`` and ``.__call__()`` to the wrapped ``run`` method — so once
+constructed, a class-based task is indistinguishable from one built by
+:func:`@task`.
+
+Class-based form is purely a convenience for code that prefers config-as-class
+over config-as-decorator-kwargs; the underlying engine sees a normal
+:class:`Task`.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from pyworkflowy._core import (
+    Backend,
+    Backoff,
+    DepFailurePolicy,
+    Task,
+    _build_task,
+)
+
+__all__ = ["TaskBase"]
+
+
+_SENTINEL = object()
+
+
+class TaskBase:
+    """Base class for class-based task definitions.
+
+    Subclass and override :meth:`run` (sync or async). Class-level attributes
+    (``name``, ``backend``, ``retries``, ``timeout``, ``backoff``,
+    ``backoff_base``, ``backoff_max``, ``retry_on``, ``on_dep_failure``)
+    configure the task. Instantiating returns a fully configured
+    :class:`pyworkflowy.Task` — the instance is *the* task, not a wrapper around
+    one::
+
+        class FetchUser(TaskBase):
+            name = "fetch-user"
+            backend = "thread"
+            retries = 2
+
+            def run(self, user_id: int) -> dict[str, Any]:
+                ...
+
+        fetch_user = FetchUser()
+        handle = fetch_user.submit(42)
+    """
+
+    name: str | None = None
+    backend: Backend = "asyncio"
+    retries: int = 0
+    timeout: float | None = None
+    backoff: Backoff = "exponential"
+    backoff_base: float = 1.0
+    backoff_max: float = 30.0
+    retry_on: type[BaseException] | tuple[type[BaseException], ...] | None = None
+    on_dep_failure: DepFailurePolicy = "fail"
+    max_attempts: int | None = None
+
+    def run(self, *args: Any, **kwargs: Any) -> Any:
+        """Override me. Sync or async ``def`` both fine."""
+        raise NotImplementedError(
+            f"{type(self).__name__}.run is not implemented — override it in your subclass."
+        )
+
+    def __new__(cls, *args: Any, **kwargs: Any) -> Task[Any]:  # type: ignore[misc]
+        if cls is TaskBase:
+            raise TypeError(
+                "TaskBase is abstract — subclass it and override `run` instead of "
+                "instantiating it directly."
+            )
+        if args or kwargs:
+            raise TypeError(
+                f"{cls.__name__} takes no constructor arguments — pass values to "
+                "`.submit(*args, **kwargs)` instead. Class-level attributes "
+                "configure the task itself; runtime args go to `run`."
+            )
+        instance = object.__new__(cls)
+        # Bind `run` so the resulting Task.fn behaves like a plain function.
+        run_method = instance.run
+        retries = cls.retries
+        if cls.max_attempts is not None:
+            if retries:
+                raise ValueError(
+                    f"{cls.__name__}: set either `retries` or `max_attempts`, not both"
+                )
+            if cls.max_attempts < 1:
+                raise ValueError(
+                    f"{cls.__name__}.max_attempts must be >= 1, got {cls.max_attempts}"
+                )
+            retries = cls.max_attempts - 1
+        return _build_task(
+            run_method,
+            name=cls.name or f"{cls.__module__}.{cls.__qualname__}",
+            backend=cls.backend,
+            retries=retries,
+            timeout=cls.timeout,
+            backoff=cls.backoff,
+            backoff_base=cls.backoff_base,
+            backoff_max=cls.backoff_max,
+            retry_on=cls.retry_on,
+            on_dep_failure=cls.on_dep_failure,
+        )