taskferry 0.2.0__py3-none-any.whl

taskferry/backends/__init__.py

@@ -0,0 +1,38 @@
+"""Backends that ship with Taskferry and need nothing installed.
+
+```mermaid
+flowchart TD
+    RT["Taskferry runtime"]
+
+    RT -->|"inline"| I["InlineExecutionBackend<br/>this thread, right now"]
+    RT -->|"task"| T["ThreadTaskBackend<br/>ThreadPoolExecutor"]
+    RT -->|"task"| P["ProcessTaskBackend<br/>ProcessPoolExecutor"]
+    RT -->|"job"| S["SubprocessJobBackend<br/>child processes"]
+```
+
+These exist so that Taskferry is useful with zero infrastructure — in a test, a
+notebook, a CLI, a script, a small application — and so that every port has at
+least one honest reference implementation to test adapters against.
+
+They are **not** a queue. There is no durability, no cross-process visibility and
+no delivery guarantee beyond "this process tried". A thread pool that dies takes
+its pending work with it. That is stated plainly rather than papered over, and it
+is exactly why the Procrastinate and Cloud Run adapters exist.
+
+Each module imports lazily from this package, so ``import taskferry`` does not
+create a process pool or touch the filesystem.
+"""
+
+from __future__ import annotations
+
+from .inline import InlineExecutionBackend
+from .process import ProcessTaskBackend
+from .subprocess import SubprocessJobBackend
+from .thread import ThreadTaskBackend
+
+__all__ = [
+    "InlineExecutionBackend",
+    "ProcessTaskBackend",
+    "SubprocessJobBackend",
+    "ThreadTaskBackend",
+]

taskferry/backends/inline.py

@@ -0,0 +1,235 @@
+"""Inline execution — run it here, run it now.
+
+The simplest backend and the one that makes Taskferry usable before any
+infrastructure exists. ``submit`` runs the callable synchronously and returns an
+already-terminal execution:
+
+```mermaid
+sequenceDiagram
+    participant App
+    participant TP as Taskferry
+    participant IB as InlineExecutionBackend
+    participant F as your function
+
+    App->>TP: inline.submit(add, 20, 22)
+    TP->>IB: submit(InlineSpec)
+    IB->>F: add(20, 22)
+    F-->>IB: 42
+    IB-->>TP: Execution(SUCCEEDED, result=42)
+    TP-->>App: ExecutionHandle (already done)
+```
+
+What it honestly does **not** do
+--------------------------------
+
+* **No timeout.** Nothing can preempt a synchronous Python call in the calling
+  thread. ``TIMEOUT`` is not advertised, so a spec asking for one is rejected
+  rather than silently ignored.
+* **No cancel.** By the time ``submit`` returns, the work is over.
+* **No delay.** Sleeping the caller's thread to honour ``delay`` would be a
+  denial of service dressed up as a feature.
+
+Retries *are* real here: this backend is the engine, so it owns the retry and
+applies the :class:`~taskferry.retry.RetryPolicy` itself, backoff included.
+
+Async callables
+---------------
+
+An ``async def`` is awaited with :func:`asyncio.run`. If a loop is already
+running in this thread, the coroutine is run on a private loop in a worker
+thread — blocking the caller either way, because ``submit`` is synchronous by
+contract. Inside async code, use ``await runtime.inline.submit_async(...)``
+instead of blocking the loop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import concurrent.futures
+import contextlib
+import time
+from collections.abc import Coroutine
+from datetime import UTC, datetime
+from typing import Any
+
+from ..capabilities import Capability, CapabilitySet
+from ..core.correlation import use_correlation
+from ..execution import (
+    Execution,
+    ExecutionId,
+    ExecutionKind,
+    ExecutionResult,
+    ExecutionState,
+    new_execution_id,
+)
+from ..functions import is_async_callable
+from ..ports import BaseBackend
+from ..retry import RetryPolicy
+from ..specs import ExecutionSpec, InlineSpec
+
+INLINE_CAPABILITIES = frozenset(
+    {
+        Capability.SUBMIT,
+        Capability.STATE,
+        Capability.RESULT,
+        Capability.RETRY,
+        Capability.ASYNC_CALLABLE,
+    }
+)
+
+
+class InlineExecutionBackend(BaseBackend):
+    """Runs :class:`~taskferry.specs.InlineSpec` callables in the calling thread.
+
+    Finished executions are kept in a bounded ring so ``get()``/``result()`` work
+    for a while after submission. The bound matters: a long-running process that
+    submits inline work in a loop must not accumulate results forever. Once an id
+    ages out, ``get()`` raises :class:`~taskferry.errors.ExecutionNotFound` — the
+    handle returned by ``submit`` still carries the terminal snapshot and the
+    result, so the common path never depends on the cache.
+    """
+
+    def __init__(self, *, history: int = 1024) -> None:
+        if history < 0:
+            raise ValueError("history must be >= 0")
+        self._history = history
+        self._executions: dict[str, Execution] = {}
+        self._results: dict[str, ExecutionResult] = {}
+        self._order: list[str] = []
+
+    @property
+    def name(self) -> str:
+        return "inline"
+
+    @property
+    def kind(self) -> ExecutionKind:
+        return ExecutionKind.INLINE
+
+    @property
+    def capabilities(self) -> CapabilitySet:
+        return CapabilitySet(INLINE_CAPABILITIES, provider=self.name)
+
+    # -- submission ---------------------------------------------------------- #
+    def _submit(self, spec: ExecutionSpec) -> Execution:
+        assert isinstance(spec, InlineSpec)  # guaranteed by BaseBackend.validate
+        execution_id = new_execution_id(ExecutionKind.INLINE)
+        started = datetime.now(UTC)
+        base = Execution(
+            id=execution_id,
+            kind=ExecutionKind.INLINE,
+            backend=self.name,
+            state=ExecutionState.RUNNING,
+            name=spec.name,
+            created_at=started,
+            started_at=started,
+            correlation=spec.correlation,
+        )
+        self.hooks.before_execute(base)
+
+        value, error, attempts = self._call_with_retries(spec, base)
+        finished = datetime.now(UTC)
+
+        if error is None:
+            result = ExecutionResult(value=value)
+            state = ExecutionState.SUCCEEDED
+        else:
+            result = ExecutionResult.from_exception(error)
+            state = ExecutionState.FAILED
+
+        execution = base.evolve(
+            state=state,
+            finished_at=finished,
+            attempt=attempts,
+            result=result,
+        )
+        self.hooks.after_execute(execution, result)
+        if error is None:
+            self.hooks.on_success(execution, result)
+        else:
+            self.hooks.on_failure(execution, error)
+        self._remember(execution, result)
+        return execution
+
+    def _call_with_retries(
+        self, spec: InlineSpec, execution: Execution
+    ) -> tuple[Any, BaseException | None, int]:
+        """Run the callable, applying the retry policy. This backend owns retries."""
+        policy = spec.retry
+        attempt = 1
+        bind = (
+            use_correlation(spec.correlation)
+            if spec.correlation is not None
+            else contextlib.nullcontext()
+        )
+        while True:
+            try:
+                with bind:
+                    return self._call(spec), None, attempt
+            except Exception as exc:
+                if not policy.should_retry(exc, attempt):
+                    return None, exc, attempt
+                self.hooks.on_retry(execution, attempt + 1, exc)
+                _sleep(policy, attempt + 1)
+                attempt += 1
+
+    def _call(self, spec: InlineSpec) -> Any:
+        if not is_async_callable(spec.func):
+            return spec.func(*spec.args, **spec.kwargs)
+        coro: Coroutine[Any, Any, Any] = spec.func(*spec.args, **spec.kwargs)
+        return _run_coroutine(coro)
+
+    # -- observation ---------------------------------------------------------- #
+    def _get(self, execution_id: ExecutionId) -> Execution:
+        execution = self._executions.get(str(execution_id))
+        if execution is None:
+            from ..errors import ExecutionNotFound
+
+            raise ExecutionNotFound(
+                f"inline execution {execution_id!r} is not in this backend's "
+                f"{self._history}-entry history",
+                backend=self.name,
+            )
+        return execution
+
+    def _result(self, execution_id: ExecutionId, *, timeout: float | None) -> ExecutionResult:
+        self._get(execution_id)  # raises ExecutionNotFound with a good message
+        return self._results[str(execution_id)]
+
+    def _remember(self, execution: Execution, result: ExecutionResult) -> None:
+        if self._history == 0:
+            return
+        key = str(execution.id)
+        self._executions[key] = execution
+        self._results[key] = result
+        self._order.append(key)
+        while len(self._order) > self._history:
+            evicted = self._order.pop(0)
+            self._executions.pop(evicted, None)
+            self._results.pop(evicted, None)
+
+    def close(self) -> None:
+        self._executions.clear()
+        self._results.clear()
+        self._order.clear()
+
+
+def _sleep(policy: RetryPolicy, attempt: int) -> None:
+    delay = policy.delay_for(attempt)
+    if delay > 0:
+        time.sleep(delay)
+
+
+def _run_coroutine(coro: Coroutine[Any, Any, Any]) -> Any:
+    """Await ``coro`` from synchronous code, whether or not a loop is running."""
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return asyncio.run(coro)
+    # A loop is already running in this thread; asyncio.run() would raise. Run
+    # the coroutine on its own loop in a worker thread instead. This blocks the
+    # caller, which is the documented contract of the synchronous submit().
+    with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
+        return pool.submit(asyncio.run, coro).result()
+
+
+__all__ = ["INLINE_CAPABILITIES", "InlineExecutionBackend"]

taskferry/backends/process.py

@@ -0,0 +1,292 @@
+"""A task backend backed by a process pool.
+
+Same shape as :mod:`taskferry.backends.thread`, one significant difference: the
+work runs in a *different interpreter*, so it is a genuine rehearsal for what a
+real worker deployment does to your code.
+
+```mermaid
+flowchart LR
+    P["parent process"]
+    C1["worker process 1"]
+    C2["worker process 2"]
+
+    P -->|"'pkg.mod:fn' + JSON args"| C1
+    P -->|"'pkg.mod:fn' + JSON args"| C2
+    C1 -->|"JSON result"| P
+    C2 -->|"JSON result"| P
+```
+
+Nothing but the function *name* and JSON arguments crosses the boundary — the
+same contract Procrastinate and Cloud Tasks impose. Code that works here works
+on a worker; code that relied on a closure, a module-level global mutated at
+import time, or a live ORM object fails here, at development time, where it is
+cheap to find. That is the whole reason this backend exists alongside the thread
+pool.
+
+Because the child imports the function by name, ``__main__``-defined functions
+and lambdas cannot be used —
+:class:`~taskferry.functions.FunctionRef` rejects them at submit time with an
+explanation rather than letting the pool fail cryptically.
+"""
+
+from __future__ import annotations
+
+import threading
+from concurrent.futures import Future, ProcessPoolExecutor
+from datetime import UTC, datetime
+from typing import Any
+
+from ..capabilities import Capability, CapabilitySet
+from ..core.typing import JSONValue
+from ..errors import ExecutionNotFound, SubmissionError, TaskferryTimeoutError
+from ..execution import (
+    Execution,
+    ExecutionId,
+    ExecutionKind,
+    ExecutionResult,
+    ExecutionState,
+    new_execution_id,
+)
+from ..functions import FunctionRef, is_async_callable
+from ..ports import BaseBackend
+from ..retry import RetryPolicy
+from ..specs import ExecutionSpec, TaskSpec
+
+PROCESS_CAPABILITIES = frozenset(
+    {
+        Capability.SUBMIT,
+        Capability.STATE,
+        Capability.RESULT,
+        Capability.CANCEL,
+        Capability.RETRY,
+        Capability.ASYNC_CALLABLE,
+    }
+)
+
+
+def _execute_in_child(
+    task: str,
+    args: list[JSONValue],
+    kwargs: dict[str, JSONValue],
+    policy: RetryPolicy,
+) -> Any:
+    """Entry point that runs in the worker process.
+
+    Module-level and picklable on purpose: a ``ProcessPoolExecutor`` can only
+    dispatch to something importable, which is the same constraint every real
+    task engine imposes.
+
+    Retries happen here, in the child, so an attempt that dies takes only its own
+    process time and the parent is not blocked coordinating them.
+    """
+    import asyncio
+
+    func = FunctionRef.parse(task).resolve()
+    attempt = 1
+    while True:
+        try:
+            if is_async_callable(func):
+                return asyncio.run(func(*args, **kwargs))
+            return func(*args, **kwargs)
+        except Exception as exc:
+            if not policy.should_retry(exc, attempt):
+                raise
+            attempt += 1
+            delay = policy.delay_for(attempt)
+            if delay:
+                import time
+
+                time.sleep(delay)
+
+
+class _Entry:
+    __slots__ = ("execution", "future", "result")
+
+    def __init__(self, execution: Execution) -> None:
+        self.execution = execution
+        self.future: Future[Any] | None = None
+        self.result: ExecutionResult | None = None
+
+
+class ProcessTaskBackend(BaseBackend):
+    """Runs tasks on a :class:`~concurrent.futures.ProcessPoolExecutor`.
+
+    Args:
+        max_workers: Pool size. ``None`` uses the interpreter default.
+        history: How many finished executions to keep for ``get``/``result``.
+
+    Like the thread backend this is **not** durable: pending work dies with the
+    parent. It does not advertise ``DELAY`` (nothing here schedules) or
+    ``PRIORITY`` (the pool is FIFO).
+
+    The pool is created lazily on first submit, so constructing this backend —
+    which the runtime does eagerly when it is configured — never forks.
+    """
+
+    def __init__(self, *, max_workers: int | None = None, history: int = 1024) -> None:
+        self._max_workers = max_workers
+        self._history = history
+        self._pool: ProcessPoolExecutor | None = None
+        self._entries: dict[str, _Entry] = {}
+        self._order: list[str] = []
+        self._lock = threading.Lock()
+        self._closed = False
+
+    @property
+    def name(self) -> str:
+        return "process"
+
+    @property
+    def kind(self) -> ExecutionKind:
+        return ExecutionKind.TASK
+
+    @property
+    def capabilities(self) -> CapabilitySet:
+        return CapabilitySet(PROCESS_CAPABILITIES, provider=self.name)
+
+    def _ensure_pool(self) -> ProcessPoolExecutor:
+        with self._lock:
+            if self._closed:
+                raise SubmissionError("this ProcessTaskBackend is closed", backend=self.name)
+            if self._pool is None:
+                self._pool = ProcessPoolExecutor(max_workers=self._max_workers)
+            return self._pool
+
+    # -- submission ------------------------------------------------------------ #
+    def _submit(self, spec: ExecutionSpec) -> Execution:
+        assert isinstance(spec, TaskSpec)
+        pool = self._ensure_pool()
+        execution = Execution(
+            id=new_execution_id(ExecutionKind.TASK),
+            kind=ExecutionKind.TASK,
+            backend=self.name,
+            state=ExecutionState.QUEUED,
+            name=spec.name,
+            created_at=datetime.now(UTC),
+            correlation=spec.correlation,
+            metadata={"queue": spec.queue},
+        )
+        key = str(execution.id)
+        entry = _Entry(execution)
+        with self._lock:
+            self._entries[key] = entry
+            self._remember_locked(key)
+        try:
+            future = pool.submit(
+                _execute_in_child,
+                spec.task,
+                list(spec.args),
+                dict(spec.kwargs),
+                spec.retry,
+            )
+        except Exception as exc:
+            raise SubmissionError(
+                f"could not dispatch {spec.task!r} to the process pool: {exc}", backend=self.name
+            ) from exc
+        entry.future = future
+        future.add_done_callback(lambda fut: self._settle(key, fut))
+        return execution
+
+    def _settle(self, key: str, future: Future[Any]) -> None:
+        """Record the outcome once the child reports back."""
+        if future.cancelled():
+            self._set_state(key, ExecutionState.CANCELLED)
+            return
+        error = future.exception()
+        if error is None:
+            result = ExecutionResult(value=future.result())
+            state = ExecutionState.SUCCEEDED
+        else:
+            result = ExecutionResult.from_exception(error)
+            state = ExecutionState.FAILED
+        with self._lock:
+            entry = self._entries.get(key)
+            if entry is None:
+                return
+            entry.result = result
+            entry.execution = entry.execution.evolve(
+                state=state, finished_at=datetime.now(UTC), result=result
+            )
+            execution = entry.execution
+        self.hooks.after_execute(execution, result)
+        if error is None:
+            self.hooks.on_success(execution, result)
+        else:
+            self.hooks.on_failure(execution, error)
+
+    def _set_state(self, key: str, state: ExecutionState) -> None:
+        with self._lock:
+            entry = self._entries.get(key)
+            if entry is None:
+                return
+            entry.execution = entry.execution.evolve(state=state, finished_at=datetime.now(UTC))
+
+    # -- observation ------------------------------------------------------------ #
+    def _entry(self, execution_id: ExecutionId) -> _Entry:
+        with self._lock:
+            entry = self._entries.get(str(execution_id))
+        if entry is None:
+            raise ExecutionNotFound(
+                f"task {execution_id!r} is not in this backend's {self._history}-entry history",
+                backend=self.name,
+            )
+        return entry
+
+    def _get(self, execution_id: ExecutionId) -> Execution:
+        entry = self._entry(execution_id)
+        # RUNNING is not observable across the pool boundary: a future is either
+        # pending or done, so QUEUED stands until the child reports.
+        if entry.future is not None and entry.future.running():
+            with self._lock:
+                if entry.execution.state is ExecutionState.QUEUED:
+                    entry.execution = entry.execution.evolve(state=ExecutionState.RUNNING)
+        return entry.execution
+
+    def _wait(self, execution_id: ExecutionId, *, timeout: float | None) -> Execution:
+        entry = self._entry(execution_id)
+        if entry.future is not None:
+            try:
+                entry.future.result(timeout=timeout)
+            except TimeoutError as exc:
+                raise TaskferryTimeoutError(
+                    f"task {execution_id!r} did not finish within {timeout}s"
+                ) from exc
+            except Exception:
+                pass  # recorded on the execution by _settle
+        return self._get(execution_id)
+
+    def _result(self, execution_id: ExecutionId, *, timeout: float | None) -> ExecutionResult:
+        self._wait(execution_id, timeout=timeout)
+        result = self._entry(execution_id).result
+        if result is None:
+            return ExecutionResult.cancelled()
+        return result
+
+    def _cancel(self, execution_id: ExecutionId) -> Execution:
+        entry = self._entry(execution_id)
+        if entry.execution.is_terminal:
+            return entry.execution
+        # Only a task that has not been picked up can be stopped; a running child
+        # is left alone rather than killed mid-write.
+        if entry.future is not None and entry.future.cancel():
+            self._set_state(str(execution_id), ExecutionState.CANCELLED)
+        return self._entry(execution_id).execution
+
+    def _remember_locked(self, key: str) -> None:
+        self._order.append(key)
+        while len(self._order) > self._history:
+            evicted = self._order.pop(0)
+            if evicted != key:
+                self._entries.pop(evicted, None)
+
+    def close(self) -> None:
+        with self._lock:
+            self._closed = True
+            pool, self._pool = self._pool, None
+            self._entries.clear()
+            self._order.clear()
+        if pool is not None:
+            pool.shutdown(wait=True, cancel_futures=True)
+
+
+__all__ = ["PROCESS_CAPABILITIES", "ProcessTaskBackend"]