taskferry 0.2.0__py3-none-any.whl

taskferry/envelope.py

@@ -0,0 +1,197 @@
+"""The portable task envelope — one wire format, every transport.
+
+```mermaid
+flowchart LR
+    SPEC["TaskSpec"]
+    ENV["envelope<br/>{taskferry, task, args, kwargs, correlation, retry}"]
+
+    ENV --> PG["Procrastinate job argument"]
+    ENV --> CT["Cloud Tasks HTTP body"]
+    ENV --> SQS["SQS message body"]
+    ENV --> SB["Service Bus message body"]
+    ENV --> DQ["Dramatiq actor argument"]
+
+    SPEC --> ENV
+```
+
+Five adapters were about to grow the same twenty lines — build a dict, read the
+correlation back, check the version, encode the retry policy. So it lives here
+once, and every transport that can carry a JSON object gets a consistent,
+inspectable payload.
+
+What it is not
+--------------
+
+This is **not** a queue protocol. It carries no routing, no priority, no
+scheduling and no delivery metadata, because every engine has its own and
+better mechanisms for those — a `queueing_lock`, a `schedule_time`, a
+`MessageGroupId`. The envelope carries only what the *worker* needs in order to
+run the right function with the right arguments. Anything the engine can express
+natively is translated by the adapter, not smuggled through here.
+
+Two rules
+---------
+
+**JSON only, never pickle.** A payload sits in a PostgreSQL table or an SQS queue
+where an operator can read it, survives a Python upgrade, and cannot become
+arbitrary code execution on the way back in. See
+[ADR-0009](../../../docs/adr/0009-serialization.md).
+
+**Version it, and reject what you do not know.** A worker running old code must
+not silently drop a field a newer producer added. :func:`read_envelope` refuses
+an unfamiliar version rather than guessing.
+"""
+
+from __future__ import annotations
+
+from typing import Any, TypedDict
+
+from .core.correlation import Correlation
+from .core.typing import JSONValue
+from .errors import SerializationError
+from .retry import NO_RETRY, Backoff, RetryPolicy
+from .specs import TaskSpec
+
+ENVELOPE_VERSION = "2"
+"""Bumped when the shape changes. Workers reject versions they do not know."""
+
+
+class Envelope(TypedDict, total=False):
+    """The on-the-wire shape of a task."""
+
+    taskferry: str
+    task: str
+    args: list[JSONValue]
+    kwargs: dict[str, JSONValue]
+    name: str
+    queue: str
+    correlation: dict[str, str] | None
+    labels: dict[str, str]
+    retry: dict[str, JSONValue] | None
+
+
+def build_envelope(spec: TaskSpec) -> Envelope:
+    """Turn a :class:`~taskferry.specs.TaskSpec` into the wire envelope.
+
+    Arguments were validated as JSON-shaped when the spec was constructed, so
+    nothing can fail here that would not already have failed at the call site.
+    """
+    return {
+        "taskferry": ENVELOPE_VERSION,
+        "task": spec.task,
+        "args": list(spec.args),
+        "kwargs": dict(spec.kwargs),
+        "name": spec.name,
+        "queue": spec.queue,
+        "correlation": spec.correlation.to_headers() if spec.correlation else None,
+        "labels": dict(spec.labels),
+        "retry": encode_retry(spec.retry),
+    }
+
+
+def read_envelope(payload: Any) -> Envelope:
+    """Validate an incoming payload and return it as an :class:`Envelope`.
+
+    Raises:
+        SerializationError: when the payload is not an envelope of a version this
+            code understands, or carries no task. Failing loudly is deliberate: a
+            malformed body means a misconfigured endpoint or a version skew, and
+            quietly doing nothing would hide both.
+    """
+    if not isinstance(payload, dict):
+        raise SerializationError(
+            f"a taskferry envelope must be a JSON object, got {type(payload).__name__}"
+        )
+    version = payload.get("taskferry")
+    if version != ENVELOPE_VERSION:
+        raise SerializationError(
+            f"unsupported taskferry envelope version {version!r} "
+            f"(this worker speaks {ENVELOPE_VERSION!r}); upgrade the worker or the producer"
+        )
+    if not payload.get("task"):
+        raise SerializationError("taskferry envelope has no 'task'")
+    return payload  # type: ignore[return-value]
+
+
+def read_correlation(envelope: Envelope) -> Correlation:
+    """Rebuild the correlation, starting a fresh flow when the envelope has none."""
+    headers = envelope.get("correlation")
+    return Correlation.from_headers(headers) if headers else Correlation.start()
+
+
+def encode_retry(policy: RetryPolicy) -> dict[str, JSONValue] | None:
+    """Serialize the portable retry intent, or ``None`` when nothing is asked for.
+
+    Only what a worker needs to compute the *next* delay travels. The policy
+    object itself is not shipped, because a queue payload must not depend on the
+    producer and the consumer running the same Taskferry version.
+    """
+    if not policy.enabled:
+        return None
+    return {
+        "max_attempts": policy.max_attempts,
+        "backoff": policy.backoff.value,
+        "initial_delay": policy.initial_delay,
+        "max_delay": policy.max_delay,
+        "retry_on": list(policy.retry_on),
+        "no_retry_on": list(policy.no_retry_on),
+    }
+
+
+def decode_retry(envelope: Envelope) -> RetryPolicy:
+    """Rebuild the policy on the worker side. Unknown fields are ignored."""
+    raw = envelope.get("retry")
+    if not raw:
+        return NO_RETRY
+    return RetryPolicy(
+        max_attempts=int(raw.get("max_attempts", 1) or 1),  # type: ignore[arg-type]
+        backoff=Backoff(str(raw.get("backoff", Backoff.EXPONENTIAL.value))),
+        initial_delay=float(raw.get("initial_delay", 1.0) or 0.0),  # type: ignore[arg-type]
+        max_delay=(
+            None if raw.get("max_delay") is None else float(raw["max_delay"])  # type: ignore[arg-type]
+        ),
+        retry_on=tuple(str(name) for name in (raw.get("retry_on") or [])),  # type: ignore[union-attr]
+        no_retry_on=tuple(str(name) for name in (raw.get("no_retry_on") or [])),  # type: ignore[union-attr]
+    )
+
+
+def execute_envelope(payload: Any, *, registry: Any = None) -> Any:
+    """Validate, resolve and run one envelope. The whole worker side, in one call.
+
+    Args:
+        payload: The decoded JSON body.
+        registry: A :class:`~taskferry.functions.FunctionRegistry`. **Give it an
+            allowlist** when envelopes arrive from a queue anything untrusted can
+            write to — resolving a name means importing a module. See
+            ``docs/security.md``.
+
+    Returns:
+        Whatever the task returned. Most engines discard it.
+    """
+    from .core.correlation import use_correlation
+    from .functions import FunctionRegistry, is_async_callable
+
+    envelope = read_envelope(payload)
+    resolver = registry if registry is not None else FunctionRegistry()
+    func = resolver.resolve(str(envelope["task"]))
+    args = list(envelope.get("args") or [])
+    kwargs = dict(envelope.get("kwargs") or {})
+
+    with use_correlation(read_correlation(envelope)):
+        if is_async_callable(func):
+            import asyncio
+
+            return asyncio.run(func(*args, **kwargs))
+        return func(*args, **kwargs)
+
+
+__all__ = [
+    "ENVELOPE_VERSION",
+    "Envelope",
+    "build_envelope",
+    "decode_retry",
+    "encode_retry",
+    "execute_envelope",
+    "read_correlation",
+    "read_envelope",
+]

taskferry/errors.py

@@ -0,0 +1,144 @@
+"""The portable Taskferry error hierarchy.
+
+Every error a caller can reasonably catch is a :class:`TaskferryError`. Adapters
+translate provider exceptions into these types and **always** chain the original
+as ``__cause__``::
+
+    raise SubmissionError("defer failed", backend="procrastinate") from exc
+
+Provider exceptions (``google.api_core.GoogleAPIError``, ``psycopg.Error``,
+``kubernetes.client.ApiException`` ...) are never part of Taskferry's public API,
+so application code never has to import a provider SDK to handle a failure.
+
+```mermaid
+flowchart TD
+    E["TaskferryError"]
+    E --> C["ConfigurationError"]
+    E --> B["BackendError"]
+    B --> S["SubmissionError"]
+    B --> X["ExecutionError"]
+    B --> NF["ExecutionNotFound"]
+    E --> U["UnsupportedCapability"]
+    E --> SER["SerializationError"]
+    E --> T["TimeoutError"]
+    E --> R["RoutingError"]
+    E --> F["FunctionResolutionError"]
+    E --> CAN["ExecutionCancelled"]
+```
+"""
+
+from __future__ import annotations
+
+from taskferry.core.errors import ConfigurationError, SerializationError, TaskferryError
+from taskferry.core.errors import ProviderError as _ProviderError
+from taskferry.core.errors import UnsupportedCapabilityError as _UnsupportedCapabilityError
+
+
+class BackendError(_ProviderError):
+    """A backend failed to carry out an operation.
+
+    Subclasses ``ProviderError`` so code written against 0.1 keeps working. The
+    offending backend is named in :attr:`backend`.
+    """
+
+    def __init__(self, message: str, *, backend: str | None = None) -> None:
+        super().__init__(message, provider=backend)
+
+    @property
+    def backend(self) -> str | None:
+        """Name of the backend that raised, when known."""
+        return self.provider
+
+
+class SubmissionError(BackendError):
+    """A spec could not be handed to the backend.
+
+    Raised at submit time — nothing was enqueued, so retrying the submission is
+    safe unless the backend documents otherwise.
+    """
+
+
+class ExecutionError(BackendError):
+    """An execution ran and failed.
+
+    :attr:`cause_repr` carries the remote exception's textual form when the
+    backend can supply it (the real exception object rarely survives a process or
+    network boundary).
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        backend: str | None = None,
+        cause_repr: str | None = None,
+    ) -> None:
+        super().__init__(message, backend=backend)
+        self.cause_repr = cause_repr
+
+
+class ExecutionNotFound(BackendError):
+    """No execution with the given id is known to the backend.
+
+    Distinct from :class:`ExecutionError`: the execution may never have existed,
+    or the backend may have expired its record.
+    """
+
+
+class ExecutionCancelled(BackendError):
+    """The execution was cancelled before it could produce a result."""
+
+
+UnsupportedCapability = _UnsupportedCapabilityError
+"""The operation needs a capability the backend does not advertise.
+
+Taskferry raises this rather than emulating the capability, because an emulated
+cancel or an emulated timeout is a correctness bug waiting for production.
+
+This is an **alias**, not a subclass, of
+:class:`taskferry.core.errors.UnsupportedCapabilityError`. It has to be: a
+subclass would mean ``CapabilitySet.require()`` raises the parent while every
+caller is told to catch the child, so the documented ``except
+UnsupportedCapability`` would silently miss the most common source of the error.
+One concept, one class, two names."""
+
+
+class RoutingError(ConfigurationError):
+    """No backend could be selected for a spec, or the selected one is unknown."""
+
+
+class FunctionResolutionError(TaskferryError):
+    """A ``package.module:function`` reference could not be resolved.
+
+    Also raised when a reference is rejected by the import allowlist — the
+    message says which, so a deployment problem is never mistaken for a typo.
+    """
+
+
+class TaskferryTimeoutError(TaskferryError, TimeoutError):
+    """A ``wait``/``result`` call exceeded its timeout.
+
+    Subclasses the built-in :class:`TimeoutError` so ``except TimeoutError``
+    keeps working for callers that do not import Taskferry's hierarchy.
+    """
+
+
+# Public alias. ``taskferry.TimeoutError`` shadows the builtin only inside an
+# explicit ``from taskferry import TimeoutError``, which is opt-in and explicit.
+TimeoutError = TaskferryTimeoutError
+
+__all__ = [
+    "BackendError",
+    "ConfigurationError",
+    "ExecutionCancelled",
+    "ExecutionError",
+    "ExecutionNotFound",
+    "FunctionResolutionError",
+    "RoutingError",
+    "SerializationError",
+    "SubmissionError",
+    "TaskferryError",
+    "TaskferryTimeoutError",
+    "TimeoutError",
+    "UnsupportedCapability",
+]

taskferry/execution.py

@@ -0,0 +1,239 @@
+"""Execution — the portable identity of one unit of work in flight.
+
+Every backend, of every kind, describes what it did in these terms. An
+:class:`Execution` is an immutable *snapshot*: what the backend knew at the
+moment it was asked. An :class:`~taskferry.handle.ExecutionHandle` (see
+``taskferry.handle``) is the live, refreshable reference built on top of it.
+
+State machine
+-------------
+
+```mermaid
+stateDiagram-v2
+    [*] --> PENDING
+    PENDING --> QUEUED
+    PENDING --> RUNNING
+    QUEUED --> RUNNING
+    QUEUED --> CANCELLED
+    RUNNING --> SUCCEEDED
+    RUNNING --> FAILED
+    RUNNING --> CANCELLED
+    RUNNING --> TIMED_OUT
+    FAILED --> QUEUED: retry (engine-owned)
+    SUCCEEDED --> [*]
+    CANCELLED --> [*]
+    TIMED_OUT --> [*]
+```
+
+Not every backend can observe every transition. An inline execution goes straight
+from ``PENDING`` to a terminal state; a fire-and-forget push queue may only ever
+report ``QUEUED`` and then ``UNKNOWN``. Adapters map what the provider actually
+reports and use :data:`ExecutionState.UNKNOWN` rather than guessing.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field, replace
+from datetime import datetime
+from enum import StrEnum
+from types import MappingProxyType
+from typing import Any
+
+from taskferry.core.correlation import Correlation
+from taskferry.core.ids import TaskferryId, new_id
+from taskferry.core.provider import ProviderMetadata
+
+ExecutionId = TaskferryId
+"""A Taskferry-owned execution id. Never the provider's id — that lives in
+:attr:`Execution.external_id`, so one execution stays followable across systems."""
+
+
+class ExecutionKind(StrEnum):
+    """Which execution primitive produced this execution."""
+
+    INLINE = "inline"
+    TASK = "task"
+    JOB = "job"
+
+
+class ExecutionState(StrEnum):
+    """Portable lifecycle state. See the diagram in the module docstring."""
+
+    PENDING = "pending"
+    """Accepted by Taskferry, not yet acknowledged by the engine."""
+
+    QUEUED = "queued"
+    """Accepted by the engine, waiting for a worker or a slot."""
+
+    RUNNING = "running"
+    CANCELLED = "cancelled"
+    SUCCEEDED = "succeeded"
+    FAILED = "failed"
+    TIMED_OUT = "timed_out"
+
+    UNKNOWN = "unknown"
+    """The backend cannot determine the state. Not an error and not terminal —
+    a later poll may resolve it."""
+
+    @property
+    def is_terminal(self) -> bool:
+        """Whether no further transition is expected (barring an engine retry)."""
+        return self in _TERMINAL
+
+    @property
+    def is_successful(self) -> bool:
+        return self is ExecutionState.SUCCEEDED
+
+
+_TERMINAL = frozenset(
+    {
+        ExecutionState.SUCCEEDED,
+        ExecutionState.FAILED,
+        ExecutionState.CANCELLED,
+        ExecutionState.TIMED_OUT,
+    }
+)
+
+#: The transitions Taskferry considers legal. Adapters are not forced through
+#: this table (providers report what they report), but ``Execution.transitions_to``
+#: lets tests and diagnostics flag a mapping that cannot be right.
+_ALLOWED: Mapping[ExecutionState, frozenset[ExecutionState]] = MappingProxyType(
+    {
+        ExecutionState.PENDING: frozenset(
+            {ExecutionState.QUEUED, ExecutionState.RUNNING, ExecutionState.UNKNOWN} | _TERMINAL
+        ),
+        # SUCCEEDED is reachable directly from QUEUED, and not as an edge case:
+        # a worker that finishes between the submit and the first poll means the
+        # observer never sees RUNNING at all. Every real engine does this
+        # constantly. A transition table that forbids it would flag correct
+        # adapters as broken.
+        ExecutionState.QUEUED: frozenset(
+            {ExecutionState.RUNNING, ExecutionState.UNKNOWN} | _TERMINAL
+        ),
+        ExecutionState.RUNNING: frozenset(
+            {ExecutionState.UNKNOWN} | _TERMINAL,
+        ),
+        # An engine-owned retry legitimately moves a failed execution back into
+        # the queue; see ADR-0011 on retry ownership.
+        ExecutionState.FAILED: frozenset({ExecutionState.QUEUED, ExecutionState.RUNNING}),
+        ExecutionState.TIMED_OUT: frozenset({ExecutionState.QUEUED}),
+        ExecutionState.SUCCEEDED: frozenset(),
+        ExecutionState.CANCELLED: frozenset(),
+        ExecutionState.UNKNOWN: frozenset(ExecutionState),
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class ExecutionResult:
+    """The outcome of a finished execution.
+
+    ``value`` is only meaningful when the backend advertises
+    :attr:`~taskferry.capabilities.Capability.RESULT`; otherwise it stays ``None``
+    and callers must not read anything into that.
+    """
+
+    value: Any = None
+    error: str | None = None
+    error_type: str | None = None
+    traceback: str | None = None
+    exit_code: int | None = None
+    logs_uri: str | None = None
+
+    @classmethod
+    def from_exception(cls, exc: BaseException) -> ExecutionResult:
+        """Build a failed result from a raised exception.
+
+        Captures the message, qualified type name and formatted traceback the
+        same way every in-process backend needs, so the construction lives here
+        once instead of being copied into each one.
+        """
+        import traceback as _traceback
+
+        return cls(
+            error=str(exc) or type(exc).__name__,
+            error_type=type(exc).__qualname__,
+            traceback="".join(_traceback.format_exception(type(exc), exc, exc.__traceback__)),
+        )
+
+    @classmethod
+    def cancelled(cls) -> ExecutionResult:
+        """The placeholder result for an execution cancelled before it produced one."""
+        return cls(error="execution was cancelled", error_type="ExecutionCancelled")
+
+
+@dataclass(frozen=True, slots=True)
+class Execution:
+    """An immutable snapshot of one unit of work.
+
+    Attributes:
+        id: Taskferry-owned id, stable for the life of the execution.
+        kind: Which primitive produced it.
+        backend: Name of the backend that owns it (``"local"``, ``"procrastinate"``).
+        state: Portable lifecycle state at snapshot time.
+        name: Human-readable name — the task path or the job name.
+        created_at: When Taskferry accepted the submission.
+        started_at: When the engine began running it, if reported.
+        finished_at: When it reached a terminal state, if reported.
+        external_id: The engine's own id (Procrastinate job id, Cloud Run
+            execution name, Cloud Tasks task name).
+        attempt: 1-based attempt counter, when the engine reports one.
+        result: Outcome, populated once terminal and if the backend can supply it.
+        correlation: Correlation propagated from the spec.
+        provider_metadata: Region/resource/labels for the underlying resource.
+        metadata: Free-form backend annotations. JSON-shaped by convention.
+    """
+
+    id: ExecutionId
+    kind: ExecutionKind
+    backend: str
+    state: ExecutionState
+    name: str = ""
+    created_at: datetime | None = None
+    started_at: datetime | None = None
+    finished_at: datetime | None = None
+    external_id: str | None = None
+    attempt: int = 1
+    result: ExecutionResult | None = None
+    correlation: Correlation | None = None
+    provider_metadata: ProviderMetadata | None = None
+    metadata: Mapping[str, str] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "metadata", MappingProxyType(dict(self.metadata)))
+
+    @property
+    def is_terminal(self) -> bool:
+        return self.state.is_terminal
+
+    @property
+    def is_successful(self) -> bool:
+        return self.state.is_successful
+
+    def transitions_to(self, state: ExecutionState) -> bool:
+        """Whether moving from this snapshot's state to ``state`` is legal.
+
+        Used by the contract suite to catch adapters that map provider states
+        onto impossible transitions (``SUCCEEDED`` → ``RUNNING``, say).
+        """
+        return state is self.state or state in _ALLOWED[self.state]
+
+    def evolve(self, **changes: Any) -> Execution:
+        """Return a copy with ``changes`` applied. The original is never mutated."""
+        return replace(self, **changes)
+
+
+def new_execution_id(kind: ExecutionKind = ExecutionKind.TASK) -> ExecutionId:
+    """Mint a fresh execution id prefixed with the kind (``task_``, ``job_``)."""
+    return new_id(kind.value)
+
+
+__all__ = [
+    "Execution",
+    "ExecutionId",
+    "ExecutionKind",
+    "ExecutionResult",
+    "ExecutionState",
+    "new_execution_id",
+]