dewey 0.4.0__py3-none-any.whl

dewey/__init__.py

@@ -0,0 +1,74 @@
+"""Dewey — Guaranteed delivery engine. Frankapps Built.
+
+Postgres is the durable scheduler and backlog; a broker is only a worker-pool
+transport. Declare a task, create work inside your own transaction, and let the
+dispatcher and the ledger handle delivery::
+
+    import dewey
+
+    @dewey.task("agent.notify", max_attempts=5, backoff=dewey.Constant(3))
+    def notify_agent(command_id: str) -> None:
+        ...
+
+Importing ``dewey`` pulls in no framework: the producer and worker APIs live in
+``dewey.sqlalchemy`` and ``dewey.django``.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from dewey.core.states import TaskStatus
+from dewey.errors import (
+    DeweyError,
+    DuplicateTaskTypeError,
+    NonRetryableError,
+    RetryAfter,
+    SerializationError,
+    TransientError,
+    UnknownTaskTypeError,
+)
+from dewey.policy import (
+    TASK_DEFAULTS,
+    BackoffPolicy,
+    Constant,
+    Custom,
+    Exponential,
+    PolicyRegistry,
+    TaskPolicy,
+    clear_project_policies,
+    configure_policies,
+    registry,
+    resolve_policy,
+    task,
+)
+from dewey.serialization import encode_args, encode_kwargs
+
+try:
+    __version__ = version("dewey")
+except PackageNotFoundError:  # pragma: no cover — source tree without an install
+    __version__ = "0.0.0.dev0"
+
+__all__ = [
+    "TASK_DEFAULTS",
+    "BackoffPolicy",
+    "Constant",
+    "Custom",
+    "DeweyError",
+    "DuplicateTaskTypeError",
+    "Exponential",
+    "NonRetryableError",
+    "PolicyRegistry",
+    "RetryAfter",
+    "SerializationError",
+    "TaskPolicy",
+    "TaskStatus",
+    "TransientError",
+    "UnknownTaskTypeError",
+    "__version__",
+    "clear_project_policies",
+    "configure_policies",
+    "encode_args",
+    "encode_kwargs",
+    "registry",
+    "resolve_policy",
+    "task",
+]

dewey/adapters/__init__.py

@@ -0,0 +1,5 @@
+"""Queue transport adapters — broker-agnostic."""
+
+from dewey.adapters.base import DispatcherAdapter, ProcessTaskFn
+
+__all__ = ["DispatcherAdapter", "ProcessTaskFn"]

dewey/adapters/base.py

@@ -0,0 +1,49 @@
+"""Abstract adapter interface for queue transports."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any, Protocol, runtime_checkable
+
+ProcessTaskFn = Callable[[str], Any]
+
+
+@runtime_checkable
+class DispatcherAdapter(Protocol):
+    """
+    Target protocol for Dewey-driven dispatch.
+
+    Producers create rows only. A Dewey dispatcher claims ready rows and
+    calls :meth:`dispatch` after commit; the transport adapter then hands
+    the task ID to its worker pool. :meth:`register` wires the worker-side
+    processor that receives the task ID.
+
+    Lifecycle
+    ---------
+    1. **Construction** — build the adapter with its transport handle
+       (Huey instance, etc.) in the worker process *and* in any
+       dispatcher process that will call :meth:`dispatch`.
+    2. **register(process_fn)** — called once per worker process, before
+       the worker pool starts consuming. The protocol does not require
+       re-registration to be idempotent; adapters are free to raise if
+       called twice. Dispatcher processes that only call :meth:`dispatch`
+       do not need to call :meth:`register`.
+    3. **dispatch(task_id)** — called by the Dewey dispatcher after the
+       claimed row has been committed to ``DISPATCHING``. Must be safe to
+       call concurrently from multiple dispatcher processes and must not
+       block on the task completing.
+
+    The protocol is :func:`runtime_checkable`, so adapters can be
+    validated with ``isinstance(adapter, DispatcherAdapter)`` at wiring
+    time. Note that runtime checks verify method presence only, not
+    signatures — see :mod:`tests.test_adapter_protocol` for signature
+    contract tests.
+    """
+
+    def register(self, process_fn: ProcessTaskFn) -> None:
+        """Register the worker-side function that processes a task ID."""
+        ...
+
+    def dispatch(self, task_id: str) -> Any:
+        """Dispatch a claimed task ID to the transport worker pool."""
+        ...

dewey/adapters/huey.py

@@ -0,0 +1,96 @@
+"""Huey adapter — transport only.
+
+Huey's job is to carry a task ID from the dispatcher to a worker. It does not
+decide when work runs, how often it retries, or what happens when it fails. That
+is Dewey's, in Postgres.
+
+Registering with ``retries=0`` is deliberate: two retry engines fighting over one
+task is how work gets run twice and how attempt counters stop meaning anything.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from huey import Huey
+
+logger = logging.getLogger(__name__)
+
+ProcessTaskFn = Callable[[str], Any]
+
+
+class HueyAdapter:
+    """Bridge Dewey's dispatcher to a Huey worker pool.
+
+    Put the wiring in a module both processes import — the Huey instance, the
+    adapter, and the ``register`` call::
+
+        # myapp/tasks.py
+        huey = RedisHuey("myapp")
+        adapter = HueyAdapter(huey)
+
+        def _process(task_id: str) -> bool:
+            with Session(engine) as session:
+                return process_task(session, task_id)
+
+        adapter.register(_process)
+
+    The worker runs ``huey_consumer myapp.tasks.huey``. The dispatcher imports the
+    same module and hands IDs over::
+
+        dispatcher = Dispatcher(SQLAlchemyDispatchBackend(engine), adapter.dispatch)
+        dispatcher.run()
+
+    Producers import none of this. They call ``create_task`` and commit.
+    """
+
+    def __init__(self, huey: Huey, *, task_name: str = "dewey_process_task") -> None:
+        self._huey = huey
+        self._task_name = task_name
+        self._process_task: Any = None
+
+    def register(self, process_fn: ProcessTaskFn) -> None:
+        """Register the callable that processes a task ID.
+
+        Call once per process, at import time, before the consumer starts.
+
+        ``retries=0`` is not an oversight: Dewey owns retry scheduling and the
+        attempt budget. A broker retrying underneath it would re-run a handler
+        Dewey had deliberately scheduled for later.
+        """
+        if self._process_task is not None:
+            raise RuntimeError(
+                "HueyAdapter.register() was called twice in this process. Register once "
+                "at import time — a second call would bind two callables to the task "
+                f"name {self._task_name!r}."
+            )
+
+        @self._huey.task(name=self._task_name, retries=0)
+        def _dewey_process_task(task_id: str) -> Any:
+            return process_fn(task_id)
+
+        self._process_task = _dewey_process_task
+        logger.info("HueyAdapter registered processor as %r", self._task_name)
+
+    def dispatch(self, task_id: str) -> Any:
+        """Hand a claimed task ID to the Huey worker pool.
+
+        Called only by the dispatcher, only after the claim is committed.
+
+        Raising when the broker is unreachable is correct and expected: the
+        dispatcher returns the row to PENDING, backs off, and tries again. Nothing
+        is lost, because Postgres — not Redis — is holding the backlog.
+        """
+        if self._process_task is None:
+            raise RuntimeError(
+                "HueyAdapter.register() must be called before dispatch(). Import the "
+                "module that wires the adapter (the same one your worker uses) in the "
+                "dispatcher process, so both agree on the task name."
+            )
+        return self._process_task(task_id)
+
+
+__all__ = ["HueyAdapter", "ProcessTaskFn"]

dewey/core/__init__.py

@@ -0,0 +1,39 @@
+"""Core module — pure Python, zero dependencies."""
+
+from dewey.core.backoff import (
+    BackoffFn,
+    default_task_backoff,
+    retry_delay,
+)
+from dewey.core.logging import (
+    TRACE_METADATA_KEY,
+    TraceContextFilter,
+    bind_to_metadata,
+    extract_trace_context,
+    get_trace_context,
+    reset_trace_context,
+    restore_trace_context,
+    set_trace_context,
+    update_trace_context,
+)
+from dewey.core.states import TaskStatus, should_die, should_retry
+from dewey.core.types import TaskEntry
+
+__all__ = [
+    "TaskStatus",
+    "should_retry",
+    "should_die",
+    "BackoffFn",
+    "default_task_backoff",
+    "retry_delay",
+    "TRACE_METADATA_KEY",
+    "TraceContextFilter",
+    "bind_to_metadata",
+    "extract_trace_context",
+    "get_trace_context",
+    "reset_trace_context",
+    "restore_trace_context",
+    "set_trace_context",
+    "update_trace_context",
+    "TaskEntry",
+]

dewey/core/backoff.py

@@ -0,0 +1,43 @@
+"""Retry delay calculation — exponential backoff with jitter and cap."""
+
+import random
+from collections.abc import Callable
+from datetime import timedelta
+
+# Defaults
+DEFAULT_BASE_DELAY_SECONDS = 120  # 2 minutes
+DEFAULT_MAX_DELAY_SECONDS = 3600  # 1 hour cap
+DEFAULT_JITTER_FRACTION = 0.25  # ±25% jitter
+
+
+# Signature for a custom backoff strategy.
+# Given the number of attempts so far, return how long to wait before retrying.
+BackoffFn = Callable[[int], timedelta]
+
+
+def default_task_backoff(attempts: int) -> timedelta:
+    """Default backoff used by process_task / process_task_async when no
+    `backoff` argument is supplied. 2 min base, 1 hr cap, ±25% jitter."""
+    return retry_delay(attempts)
+
+
+def retry_delay(
+    attempts: int,
+    base_delay: int = DEFAULT_BASE_DELAY_SECONDS,
+    max_delay: int = DEFAULT_MAX_DELAY_SECONDS,
+    jitter: float = DEFAULT_JITTER_FRACTION,
+) -> timedelta:
+    """
+    Exponential backoff with jitter: base * 2^attempts ± jitter%, capped at max_delay.
+
+    Jitter prevents thundering herd when many tasks fail simultaneously.
+    Set jitter=0 for deterministic behavior (e.g. in tests).
+
+    attempts=0 → ~2min, attempts=1 → ~4min, attempts=2 → ~8min, ...
+    """
+    delay = min(base_delay * (2**attempts), max_delay)
+    if jitter > 0:
+        jitter_amount = delay * jitter
+        delay = delay + random.uniform(-jitter_amount, jitter_amount)
+        delay = max(0, min(delay, max_delay))  # Never negative, never above cap
+    return timedelta(seconds=delay)

dewey/core/execution.py

@@ -0,0 +1,102 @@
+"""Shared execution decisions — one implementation for every backend.
+
+The SQLAlchemy sync, SQLAlchemy async, and Django executors all differ in how
+they load and save rows, and not at all in what a failure *means*. That decision
+lives here so the three paths cannot drift apart.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Any
+
+from dewey.core.backoff import BackoffFn
+from dewey.core.states import TaskStatus, should_die
+from dewey.errors import RetryAfter, UnknownTaskTypeError
+from dewey.policy import TaskPolicy, resolve_policy
+
+
+@dataclass(frozen=True)
+class FailureOutcome:
+    """What to write to the row after a handler raised."""
+
+    status: TaskStatus
+    retry_at: datetime | None
+    reason: str
+
+    @property
+    def is_dead(self) -> bool:
+        return self.status == TaskStatus.DEAD
+
+
+def classify_failure(
+    exc: BaseException,
+    *,
+    policy: TaskPolicy,
+    attempts: int,
+    max_attempts: int,
+    now: datetime,
+    backoff: BackoffFn | None = None,
+) -> FailureOutcome:
+    """Decide whether a failed attempt retries, and when.
+
+    ``attempts`` is the number of attempts *including* the one that just failed.
+
+    Precedence:
+
+    1. ``fail_fast_on`` (and anything outside ``retry_on``) dead-letters at once,
+       however much of the attempt budget is left — retrying a malformed request
+       just burns capacity.
+    2. A spent attempt budget dead-letters.
+    3. Otherwise retry, no earlier than the policy backoff. A handler raising
+       ``RetryAfter(n)`` can ask for *later* than the policy would, never
+       earlier, so a misbehaving provider cannot pull retries in past your own
+       rate budget.
+    """
+    requested_delay: timedelta | None = None
+    if isinstance(exc, RetryAfter):
+        requested_delay = timedelta(seconds=exc.seconds)
+    elif not policy.is_retryable(exc):
+        return FailureOutcome(TaskStatus.DEAD, None, "non-retryable")
+
+    if should_die(attempts, max_attempts):
+        return FailureOutcome(TaskStatus.DEAD, None, "attempts exhausted")
+
+    delay = (backoff or policy.delay_for)(attempts)
+    if requested_delay is not None and requested_delay > delay:
+        delay = requested_delay
+    return FailureOutcome(TaskStatus.FAILED, now + delay, "retry")
+
+
+def resolve_handler(
+    task_type: str,
+    explicit: Callable[..., Any] | None,
+    *,
+    policy: TaskPolicy | None = None,
+) -> Callable[..., Any]:
+    """Return the callable for ``task_type``.
+
+    An explicitly passed handler wins — that is the escape hatch for tests and
+    for in-process use. Otherwise the registered handler is used.
+
+    Raises:
+        UnknownTaskTypeError: nothing is registered and nothing was passed. The
+            executor treats this as an ordinary failed attempt rather than a
+            crash, so a worker that predates the handler's deploy retries with
+            backoff instead of destroying the work.
+    """
+    if explicit is not None:
+        return explicit
+    policy = policy or resolve_policy(task_type)
+    if policy.handler is None:
+        raise UnknownTaskTypeError(
+            f"No handler registered for task type {task_type!r}. Declare one with "
+            f"@dewey.task({task_type!r}) and make sure the worker imports that "
+            f"module, or pass handler= explicitly."
+        )
+    return policy.handler
+
+
+__all__ = ["FailureOutcome", "classify_failure", "resolve_handler"]

dewey/core/logging.py

@@ -0,0 +1,186 @@
+"""Logging + trace-context helpers.
+
+Dewey is a library, not a framework — it doesn't ship a logging config.
+Every Dewey log goes through a logger under the ``dewey.*`` namespace
+(``dewey.core``, ``dewey.sqlalchemy.executor``, ``dewey.django.executor``,
+etc.), so consumers can wire them up like any other library:
+
+.. code-block:: python
+
+    # Django LOGGING dict
+    LOGGING = {
+        "version": 1,
+        "loggers": {"dewey": {"level": "INFO", "handlers": ["console"]}},
+        ...
+    }
+
+    # stdlib
+    logging.getLogger("dewey").setLevel("INFO")
+
+Trace context
+-------------
+
+Dewey doesn't know what a "trace ID" means — it just round-trips a small
+dict of correlation IDs from the task creator, through the
+ledger row's ``metadata["trace"]`` field, into the worker, and makes that
+dict available to every log record produced while the handler runs.
+
+Producers (typically your API request handler) push values into the
+context with :func:`set_trace_context` or merge them into ``metadata``
+with :func:`bind_to_metadata` before calling ``create_task``. The
+``process_*`` functions automatically restore the trace context from
+``metadata["trace"]`` before invoking the handler and reset it after.
+
+Consumers (anyone configuring logs) attach :class:`TraceContextFilter`
+to their handler so every log record gains ``record.dewey_trace`` plus
+flattened ``record.dewey_<field>`` attributes:
+
+.. code-block:: python
+
+    handler = logging.StreamHandler()
+    handler.addFilter(TraceContextFilter())
+    handler.setFormatter(JsonFormatter())  # python-json-logger, logfire, etc.
+
+The carrier is intentionally library-agnostic. Put W3C ``traceparent``,
+an OpenTelemetry context, an AWS X-Ray trace ID, a plain ``request_id``,
+or all of the above into the dict. Dewey will round-trip it untouched.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterator
+from contextlib import contextmanager
+from contextvars import ContextVar, Token
+from typing import Any
+
+#: Key inside a task's ``metadata`` dict used to carry
+#: the trace context across the worker boundary.
+TRACE_METADATA_KEY = "trace"
+
+
+# Default is ``None`` (not ``{}``) to avoid the well-known ContextVar
+# mutable-default footgun; callers always go through get_trace_context()
+# which normalizes to an empty dict.
+_trace_context: ContextVar[dict[str, Any] | None] = ContextVar("dewey_trace_context", default=None)
+
+
+def get_trace_context() -> dict[str, Any]:
+    """Return a copy of the current trace context dict."""
+    ctx = _trace_context.get()
+    return dict(ctx) if ctx else {}
+
+
+def set_trace_context(ctx: dict[str, Any] | None) -> Token[dict[str, Any] | None]:
+    """Replace the trace context. Returns a token usable with :func:`reset_trace_context`.
+
+    Pass ``None`` or ``{}`` to clear the context for the current frame.
+    """
+    return _trace_context.set(dict(ctx) if ctx else None)
+
+
+def reset_trace_context(token: Token[dict[str, Any] | None]) -> None:
+    """Restore the trace context to its previous value (paired with :func:`set_trace_context`)."""
+    _trace_context.reset(token)
+
+
+def update_trace_context(**fields: Any) -> Token[dict[str, Any] | None]:
+    """Shallow-merge fields into the current trace context.
+
+    Returns a token so the caller can restore the previous context.
+    """
+    current = get_trace_context()
+    current.update(fields)
+    return _trace_context.set(current)
+
+
+def bind_to_metadata(
+    metadata: dict[str, Any] | None = None,
+    *,
+    extra: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build a ``metadata`` dict for ``create_task``
+    that carries the current trace context.
+
+    Merges, in order: ``metadata`` (caller-supplied), the current trace
+    context (under ``"trace"``), and any ``extra`` fields the caller wants
+    on top.
+
+    .. code-block:: python
+
+        await create_task_async(
+            session,
+            task_type="order.confirmed",
+            payload=body,
+            metadata=bind_to_metadata({"source": "webhook"}),
+        )
+    """
+    out: dict[str, Any] = dict(metadata) if metadata else {}
+    ctx = get_trace_context()
+    if ctx:
+        existing = dict(out.get(TRACE_METADATA_KEY) or {})
+        existing.update(ctx)
+        out[TRACE_METADATA_KEY] = existing
+    if extra:
+        out.update(extra)
+    return out
+
+
+def extract_trace_context(metadata: dict[str, Any] | None) -> dict[str, Any]:
+    """Pull the trace context dict out of a metadata blob (or ``{}``)."""
+    if not metadata:
+        return {}
+    ctx = metadata.get(TRACE_METADATA_KEY)
+    return dict(ctx) if isinstance(ctx, dict) else {}
+
+
+@contextmanager
+def restore_trace_context(metadata: dict[str, Any] | None) -> Iterator[dict[str, Any]]:
+    """Context manager: temporarily set the trace context from a task or
+    task's ``metadata`` blob, restoring the previous context on exit.
+
+    Used by Dewey's ``process_*`` functions to make trace IDs that were
+    captured at task-creation time available to logs produced inside the
+    handler.
+    """
+    ctx = extract_trace_context(metadata)
+    token = _trace_context.set(ctx)
+    try:
+        yield ctx
+    finally:
+        _trace_context.reset(token)
+
+
+class TraceContextFilter(logging.Filter):
+    """Logging filter that attaches the current Dewey trace context to every record.
+
+    Attach to a handler in your logging configuration; every record routed
+    through that handler will gain:
+
+    - ``record.dewey_trace`` — a ``dict`` of the full trace context
+    - ``record.dewey_<field>`` — one attribute per key (e.g. ``record.dewey_request_id``)
+
+    JSON formatters (logfire, python-json-logger, structlog renderers) pick
+    these up automatically. Plain text formatters can reference them in the
+    format string, e.g. ``"%(asctime)s [%(dewey_request_id)s] %(message)s"``.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        ctx = get_trace_context()
+        record.dewey_trace = ctx  # type: ignore[attr-defined]
+        for key, value in ctx.items():
+            setattr(record, f"dewey_{key}", value)
+        return True
+
+
+__all__ = [
+    "TRACE_METADATA_KEY",
+    "TraceContextFilter",
+    "bind_to_metadata",
+    "extract_trace_context",
+    "get_trace_context",
+    "reset_trace_context",
+    "restore_trace_context",
+    "set_trace_context",
+    "update_trace_context",
+]

dewey/core/states.py

@@ -0,0 +1,66 @@
+"""Task status state machine — the heart of dewey."""
+
+from enum import Enum
+
+
+class TaskStatus(str, Enum):  # noqa: UP042 — keeping (str, Enum) intentional; StrEnum changes str() repr
+    """Task lifecycle states."""
+
+    PENDING = "pending"
+    #: Claimed by a dispatcher and handed to the transport, but not yet picked up
+    #: by a worker. A row that sits here past the dispatch timeout is reclaimed by
+    #: the sweep — that is what makes a dispatcher crash survivable.
+    DISPATCHING = "dispatching"
+    PROCESSING = "processing"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    DEAD = "dead"
+
+    @property
+    def is_terminal(self) -> bool:
+        """Terminal for auto-processing — sweep and process_task skip these.
+
+        Note: DEAD is terminal but allows manual retry (DEAD → PENDING).
+        COMPLETED is fully terminal — no transitions out.
+        """
+        return self in _TERMINAL_STATES
+
+    def can_transition_to(self, target: "TaskStatus") -> bool:
+        """Check if transitioning to ``target`` is allowed by the state machine."""
+        return target in _ALLOWED_TRANSITIONS.get(self, set())
+
+
+# States where automatic processing won't pick up the task.
+_TERMINAL_STATES = {TaskStatus.COMPLETED, TaskStatus.DEAD}
+
+_ALLOWED_TRANSITIONS: dict[TaskStatus, set[TaskStatus]] = {
+    TaskStatus.PENDING: {
+        TaskStatus.DISPATCHING,  # a dispatcher claimed it
+        TaskStatus.PROCESSING,  # in-process execution, with no broker in the path
+        TaskStatus.DEAD,  # manual kill
+    },
+    TaskStatus.DISPATCHING: {
+        TaskStatus.PROCESSING,  # a worker picked it up off the transport
+        TaskStatus.PENDING,  # dispatch failed, or the dispatch timeout swept it
+        TaskStatus.DEAD,  # manual kill, or attempts exhausted while stuck
+    },
+    TaskStatus.PROCESSING: {
+        TaskStatus.COMPLETED,
+        TaskStatus.FAILED,
+        TaskStatus.DEAD,
+        TaskStatus.PENDING,  # sweep_stuck resets abandoned tasks
+    },
+    TaskStatus.FAILED: {TaskStatus.PENDING, TaskStatus.DEAD},
+    TaskStatus.DEAD: {TaskStatus.PENDING},  # manual retry only
+    # COMPLETED is fully terminal — no transitions out.
+}
+
+
+def should_retry(attempts: int, max_attempts: int) -> bool:
+    """Should the task be retried (FAILED → PENDING)?"""
+    return attempts < max_attempts
+
+
+def should_die(attempts: int, max_attempts: int) -> bool:
+    """Should the task be dead-lettered (FAILED → DEAD)?"""
+    return attempts >= max_attempts