pgdispatch 0.1.0__py3-none-any.whl

pgdispatch/config.py

@@ -0,0 +1,223 @@
+"""Library configuration.
+
+All settings live in one immutable :class:`WebhooksConfig`. Values come from, in
+order of precedence: keyword arguments, environment variables prefixed with
+``PGDISPATCH_`` (``PGDISPATCH_CONCURRENCY=100``), and the defaults below. Django
+projects can also use ``settings.PGDISPATCH``.
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import timedelta
+from typing import Any
+
+from pydantic import Field, SecretStr, field_validator, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from pgdispatch._version import __version__
+from pgdispatch.core.retry import DEFAULT_RETRY_SCHEDULE
+
+_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]{0,62}$")
+_NUMBER = re.compile(r"^\d+(\.\d+)?$")
+_DURATION_PART = re.compile(r"(\d+)(ms|s|m|h|d|w)")
+_DURATION_UNITS = {
+    "ms": "milliseconds",
+    "s": "seconds",
+    "m": "minutes",
+    "h": "hours",
+    "d": "days",
+    "w": "weeks",
+}
+_DURATION_FIELDS = (
+    "cleanup_interval",
+    "poll_interval",
+    "lease_duration",
+    "heartbeat_interval",
+    "shutdown_timeout",
+    "request_timeout",
+    "circuit_cooldown",
+    "circuit_max_cooldown",
+    "auto_disable_after",
+)
+
+
+def parse_duration(value: str) -> timedelta:
+    """Parse a duration: ``"30"`` and ``"0.5"`` are seconds; units: ``ms s m h d w``.
+
+    Examples: ``"250ms"``, ``"30s"``, ``"5m"``, ``"1h30m"``, ``"14d"``.
+    """
+    text = value.strip()
+    if _NUMBER.match(text):
+        return timedelta(seconds=float(text))
+    parts = _DURATION_PART.findall(text)
+    if not parts or "".join(number + unit for number, unit in parts) != text:
+        msg = f"invalid duration {value!r}; use seconds or e.g. 500ms, 30s, 5m, 1h30m, 14d"
+        raise ValueError(msg)
+    return sum((timedelta(**{_DURATION_UNITS[unit]: int(n)}) for n, unit in parts), timedelta())
+
+
+def _coerce_duration(value: Any) -> Any:
+    # ISO 8601 strings ("PT5M") are left to pydantic.
+    if isinstance(value, str) and not value.strip().upper().startswith("P"):
+        return parse_duration(value)
+    return value
+
+
+class WebhooksConfig(BaseSettings):
+    """Tunable behaviour of publishing, delivery and housekeeping.
+
+    The defaults are production-ready; most applications never change them.
+    Durations accept :class:`~datetime.timedelta`, a number of seconds, a
+    short string such as ``"500ms"``, ``"30s"``, ``"5m"``, ``"1h30m"`` or
+    ``"14d"``, or an ISO 8601 duration (``"PT5M"``).
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="PGDISPATCH_",
+        frozen=True,
+        extra="forbid",
+    )
+
+    # --- storage -------------------------------------------------------------
+    db_schema: str = Field(
+        default="pgdispatch",
+        description="PostgreSQL schema that holds all pgdispatch tables.",
+    )
+    retention_days: int = Field(
+        default=14,
+        ge=1,
+        description="How long events, deliveries and attempts are kept.",
+    )
+    cleanup_interval: timedelta = Field(
+        default=timedelta(minutes=10),
+        description="How often a worker purges data older than the retention period.",
+    )
+    cleanup_batch_size: int = Field(default=5_000, ge=1)
+
+    # --- publishing ----------------------------------------------------------
+    max_payload_bytes: int = Field(
+        default=256 * 1024,
+        ge=1,
+        description="Upper bound for the JSON-serialized event payload.",
+    )
+
+    # --- worker --------------------------------------------------------------
+    concurrency: int = Field(default=50, ge=1, description="Simultaneous HTTP requests per worker.")
+    endpoint_concurrency: int = Field(
+        default=10,
+        ge=1,
+        description="Simultaneous HTTP requests per worker to a single endpoint.",
+    )
+    batch_size: int = Field(default=100, ge=1, description="Deliveries claimed per database query.")
+    poll_interval: timedelta = Field(
+        default=timedelta(seconds=1),
+        description="Fallback polling period in case a NOTIFY wake-up is missed.",
+    )
+    lease_duration: timedelta = Field(
+        default=timedelta(seconds=30),
+        description="How long a claimed delivery stays reserved for a worker.",
+    )
+    heartbeat_interval: timedelta = Field(default=timedelta(seconds=5))
+    shutdown_timeout: timedelta = Field(
+        default=timedelta(seconds=25),
+        description="Grace period for in-flight requests when the worker stops.",
+    )
+
+    # --- HTTP ----------------------------------------------------------------
+    request_timeout: timedelta = Field(default=timedelta(seconds=15))
+    user_agent: str = Field(default=f"pgdispatch/{__version__}")
+    store_response_body: bool = Field(default=True)
+    max_response_body_bytes: int = Field(default=4096, ge=0)
+    outbound_proxy: str | None = Field(
+        default=None,
+        description="Send all requests through this proxy; SSRF filtering is then the proxy's job.",
+    )
+
+    # --- retries -------------------------------------------------------------
+    retry_schedule: tuple[timedelta, ...] = Field(
+        default=DEFAULT_RETRY_SCHEDULE,
+        min_length=1,
+        description="Delay before each attempt; its length is the maximum number of attempts.",
+    )
+    retry_jitter: float = Field(default=0.2, ge=0.0, lt=1.0)
+
+    # --- circuit breaker -----------------------------------------------------
+    failure_threshold: int = Field(default=10, ge=1)
+    circuit_cooldown: timedelta = Field(default=timedelta(minutes=5))
+    circuit_max_cooldown: timedelta = Field(default=timedelta(hours=1))
+    auto_disable_after: timedelta | None = Field(
+        default=timedelta(days=5),
+        description="Disable an endpoint that has been failing continuously for this long.",
+    )
+
+    # --- security ------------------------------------------------------------
+    allow_http: bool = Field(default=False, description="Allow plain-HTTP endpoints (development).")
+    allow_private_ips: bool = Field(
+        default=False,
+        description="Disable SSRF protection. Never enable this in production.",
+    )
+    secret_encryption_keys: tuple[SecretStr, ...] = Field(
+        default=(),
+        description="Fernet keys used to encrypt signing secrets at rest; the first one encrypts.",
+    )
+
+    @field_validator(*_DURATION_FIELDS, mode="before")
+    @classmethod
+    def _parse_durations(cls, value: Any) -> Any:
+        return _coerce_duration(value)
+
+    @field_validator("retry_schedule", mode="before")
+    @classmethod
+    def _parse_schedule(cls, value: Any) -> Any:
+        if isinstance(value, (list, tuple)):
+            return [_coerce_duration(item) for item in value]
+        return value
+
+    @field_validator("db_schema")
+    @classmethod
+    def _check_schema(cls, value: str) -> str:
+        if not _IDENTIFIER.match(value):
+            msg = "db_schema must be a lowercase SQL identifier, e.g. 'pgdispatch'"
+            raise ValueError(msg)
+        return value
+
+    @field_validator("retry_schedule")
+    @classmethod
+    def _check_retry_schedule(cls, value: tuple[timedelta, ...]) -> tuple[timedelta, ...]:
+        if any(delay < timedelta(0) for delay in value):
+            msg = "retry_schedule delays must not be negative"
+            raise ValueError(msg)
+        return value
+
+    @model_validator(mode="after")
+    def _check_timeouts(self) -> WebhooksConfig:
+        positive = {
+            "poll_interval": self.poll_interval,
+            "lease_duration": self.lease_duration,
+            "heartbeat_interval": self.heartbeat_interval,
+            "request_timeout": self.request_timeout,
+            "circuit_cooldown": self.circuit_cooldown,
+            "cleanup_interval": self.cleanup_interval,
+        }
+        for name, value in positive.items():
+            if value <= timedelta(0):
+                msg = f"{name} must be positive"
+                raise ValueError(msg)
+        if self.lease_duration <= self.request_timeout:
+            msg = "lease_duration must be longer than request_timeout"
+            raise ValueError(msg)
+        if self.circuit_max_cooldown < self.circuit_cooldown:
+            msg = "circuit_max_cooldown must not be shorter than circuit_cooldown"
+            raise ValueError(msg)
+        return self
+
+    @property
+    def notify_channel(self) -> str:
+        """``LISTEN``/``NOTIFY`` channel used to wake workers up."""
+        return f"{self.db_schema}_new"
+
+    @property
+    def max_attempts(self) -> int:
+        """Number of attempts before a delivery is marked as failed."""
+        return len(self.retry_schedule)

pgdispatch/core/__init__.py

@@ -0,0 +1 @@
+"""Framework-independent logic: matching, signing, retries, circuit breaking and SQL operations."""

pgdispatch/core/circuit.py

@@ -0,0 +1,94 @@
+"""Per-endpoint circuit breaker.
+
+The breaker protects the worker (and the receiver) from hammering an endpoint
+that is down::
+
+    closed ──(failure_threshold failures in a row)──▶ open
+    open ──(cooldown elapsed, a worker claims a probe)──▶ half_open
+    half_open ──(probe succeeds)──▶ closed
+    half_open ──(probe fails)──▶ open, with a longer cooldown
+
+While a breaker is open its deliveries are not claimed at all, so they keep
+their remaining attempts. The cooldown doubles with every consecutive trip, up
+to ``max_cooldown``. Independently of the breaker, an endpoint that has been
+failing continuously for ``auto_disable_after`` is disabled.
+
+This module is pure: it computes the next state, and the worker persists it.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, replace
+from datetime import datetime, timedelta
+
+from pgdispatch.models import CircuitState
+
+__all__ = ["BreakerPolicy", "BreakerState", "BreakerUpdate", "record_failure", "record_success"]
+
+
+@dataclass(frozen=True, slots=True)
+class BreakerPolicy:
+    failure_threshold: int
+    cooldown: timedelta
+    max_cooldown: timedelta
+    auto_disable_after: timedelta | None
+
+    def cooldown_for(self, trips: int) -> timedelta:
+        """Cooldown after the ``trips``-th consecutive opening (1-based)."""
+        # Cap the exponent: the cooldown hits max_cooldown long before 2**20.
+        factor: int = 2 ** min(trips - 1, 20)
+        return min(self.cooldown * factor, self.max_cooldown)
+
+
+@dataclass(frozen=True, slots=True)
+class BreakerState:
+    state: CircuitState = CircuitState.CLOSED
+    consecutive_failures: int = 0
+    failing_since: datetime | None = None
+    opened_at: datetime | None = None
+    retry_at: datetime | None = None
+    trips: int = 0
+
+
+@dataclass(frozen=True, slots=True)
+class BreakerUpdate:
+    state: BreakerState
+    opened: bool = False
+    """The breaker transitioned to ``open`` with this result."""
+    disable: bool = False
+    """The endpoint has been failing for longer than ``auto_disable_after``."""
+
+
+def record_success(state: BreakerState) -> BreakerUpdate:
+    """Any successful delivery proves the endpoint works and closes the breaker."""
+    del state  # the outcome does not depend on the previous state
+    return BreakerUpdate(BreakerState())
+
+
+def record_failure(state: BreakerState, *, now: datetime, policy: BreakerPolicy) -> BreakerUpdate:
+    """Account for a failed attempt."""
+    failures = state.consecutive_failures + 1
+    failing_since = state.failing_since or now
+    disable = (
+        policy.auto_disable_after is not None and now - failing_since >= policy.auto_disable_after
+    )
+
+    should_open = state.state is CircuitState.HALF_OPEN or (
+        state.state is CircuitState.CLOSED and failures >= policy.failure_threshold
+    )
+    if not should_open:
+        # Closed below the threshold, or already open (a request that was in
+        # flight when the breaker opened): keep the current schedule.
+        next_state = replace(state, consecutive_failures=failures, failing_since=failing_since)
+        return BreakerUpdate(next_state, disable=disable)
+
+    trips = state.trips + 1
+    next_state = BreakerState(
+        state=CircuitState.OPEN,
+        consecutive_failures=failures,
+        failing_since=failing_since,
+        opened_at=now,
+        retry_at=now + policy.cooldown_for(trips),
+        trips=trips,
+    )
+    return BreakerUpdate(next_state, opened=True, disable=disable)

pgdispatch/core/common.py

@@ -0,0 +1,87 @@
+"""Building blocks shared by the database operations."""
+
+from __future__ import annotations
+
+import enum
+import uuid
+from typing import Any, Final
+
+from pgdispatch.backends.base import Row
+from pgdispatch.errors import NotFoundError, ValidationError
+
+__all__ = [
+    "DELIVERY_SELECT",
+    "ENDPOINT_COLUMNS",
+    "UNSET",
+    "OwnerFilter",
+    "Unset",
+    "first_or_not_found",
+    "owner_condition",
+    "validate_key",
+    "validate_limit",
+]
+
+MAX_KEY_LENGTH = 255
+MAX_PAGE_SIZE = 1000
+
+
+class Unset(enum.Enum):
+    """Marker for "argument not given", distinct from an explicit ``None``."""
+
+    UNSET = "UNSET"
+
+    def __repr__(self) -> str:
+        return "UNSET"
+
+
+UNSET: Final = Unset.UNSET
+
+OwnerFilter = str | None | Unset
+"""``UNSET`` (default) ignores ownership; ``None`` or a string restricts results to that owner."""
+
+
+ENDPOINT_COLUMNS = """
+    e.id, e.owner_id, e.url, e.description, e.event_types, e.headers::text AS headers,
+    e.rate_limit_rps, e.is_enabled, e.disabled_reason, e.circuit_state,
+    e.consecutive_failures, e.failing_since, e.created_at, e.updated_at
+"""
+
+DELIVERY_SELECT = """
+    SELECT d.id, d.event_id, d.endpoint_id, ev.event_type, d.ordering_key, d.status,
+           d.attempt_count, d.next_attempt_at, d.last_status_code, d.last_error,
+           d.created_at, d.finished_at
+    FROM {schema}.deliveries AS d
+    JOIN {schema}.events AS ev ON ev.id = d.event_id
+    JOIN {schema}.endpoints AS e ON e.id = d.endpoint_id
+"""
+
+
+def owner_condition(column: str, owner_id: OwnerFilter, params: dict[str, Any]) -> str:
+    """SQL condition restricting ``column`` to ``owner_id``; adds the bound parameter."""
+    if owner_id is UNSET:
+        return "TRUE"
+    if owner_id is None:
+        return f"{column} IS NULL"
+    params["owner_id"] = validate_key(owner_id, "owner_id")
+    return f"{column} = CAST(:owner_id AS text)"
+
+
+def validate_key(value: str, name: str) -> str:
+    if not isinstance(value, str) or not value or len(value) > MAX_KEY_LENGTH:
+        msg = f"{name} must be a non-empty string of at most {MAX_KEY_LENGTH} characters"
+        raise ValidationError(msg)
+    return value
+
+
+def validate_limit(limit: int) -> int:
+    if not 1 <= limit <= MAX_PAGE_SIZE:
+        msg = f"limit must be between 1 and {MAX_PAGE_SIZE}"
+        raise ValidationError(msg)
+    return limit
+
+
+def first_or_not_found(rows: list[Row], what: str, identifier: uuid.UUID | int | str) -> Row:
+    if not rows:
+        msg = f"{what} {identifier} not found"
+        raise NotFoundError(msg)
+    return rows[0]

pgdispatch/core/deliveries.py

@@ -0,0 +1,216 @@
+"""Inspecting and retrying deliveries."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+from typing import Any
+
+from pgdispatch.backends.base import Operation, Query
+from pgdispatch.core.common import (
+    DELIVERY_SELECT,
+    OwnerFilter,
+    first_or_not_found,
+    owner_condition,
+    validate_limit,
+)
+from pgdispatch.errors import ConflictError, ValidationError
+from pgdispatch.models import Attempt, Delivery, DeliveryStatus
+
+__all__ = [
+    "cancel_delivery",
+    "get_delivery",
+    "list_attempts",
+    "list_deliveries",
+    "retry_deliveries",
+    "retry_delivery",
+]
+
+
+def list_deliveries(
+    *,
+    endpoint_id: uuid.UUID | None,
+    event_id: uuid.UUID | None,
+    status: DeliveryStatus | str | None,
+    since: datetime | None,
+    before: int | None,
+    limit: int,
+    owner_id: OwnerFilter,
+) -> Operation[list[Delivery]]:
+    """Newest deliveries first; pass the last id as ``before`` to get the next page."""
+    params: dict[str, Any] = {"limit": validate_limit(limit)}
+    conditions = [owner_condition("e.owner_id", owner_id, params)]
+    conditions.extend(_filters(params, endpoint_id, event_id, status, since))
+    if before is not None:
+        params["before"] = before
+        conditions.append("d.id < CAST(:before AS bigint)")
+    rows = yield Query(
+        f"""
+        {DELIVERY_SELECT}
+        WHERE {" AND ".join(conditions)}
+        ORDER BY d.id DESC
+        LIMIT CAST(:limit AS integer)
+        """,
+        params,
+    )
+    return [Delivery.from_row(row) for row in rows]
+
+
+def get_delivery(delivery_id: int, *, owner_id: OwnerFilter) -> Operation[Delivery]:
+    params: dict[str, Any] = {"id": delivery_id}
+    owner_sql = owner_condition("e.owner_id", owner_id, params)
+    rows = yield Query(
+        f"{DELIVERY_SELECT} WHERE d.id = CAST(:id AS bigint) AND {owner_sql}",
+        params,
+    )
+    return Delivery.from_row(first_or_not_found(rows, "delivery", delivery_id))
+
+
+def list_attempts(delivery_id: int, *, owner_id: OwnerFilter) -> Operation[list[Attempt]]:
+    """All attempts of a delivery, oldest first."""
+    yield from get_delivery(delivery_id, owner_id=owner_id)  # ownership check
+    rows = yield Query(
+        """
+        SELECT a.id, a.delivery_id, a.attempt_number, a.request_headers::text AS request_headers,
+               a.response_status, a.response_body, a.duration_ms, a.error, a.created_at
+        FROM {schema}.attempts AS a
+        WHERE a.delivery_id = CAST(:id AS bigint)
+        ORDER BY a.id
+        """,
+        {"id": delivery_id},
+    )
+    return [Attempt.from_row(row) for row in rows]
+
+
+# A retry restarts the schedule: ``schedule_offset`` marks where the new round begins.
+_RETRY_ASSIGNMENTS = """
+    status = 'pending', next_attempt_at = now(), locked_until = NULL,
+    schedule_offset = d.attempt_count, finished_at = NULL, updated_at = now()
+"""
+
+
+def retry_delivery(delivery_id: int, *, owner_id: OwnerFilter, channel: str) -> Operation[Delivery]:
+    """Schedule a delivery for an immediate attempt with a fresh retry schedule.
+
+    Works for failed, cancelled and even succeeded deliveries (to resend).
+    A delivery that a worker is sending right now cannot be retried.
+    """
+    params: dict[str, Any] = {"id": delivery_id, "channel": channel}
+    owner_sql = owner_condition("e.owner_id", owner_id, params)
+    rows = yield Query(
+        f"""
+        WITH target AS (
+            SELECT d.id, d.status FROM {{schema}}.deliveries AS d
+            JOIN {{schema}}.endpoints AS e ON e.id = d.endpoint_id
+            WHERE d.id = CAST(:id AS bigint) AND {owner_sql}
+            FOR UPDATE OF d
+        ), updated AS (
+            UPDATE {{schema}}.deliveries AS d SET {_RETRY_ASSIGNMENTS}
+            FROM target
+            WHERE d.id = target.id AND target.status <> 'in_flight'
+            RETURNING d.id
+        ), wake AS (
+            SELECT pg_notify(CAST(:channel AS text), '') FROM updated
+        )
+        SELECT target.status, (SELECT count(*) FROM updated) AS updated,
+               (SELECT count(*) FROM wake) AS notified
+        FROM target
+        """,
+        params,
+    )
+    row = first_or_not_found(rows, "delivery", delivery_id)
+    if not row["updated"]:
+        msg = f"delivery {delivery_id} is being sent right now and cannot be retried"
+        raise ConflictError(msg)
+    return (yield from get_delivery(delivery_id, owner_id=owner_id))
+
+
+def retry_deliveries(
+    *,
+    endpoint_id: uuid.UUID | None,
+    status: DeliveryStatus | str,
+    since: datetime | None,
+    owner_id: OwnerFilter,
+    channel: str,
+) -> Operation[int]:
+    """Retry every matching delivery (``failed`` by default); return how many were scheduled."""
+    params: dict[str, Any] = {"channel": channel}
+    conditions = [owner_condition("e.owner_id", owner_id, params)]
+    conditions.extend(_filters(params, endpoint_id, None, status, since))
+    if DeliveryStatus(status) is DeliveryStatus.IN_FLIGHT:
+        msg = "deliveries that are in flight cannot be retried"
+        raise ValidationError(msg)
+    rows = yield Query(
+        f"""
+        WITH updated AS (
+            UPDATE {{schema}}.deliveries AS d SET {_RETRY_ASSIGNMENTS}
+            FROM {{schema}}.endpoints AS e
+            WHERE e.id = d.endpoint_id AND {" AND ".join(conditions)}
+            RETURNING d.id
+        ), wake AS (
+            SELECT pg_notify(CAST(:channel AS text), '') FROM updated LIMIT 1
+        )
+        SELECT (SELECT count(*) FROM updated) AS updated, (SELECT count(*) FROM wake) AS notified
+        """,
+        params,
+    )
+    count: int = rows[0]["updated"]
+    return count
+
+
+def cancel_delivery(delivery_id: int, *, owner_id: OwnerFilter) -> Operation[Delivery]:
+    """Give up on a pending delivery."""
+    params: dict[str, Any] = {"id": delivery_id}
+    owner_sql = owner_condition("e.owner_id", owner_id, params)
+    rows = yield Query(
+        f"""
+        WITH target AS (
+            SELECT d.id, d.status FROM {{schema}}.deliveries AS d
+            JOIN {{schema}}.endpoints AS e ON e.id = d.endpoint_id
+            WHERE d.id = CAST(:id AS bigint) AND {owner_sql}
+            FOR UPDATE OF d
+        ), updated AS (
+            UPDATE {{schema}}.deliveries AS d
+            SET status = 'cancelled', finished_at = now(), locked_until = NULL,
+                last_error = coalesce(d.last_error, 'cancelled manually'), updated_at = now()
+            FROM target
+            WHERE d.id = target.id AND target.status = 'pending'
+            RETURNING d.id
+        )
+        SELECT target.status, (SELECT count(*) FROM updated) AS updated FROM target
+        """,
+        params,
+    )
+    row = first_or_not_found(rows, "delivery", delivery_id)
+    if not row["updated"]:
+        msg = f"only pending deliveries can be cancelled; delivery {delivery_id} is {row['status']}"
+        raise ConflictError(msg)
+    return (yield from get_delivery(delivery_id, owner_id=owner_id))
+
+
+def _filters(
+    params: dict[str, Any],
+    endpoint_id: uuid.UUID | None,
+    event_id: uuid.UUID | None,
+    status: DeliveryStatus | str | None,
+    since: datetime | None,
+) -> list[str]:
+    conditions = []
+    if endpoint_id is not None:
+        params["endpoint_id"] = str(endpoint_id)
+        conditions.append("d.endpoint_id = CAST(:endpoint_id AS uuid)")
+    if event_id is not None:
+        params["event_id"] = str(event_id)
+        conditions.append("d.event_id = CAST(:event_id AS uuid)")
+    if status is not None:
+        try:
+            params["status"] = DeliveryStatus(status).value
+        except ValueError:
+            allowed = ", ".join(member.value for member in DeliveryStatus)
+            msg = f"invalid status {status!r}; expected one of: {allowed}"
+            raise ValidationError(msg) from None
+        conditions.append("d.status = CAST(:status AS text)")
+    if since is not None:
+        params["since"] = since
+        conditions.append("d.created_at >= CAST(:since AS timestamptz)")
+    return conditions