pgdispatch 0.1.0__py3-none-any.whl

pgdispatch/core/signing.py

@@ -0,0 +1,141 @@
+"""Request signing according to the `Standard Webhooks`_ specification.
+
+Every request carries three headers:
+
+``webhook-id``
+    The event id. It is identical across retries, so receivers can deduplicate.
+``webhook-timestamp``
+    Unix time of this attempt, in seconds.
+``webhook-signature``
+    ``v1,<base64 HMAC-SHA256(secret, "{id}.{timestamp}.{body}")>``. During a
+    secret rotation several space-separated signatures are sent, one per
+    active secret, and a receiver accepts the request if any of them matches.
+
+The implementation is a few lines of standard-library code; the test-suite
+checks it against the reference ``standardwebhooks`` package, so receivers can
+use any official Standard Webhooks SDK.
+
+.. _Standard Webhooks: https://www.standardwebhooks.com/
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import hashlib
+import hmac
+import secrets
+import time
+from collections.abc import Iterable, Mapping
+
+from pgdispatch.errors import InvalidSignatureError, ValidationError
+
+__all__ = [
+    "SECRET_PREFIX",
+    "generate_secret",
+    "sign",
+    "signature_header",
+    "validate_secret",
+    "verify",
+]
+
+SECRET_PREFIX = "whsec_"  # noqa: S105 - a prefix, not a password
+SIGNATURE_VERSION = "v1"
+DEFAULT_TOLERANCE_SECONDS = 5 * 60
+
+_MIN_SECRET_BYTES = 24
+_MAX_SECRET_BYTES = 64
+
+
+def generate_secret(num_bytes: int = 32) -> str:
+    """Create a new random signing secret such as ``whsec_MfKQ9r8GKYqrTwjUPD8ILPZIo2LaLaSw``."""
+    if not _MIN_SECRET_BYTES <= num_bytes <= _MAX_SECRET_BYTES:
+        msg = f"secrets must be {_MIN_SECRET_BYTES}-{_MAX_SECRET_BYTES} bytes long"
+        raise ValueError(msg)
+    return SECRET_PREFIX + base64.b64encode(secrets.token_bytes(num_bytes)).decode("ascii")
+
+
+def validate_secret(secret: str) -> str:
+    """Return ``secret`` if it is a well-formed ``whsec_`` secret.
+
+    :raises ValidationError: if it is not.
+    """
+    try:
+        key = _decode_secret(secret)
+    except ValueError:
+        key = b""
+    if (
+        not secret.startswith(SECRET_PREFIX)
+        or not _MIN_SECRET_BYTES <= len(key) <= _MAX_SECRET_BYTES
+    ):
+        msg = (
+            f"a signing secret must look like '{SECRET_PREFIX}<base64>' and encode "
+            f"{_MIN_SECRET_BYTES}-{_MAX_SECRET_BYTES} random bytes"
+        )
+        raise ValidationError(msg)
+    return secret
+
+
+def sign(secret: str, msg_id: str, timestamp: int, body: bytes) -> str:
+    """Return a single ``v1,<signature>`` value for ``body``."""
+    signed_content = f"{msg_id}.{timestamp}.".encode() + body
+    digest = hmac.new(_decode_secret(secret), signed_content, hashlib.sha256).digest()
+    return f"{SIGNATURE_VERSION},{base64.b64encode(digest).decode('ascii')}"
+
+
+def signature_header(secrets_: Iterable[str], msg_id: str, timestamp: int, body: bytes) -> str:
+    """Build the ``webhook-signature`` header value, one signature per active secret."""
+    return " ".join(sign(secret, msg_id, timestamp, body) for secret in secrets_)
+
+
+def verify(
+    secret: str,
+    headers: Mapping[str, str],
+    body: bytes,
+    *,
+    tolerance: int = DEFAULT_TOLERANCE_SECONDS,
+    now: float | None = None,
+) -> None:
+    """Check a received webhook; raise :class:`InvalidSignatureError` if it is not authentic.
+
+    Useful in tests and in Python receivers::
+
+        from pgdispatch import verify_signature
+
+        verify_signature(secret, request.headers, await request.body())
+    """
+    lowered = {name.lower(): value for name, value in headers.items()}
+    try:
+        msg_id = lowered["webhook-id"]
+        raw_timestamp = lowered["webhook-timestamp"]
+        received = lowered["webhook-signature"]
+    except KeyError as exc:
+        msg = f"missing required header {exc.args[0]!r}"
+        raise InvalidSignatureError(msg) from None
+
+    try:
+        timestamp = int(raw_timestamp)
+    except ValueError:
+        msg = "invalid webhook-timestamp header"
+        raise InvalidSignatureError(msg) from None
+    current = time.time() if now is None else now
+    if abs(current - timestamp) > tolerance:
+        msg = "webhook timestamp is outside the allowed tolerance"
+        raise InvalidSignatureError(msg)
+
+    expected = sign(secret, msg_id, timestamp, body).encode()
+    for candidate in received.split():
+        version, _, _ = candidate.partition(",")
+        if version == SIGNATURE_VERSION and hmac.compare_digest(candidate.encode(), expected):
+            return
+    msg = "no matching signature found"
+    raise InvalidSignatureError(msg)
+
+
+def _decode_secret(secret: str) -> bytes:
+    encoded = secret.removeprefix(SECRET_PREFIX)
+    try:
+        return base64.b64decode(encoded, validate=True)
+    except binascii.Error as exc:
+        msg = "secret is not valid base64"
+        raise ValueError(msg) from exc

pgdispatch/django/__init__.py

@@ -0,0 +1,37 @@
+"""Django integration.
+
+Add the app and publish events inside ``transaction.atomic()``::
+
+    INSTALLED_APPS = [..., "pgdispatch.django"]
+    PGDISPATCH = {"concurrency": 50}  # optional, any WebhooksConfig field
+
+    from pgdispatch.django import webhooks
+
+    with transaction.atomic():
+        order.save()
+        webhooks.publish("order.shipped", {"id": order.id}, owner_id=order.shop_id)
+
+``python manage.py migrate`` creates the tables and ``python manage.py
+pgdispatch_worker`` delivers the webhooks.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from pgdispatch.django.client import DjangoWebhooks
+
+    webhooks: DjangoWebhooks
+
+__all__ = ["webhooks"]
+
+
+def __getattr__(name: str) -> Any:
+    # Lazy, so that importing this package does not require configured settings.
+    if name == "webhooks":
+        from pgdispatch.django.client import get_webhooks
+
+        return get_webhooks()
+    msg = f"module {__name__!r} has no attribute {name!r}"
+    raise AttributeError(msg)

pgdispatch/django/apps.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from django.apps import AppConfig
+
+
+class PgDispatchConfig(AppConfig):
+    name = "pgdispatch.django"
+    label = "pgdispatch"
+    verbose_name = "pgdispatch webhooks"
+    default_auto_field = "django.db.models.BigAutoField"

pgdispatch/django/client.py

@@ -0,0 +1,402 @@
+"""The synchronous Django facade: ``pgdispatch.django.webhooks``."""
+
+from __future__ import annotations
+
+import builtins
+import uuid
+from collections.abc import Iterable, Mapping
+from datetime import datetime, timedelta
+from functools import cache
+from typing import Any, TypeVar
+
+import asyncpg
+from django.conf import settings
+from django.db import connections, transaction
+from django.db.backends.base.base import BaseDatabaseWrapper
+
+from pgdispatch.backends.asyncpg import WORKER_POOL_SIZE
+from pgdispatch.backends.base import Operation, Row, run_sync, to_pyformat_style
+from pgdispatch.config import WebhooksConfig
+from pgdispatch.core import deliveries as delivery_ops
+from pgdispatch.core import endpoints as endpoint_ops
+from pgdispatch.core import events as event_ops
+from pgdispatch.core import publisher
+from pgdispatch.core.common import UNSET, OwnerFilter, Unset
+from pgdispatch.core.maintenance import QueueStats, queue_stats
+from pgdispatch.core.secrets import SecretStore, fernet_cipher
+from pgdispatch.errors import ConfigurationError
+from pgdispatch.hooks import Hooks
+from pgdispatch.metrics import get_metrics
+from pgdispatch.migrations import drop_schema as drop_schema_operation
+from pgdispatch.migrations import migrate
+from pgdispatch.models import Attempt, Delivery, DeliveryStatus, Endpoint, Event, PublishResult
+from pgdispatch.worker.worker import Worker
+
+__all__ = ["DjangoWebhooks", "drop_schema", "get_webhooks", "migrate_schema"]
+
+T = TypeVar("T")
+
+
+class DjangoExecutor:
+    """Runs statements through a Django database connection."""
+
+    def __init__(self, connection: BaseDatabaseWrapper) -> None:
+        if connection.vendor != "postgresql":
+            msg = (
+                f"pgdispatch requires PostgreSQL, "
+                f"but database {connection.alias!r} is {connection.vendor}"
+            )
+            raise ConfigurationError(msg)
+        self._connection = connection
+
+    def fetch(self, sql: str, params: Mapping[str, Any]) -> list[Row]:
+        with self._connection.cursor() as cursor:
+            cursor.execute(to_pyformat_style(sql), dict(params))
+            if cursor.description is None:
+                return []
+            names = [column[0] for column in cursor.description]
+            return [dict(zip(names, row, strict=True)) for row in cursor.fetchall()]
+
+
+class DjangoWebhooks:
+    """Blocking counterpart of :class:`pgdispatch.Webhooks` for Django projects.
+
+    Every call runs in ``transaction.atomic(using=...)``, which becomes a
+    savepoint when you are already inside a transaction, so ``publish`` joins
+    your business transaction.
+    """
+
+    def __init__(self, config: WebhooksConfig | None = None, *, using: str = "default") -> None:
+        self.config = config or WebhooksConfig()
+        self.using = using
+        self.hooks = Hooks()
+        keys = [key.get_secret_value() for key in self.config.secret_encryption_keys]
+        self._secrets = SecretStore(fernet_cipher(keys) if keys else None)
+        self._metrics = get_metrics()
+        self._url_policy = endpoint_ops.UrlPolicy(
+            allow_http=self.config.allow_http, allow_private_ips=self.config.allow_private_ips
+        )
+        self.endpoints = DjangoEndpoints(self)
+        self.deliveries = DjangoDeliveries(self)
+        self.events = DjangoEvents(self)
+
+    def publish(
+        self,
+        event_type: str,
+        payload: Any,
+        *,
+        owner_id: str | None = None,
+        ordering_key: str | None = None,
+        idempotency_key: str | None = None,
+        using: str | None = None,
+    ) -> PublishResult:
+        """Record an event in the current transaction; see :meth:`pgdispatch.Webhooks.publish`."""
+        result = self._run(
+            publisher.publish(
+                event_type=event_type,
+                payload=payload,
+                owner_id=owner_id,
+                ordering_key=ordering_key,
+                idempotency_key=idempotency_key,
+                channel=self.config.notify_channel,
+                max_payload_bytes=self.config.max_payload_bytes,
+                initial_delay=self.config.retry_schedule[0],
+            ),
+            using=using,
+        )
+        if not result.duplicate:
+            self._metrics.events_published.labels(event_type).inc()
+        return result
+
+    def status(self) -> QueueStats:
+        return self._run(queue_stats(worker_ttl=self.config.heartbeat_interval * 3))
+
+    def worker(self, **overrides: Any) -> Worker:
+        """A :class:`Worker` connected to the same database (used by ``pgdispatch_worker``)."""
+        config = (
+            WebhooksConfig(**{**self.config.model_dump(), **overrides})
+            if overrides
+            else self.config
+        )
+        connect_kwargs = asyncpg_connect_kwargs(settings.DATABASES[self.using])
+
+        async def create_pool() -> asyncpg.Pool:
+            pool: asyncpg.Pool = await asyncpg.create_pool(
+                min_size=1, max_size=WORKER_POOL_SIZE, **connect_kwargs
+            )
+            return pool
+
+        return Worker(
+            create_pool, config, hooks=self.hooks, metrics=self._metrics, secret_store=self._secrets
+        )
+
+    def _run(self, operation: Operation[T], *, using: str | None = None) -> T:
+        alias = using or self.using
+        with transaction.atomic(using=alias):
+            return run_sync(
+                operation, DjangoExecutor(connections[alias]), schema=self.config.db_schema
+            )
+
+
+class DjangoEndpoints:
+    def __init__(self, webhooks: DjangoWebhooks) -> None:
+        self._webhooks = webhooks
+
+    def create(
+        self,
+        url: str,
+        *,
+        event_types: Iterable[str] = ("*",),
+        owner_id: str | None = None,
+        description: str | None = None,
+        headers: Mapping[str, str] | None = None,
+        rate_limit_rps: float | None = None,
+        secret: str | None = None,
+        enabled: bool = True,
+    ) -> Endpoint:
+        w = self._webhooks
+        return w._run(
+            endpoint_ops.create_endpoint(
+                url=url,
+                event_types=event_types,
+                owner_id=owner_id,
+                description=description,
+                headers=headers,
+                rate_limit_rps=rate_limit_rps,
+                secret=secret,
+                enabled=enabled,
+                url_policy=w._url_policy,
+                secret_store=w._secrets,
+            )
+        )
+
+    def get(self, endpoint_id: uuid.UUID, *, owner_id: OwnerFilter = UNSET) -> Endpoint:
+        return self._webhooks._run(endpoint_ops.get_endpoint(endpoint_id, owner_id=owner_id))
+
+    def list(
+        self,
+        *,
+        owner_id: OwnerFilter = UNSET,
+        enabled: bool | None = None,
+        limit: int = 100,
+        after: uuid.UUID | None = None,
+    ) -> builtins.list[Endpoint]:
+        return self._webhooks._run(
+            endpoint_ops.list_endpoints(
+                owner_id=owner_id, enabled=enabled, limit=limit, after=after
+            )
+        )
+
+    def update(
+        self,
+        endpoint_id: uuid.UUID,
+        *,
+        url: str | Unset = UNSET,
+        event_types: Iterable[str] | Unset = UNSET,
+        description: str | Unset | None = UNSET,
+        headers: Mapping[str, str] | Unset = UNSET,
+        rate_limit_rps: float | Unset | None = UNSET,
+        owner_id: OwnerFilter = UNSET,
+    ) -> Endpoint:
+        w = self._webhooks
+        return w._run(
+            endpoint_ops.update_endpoint(
+                endpoint_id,
+                owner_id=owner_id,
+                url=url,
+                event_types=event_types,
+                description=description,
+                headers=headers,
+                rate_limit_rps=rate_limit_rps,
+                url_policy=w._url_policy,
+            )
+        )
+
+    def enable(self, endpoint_id: uuid.UUID, *, owner_id: OwnerFilter = UNSET) -> Endpoint:
+        return self._webhooks._run(endpoint_ops.enable_endpoint(endpoint_id, owner_id=owner_id))
+
+    def disable(
+        self, endpoint_id: uuid.UUID, *, reason: str | None = None, owner_id: OwnerFilter = UNSET
+    ) -> Endpoint:
+        return self._webhooks._run(
+            endpoint_ops.disable_endpoint(endpoint_id, reason=reason, owner_id=owner_id)
+        )
+
+    def delete(self, endpoint_id: uuid.UUID, *, owner_id: OwnerFilter = UNSET) -> None:
+        self._webhooks._run(endpoint_ops.delete_endpoint(endpoint_id, owner_id=owner_id))
+
+    def rotate_secret(
+        self,
+        endpoint_id: uuid.UUID,
+        *,
+        grace_period: timedelta = timedelta(hours=24),
+        secret: str | None = None,
+        owner_id: OwnerFilter = UNSET,
+    ) -> Endpoint:
+        w = self._webhooks
+        return w._run(
+            endpoint_ops.rotate_secret(
+                endpoint_id,
+                grace_period=grace_period,
+                secret=secret,
+                owner_id=owner_id,
+                secret_store=w._secrets,
+            )
+        )
+
+    def get_secret(self, endpoint_id: uuid.UUID, *, owner_id: OwnerFilter = UNSET) -> str:
+        w = self._webhooks
+        return w._run(
+            endpoint_ops.get_secret(endpoint_id, owner_id=owner_id, secret_store=w._secrets)
+        )
+
+    def send_test(self, endpoint_id: uuid.UUID, *, owner_id: OwnerFilter = UNSET) -> PublishResult:
+        w = self._webhooks
+        return w._run(
+            publisher.send_test(endpoint_id, owner_id=owner_id, channel=w.config.notify_channel)
+        )
+
+
+class DjangoDeliveries:
+    def __init__(self, webhooks: DjangoWebhooks) -> None:
+        self._webhooks = webhooks
+
+    def list(
+        self,
+        *,
+        endpoint_id: uuid.UUID | None = None,
+        event_id: uuid.UUID | None = None,
+        status: DeliveryStatus | str | None = None,
+        since: datetime | None = None,
+        before: int | None = None,
+        limit: int = 50,
+        owner_id: OwnerFilter = UNSET,
+    ) -> builtins.list[Delivery]:
+        return self._webhooks._run(
+            delivery_ops.list_deliveries(
+                endpoint_id=endpoint_id,
+                event_id=event_id,
+                status=status,
+                since=since,
+                before=before,
+                limit=limit,
+                owner_id=owner_id,
+            )
+        )
+
+    def get(self, delivery_id: int, *, owner_id: OwnerFilter = UNSET) -> Delivery:
+        return self._webhooks._run(delivery_ops.get_delivery(delivery_id, owner_id=owner_id))
+
+    def attempts(
+        self, delivery_id: int, *, owner_id: OwnerFilter = UNSET
+    ) -> builtins.list[Attempt]:
+        return self._webhooks._run(delivery_ops.list_attempts(delivery_id, owner_id=owner_id))
+
+    def retry(self, delivery_id: int, *, owner_id: OwnerFilter = UNSET) -> Delivery:
+        w = self._webhooks
+        return w._run(
+            delivery_ops.retry_delivery(
+                delivery_id, owner_id=owner_id, channel=w.config.notify_channel
+            )
+        )
+
+    def retry_all(
+        self,
+        *,
+        endpoint_id: uuid.UUID | None = None,
+        status: DeliveryStatus | str = DeliveryStatus.FAILED,
+        since: datetime | None = None,
+        owner_id: OwnerFilter = UNSET,
+    ) -> int:
+        w = self._webhooks
+        return w._run(
+            delivery_ops.retry_deliveries(
+                endpoint_id=endpoint_id,
+                status=status,
+                since=since,
+                owner_id=owner_id,
+                channel=w.config.notify_channel,
+            )
+        )
+
+    def cancel(self, delivery_id: int, *, owner_id: OwnerFilter = UNSET) -> Delivery:
+        return self._webhooks._run(delivery_ops.cancel_delivery(delivery_id, owner_id=owner_id))
+
+
+class DjangoEvents:
+    def __init__(self, webhooks: DjangoWebhooks) -> None:
+        self._webhooks = webhooks
+
+    def get(self, event_id: uuid.UUID, *, owner_id: OwnerFilter = UNSET) -> Event:
+        return self._webhooks._run(event_ops.get_event(event_id, owner_id=owner_id))
+
+    def list(
+        self,
+        *,
+        owner_id: OwnerFilter = UNSET,
+        event_type: str | None = None,
+        since: datetime | None = None,
+        before: uuid.UUID | None = None,
+        limit: int = 50,
+    ) -> builtins.list[Event]:
+        return self._webhooks._run(
+            event_ops.list_events(
+                owner_id=owner_id, event_type=event_type, since=since, before=before, limit=limit
+            )
+        )
+
+    def replay(
+        self,
+        event_id: uuid.UUID,
+        *,
+        endpoint_id: uuid.UUID | None = None,
+        owner_id: OwnerFilter = UNSET,
+    ) -> builtins.list[int]:
+        w = self._webhooks
+        return w._run(
+            publisher.replay(
+                event_id,
+                endpoint_id=endpoint_id,
+                owner_id=owner_id,
+                channel=w.config.notify_channel,
+            )
+        )
+
+
+@cache
+def get_webhooks() -> DjangoWebhooks:
+    """The project-wide instance, configured from ``settings.PGDISPATCH``."""
+    options = dict(getattr(settings, "PGDISPATCH", {}))
+    using = options.pop("database", "default")
+    return DjangoWebhooks(WebhooksConfig(**options), using=using)
+
+
+def migrate_schema(connection: BaseDatabaseWrapper, *, target: int | None = None) -> list[int]:
+    """Apply pgdispatch SQL migrations through a Django connection."""
+    return run_sync(
+        migrate(target=target), DjangoExecutor(connection), schema=get_webhooks().config.db_schema
+    )
+
+
+def drop_schema(connection: BaseDatabaseWrapper) -> None:
+    run_sync(
+        drop_schema_operation(), DjangoExecutor(connection), schema=get_webhooks().config.db_schema
+    )
+
+
+def asyncpg_connect_kwargs(database: Mapping[str, Any]) -> dict[str, Any]:
+    """Translate a Django ``DATABASES`` entry into asyncpg connection arguments."""
+    if "postgresql" not in database.get("ENGINE", ""):
+        msg = f"pgdispatch requires PostgreSQL, got ENGINE={database.get('ENGINE')!r}"
+        raise ConfigurationError(msg)
+    options = database.get("OPTIONS", {})
+    kwargs: dict[str, Any] = {
+        "host": database.get("HOST") or None,
+        "port": int(database["PORT"]) if database.get("PORT") else None,
+        "user": database.get("USER") or None,
+        "password": database.get("PASSWORD") or None,
+        "database": database.get("NAME") or None,
+    }
+    if options.get("sslmode"):
+        kwargs["ssl"] = options["sslmode"]
+    return {key: value for key, value in kwargs.items() if value is not None}

pgdispatch/django/management/commands/pgdispatch_worker.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandParser
+
+from pgdispatch.django.client import get_webhooks
+from pgdispatch.worker.runner import run_until_signalled
+
+
+class Command(BaseCommand):
+    help = "Deliver pgdispatch webhooks until interrupted (SIGINT/SIGTERM)."
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument("--concurrency", type=int, help="Simultaneous HTTP requests.")
+        parser.add_argument("--batch-size", type=int, help="Deliveries claimed per query.")
+        parser.add_argument(
+            "--metrics-port", type=int, help="Serve Prometheus metrics on this port."
+        )
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        if not logging.getLogger().handlers:
+            logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+        overrides = {
+            name: options[name]
+            for name in ("concurrency", "batch_size")
+            if options.get(name) is not None
+        }
+        worker = get_webhooks().worker(**overrides)
+        asyncio.run(run_until_signalled(worker, metrics_port=options.get("metrics_port")))

pgdispatch/django/migrations/0001_initial.py

@@ -0,0 +1,30 @@
+"""Create the pgdispatch schema.
+
+The tables are defined once, in pgdispatch's own SQL migrations; this Django
+migration simply runs them, so ``migrate`` and ``pgdispatch migrate`` always
+produce the same schema. Future versions add a Django migration per SQL one.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from django.db import migrations
+
+
+def forwards(apps: Any, schema_editor: Any) -> None:
+    from pgdispatch.django.client import migrate_schema
+
+    migrate_schema(schema_editor.connection, target=1)
+
+
+def backwards(apps: Any, schema_editor: Any) -> None:
+    from pgdispatch.django.client import drop_schema
+
+    drop_schema(schema_editor.connection)
+
+
+class Migration(migrations.Migration):
+    initial = True
+    dependencies = []
+    operations = [migrations.RunPython(forwards, backwards, elidable=False)]