pgdispatch 0.1.0__py3-none-any.whl

pgdispatch/__init__.py

@@ -0,0 +1,69 @@
+"""pgdispatch: reliable webhook delivery using only the PostgreSQL you already have.
+
+Publish events inside your database transaction, and a background worker
+delivers them to every subscribed endpoint, signed according to Standard
+Webhooks, with retries, circuit breaking and a full attempt log.
+"""
+
+from pgdispatch._version import __version__
+from pgdispatch.backends.asyncpg import AsyncpgBackend
+from pgdispatch.client import Webhooks
+from pgdispatch.config import WebhooksConfig
+from pgdispatch.core.common import UNSET
+from pgdispatch.core.maintenance import QueueStats
+from pgdispatch.core.retry import DEFAULT_RETRY_SCHEDULE, exponential_backoff
+from pgdispatch.core.signing import generate_secret
+from pgdispatch.core.signing import verify as verify_signature
+from pgdispatch.errors import (
+    ConfigurationError,
+    ConflictError,
+    InvalidSignatureError,
+    MigrationError,
+    NotFoundError,
+    PayloadTooLargeError,
+    PgDispatchError,
+    UnsafeURLError,
+    ValidationError,
+)
+from pgdispatch.hooks import Hooks
+from pgdispatch.models import (
+    Attempt,
+    CircuitState,
+    Delivery,
+    DeliveryStatus,
+    Endpoint,
+    Event,
+    PublishResult,
+)
+from pgdispatch.worker.worker import Worker
+
+__all__ = [
+    "DEFAULT_RETRY_SCHEDULE",
+    "UNSET",
+    "AsyncpgBackend",
+    "Attempt",
+    "CircuitState",
+    "ConfigurationError",
+    "ConflictError",
+    "Delivery",
+    "DeliveryStatus",
+    "Endpoint",
+    "Event",
+    "Hooks",
+    "InvalidSignatureError",
+    "MigrationError",
+    "NotFoundError",
+    "PayloadTooLargeError",
+    "PgDispatchError",
+    "PublishResult",
+    "QueueStats",
+    "UnsafeURLError",
+    "ValidationError",
+    "Webhooks",
+    "WebhooksConfig",
+    "Worker",
+    "__version__",
+    "exponential_backoff",
+    "generate_secret",
+    "verify_signature",
+]

pgdispatch/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from pgdispatch.cli import main
+
+sys.exit(main())

pgdispatch/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

pgdispatch/backends/__init__.py

@@ -0,0 +1,6 @@
+"""Adapters between pgdispatch and the application's database driver."""
+
+from pgdispatch.backends.asyncpg import AsyncpgBackend
+from pgdispatch.backends.base import Backend
+
+__all__ = ["AsyncpgBackend", "Backend"]

pgdispatch/backends/asyncpg.py

@@ -0,0 +1,103 @@
+"""Backend for applications that talk to PostgreSQL with plain asyncpg."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Mapping
+from contextlib import asynccontextmanager
+from typing import Any
+
+import asyncpg
+from asyncpg.pool import PoolConnectionProxy
+
+from pgdispatch.backends.base import Row, to_numeric_style
+from pgdispatch.errors import ConfigurationError
+
+__all__ = ["AsyncpgBackend", "AsyncpgConnection", "AsyncpgExecutor", "normalize_dsn"]
+
+AsyncpgConnection = asyncpg.Connection | PoolConnectionProxy
+"""A plain connection or one acquired from a pool."""
+
+WORKER_POOL_SIZE = 5
+"""Connections used by a worker: LISTEN, claiming, recording results and housekeeping."""
+
+
+class AsyncpgExecutor:
+    """Runs statements on an asyncpg connection (or pool)."""
+
+    def __init__(self, connection: AsyncpgConnection | asyncpg.Pool) -> None:
+        self._connection = connection
+
+    async def fetch(self, sql: str, params: Mapping[str, Any]) -> list[Row]:
+        statement, names = to_numeric_style(sql)
+        records = await self._connection.fetch(statement, *(params[name] for name in names))
+        return [dict(record) for record in records]
+
+
+class AsyncpgBackend:
+    """Use pgdispatch with asyncpg.
+
+    ``AsyncpgBackend("postgresql://...")`` creates and owns a pool. Pass an
+    existing pool with ``AsyncpgBackend(pool=pool)`` to share yours; the worker
+    always opens a small pool of its own because it keeps a connection busy
+    with ``LISTEN``.
+    """
+
+    def __init__(
+        self,
+        dsn: str | None = None,
+        *,
+        pool: asyncpg.Pool | None = None,
+        **connect_kwargs: Any,
+    ) -> None:
+        if dsn is None and pool is None and not connect_kwargs:
+            msg = "pass a DSN, an asyncpg pool or asyncpg connection arguments"
+            raise ValueError(msg)
+        self._dsn = normalize_dsn(dsn) if dsn else None
+        self._connect_kwargs = connect_kwargs
+        self._pool = pool
+        self._owns_pool = pool is None
+
+    def executor(self, connection: Any) -> AsyncpgExecutor:
+        if not isinstance(connection, (asyncpg.Connection, PoolConnectionProxy, asyncpg.Pool)):
+            kind = type(connection).__name__
+            msg = f"AsyncpgBackend.publish() expects an asyncpg connection, got {kind}"
+            raise TypeError(msg)
+        return AsyncpgExecutor(connection)
+
+    @asynccontextmanager
+    async def transaction(self) -> AsyncIterator[AsyncpgExecutor]:
+        pool = await self._get_pool()
+        async with pool.acquire() as connection, connection.transaction():
+            yield AsyncpgExecutor(connection)
+
+    async def create_worker_pool(self) -> asyncpg.Pool:
+        if self._dsn is None and not self._connect_kwargs:
+            msg = (
+                "the worker needs its own connections: pass the DSN as well, "
+                "e.g. AsyncpgBackend(dsn, pool=pool)"
+            )
+            raise ConfigurationError(msg)
+        kwargs = {"min_size": 1, "max_size": WORKER_POOL_SIZE, **self._connect_kwargs}
+        pool: asyncpg.Pool = await asyncpg.create_pool(self._dsn, **kwargs)
+        return pool
+
+    async def close(self) -> None:
+        if self._owns_pool and self._pool is not None:
+            await self._pool.close()
+            self._pool = None
+
+    async def _get_pool(self) -> asyncpg.Pool:
+        if self._pool is None:
+            self._pool = await asyncpg.create_pool(self._dsn, **self._connect_kwargs)
+        return self._pool
+
+
+def normalize_dsn(dsn: str) -> str:
+    """Accept SQLAlchemy-style URLs such as ``postgresql+asyncpg://`` as well."""
+    scheme, separator, rest = dsn.partition("://")
+    if not separator:
+        return dsn
+    base = scheme.split("+", 1)[0]
+    if base == "postgres":
+        base = "postgresql"
+    return f"{base}://{rest}"

pgdispatch/backends/base.py

@@ -0,0 +1,142 @@
+"""Driver-independent database access.
+
+pgdispatch has to run the same SQL through asyncpg, SQLAlchemy (any async
+driver) and Django (psycopg, synchronously). Instead of writing every operation
+three times, operations are written once as *generators* that yield
+:class:`Query` objects and receive the resulting rows::
+
+    def get_endpoint(endpoint_id: UUID) -> Operation[Endpoint]:
+        rows = yield Query("SELECT ... WHERE id = CAST(:id AS uuid)", {"id": str(endpoint_id)})
+        return Endpoint.from_row(rows[0])
+
+A small runner (:func:`run_async` or :func:`run_sync`) drives the generator
+against an executor. Executors only know how to send one statement to their
+driver; they never contain business logic.
+
+SQL conventions:
+
+* Parameters use the ``:name`` style and are always wrapped in ``CAST`` so that
+  every driver binds them with the same PostgreSQL type.
+* ``{schema}`` is replaced with the configured, quoted schema name.
+* ``jsonb`` values are selected as ``text`` and decoded in Python.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Generator, Mapping
+from contextlib import AbstractAsyncContextManager
+from dataclasses import dataclass, field
+from functools import lru_cache
+from typing import TYPE_CHECKING, Any, Protocol, TypeVar
+
+if TYPE_CHECKING:
+    import asyncpg
+
+__all__ = [
+    "AsyncExecutor",
+    "Backend",
+    "Operation",
+    "Query",
+    "Row",
+    "SyncExecutor",
+    "run_async",
+    "run_sync",
+    "to_numeric_style",
+    "to_pyformat_style",
+]
+
+T = TypeVar("T")
+Row = dict[str, Any]
+
+
+@dataclass(frozen=True, slots=True)
+class Query:
+    sql: str
+    params: Mapping[str, Any] = field(default_factory=dict)
+
+
+Operation = Generator[Query, list[Row], T]
+"""A database operation: yields queries, receives their rows, returns a result."""
+
+
+class AsyncExecutor(Protocol):
+    async def fetch(self, sql: str, params: Mapping[str, Any]) -> list[Row]:
+        """Execute one statement and return its rows (an empty list if it returns none)."""
+
+
+class SyncExecutor(Protocol):
+    def fetch(self, sql: str, params: Mapping[str, Any]) -> list[Row]:
+        """Execute one statement and return its rows (an empty list if it returns none)."""
+
+
+class Backend(Protocol):
+    """Connects :class:`~pgdispatch.Webhooks` to the application's database layer."""
+
+    def executor(self, connection: Any) -> AsyncExecutor:
+        """Wrap a connection or session owned by the application (used by ``publish``)."""
+
+    def transaction(self) -> AbstractAsyncContextManager[AsyncExecutor]:
+        """Run management operations in a new transaction."""
+
+    async def create_worker_pool(self) -> asyncpg.Pool:
+        """Create the asyncpg pool used by the delivery worker."""
+
+    async def close(self) -> None:
+        """Release resources owned by the backend."""
+
+
+async def run_async(operation: Operation[T], executor: AsyncExecutor, *, schema: str) -> T:
+    """Drive ``operation`` to completion against an async executor."""
+    try:
+        query = next(operation)
+        while True:
+            rows = await executor.fetch(render_schema(query.sql, schema), query.params)
+            query = operation.send(rows)
+    except StopIteration as stop:
+        result: T = stop.value
+        return result
+
+
+def run_sync(operation: Operation[T], executor: SyncExecutor, *, schema: str) -> T:
+    """Drive ``operation`` to completion against a blocking executor."""
+    try:
+        query = next(operation)
+        while True:
+            rows = executor.fetch(render_schema(query.sql, schema), query.params)
+            query = operation.send(rows)
+    except StopIteration as stop:
+        result: T = stop.value
+        return result
+
+
+# ``:name`` placeholders; ``::type`` casts are left alone.
+_PARAMETER = re.compile(r"(?<![:\w]):(\w+)(?!:)")
+
+
+@lru_cache(maxsize=512)
+def render_schema(sql: str, schema: str) -> str:
+    return sql.replace("{schema}", f'"{schema}"')
+
+
+@lru_cache(maxsize=512)
+def to_numeric_style(sql: str) -> tuple[str, tuple[str, ...]]:
+    """Convert ``:name`` placeholders to asyncpg's ``$1`` style.
+
+    Returns the new SQL and the parameter names in positional order.
+    """
+    names: list[str] = []
+
+    def replace(match: re.Match[str]) -> str:
+        name = match.group(1)
+        if name not in names:
+            names.append(name)
+        return f"${names.index(name) + 1}"
+
+    return _PARAMETER.sub(replace, sql), tuple(names)
+
+
+@lru_cache(maxsize=512)
+def to_pyformat_style(sql: str) -> str:
+    """Convert ``:name`` placeholders to the DB-API ``%(name)s`` style used by psycopg."""
+    return _PARAMETER.sub(r"%(\1)s", sql.replace("%", "%%"))