fastapi-microservices-platform-sdk 0.1.1__py3-none-any.whl

webhook_platform_sdk/cli.py

@@ -0,0 +1,371 @@
+"""`webhookctl` control-plane and test-event CLI."""
+
+from __future__ import annotations
+
+import argparse
+from getpass import getpass
+import json
+import os
+import stat
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from webhook_platform_sdk.producer import (
+    AuthClient,
+    ManagementClient,
+    Producer,
+    WebhookPlatformError,
+)
+
+
+def _add_project_id(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--project", required=True)
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="webhookctl")
+    parser.add_argument(
+        "--base-url",
+        default=os.getenv("WEBHOOK_PLATFORM_URL", "http://localhost:8000"),
+    )
+    commands = parser.add_subparsers(dest="resource", required=True)
+
+    auth = commands.add_parser("auth")
+    auth_commands = auth.add_subparsers(dest="action", required=True)
+    auth_register = auth_commands.add_parser("register")
+    auth_register.add_argument("--email", required=True)
+    auth_register.add_argument("--name", required=True)
+    auth_login = auth_commands.add_parser("login")
+    auth_login.add_argument("--email", required=True)
+
+    organizations = commands.add_parser("organizations")
+    organization_commands = organizations.add_subparsers(
+        dest="action", required=True
+    )
+    organization_commands.add_parser("list")
+    organization_create = organization_commands.add_parser("create")
+    organization_create.add_argument("--name", required=True)
+
+    projects = commands.add_parser("projects")
+    project_commands = projects.add_subparsers(dest="action", required=True)
+    project_list = project_commands.add_parser("list")
+    project_list.add_argument("--organization", required=True)
+    project_create = project_commands.add_parser("create")
+    project_create.add_argument("--organization", required=True)
+    project_create.add_argument("--name", required=True)
+
+    api_keys = commands.add_parser("api-keys")
+    api_key_commands = api_keys.add_subparsers(dest="action", required=True)
+    api_key_list = api_key_commands.add_parser("list")
+    _add_project_id(api_key_list)
+    api_key_create = api_key_commands.add_parser("create")
+    _add_project_id(api_key_create)
+    api_key_create.add_argument("--name", required=True)
+    api_key_create.add_argument("--expires-in-days", type=int)
+    api_key_revoke = api_key_commands.add_parser("revoke")
+    _add_project_id(api_key_revoke)
+    api_key_revoke.add_argument("--api-key", required=True)
+
+    endpoints = commands.add_parser("endpoints")
+    endpoint_commands = endpoints.add_subparsers(dest="action", required=True)
+    endpoint_list = endpoint_commands.add_parser("list")
+    _add_project_id(endpoint_list)
+    endpoint_create = endpoint_commands.add_parser("create")
+    _add_project_id(endpoint_create)
+    endpoint_create.add_argument("--url", required=True)
+    endpoint_create.add_argument("--description")
+    endpoint_create.add_argument(
+        "--event-type",
+        action="append",
+        dest="event_types",
+        metavar="TYPE",
+        help=(
+            "subscribe only to this event type; repeatable; supports a "
+            "trailing 'prefix.*' wildcard; omit to receive every event"
+        ),
+    )
+    endpoint_create.add_argument(
+        "--signature-scheme",
+        choices=("legacy", "standard"),
+        dest="signature_scheme",
+        help="wire signature scheme; 'standard' emits Standard Webhooks",
+    )
+    endpoint_update = endpoint_commands.add_parser("update")
+    _add_project_id(endpoint_update)
+    endpoint_update.add_argument("--endpoint", required=True)
+    endpoint_update.add_argument("--url")
+    endpoint_update.add_argument("--description")
+    endpoint_activation = endpoint_update.add_mutually_exclusive_group()
+    endpoint_activation.add_argument(
+        "--activate", action="store_true", help="reactivate the endpoint"
+    )
+    endpoint_activation.add_argument(
+        "--deactivate", action="store_true", help="deactivate the endpoint"
+    )
+    endpoint_filter = endpoint_update.add_mutually_exclusive_group()
+    endpoint_filter.add_argument(
+        "--event-type",
+        action="append",
+        dest="event_types",
+        metavar="TYPE",
+        help="replace the subscription list with these types; repeatable",
+    )
+    endpoint_filter.add_argument(
+        "--all-events",
+        action="store_true",
+        help="clear the subscription filter and receive every event",
+    )
+    endpoint_update.add_argument(
+        "--signature-scheme",
+        choices=("legacy", "standard"),
+        dest="signature_scheme",
+        help=(
+            "change the wire signature scheme; switching to 'standard' "
+            "prints a one-time signing secret"
+        ),
+    )
+
+    events = commands.add_parser("events")
+    event_commands = events.add_subparsers(dest="action", required=True)
+    event_send = event_commands.add_parser("send")
+    event_send.add_argument("--type", required=True)
+    event_send.add_argument("--idempotency-key", required=True)
+    event_send.add_argument("--payload-file", required=True)
+    event_send.add_argument(
+        "--envelope-mode", choices=("native", "cloudevents"), default="native"
+    )
+
+    deliveries = commands.add_parser("deliveries")
+    delivery_commands = deliveries.add_subparsers(dest="action", required=True)
+    delivery_list = delivery_commands.add_parser("list")
+    _add_project_id(delivery_list)
+    delivery_list.add_argument("--offset", type=int, default=0)
+    delivery_list.add_argument("--limit", type=int, default=50)
+    for action in ("get", "attempts"):
+        command = delivery_commands.add_parser(action)
+        _add_project_id(command)
+        command.add_argument("--delivery", required=True)
+
+    replay = commands.add_parser("replay")
+    replay_commands = replay.add_subparsers(dest="action", required=True)
+    replay_one = replay_commands.add_parser("one")
+    _add_project_id(replay_one)
+    replay_one.add_argument("--delivery", required=True)
+    replay_bulk = replay_commands.add_parser("bulk")
+    _add_project_id(replay_bulk)
+    replay_bulk.add_argument("--delivery", action="append", required=True)
+    replay_bulk.add_argument("--idempotency-key", required=True)
+    return parser
+
+
+def _credentials_path() -> Path:
+    configured = os.getenv("WEBHOOK_PLATFORM_CREDENTIALS")
+    if configured:
+        return Path(configured).expanduser()
+    return Path.home() / ".config/webhook-platform/credentials.json"
+
+
+def _save_credentials(base_url: str, token: str) -> Path:
+    path = _credentials_path()
+    path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    descriptor, temporary_name = tempfile.mkstemp(
+        prefix=f".{path.name}.", suffix=".tmp", dir=path.parent
+    )
+    temporary = Path(temporary_name)
+    try:
+        os.chmod(temporary, 0o600)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            descriptor = -1
+            json.dump(
+                {"base_url": base_url.rstrip("/"), "token": token},
+                handle,
+            )
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, path)
+        os.chmod(path, 0o600)
+    finally:
+        if descriptor >= 0:
+            os.close(descriptor)
+        temporary.unlink(missing_ok=True)
+    return path
+
+
+def _stored_token(base_url: str) -> str | None:
+    path = _credentials_path()
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        descriptor = os.open(path, flags)
+    except FileNotFoundError:
+        return None
+    except OSError as exc:
+        raise ValueError("credential file cannot be read safely") from exc
+
+    try:
+        metadata = os.fstat(descriptor)
+        if not stat.S_ISREG(metadata.st_mode):
+            raise ValueError("credential file must be a regular file")
+        if stat.S_IMODE(metadata.st_mode) != 0o600:
+            raise ValueError("credential file permissions must be 0600")
+        with os.fdopen(descriptor, encoding="utf-8") as handle:
+            descriptor = -1
+            document = json.load(handle)
+    finally:
+        if descriptor >= 0:
+            os.close(descriptor)
+
+    if not isinstance(document, dict):
+        raise ValueError("credential file is invalid")
+    stored_url = document.get("base_url")
+    token = document.get("token")
+    if stored_url != base_url.rstrip("/"):
+        raise ValueError("credential file belongs to a different base URL")
+    if not isinstance(token, str) or not token:
+        raise ValueError("credential file is invalid")
+    return token
+
+
+def _management(base_url: str) -> ManagementClient:
+    token = os.getenv("WEBHOOK_PLATFORM_TOKEN") or _stored_token(base_url)
+    if not token:
+        raise ValueError(
+            "run `webhookctl auth login` or set WEBHOOK_PLATFORM_TOKEN"
+        )
+    return ManagementClient(base_url, token)
+
+
+def _producer(base_url: str) -> Producer:
+    api_key = os.getenv("WEBHOOK_PLATFORM_API_KEY")
+    if not api_key:
+        raise ValueError("WEBHOOK_PLATFORM_API_KEY is required")
+    return Producer(base_url, api_key)
+
+
+def _payload(path: str) -> object:
+    if path == "-":
+        return json.load(sys.stdin)
+    with Path(path).open(encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def _auth_command(args: argparse.Namespace) -> dict[str, Any]:
+    password = getpass("Password: ")
+    if args.action == "register":
+        confirmation = getpass("Confirm password: ")
+        if password != confirmation:
+            raise ValueError("passwords do not match")
+        with AuthClient(args.base_url) as client:
+            return client.register(args.email, args.name, password)
+
+    with AuthClient(args.base_url) as client:
+        token = client.login(args.email, password)
+    path = _save_credentials(args.base_url, token)
+    return {"status": "stored", "credential_file": str(path)}
+
+
+def _management_command(args: argparse.Namespace) -> Any:
+    with _management(args.base_url) as client:
+        if args.resource == "organizations":
+            if args.action == "list":
+                return client.list_organizations()
+            return client.create_organization(args.name)
+        if args.resource == "projects":
+            if args.action == "list":
+                return client.list_projects(args.organization)
+            return client.create_project(args.organization, args.name)
+        if args.resource == "api-keys":
+            if args.action == "list":
+                return client.list_api_keys(args.project)
+            if args.action == "create":
+                return client.create_api_key(
+                    args.project,
+                    args.name,
+                    expires_in_days=args.expires_in_days,
+                )
+            return client.revoke_api_key(args.project, args.api_key)
+        if args.resource == "endpoints":
+            if args.action == "list":
+                return client.list_endpoints(args.project)
+            if args.action == "create":
+                return client.create_endpoint(
+                    args.project,
+                    args.url,
+                    args.description,
+                    event_types=args.event_types,
+                    signature_scheme=args.signature_scheme,
+                )
+            changes: dict[str, Any] = {}
+            if args.url is not None:
+                changes["url"] = args.url
+            if args.description is not None:
+                changes["description"] = args.description
+            if args.activate:
+                changes["is_active"] = True
+            if args.deactivate:
+                changes["is_active"] = False
+            if args.all_events:
+                changes["event_types"] = None
+            elif args.event_types is not None:
+                changes["event_types"] = args.event_types
+            if args.signature_scheme is not None:
+                changes["signature_scheme"] = args.signature_scheme
+            if not changes:
+                raise ValueError(
+                    "endpoints update requires at least one change"
+                )
+            return client.update_endpoint(
+                args.project, args.endpoint, **changes
+            )
+        if args.resource == "deliveries":
+            if args.action == "list":
+                return client.list_deliveries(
+                    args.project, offset=args.offset, limit=args.limit
+                )
+            delivery = client.get_delivery(args.project, args.delivery)
+            if args.action == "attempts":
+                return delivery.get("attempts", [])
+            return delivery
+        if args.resource == "replay":
+            if args.action == "one":
+                return client.replay_delivery(args.project, args.delivery)
+            return client.replay_deliveries(
+                args.project, args.delivery, args.idempotency_key
+            )
+    raise ValueError("unsupported command")
+
+
+def _run(args: argparse.Namespace) -> Any:
+    if args.resource == "auth":
+        return _auth_command(args)
+    if args.resource == "events":
+        with _producer(args.base_url) as producer:
+            return producer.send_event(
+                args.type,
+                _payload(args.payload_file),
+                args.idempotency_key,
+                envelope_mode=args.envelope_mode,
+            )
+    return _management_command(args)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _parser()
+    args = parser.parse_args(argv)
+    try:
+        result = _run(args)
+    except (
+        WebhookPlatformError,
+        ValueError,
+        OSError,
+        json.JSONDecodeError,
+    ) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 2
+    print(json.dumps(result, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

webhook_platform_sdk/outbox.py

@@ -0,0 +1,281 @@
+"""Optional SQLAlchemy transactional outbox with at-least-once relay."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+from random import SystemRandom
+from typing import Any, Literal, Protocol
+from uuid import uuid4
+
+from sqlalchemy import (
+    JSON,
+    Column,
+    DateTime,
+    Integer,
+    MetaData,
+    String,
+    Table,
+    or_,
+    select,
+    update,
+)
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
+
+from webhook_platform_sdk.producer import ApiError, WebhookPlatformError
+
+EnvelopeMode = Literal["native", "cloudevents"]
+metadata = MetaData()
+outbox_events = Table(
+    "webhook_outbox",
+    metadata,
+    Column("id", String(36), primary_key=True),
+    Column("event_type", String(150), nullable=False),
+    Column("payload", JSON, nullable=False),
+    Column("idempotency_key", String(255), nullable=False, unique=True),
+    Column("envelope_mode", String(16), nullable=False, default="native"),
+    Column("lease_token", String(36)),
+    Column("locked_until", DateTime(timezone=True)),
+    Column("attempt_count", Integer, nullable=False, default=0),
+    Column("next_attempt_at", DateTime(timezone=True)),
+    Column("last_error_code", String(64)),
+    Column("created_at", DateTime(timezone=True), nullable=False),
+    Column("sent_at", DateTime(timezone=True)),
+    Column("failed_at", DateTime(timezone=True)),
+)
+
+
+class EventProducer(Protocol):
+    def send_event(
+        self,
+        event_type: str,
+        payload: object,
+        idempotency_key: str,
+        *,
+        envelope_mode: EnvelopeMode = "native",
+    ) -> dict[str, Any]: ...
+
+
+def utcnow() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def create_outbox_schema(engine: Engine) -> None:
+    """Create the SDK-owned table; production users should use a migration."""
+    metadata.create_all(engine, tables=[outbox_events])
+
+
+def enqueue_outbox(
+    session: Session,
+    event_type: str,
+    payload: object,
+    *,
+    idempotency_key: str | None = None,
+    envelope_mode: EnvelopeMode = "native",
+) -> str:
+    """Insert into the caller's current transaction without committing it."""
+    if not event_type:
+        raise ValueError("event_type is required")
+    if envelope_mode not in {"native", "cloudevents"}:
+        raise ValueError("envelope_mode is invalid")
+    key = idempotency_key or str(uuid4())
+    created_at = utcnow()
+    session.execute(
+        outbox_events.insert().values(
+            id=str(uuid4()),
+            event_type=event_type,
+            payload=payload,
+            idempotency_key=key,
+            envelope_mode=envelope_mode,
+            attempt_count=0,
+            next_attempt_at=created_at,
+            created_at=created_at,
+        )
+    )
+    return key
+
+
+@dataclass(frozen=True)
+class OutboxMessage:
+    id: str
+    event_type: str
+    payload: object
+    idempotency_key: str
+    envelope_mode: EnvelopeMode
+    lease_token: str
+    attempt_number: int
+
+
+class OutboxRelay:
+    """Claim briefly, send outside the transaction, then guard finalization."""
+
+    def __init__(
+        self,
+        session_factory: Callable[[], Session],
+        producer: EventProducer,
+        *,
+        batch_size: int = 50,
+        lease_seconds: int = 60,
+        max_attempts: int = 20,
+        retry_base_seconds: float = 1.0,
+        retry_cap_seconds: float = 300.0,
+    ) -> None:
+        if not 1 <= batch_size <= 1_000:
+            raise ValueError("batch_size must be between 1 and 1000")
+        if lease_seconds < 1:
+            raise ValueError("lease_seconds must be positive")
+        if max_attempts < 1:
+            raise ValueError("max_attempts must be positive")
+        if not 0 <= retry_base_seconds <= retry_cap_seconds:
+            raise ValueError("retry delay bounds are invalid")
+        self._session_factory = session_factory
+        self._producer = producer
+        self._batch_size = batch_size
+        self._lease_seconds = lease_seconds
+        self._max_attempts = max_attempts
+        self._retry_base_seconds = retry_base_seconds
+        self._retry_cap_seconds = retry_cap_seconds
+        self._random = SystemRandom()
+
+    def _claim(self) -> list[OutboxMessage]:
+        now = utcnow()
+        claimed: list[OutboxMessage] = []
+        with self._session_factory() as session, session.begin():
+            rows = session.execute(
+                select(outbox_events)
+                .where(
+                    outbox_events.c.sent_at.is_(None),
+                    outbox_events.c.failed_at.is_(None),
+                    outbox_events.c.next_attempt_at <= now,
+                    or_(
+                        outbox_events.c.locked_until.is_(None),
+                        outbox_events.c.locked_until <= now,
+                    ),
+                )
+                .order_by(outbox_events.c.created_at, outbox_events.c.id)
+                .limit(self._batch_size)
+                .with_for_update(skip_locked=True)
+            ).mappings()
+            for row in rows:
+                token = str(uuid4())
+                session.execute(
+                    update(outbox_events)
+                    .where(outbox_events.c.id == row["id"])
+                    .values(
+                        lease_token=token,
+                        locked_until=now
+                        + timedelta(seconds=self._lease_seconds),
+                        attempt_count=outbox_events.c.attempt_count + 1,
+                    )
+                )
+                claimed.append(
+                    OutboxMessage(
+                        id=row["id"],
+                        event_type=row["event_type"],
+                        payload=row["payload"],
+                        idempotency_key=row["idempotency_key"],
+                        envelope_mode=row["envelope_mode"],
+                        lease_token=token,
+                        attempt_number=row["attempt_count"] + 1,
+                    )
+                )
+        return claimed
+
+    def _finish(
+        self,
+        message: OutboxMessage,
+        *,
+        sent: bool,
+        error_code: str | None = None,
+        retry_at: datetime | None = None,
+        terminal: bool = False,
+    ) -> None:
+        values: dict[str, object | None] = {
+            "lease_token": None,
+            "locked_until": None,
+            "last_error_code": error_code,
+            "next_attempt_at": retry_at,
+        }
+        if sent:
+            values["sent_at"] = utcnow()
+        elif terminal:
+            values["failed_at"] = utcnow()
+        with self._session_factory() as session, session.begin():
+            session.execute(
+                update(outbox_events)
+                .where(
+                    outbox_events.c.id == message.id,
+                    outbox_events.c.lease_token == message.lease_token,
+                    outbox_events.c.sent_at.is_(None),
+                )
+                .values(**values)
+            )
+
+    def _record_failure(
+        self,
+        message: OutboxMessage,
+        code: str,
+        *,
+        retryable: bool,
+        retry_after: int | None = None,
+    ) -> None:
+        terminal = (
+            not retryable or message.attempt_number >= self._max_attempts
+        )
+        retry_at: datetime | None = None
+        if not terminal:
+            cap = min(
+                self._retry_cap_seconds,
+                self._retry_base_seconds * 2 ** (message.attempt_number - 1),
+            )
+            delay = self._random.uniform(0, cap)
+            if retry_after is not None:
+                delay = max(delay, min(retry_after, self._retry_cap_seconds))
+            retry_at = utcnow() + timedelta(seconds=delay)
+        self._finish(
+            message,
+            sent=False,
+            error_code=code[:64],
+            retry_at=retry_at,
+            terminal=terminal,
+        )
+
+    def run_once(self) -> tuple[int, int]:
+        """Return successful and failed counts for one bounded batch.
+
+        A crash after API acceptance but before `_finish` causes another relay
+        attempt with the same idempotency key. Transient retries use bounded
+        full jitter and stop at `max_attempts`; terminal API responses stop
+        immediately.
+        """
+        succeeded = 0
+        failed = 0
+        for message in self._claim():
+            try:
+                self._producer.send_event(
+                    message.event_type,
+                    message.payload,
+                    message.idempotency_key,
+                    envelope_mode=message.envelope_mode,
+                )
+            except ApiError as exc:
+                retryable = exc.status_code in {408, 425, 429, 503}
+                retryable = retryable or exc.status_code >= 500
+                self._record_failure(
+                    message,
+                    exc.code,
+                    retryable=retryable,
+                    retry_after=exc.retry_after,
+                )
+                failed += 1
+            except WebhookPlatformError as exc:
+                self._record_failure(
+                    message, type(exc).__name__, retryable=True
+                )
+                failed += 1
+            else:
+                self._finish(message, sent=True)
+                succeeded += 1
+        return succeeded, failed