idemkit 0.1.0__py3-none-any.whl

idemkit/contrib/pubsub.py

@@ -0,0 +1,98 @@
+"""Google Cloud Pub/Sub glue for idemkit's queue surface.
+
+Pub/Sub is at-least-once: if an ack doesn't reach the server before the ack
+deadline (or you nack), the message is redelivered. Even with exactly-once
+delivery enabled it only dedups *subscriber-side* redeliveries — a publisher that
+retries a lost publish produces a new ``message_id``, so a duplicate can still
+reach you. :func:`pubsub_consumer` presets an
+:class:`~idemkit.IdempotentConsumer` so the side effect runs once per message.
+
+Two Pub/Sub specifics:
+
+* **Dedup id = ``message_id``.** Stable across subscriber redeliveries of the same
+  message. It does *not* cover publisher-side duplicates (those carry different
+  ids); if your producer can double-publish, dedup on a business id in the payload
+  instead via a custom ``QueueConfig.dedup_id``.
+* **Visibility-timeout analogue = the ack deadline.** The lease is derived from it
+  (``ack_deadline_seconds``); for slow handlers, extend the deadline with the
+  client's lease-management or raise it on the subscription.
+
+Streaming pull runs your callback in a background thread per message. Wrap it with
+:func:`pubsub_callback`: it dispatches through idemkit and acks on ``ACK`` / nacks
+on ``RETRY``.
+
+Example (google-cloud-pubsub)::
+
+    from google.cloud import pubsub_v1
+    from idemkit import RedisBackend
+    from idemkit.contrib.pubsub import pubsub_consumer, pubsub_callback
+
+    consumer = pubsub_consumer(
+        backend=RedisBackend.from_url("redis://localhost:6379"),
+        ack_deadline_seconds=60,
+    )
+
+
+    @consumer.handle
+    def process(message) -> None:
+        charge_customer(message.data)  # runs once per message_id
+
+
+    subscriber = pubsub_v1.SubscriberClient()
+    subscriber.subscribe(subscription_path, callback=pubsub_callback(consumer))
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from collections.abc import Callable
+from typing import Any
+
+from idemkit.adapters.queue import ConsumerAction, IdempotentConsumer
+from idemkit.backends.base import IdempotencyBackend
+from idemkit.core.policy import QueueConfig
+
+
+def pubsub_dedup_id(message: Any) -> str:
+    """The Pub/Sub ``message_id`` (the dedup id)."""
+    return str(message.message_id)
+
+
+def pubsub_consumer(
+    *,
+    backend: IdempotencyBackend,
+    ack_deadline_seconds: float = 60.0,
+    config: QueueConfig | None = None,
+) -> IdempotentConsumer:
+    """Build an :class:`~idemkit.IdempotentConsumer` wired for Pub/Sub messages.
+
+    Presets the dedup id (``message_id``) and the lease (from the ack deadline).
+    Pass a :class:`~idemkit.QueueConfig` for behaviour (``max_attempts``,
+    ``on_exhausted``, ``scope`` per subscription, ...).
+    """
+    cfg = config or QueueConfig()
+    cfg = dataclasses.replace(
+        cfg,
+        dedup_id=pubsub_dedup_id,
+        visibility_timeout_seconds=ack_deadline_seconds,
+    )
+    return IdempotentConsumer(backend=backend, config=cfg)
+
+
+def pubsub_callback(consumer: IdempotentConsumer) -> Callable[[Any], None]:
+    """Wrap ``consumer`` as a streaming-pull callback that acks/nacks the message.
+
+    Returns a function you pass to ``subscriber.subscribe(..., callback=...)``. It
+    dispatches the message through idemkit (``dispatch_sync``, since the callback
+    runs on the client's thread) and calls ``message.ack()`` on ``ACK`` or
+    ``message.nack()`` on ``RETRY`` (redeliver after the ack deadline).
+    """
+
+    def _callback(message: Any) -> None:
+        result = consumer.dispatch_sync(message)
+        if result.action is ConsumerAction.ACK:
+            message.ack()
+        else:  # RETRY
+            message.nack()
+
+    return _callback

idemkit/contrib/rabbitmq.py

@@ -0,0 +1,138 @@
+"""RabbitMQ (pika) glue for idemkit's queue surface.
+
+RabbitMQ is at-least-once under manual acks: an unacked delivery is requeued when
+the channel or connection drops, and you can requeue explicitly with ``basic_nack``.
+So a consumer sees the same message again after a network blip, a crash, or a
+transient failure. :func:`rabbitmq_consumer` presets an
+:class:`~idemkit.IdempotentConsumer` so the side effect runs once per message even
+across those redeliveries.
+
+Two things are different from SQS/Kafka, and worth stating plainly:
+
+* **There is no dedup id built in.** RabbitMQ does not stamp a stable per-message
+  id for you. The clean choice is the AMQP ``message_id`` property, which the
+  *publisher* must set (a UUID, or your own business id). If your producer doesn't
+  set one, dedup has nothing to key on — see :class:`RabbitMessage`.
+* **There is no visibility timeout.** An unacked message is not redelivered on a
+  timer; it comes back only on nack/requeue or a channel/connection loss. So the
+  lease here is just how long idemkit holds the claim before assuming a crash — you
+  pick it (``lease_seconds``), it isn't derived from the broker.
+
+The ``redelivered`` flag RabbitMQ sets on a re-delivery is carried through on
+:attr:`RabbitMessage.redelivered` if you want to log or branch on it.
+
+Example (pika)::
+
+    import pika
+    from idemkit import RedisBackend
+    from idemkit.contrib.rabbitmq import rabbitmq_consumer, run_forever
+
+    consumer = rabbitmq_consumer(
+        backend=RedisBackend.from_url("redis://localhost:6379"),
+        lease_seconds=300,
+    )
+
+
+    @consumer.handle
+    def process(msg) -> None:
+        charge_customer(msg.body)  # runs once per message_id
+
+
+    conn = pika.BlockingConnection(pika.ConnectionParameters("localhost"))
+    run_forever(consumer, channel=conn.channel(), queue="charges", lease_seconds=300)
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from collections.abc import Callable
+from typing import Any
+
+from idemkit.adapters.queue import ConsumerAction, IdempotentConsumer
+from idemkit.backends.base import IdempotencyBackend
+from idemkit.core.policy import QueueConfig
+
+
+@dataclasses.dataclass(frozen=True)
+class RabbitMessage:
+    """One RabbitMQ delivery, in the shape idemkit dedupes on.
+
+    ``message_id`` is the AMQP ``message_id`` property the publisher set; it is the
+    dedup id. ``run_forever`` builds this for you from pika's delivery; if you drive
+    the consumer yourself, construct it from ``properties.message_id`` and ``body``.
+    """
+
+    message_id: str
+    body: bytes
+    redelivered: bool = False
+    delivery_tag: int | None = None
+
+
+def rabbitmq_dedup_id(message: RabbitMessage) -> str:
+    """The message's AMQP ``message_id`` (the dedup id)."""
+    return message.message_id
+
+
+def rabbitmq_consumer(
+    *,
+    backend: IdempotencyBackend,
+    lease_seconds: float = 300.0,
+    config: QueueConfig | None = None,
+) -> IdempotentConsumer:
+    """Build an :class:`~idemkit.IdempotentConsumer` wired for RabbitMQ deliveries.
+
+    Presets the dedup id (the AMQP ``message_id``) and the lease (``lease_seconds``,
+    since RabbitMQ has no visibility timeout to derive it from). Pass a
+    :class:`~idemkit.QueueConfig` for behaviour (``max_attempts``, ``on_exhausted``,
+    ``scope`` per queue, ...).
+    """
+    cfg = config or QueueConfig()
+    cfg = dataclasses.replace(
+        cfg,
+        dedup_id=rabbitmq_dedup_id,
+        visibility_timeout_seconds=lease_seconds,
+    )
+    return IdempotentConsumer(backend=backend, config=cfg)
+
+
+def run_forever(
+    consumer: IdempotentConsumer,
+    *,
+    channel: Any,
+    queue: str,
+    lease_seconds: float = 300.0,
+    prefetch: int = 10,
+    stop: Callable[[], bool] | None = None,
+) -> None:
+    """Pull from ``queue`` and dispatch each delivery, acking on ``ACK``.
+
+    A blocking loop for a synchronous worker (it uses ``dispatch_sync`` and pika's
+    ``basic_get``). On ``ACK`` the delivery is acked; on ``RETRY`` it is nacked with
+    ``requeue=True`` so RabbitMQ redelivers it. The publisher MUST set the AMQP
+    ``message_id`` property; a delivery without one raises ``ValueError`` rather than
+    silently skipping dedup. Pass ``stop=lambda: flag`` for graceful shutdown.
+    """
+    channel.basic_qos(prefetch_count=prefetch)
+    while stop is None or not stop():
+        method, properties, body = channel.basic_get(queue=queue, auto_ack=False)
+        if method is None:  # queue empty
+            continue
+        message_id = getattr(properties, "message_id", None)
+        if not message_id:
+            channel.basic_nack(delivery_tag=method.delivery_tag, requeue=False)
+            raise ValueError(
+                "idemkit: RabbitMQ delivery has no message_id property to dedup on. "
+                "Publish with a message_id (a UUID or your business id), or key on a "
+                "header via a custom QueueConfig.dedup_id."
+            )
+        message = RabbitMessage(
+            message_id=message_id,
+            body=body,
+            redelivered=bool(getattr(method, "redelivered", False)),
+            delivery_tag=method.delivery_tag,
+        )
+        result = consumer.dispatch_sync(message)
+        if result.action is ConsumerAction.ACK:
+            channel.basic_ack(delivery_tag=method.delivery_tag)
+        else:  # RETRY: leave it for redelivery
+            channel.basic_nack(delivery_tag=method.delivery_tag, requeue=True)

idemkit/contrib/reconciliation.py

@@ -0,0 +1,86 @@
+"""Turn idemkit's uncertain-completion events into a reconciliation signal.
+
+Most of the time a key completes cleanly and there is nothing to reconcile. A few
+decisions mean something else happened: the side effect ran, but idemkit could not
+record its result, or ran without protection during a storage outage. In those
+cases the record and reality may disagree, and only the downstream system (your
+payment provider, your mailer) knows the truth.
+
+idemkit cannot ask the provider for you. It does not know your business ids and it
+holds only a hash of the key. What it can do is tell you *which* keys landed in the
+uncertain zone, so your own reconciliation can go and check them. That is the
+fourth assumption from the design guide (docs/the-four-assumptions.md): the boundary
+between your record and a system you do not control.
+
+Wire it like any other event handler::
+
+    from idemkit import MethodConfig, idempotent
+    from idemkit.contrib.reconciliation import reconciliation_handler
+
+
+    def needs_check(event):
+        # Correlate back to a business operation via your own pending record,
+        # or trust the key you passed downstream, then ask the provider.
+        reconcile_queue.put(event.effective_key)
+
+
+    @idempotent(
+        backend=backend,
+        config=MethodConfig(
+            key_fields=["order_id"],
+            event_handlers=(reconciliation_handler(needs_check),),
+        ),
+    )
+    async def charge(*, order_id): ...
+
+The handler fires only for the decisions below. Everything else (a clean run, a
+replay, a rejected payload) is left alone, so a quiet queue means nothing to do.
+"""
+
+from __future__ import annotations
+
+from idemkit.core.events import EventHandler, IdempotencyEvent
+from idemkit.core.state import Decision
+
+#: The decisions where a side effect may have fired without a recorded result, so
+#: the record and the downstream system can disagree. Each is emitted at most once
+#: per operation; see spec §4.15 and docs/the-four-assumptions.md.
+NEEDS_RECONCILIATION: frozenset[Decision] = frozenset(
+    {
+        # The side effect ran, but writing its result failed. A retry re-runs it.
+        Decision.COMPLETE_FAILED,
+        # Our lease was reclaimed mid-run; our completion is fenced, but the side
+        # effect may already have fired before we lost the key.
+        Decision.LEASE_RECLAIMED_LOSS,
+        # The lease lapsed and the handler was cancelled part-way.
+        Decision.LEASE_LOST,
+        # fail_open ran the operation without dedup during a storage outage.
+        Decision.RAN_UNPROTECTED,
+    }
+)
+
+# Decision.CORRUPT_RECORD is left out on purpose. It fires when a stored record cannot be
+# deserialized, which is a genuine duplicate-risk case, but a codec or schema change can
+# emit it in bulk for records that are only unreadable, not wrong. Reconcile those on
+# their own track so a migration does not flood this signal.
+
+
+def reconciliation_handler(callback: EventHandler) -> EventHandler:
+    """Build an event handler that calls ``callback`` only for uncertain completions.
+
+    ``callback`` receives the :class:`~idemkit.IdempotencyEvent` for each decision in
+    :data:`NEEDS_RECONCILIATION` and nothing else. Use it to enqueue the key for a
+    reconciliation sweep, page an operator, or bump a metric. The event carries the
+    hashed ``effective_key`` (safe to log), the ``decision``, and the ``surface``; it
+    does not carry your business id, so correlate through your own pending record or
+    the key you passed downstream.
+
+    Like every event handler, a raised exception here is caught and suppressed by the
+    emitter, so a slow or broken sink never touches the request path.
+    """
+
+    def handle(event: IdempotencyEvent) -> None:
+        if event.decision in NEEDS_RECONCILIATION:
+            callback(event)
+
+    return handle

idemkit/contrib/sqs.py

@@ -0,0 +1,117 @@
+"""Amazon SQS glue for idemkit's queue surface.
+
+SQS is at-least-once: it redelivers a message whenever a consumer doesn't delete
+it within the visibility timeout. :func:`sqs_consumer` presets an
+:class:`~idemkit.IdempotentConsumer` for the standard boto3 message shape so the
+side effect runs once per SQS ``MessageId`` even across redeliveries, concurrent
+consumers, and crashes.
+
+Two SQS-specific defaults it gets right for you:
+
+* **Dedup id = ``MessageId``.** Stable across redeliveries of the same message.
+* **Attempt count = ``ApproximateReceiveCount``.** SQS's own receive count, so
+  ``max_attempts`` is enforced durably (no in-process counter). Request it when
+  you receive: ``AttributeNames=["ApproximateReceiveCount"]`` — :func:`run_forever`
+  does this for you.
+
+Ack = *delete the message*. :func:`run_forever` deletes on ``ACK`` and leaves the
+message for redelivery on ``RETRY``.
+
+Example::
+
+    import boto3
+    from idemkit import RedisBackend
+    from idemkit.contrib.sqs import sqs_consumer, run_forever
+
+    sqs = boto3.client("sqs")
+    consumer = sqs_consumer(
+        backend=RedisBackend.from_url("redis://localhost:6379"),
+        visibility_timeout_seconds=30,
+        config=QueueConfig(
+            max_attempts=5,
+            on_exhausted=lambda msg, exc: sqs.send_message(QueueUrl=DLQ, MessageBody=msg["Body"]),
+        ),
+    )
+
+
+    @consumer.handle
+    def process(msg) -> None:
+        charge_customer(msg["Body"])  # runs once per MessageId
+
+
+    run_forever(consumer, sqs_client=sqs, queue_url=QUEUE_URL, visibility_timeout=30)
+"""
+
+from __future__ import annotations
+
+import dataclasses
+from collections.abc import Callable
+from typing import Any
+
+from idemkit.adapters.queue import ConsumerAction, IdempotentConsumer
+from idemkit.backends.base import IdempotencyBackend
+from idemkit.core.policy import QueueConfig
+
+
+def sqs_receive_count(message: dict[str, Any]) -> int | None:
+    """Read SQS ``ApproximateReceiveCount`` from a boto3 message dict."""
+    attrs = message.get("Attributes") or {}
+    value = attrs.get("ApproximateReceiveCount")
+    return int(value) if value is not None else None
+
+
+def sqs_consumer(
+    *,
+    backend: IdempotencyBackend,
+    visibility_timeout_seconds: float,
+    config: QueueConfig | None = None,
+) -> IdempotentConsumer:
+    """Build an :class:`~idemkit.IdempotentConsumer` wired for boto3 SQS messages.
+
+    SQS presets the dedup id (``MessageId``) and the attempt count
+    (``ApproximateReceiveCount``). Pass a :class:`~idemkit.QueueConfig` for
+    behaviour (``max_attempts``, ``on_exhausted``, ``scope`` per queue, ...).
+    """
+    cfg = config or QueueConfig()
+    presets: dict[str, Any] = {
+        "dedup_id": lambda m: m["MessageId"],
+        "visibility_timeout_seconds": visibility_timeout_seconds,
+    }
+    if cfg.receive_count is None:
+        presets["receive_count"] = sqs_receive_count
+    cfg = dataclasses.replace(cfg, **presets)
+    return IdempotentConsumer(backend=backend, config=cfg)
+
+
+def run_forever(
+    consumer: IdempotentConsumer,
+    *,
+    sqs_client: Any,
+    queue_url: str,
+    visibility_timeout: int,
+    wait_time_seconds: int = 20,
+    max_messages: int = 10,
+    stop: Callable[[], bool] | None = None,
+) -> None:
+    """Poll SQS and dispatch each message, deleting it on ``ACK``.
+
+    A blocking loop for a synchronous worker (it uses ``dispatch_sync``). Long-polls
+    with ``wait_time_seconds`` and requests ``ApproximateReceiveCount`` so attempt
+    counting is durable. On ``ACK`` the message is deleted; on ``RETRY`` it is left
+    for redelivery (the visibility timeout lapses). Pass ``stop=lambda: flag`` to
+    break the loop for graceful shutdown.
+    """
+    while stop is None or not stop():
+        resp = sqs_client.receive_message(
+            QueueUrl=queue_url,
+            MaxNumberOfMessages=max_messages,
+            WaitTimeSeconds=wait_time_seconds,
+            VisibilityTimeout=visibility_timeout,
+            AttributeNames=["ApproximateReceiveCount"],
+        )
+        for message in resp.get("Messages", []):
+            result = consumer.dispatch_sync(message)
+            if result.action is ConsumerAction.ACK:
+                sqs_client.delete_message(
+                    QueueUrl=queue_url, ReceiptHandle=message["ReceiptHandle"]
+                )

idemkit/core/__init__.py

@@ -0,0 +1,36 @@
+from idemkit.core.config import IdempotencyConfig
+from idemkit.core.events import EventEmitter, IdempotencyEvent
+from idemkit.core.exceptions import (
+    ConfigurationError,
+    IdempotencyError,
+    StorageError,
+)
+from idemkit.core.fingerprint import (
+    FINGERPRINT_VERSION,
+    compute_effective_key,
+    compute_fingerprint,
+)
+from idemkit.core.state import (
+    ClaimResult,
+    ClaimResultType,
+    Decision,
+    State,
+    StoredRecord,
+)
+
+__all__ = [
+    "FINGERPRINT_VERSION",
+    "ClaimResult",
+    "ClaimResultType",
+    "ConfigurationError",
+    "Decision",
+    "EventEmitter",
+    "IdempotencyConfig",
+    "IdempotencyError",
+    "IdempotencyEvent",
+    "State",
+    "StorageError",
+    "StoredRecord",
+    "compute_effective_key",
+    "compute_fingerprint",
+]