idemkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- idemkit/__init__.py +101 -0
- idemkit/_version.py +1 -0
- idemkit/adapters/__init__.py +10 -0
- idemkit/adapters/ai.py +723 -0
- idemkit/adapters/asgi.py +533 -0
- idemkit/adapters/queue.py +413 -0
- idemkit/adapters/route.py +273 -0
- idemkit/adapters/wsgi.py +393 -0
- idemkit/backends/__init__.py +30 -0
- idemkit/backends/base.py +116 -0
- idemkit/backends/dynamodb.py +489 -0
- idemkit/backends/memory.py +325 -0
- idemkit/backends/mongo.py +395 -0
- idemkit/backends/postgres.py +682 -0
- idemkit/backends/redis.py +547 -0
- idemkit/cli.py +165 -0
- idemkit/conformance/__init__.py +25 -0
- idemkit/conformance/backend.py +257 -0
- idemkit/conformance/report.py +54 -0
- idemkit/contrib/__init__.py +30 -0
- idemkit/contrib/drf.py +181 -0
- idemkit/contrib/fastapi.py +141 -0
- idemkit/contrib/kafka.py +96 -0
- idemkit/contrib/logging.py +61 -0
- idemkit/contrib/mcp.py +113 -0
- idemkit/contrib/prometheus.py +79 -0
- idemkit/contrib/pubsub.py +98 -0
- idemkit/contrib/rabbitmq.py +138 -0
- idemkit/contrib/reconciliation.py +86 -0
- idemkit/contrib/sqs.py +117 -0
- idemkit/core/__init__.py +36 -0
- idemkit/core/codecs.py +229 -0
- idemkit/core/config.py +255 -0
- idemkit/core/engine.py +331 -0
- idemkit/core/events.py +63 -0
- idemkit/core/exception_cache.py +88 -0
- idemkit/core/exceptions.py +79 -0
- idemkit/core/fingerprint.py +153 -0
- idemkit/core/policy.py +143 -0
- idemkit/core/runner.py +664 -0
- idemkit/core/state.py +95 -0
- idemkit/core/sync_bridge.py +115 -0
- idemkit/problem_details.py +119 -0
- idemkit/py.typed +0 -0
- idemkit-0.1.0.dist-info/METADATA +446 -0
- idemkit-0.1.0.dist-info/RECORD +48 -0
- idemkit-0.1.0.dist-info/WHEEL +4 -0
- idemkit-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
"""Queue consumer idempotency, per spec §7.
|
|
2
|
+
|
|
3
|
+
``IdempotentConsumer`` wraps a message handler so its side effect runs once per
|
|
4
|
+
broker dedup id even under at-least-once delivery, concurrent consumers, and
|
|
5
|
+
consumer crashes. It is broker-agnostic: you tell it how to read the dedup id,
|
|
6
|
+
the scope, and (ideally) the broker's native receive count; it tells you whether
|
|
7
|
+
to ack the message or leave it for redelivery.
|
|
8
|
+
|
|
9
|
+
What this closes that ad-hoc dedup does not (spec §7.2):
|
|
10
|
+
|
|
11
|
+
* **Lease vs visibility-timeout coupling.** The lease is derived from, and kept
|
|
12
|
+
strictly shorter than, the broker's visibility timeout, and the heartbeat
|
|
13
|
+
(spec §5.3.1) renews it so a slow handler keeps its claim without ever letting
|
|
14
|
+
the lease approach the visibility window. A lease >= visibility is rejected at
|
|
15
|
+
construction.
|
|
16
|
+
* **Crash-safe claim.** A consumer that dies after claiming but before acking
|
|
17
|
+
does not wedge the message: the storage-clock lease lapses and the redelivery
|
|
18
|
+
is reclaimed and processed once; the dead consumer's late completion is fenced.
|
|
19
|
+
* **Side-effect-only vs result-bearing.** A handler that returns ``None`` records
|
|
20
|
+
a "processed" marker (replay = skip). A handler that returns a value caches it
|
|
21
|
+
and replays it on redelivery.
|
|
22
|
+
* **Poison / DLQ boundary.** Idempotency releases the claim on failure so the
|
|
23
|
+
broker can redeliver — but bounded. After ``max_attempts`` the consumer stops
|
|
24
|
+
retrying and calls ``on_exhausted`` (route to a DLQ, alert, ...).
|
|
25
|
+
* **Attempt counting that survives release** (spec §7.2 #6). The claim record is
|
|
26
|
+
gone after a release, so attempts are counted from the broker's receive count
|
|
27
|
+
when available, else a separate attempt store.
|
|
28
|
+
|
|
29
|
+
You write: how to read the dedup id and visibility timeout, and your handler.
|
|
30
|
+
idemkit guarantees the side effect runs once per dedup id and keeps the lease
|
|
31
|
+
under the visibility timeout. You decide ``max_attempts`` and ``on_exhausted``.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import asyncio
|
|
37
|
+
import enum
|
|
38
|
+
import hashlib
|
|
39
|
+
import inspect
|
|
40
|
+
import logging
|
|
41
|
+
from collections.abc import Callable
|
|
42
|
+
from dataclasses import dataclass
|
|
43
|
+
from typing import Any, Protocol
|
|
44
|
+
|
|
45
|
+
from idemkit.backends.base import IdempotencyBackend
|
|
46
|
+
from idemkit.core.codecs import JsonResultCodec, ResultCodec, SideEffectCodec
|
|
47
|
+
from idemkit.core.events import EventEmitter
|
|
48
|
+
from idemkit.core.exceptions import ConfigurationError, PayloadMismatch, StorageError
|
|
49
|
+
from idemkit.core.fingerprint import compose_key
|
|
50
|
+
from idemkit.core.policy import QueueConfig
|
|
51
|
+
from idemkit.core.runner import IdempotentCore, RunStatus, StoredResult
|
|
52
|
+
from idemkit.core.sync_bridge import register_closable, run_sync
|
|
53
|
+
|
|
54
|
+
_logger = logging.getLogger(__name__)
|
|
55
|
+
|
|
56
|
+
# A queue message's identity is its dedup id (carried in the effective key), so
|
|
57
|
+
# there is no payload fingerprint to compare — a fixed value means the core's
|
|
58
|
+
# fingerprint check never trips for the queue surface (spec §5.1: the broker's
|
|
59
|
+
# dedup id IS the "same operation" declaration).
|
|
60
|
+
QUEUE_FINGERPRINT = "idemkit-queue-v1"
|
|
61
|
+
|
|
62
|
+
# By default the lease is half the visibility timeout: comfortably shorter (so a
|
|
63
|
+
# redelivery can't race a still-running handler) while leaving room for the
|
|
64
|
+
# heartbeat to renew it. Override with an explicit ``lease_ttl_seconds``.
|
|
65
|
+
_DEFAULT_LEASE_FRACTION = 0.5
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _default_scope(_m: Any) -> tuple[()]:
|
|
69
|
+
"""Default queue scope: no scope (a single shared namespace)."""
|
|
70
|
+
return ()
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class ConsumerAction(enum.Enum):
|
|
74
|
+
"""What the broker glue should do with the message after ``dispatch``."""
|
|
75
|
+
|
|
76
|
+
ACK = "ack"
|
|
77
|
+
"""Remove it from the broker. Success, a deduplicated skip, or DLQ'd."""
|
|
78
|
+
|
|
79
|
+
RETRY = "retry"
|
|
80
|
+
"""Leave it for redelivery. A transient failure (within ``max_attempts``), a
|
|
81
|
+
sibling consumer still processing it, or a lost lease."""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass(frozen=True)
|
|
85
|
+
class ConsumerResult:
|
|
86
|
+
"""Outcome of dispatching one message."""
|
|
87
|
+
|
|
88
|
+
action: ConsumerAction
|
|
89
|
+
status: RunStatus | None
|
|
90
|
+
attempt: int
|
|
91
|
+
result: Any = None
|
|
92
|
+
exhausted: bool = False
|
|
93
|
+
error: BaseException | None = None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class AttemptStore(Protocol):
|
|
97
|
+
"""Durable per-dedup-id attempt counter (spec §7.2 #6 fallback).
|
|
98
|
+
|
|
99
|
+
Used only when the broker does not expose a receive count. Implementations
|
|
100
|
+
increment atomically per delivery and expire entries after the redelivery
|
|
101
|
+
window. The claim record cannot hold this count because it is deleted on
|
|
102
|
+
release between redeliveries.
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
async def incr(self, dedup_id: str, ttl_seconds: float) -> int:
|
|
106
|
+
"""Increment and return the attempt number for ``dedup_id`` (1-based)."""
|
|
107
|
+
...
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class InMemoryAttemptStore:
|
|
111
|
+
"""Single-process attempt counter.
|
|
112
|
+
|
|
113
|
+
Honest limitation: this lives in one process, so it does NOT count attempts
|
|
114
|
+
across multiple consumer processes or survive a restart. In a multi-consumer
|
|
115
|
+
deployment, prefer the broker's native receive count (``receive_count=``) or
|
|
116
|
+
a durable ``AttemptStore``. idemkit falls back to this only when neither is
|
|
117
|
+
given, and warns once.
|
|
118
|
+
"""
|
|
119
|
+
|
|
120
|
+
def __init__(self) -> None:
|
|
121
|
+
import asyncio
|
|
122
|
+
|
|
123
|
+
self._counts: dict[str, int] = {}
|
|
124
|
+
self._lock = asyncio.Lock()
|
|
125
|
+
|
|
126
|
+
async def incr(self, dedup_id: str, ttl_seconds: float) -> int:
|
|
127
|
+
async with self._lock:
|
|
128
|
+
self._counts[dedup_id] = self._counts.get(dedup_id, 0) + 1
|
|
129
|
+
return self._counts[dedup_id]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class IdempotentConsumer:
|
|
133
|
+
"""Idempotent queue-consumer wrapper (spec §7.3)."""
|
|
134
|
+
|
|
135
|
+
def __init__(
|
|
136
|
+
self,
|
|
137
|
+
*,
|
|
138
|
+
backend: IdempotencyBackend,
|
|
139
|
+
config: QueueConfig,
|
|
140
|
+
handler: Callable[[Any], Any] | None = None,
|
|
141
|
+
) -> None:
|
|
142
|
+
# backend is WHERE state lives; everything else is on the QueueConfig.
|
|
143
|
+
c = config
|
|
144
|
+
if c.dedup_id is None:
|
|
145
|
+
raise ConfigurationError(
|
|
146
|
+
"idemkit: QueueConfig.dedup_id is required — tell idemkit how to read "
|
|
147
|
+
"the broker's dedup id from a message (e.g. dedup_id=lambda m: m.message_id). "
|
|
148
|
+
"The contrib sqs/kafka helpers preset it for you."
|
|
149
|
+
)
|
|
150
|
+
if c.visibility_timeout_seconds is None:
|
|
151
|
+
raise ConfigurationError(
|
|
152
|
+
"idemkit: QueueConfig.visibility_timeout_seconds is required (the "
|
|
153
|
+
"broker's visibility window; the lease is derived from it)."
|
|
154
|
+
)
|
|
155
|
+
dedup_id = c.dedup_id
|
|
156
|
+
visibility_timeout_seconds = c.visibility_timeout_seconds
|
|
157
|
+
scope = c.scope if c.scope is not None else _default_scope
|
|
158
|
+
max_attempts = c.max_attempts
|
|
159
|
+
on_exhausted = c.on_exhausted
|
|
160
|
+
receive_count = c.receive_count
|
|
161
|
+
attempt_store = c.attempt_store
|
|
162
|
+
cache_result = c.cache_result
|
|
163
|
+
result_codec = c.result_codec
|
|
164
|
+
validation_fingerprint = c.validation_fingerprint
|
|
165
|
+
lease_ttl_seconds = c.lease_ttl_seconds
|
|
166
|
+
expires_after_seconds = c.expires_after_seconds
|
|
167
|
+
wait_timeout_seconds = c.wait_timeout_seconds if c.wait_timeout_seconds is not None else 5.0
|
|
168
|
+
on_storage_error = c.on_storage_error
|
|
169
|
+
use_local_cache = c.use_local_cache
|
|
170
|
+
local_cache_max_items = c.local_cache_max_items
|
|
171
|
+
event_handlers = list(c.event_handlers)
|
|
172
|
+
|
|
173
|
+
# Lease vs visibility timeout (§7.2 #1): the lease MUST be shorter than
|
|
174
|
+
# the broker's visibility timeout, or a redelivery can race a handler
|
|
175
|
+
# that is still legitimately running.
|
|
176
|
+
if lease_ttl_seconds is None:
|
|
177
|
+
lease_ttl_seconds = visibility_timeout_seconds * _DEFAULT_LEASE_FRACTION
|
|
178
|
+
if lease_ttl_seconds >= visibility_timeout_seconds:
|
|
179
|
+
raise ConfigurationError(
|
|
180
|
+
f"idemkit: lease_ttl_seconds ({lease_ttl_seconds}) must be shorter "
|
|
181
|
+
f"than visibility_timeout_seconds ({visibility_timeout_seconds}). "
|
|
182
|
+
"A lease at or beyond the visibility timeout lets the broker "
|
|
183
|
+
"redeliver while the first consumer is still running, so both "
|
|
184
|
+
"execute (spec §7.2 #1). Leave lease_ttl_seconds unset to derive "
|
|
185
|
+
"a safe default."
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
self.dedup_id = dedup_id
|
|
189
|
+
self.scope = scope
|
|
190
|
+
self.visibility_timeout_seconds = visibility_timeout_seconds
|
|
191
|
+
self.lease_ttl_seconds = lease_ttl_seconds
|
|
192
|
+
self.max_attempts = max_attempts
|
|
193
|
+
self.on_exhausted = on_exhausted
|
|
194
|
+
self.receive_count = receive_count
|
|
195
|
+
self.expires_after_seconds = expires_after_seconds
|
|
196
|
+
self._cache_result = cache_result
|
|
197
|
+
self._validation_fingerprint = validation_fingerprint
|
|
198
|
+
self._backend = backend
|
|
199
|
+
self._handler: Callable[[Any], Any] | None = None
|
|
200
|
+
if handler is not None:
|
|
201
|
+
self._set_handler(handler)
|
|
202
|
+
self._warned_inprocess_attempts = False
|
|
203
|
+
|
|
204
|
+
# A fallback attempt counter is ALWAYS available, so max_attempts stays
|
|
205
|
+
# enforced even when a configured receive_count returns None for some
|
|
206
|
+
# delivery (a broker that didn't populate the count). The in-process
|
|
207
|
+
# default is warned at first use; prefer a durable store or the broker's
|
|
208
|
+
# receive count in a multi-consumer deployment (§7.2 #6).
|
|
209
|
+
self._attempt_store: AttemptStore = attempt_store or InMemoryAttemptStore()
|
|
210
|
+
|
|
211
|
+
if result_codec is not None:
|
|
212
|
+
self._codec: ResultCodec[Any, Any] = result_codec
|
|
213
|
+
elif cache_result:
|
|
214
|
+
self._codec = JsonResultCodec()
|
|
215
|
+
else:
|
|
216
|
+
self._codec = SideEffectCodec()
|
|
217
|
+
|
|
218
|
+
self.core = IdempotentCore(
|
|
219
|
+
backend,
|
|
220
|
+
emitter=EventEmitter(event_handlers or []),
|
|
221
|
+
backend_name=type(backend).__name__,
|
|
222
|
+
surface="queue",
|
|
223
|
+
on_storage_error=on_storage_error,
|
|
224
|
+
lease_ttl_seconds=lease_ttl_seconds,
|
|
225
|
+
wait_timeout_seconds=wait_timeout_seconds,
|
|
226
|
+
expires_after_seconds=expires_after_seconds,
|
|
227
|
+
use_local_cache=use_local_cache,
|
|
228
|
+
local_cache_max_items=local_cache_max_items,
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
def handle(self, fn: Callable[[Any], Any]) -> Callable[[Any], Any]:
|
|
232
|
+
"""Register the message handler. Usable as ``@consumer.handle``.
|
|
233
|
+
|
|
234
|
+
The handler may be ``async def`` (awaited) or a plain ``def`` (run on a
|
|
235
|
+
worker thread so a blocking call doesn't stall the event loop). Drive an
|
|
236
|
+
async handler with ``await dispatch(...)``; a synchronous poll loop calls
|
|
237
|
+
``dispatch_sync(...)``.
|
|
238
|
+
"""
|
|
239
|
+
self._set_handler(fn)
|
|
240
|
+
return fn
|
|
241
|
+
|
|
242
|
+
def _set_handler(self, fn: Callable[[Any], Any]) -> None:
|
|
243
|
+
self._handler = fn
|
|
244
|
+
self._handler_is_async = inspect.iscoroutinefunction(fn)
|
|
245
|
+
|
|
246
|
+
def effective_key(self, message: Any) -> str:
|
|
247
|
+
"""The effective key for a message — exposed for tests that need to set
|
|
248
|
+
up a record by hand (spec §9.4 testability)."""
|
|
249
|
+
dedup_id = self.dedup_id(message)
|
|
250
|
+
return self._effective_key(message, dedup_id)
|
|
251
|
+
|
|
252
|
+
def _fingerprint(self, message: Any) -> str:
|
|
253
|
+
"""Payload fingerprint compared on a dedup-id hit (spec §5.1).
|
|
254
|
+
|
|
255
|
+
With no ``validation_fingerprint`` it is a constant, so two deliveries of
|
|
256
|
+
the same dedup id always replay: the broker's id is authoritative. With one
|
|
257
|
+
set, a redelivery whose fingerprint bytes differ is a ``PayloadMismatch`` —
|
|
258
|
+
this catches a producer that reused a message id with a different body."""
|
|
259
|
+
if self._validation_fingerprint is None:
|
|
260
|
+
return QUEUE_FINGERPRINT
|
|
261
|
+
return hashlib.sha256(self._validation_fingerprint(message)).hexdigest()
|
|
262
|
+
|
|
263
|
+
async def dispatch(self, message: Any) -> ConsumerResult:
|
|
264
|
+
"""Process one delivery: dedup, run the handler at most once, and report
|
|
265
|
+
whether to ack or leave it for redelivery."""
|
|
266
|
+
if self._handler is None:
|
|
267
|
+
raise RuntimeError(
|
|
268
|
+
"idemkit: no handler registered. Use @consumer.handle or pass "
|
|
269
|
+
"handler= to IdempotentConsumer."
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
dedup_id = self.dedup_id(message)
|
|
273
|
+
effective_key = self._effective_key(message, dedup_id)
|
|
274
|
+
attempt = await self._attempt_number(message, dedup_id)
|
|
275
|
+
handler = self._handler
|
|
276
|
+
|
|
277
|
+
async def invoke() -> Any:
|
|
278
|
+
# Return the raw handler value; serialization (for result-bearing
|
|
279
|
+
# mode) happens AFTER the protected run so an unserializable result
|
|
280
|
+
# fails closed instead of releasing the claim and re-running (§5.4).
|
|
281
|
+
if self._handler_is_async:
|
|
282
|
+
return await handler(message)
|
|
283
|
+
# Synchronous handler: run it on a worker thread so a blocking call
|
|
284
|
+
# doesn't stall the loop (and the heartbeat can keep renewing).
|
|
285
|
+
return await asyncio.to_thread(handler, message)
|
|
286
|
+
|
|
287
|
+
encode = self._codec.encode if self._cache_result else None
|
|
288
|
+
|
|
289
|
+
try:
|
|
290
|
+
outcome = await self.core.run_once(
|
|
291
|
+
effective_key, self._fingerprint(message), invoke, encode=encode, heartbeat=True
|
|
292
|
+
)
|
|
293
|
+
except StorageError as exc:
|
|
294
|
+
# The idempotency backend is unavailable — an INFRASTRUCTURE problem,
|
|
295
|
+
# not a poison message. Leave it for redelivery WITHOUT counting it
|
|
296
|
+
# against max_attempts, so an outage doesn't dead-letter healthy
|
|
297
|
+
# messages the handler never even ran.
|
|
298
|
+
return ConsumerResult(ConsumerAction.RETRY, None, attempt, error=exc)
|
|
299
|
+
except Exception as exc:
|
|
300
|
+
# The handler itself raised; run_once released the claim so the broker
|
|
301
|
+
# may redeliver. Bounded by max_attempts (§7.2 #4).
|
|
302
|
+
if attempt >= self.max_attempts:
|
|
303
|
+
await self._exhaust(message, exc)
|
|
304
|
+
return ConsumerResult(ConsumerAction.ACK, None, attempt, exhausted=True, error=exc)
|
|
305
|
+
return ConsumerResult(ConsumerAction.RETRY, None, attempt, error=exc)
|
|
306
|
+
|
|
307
|
+
status = outcome.status
|
|
308
|
+
if status in (RunStatus.EXECUTED, RunStatus.REPLAYED, RunStatus.UNPROTECTED):
|
|
309
|
+
result_value = self._decode_result(outcome.stored)
|
|
310
|
+
return ConsumerResult(ConsumerAction.ACK, status, attempt, result=result_value)
|
|
311
|
+
|
|
312
|
+
if status is RunStatus.MISMATCH:
|
|
313
|
+
# Same dedup id, DIFFERENT validation payload (validation_fingerprint is
|
|
314
|
+
# set and the bytes changed): a producer reused a message id with a new
|
|
315
|
+
# body, or an id collision. Redelivery can't fix a body mismatch, so do
|
|
316
|
+
# NOT retry forever — hand it to on_exhausted (DLQ) if set, then ack.
|
|
317
|
+
mismatch = PayloadMismatch(
|
|
318
|
+
f"idemkit: message dedup id {dedup_id!r} was already processed with a "
|
|
319
|
+
"different validation payload; refusing to replay a mismatched body."
|
|
320
|
+
)
|
|
321
|
+
_logger.error("%s", mismatch)
|
|
322
|
+
await self._exhaust(message, mismatch)
|
|
323
|
+
return ConsumerResult(
|
|
324
|
+
ConsumerAction.ACK, status, attempt, exhausted=True, error=mismatch
|
|
325
|
+
)
|
|
326
|
+
|
|
327
|
+
if status is RunStatus.ENCODE_FAILED:
|
|
328
|
+
# The side effect ran once but its result couldn't be serialized. Ack
|
|
329
|
+
# so the message is NOT redelivered and re-run; the result is simply
|
|
330
|
+
# not cached. Loud because it's an application bug to fix.
|
|
331
|
+
_logger.error(
|
|
332
|
+
"idemkit: handler result for dedup id %r could not be serialized; "
|
|
333
|
+
"acking so it is not re-run, but the result was not cached. Return "
|
|
334
|
+
"a serializable value or set a matching result_codec (§5.4).",
|
|
335
|
+
dedup_id,
|
|
336
|
+
)
|
|
337
|
+
return ConsumerResult(ConsumerAction.ACK, status, attempt)
|
|
338
|
+
|
|
339
|
+
# CONFLICT or LEASE_LOST: a sibling consumer holds (or held) the claim
|
|
340
|
+
# and we couldn't confirm completion. Decline WITHOUT acking — acking
|
|
341
|
+
# would risk losing the message if the sibling then fails. Let the broker
|
|
342
|
+
# redeliver; by then the record is likely COMPLETED and we replay.
|
|
343
|
+
return ConsumerResult(ConsumerAction.RETRY, status, attempt)
|
|
344
|
+
|
|
345
|
+
def dispatch_sync(self, message: Any) -> ConsumerResult:
|
|
346
|
+
"""Synchronous :meth:`dispatch` for a non-async poll loop.
|
|
347
|
+
|
|
348
|
+
Runs the same dedup/claim/lease flow on a shared background event loop, so
|
|
349
|
+
a Celery task or a threaded SQS/Kafka worker can call it like a normal
|
|
350
|
+
function. The handler (async or sync) is driven there too. Do not call
|
|
351
|
+
this from inside a running event loop — use ``await dispatch(...)``.
|
|
352
|
+
"""
|
|
353
|
+
register_closable(self._backend)
|
|
354
|
+
return run_sync(self.dispatch(message))
|
|
355
|
+
|
|
356
|
+
# ----- internals -----
|
|
357
|
+
|
|
358
|
+
def _effective_key(self, message: Any, dedup_id: str) -> str:
|
|
359
|
+
scope = self.scope(message)
|
|
360
|
+
if scope is None:
|
|
361
|
+
scope_parts: list[str] = []
|
|
362
|
+
elif isinstance(scope, str):
|
|
363
|
+
scope_parts = [scope]
|
|
364
|
+
else:
|
|
365
|
+
scope_parts = [str(part) for part in scope]
|
|
366
|
+
return compose_key([dedup_id, *scope_parts])
|
|
367
|
+
|
|
368
|
+
async def _attempt_number(self, message: Any, dedup_id: str) -> int:
|
|
369
|
+
# (a) broker-native receive count — the simplest, most honest source.
|
|
370
|
+
if self.receive_count is not None:
|
|
371
|
+
count = self.receive_count(message)
|
|
372
|
+
if count is not None:
|
|
373
|
+
return int(count)
|
|
374
|
+
# (b) the separate attempts record (always present) — so max_attempts is
|
|
375
|
+
# enforced even when (a) is configured but returned None this delivery.
|
|
376
|
+
if isinstance(self._attempt_store, InMemoryAttemptStore):
|
|
377
|
+
self._warn_inprocess_attempts()
|
|
378
|
+
return await self._attempt_store.incr(dedup_id, self.expires_after_seconds)
|
|
379
|
+
|
|
380
|
+
def _decode_result(self, stored: StoredResult | None) -> Any:
|
|
381
|
+
if not self._cache_result or stored is None:
|
|
382
|
+
return None
|
|
383
|
+
return self._codec.decode(stored)
|
|
384
|
+
|
|
385
|
+
async def _exhaust(self, message: Any, error: BaseException | None) -> None:
|
|
386
|
+
if self.on_exhausted is None:
|
|
387
|
+
_logger.warning(
|
|
388
|
+
"idemkit: message hit max_attempts=%d with no on_exhausted hook "
|
|
389
|
+
"configured; it will be acked and dropped. Set on_exhausted to "
|
|
390
|
+
"route poison messages to a DLQ (spec §7.2 #4).",
|
|
391
|
+
self.max_attempts,
|
|
392
|
+
)
|
|
393
|
+
return
|
|
394
|
+
outcome = self.on_exhausted(message, error)
|
|
395
|
+
if inspect.isawaitable(outcome):
|
|
396
|
+
await outcome
|
|
397
|
+
|
|
398
|
+
def _warn_inprocess_attempts(self) -> None:
|
|
399
|
+
if self._warned_inprocess_attempts:
|
|
400
|
+
return
|
|
401
|
+
# With InMemoryBackend the whole consumer is single-process by definition
|
|
402
|
+
# (its own warning already says so), so an in-process attempt counter is
|
|
403
|
+
# perfectly correct — no need to add noise about multi-process here.
|
|
404
|
+
if self.core.backend_name == "InMemoryBackend":
|
|
405
|
+
self._warned_inprocess_attempts = True
|
|
406
|
+
return
|
|
407
|
+
self._warned_inprocess_attempts = True
|
|
408
|
+
_logger.warning(
|
|
409
|
+
"idemkit: counting delivery attempts in-process. This does NOT survive "
|
|
410
|
+
"across consumer processes or restarts, so max_attempts may not be "
|
|
411
|
+
"enforced correctly in a multi-consumer deployment. Pass receive_count "
|
|
412
|
+
"(broker-native) or a durable AttemptStore (spec §7.2 #6)."
|
|
413
|
+
)
|
|
@@ -0,0 +1,273 @@
|
|
|
1
|
+
"""Per-route idempotency for Starlette / FastAPI.
|
|
2
|
+
|
|
3
|
+
Where ``IdempotencyMiddleware`` applies idempotency to a whole app, ``Idempotency``
|
|
4
|
+
attaches it to individual endpoints with a decorator. Bind it to a backend and
|
|
5
|
+
config once, then decorate the routes you want protected::
|
|
6
|
+
|
|
7
|
+
from idemkit import HttpConfig, Idempotency, RedisBackend
|
|
8
|
+
|
|
9
|
+
idem = Idempotency(
|
|
10
|
+
backend=RedisBackend.from_url("redis://localhost:6379"),
|
|
11
|
+
config=HttpConfig(scope=lambda req: req.state.user.id),
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@app.post("/charge")
|
|
16
|
+
@idem.protect
|
|
17
|
+
async def charge(request: Request) -> Response:
|
|
18
|
+
...
|
|
19
|
+
return JSONResponse({"ok": True}, status_code=201)
|
|
20
|
+
|
|
21
|
+
Two requirements for a decorated endpoint:
|
|
22
|
+
|
|
23
|
+
1. It must declare a ``request: Request`` parameter (this is how the decorator
|
|
24
|
+
gets the request; it is also what your ``scope`` / ``key``
|
|
25
|
+
receive — the real Starlette ``Request``, so ``request.state`` and
|
|
26
|
+
``request.client`` work).
|
|
27
|
+
2. When idempotency applies, it must return a Starlette/FastAPI ``Response``
|
|
28
|
+
(``JSONResponse``, ``PlainTextResponse``, ...). A handler that returns a dict
|
|
29
|
+
relies on FastAPI's serialization, which happens *after* this decorator runs
|
|
30
|
+
and cannot be captured here. For dict/model returns, use the middleware
|
|
31
|
+
instead, or return a ``Response`` explicitly.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import functools
|
|
37
|
+
import logging
|
|
38
|
+
from collections.abc import Awaitable, Callable
|
|
39
|
+
from typing import Any
|
|
40
|
+
|
|
41
|
+
from idemkit.adapters.asgi import _unwrap_sf_string
|
|
42
|
+
from idemkit.backends.base import IdempotencyBackend
|
|
43
|
+
from idemkit.core.config import resolve_http_config
|
|
44
|
+
from idemkit.core.engine import (
|
|
45
|
+
EngineOutcome,
|
|
46
|
+
IdempotencyEngine,
|
|
47
|
+
NeutralRequest,
|
|
48
|
+
NeutralResponse,
|
|
49
|
+
)
|
|
50
|
+
from idemkit.core.exceptions import (
|
|
51
|
+
ConfigurationError,
|
|
52
|
+
IdempotencyConflict,
|
|
53
|
+
IdempotencyError,
|
|
54
|
+
IdempotencyKeyMissing,
|
|
55
|
+
PayloadMismatch,
|
|
56
|
+
StorageUnavailable,
|
|
57
|
+
)
|
|
58
|
+
from idemkit.core.policy import HttpConfig
|
|
59
|
+
|
|
60
|
+
_logger = logging.getLogger(__name__)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class Idempotency:
|
|
64
|
+
"""Decorator-based idempotency bound to a backend + config."""
|
|
65
|
+
|
|
66
|
+
def __init__(
|
|
67
|
+
self,
|
|
68
|
+
*,
|
|
69
|
+
backend: IdempotencyBackend,
|
|
70
|
+
config: HttpConfig | None = None,
|
|
71
|
+
) -> None:
|
|
72
|
+
resolved = resolve_http_config(config)
|
|
73
|
+
self.config = resolved
|
|
74
|
+
self.engine = IdempotencyEngine(backend=backend, config=resolved)
|
|
75
|
+
|
|
76
|
+
def protect(self, func: Callable[..., Awaitable[Any]]) -> Callable[..., Awaitable[Any]]:
|
|
77
|
+
"""Decorate an async endpoint so duplicate requests are deduplicated."""
|
|
78
|
+
|
|
79
|
+
@functools.wraps(func)
|
|
80
|
+
async def wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
81
|
+
request = _find_request(args, kwargs)
|
|
82
|
+
if request is None:
|
|
83
|
+
raise TypeError(
|
|
84
|
+
"idemkit: @Idempotency.protect requires the endpoint to "
|
|
85
|
+
"declare a `request: Request` parameter."
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
body = await request.body()
|
|
89
|
+
max_request = self.config.max_request_body_bytes
|
|
90
|
+
if max_request is not None and len(body) > max_request:
|
|
91
|
+
# Too large to fingerprint; bypass idempotency entirely.
|
|
92
|
+
_logger.warning(
|
|
93
|
+
"idemkit: request body exceeded max_request_body_bytes=%s; "
|
|
94
|
+
"bypassing idempotency for this request.",
|
|
95
|
+
max_request,
|
|
96
|
+
)
|
|
97
|
+
return await func(*args, **kwargs)
|
|
98
|
+
|
|
99
|
+
neutral = self._neutral_request(request, body)
|
|
100
|
+
outcome = await self.engine.decide(neutral)
|
|
101
|
+
|
|
102
|
+
if outcome.kind == "replay":
|
|
103
|
+
assert outcome.response is not None
|
|
104
|
+
return _to_response(outcome.response)
|
|
105
|
+
|
|
106
|
+
if outcome.kind == "reject":
|
|
107
|
+
# The per-route decorator raises typed exceptions (§6.3) so your
|
|
108
|
+
# code can catch and branch on them. Register
|
|
109
|
+
# ``idempotency_problem_handler`` to render them as the same
|
|
110
|
+
# problem+json the middleware would return.
|
|
111
|
+
_raise_for_reject(outcome)
|
|
112
|
+
|
|
113
|
+
if outcome.effective_key is None or outcome.claim_token is None:
|
|
114
|
+
# Idempotency does not apply (method not in scope, or no key).
|
|
115
|
+
return await func(*args, **kwargs)
|
|
116
|
+
|
|
117
|
+
try:
|
|
118
|
+
result = await func(*args, **kwargs)
|
|
119
|
+
except Exception:
|
|
120
|
+
await self.engine.on_error(outcome.effective_key, outcome.claim_token)
|
|
121
|
+
raise
|
|
122
|
+
|
|
123
|
+
response = _require_response(result)
|
|
124
|
+
captured, bypass = _capture(response, self.config.max_body_bytes)
|
|
125
|
+
if captured is None:
|
|
126
|
+
# Streaming or oversized response: deliver it, don't cache it.
|
|
127
|
+
if bypass is not None:
|
|
128
|
+
response.headers["idempotency-replay-unavailable"] = bypass
|
|
129
|
+
await self.engine.on_error(outcome.effective_key, outcome.claim_token)
|
|
130
|
+
return response
|
|
131
|
+
|
|
132
|
+
await self.engine.on_complete(outcome.effective_key, outcome.claim_token, captured)
|
|
133
|
+
return response
|
|
134
|
+
|
|
135
|
+
return wrapper
|
|
136
|
+
|
|
137
|
+
# Allow `@idem` as shorthand for `@idem.protect`.
|
|
138
|
+
__call__ = protect
|
|
139
|
+
|
|
140
|
+
def _neutral_request(self, request: Any, body: bytes) -> NeutralRequest:
|
|
141
|
+
if self.config.key is not None:
|
|
142
|
+
try:
|
|
143
|
+
key = self.config.key(request)
|
|
144
|
+
except Exception:
|
|
145
|
+
_logger.exception("idemkit: key raised; treating as no key")
|
|
146
|
+
key = None
|
|
147
|
+
else:
|
|
148
|
+
key = _unwrap_sf_string(request.headers.get("idempotency-key"))
|
|
149
|
+
|
|
150
|
+
if self.config.scope is None:
|
|
151
|
+
identity: str | None = None # only reachable in optional mode
|
|
152
|
+
else:
|
|
153
|
+
try:
|
|
154
|
+
identity = self.config.scope(request) or None
|
|
155
|
+
except Exception:
|
|
156
|
+
_logger.exception("idemkit: scope raised; treating as anonymous")
|
|
157
|
+
identity = None
|
|
158
|
+
|
|
159
|
+
return NeutralRequest(
|
|
160
|
+
idempotency_key=key,
|
|
161
|
+
scope=identity,
|
|
162
|
+
method=request.method,
|
|
163
|
+
path=request.url.path,
|
|
164
|
+
query=request.url.query,
|
|
165
|
+
body=body,
|
|
166
|
+
content_type=request.headers.get("content-type"),
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _raise_for_reject(outcome: EngineOutcome) -> None:
|
|
171
|
+
"""Map a reject outcome to a typed exception carrying its problem+json (§6.3)."""
|
|
172
|
+
assert outcome.response is not None
|
|
173
|
+
problem = (
|
|
174
|
+
outcome.response.status,
|
|
175
|
+
dict(outcome.response.headers),
|
|
176
|
+
outcome.response.body,
|
|
177
|
+
)
|
|
178
|
+
reason = outcome.reject_reason
|
|
179
|
+
if reason == "payload_mismatch":
|
|
180
|
+
exc: IdempotencyError = PayloadMismatch(
|
|
181
|
+
"Idempotency key was reused with a different payload. Resend the "
|
|
182
|
+
"original payload or use a new key; the stored result is not replayed."
|
|
183
|
+
)
|
|
184
|
+
elif reason == "in_progress":
|
|
185
|
+
exc = IdempotencyConflict(
|
|
186
|
+
"Another request with this idempotency key is in progress and did not "
|
|
187
|
+
"finish within the wait timeout. Retry with bounded backoff."
|
|
188
|
+
)
|
|
189
|
+
elif reason == "missing_key":
|
|
190
|
+
exc = IdempotencyKeyMissing("This endpoint requires a valid Idempotency-Key header.")
|
|
191
|
+
elif reason == "storage_error":
|
|
192
|
+
exc = StorageUnavailable(
|
|
193
|
+
"The idempotency backend is unavailable and the policy is fail_closed. "
|
|
194
|
+
"Retry after a short backoff."
|
|
195
|
+
)
|
|
196
|
+
else: # identity_unavailable — a server-side configuration problem
|
|
197
|
+
exc = ConfigurationError(
|
|
198
|
+
"The scope extractor produced no identity, so idempotency "
|
|
199
|
+
"was refused to preserve cross-tenant isolation. Fix the extractor."
|
|
200
|
+
)
|
|
201
|
+
exc.problem = problem
|
|
202
|
+
raise exc
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def idempotency_problem_handler(request: Any, exc: Exception) -> Any:
|
|
206
|
+
"""Starlette/FastAPI exception handler that renders an idemkit typed
|
|
207
|
+
exception as the same RFC 9457 problem+json the middleware would return.
|
|
208
|
+
|
|
209
|
+
Install it once so the decorator's typed exceptions still reach the client
|
|
210
|
+
as proper HTTP responses::
|
|
211
|
+
|
|
212
|
+
from idemkit.adapters.route import idempotency_problem_handler
|
|
213
|
+
from idemkit.core.exceptions import IdempotencyError
|
|
214
|
+
|
|
215
|
+
app.add_exception_handler(IdempotencyError, idempotency_problem_handler)
|
|
216
|
+
|
|
217
|
+
Exceptions without a wire mapping (``problem is None``) are re-raised so the
|
|
218
|
+
framework's default handling applies.
|
|
219
|
+
"""
|
|
220
|
+
from starlette.responses import Response
|
|
221
|
+
|
|
222
|
+
problem = getattr(exc, "problem", None)
|
|
223
|
+
if problem is None:
|
|
224
|
+
raise exc
|
|
225
|
+
status, headers, body = problem
|
|
226
|
+
return Response(content=body, status_code=status, headers=dict(headers))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _find_request(args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any | None:
|
|
230
|
+
from starlette.requests import Request
|
|
231
|
+
|
|
232
|
+
for value in (*kwargs.values(), *args):
|
|
233
|
+
if isinstance(value, Request):
|
|
234
|
+
return value
|
|
235
|
+
return None
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _require_response(result: Any) -> Any:
|
|
239
|
+
from starlette.responses import Response
|
|
240
|
+
|
|
241
|
+
if isinstance(result, Response):
|
|
242
|
+
return result
|
|
243
|
+
raise TypeError(
|
|
244
|
+
"idemkit: an idempotency-protected endpoint must return a "
|
|
245
|
+
"Starlette/FastAPI Response (e.g. JSONResponse(...)), not "
|
|
246
|
+
f"{type(result).__name__}. Returning a dict/model relies on FastAPI "
|
|
247
|
+
"serialization that runs after this decorator and can't be captured. "
|
|
248
|
+
"Return a Response here, or use IdempotencyMiddleware app-wide."
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _capture(response: Any, max_body_bytes: int) -> tuple[NeutralResponse | None, str | None]:
|
|
253
|
+
body = getattr(response, "body", None)
|
|
254
|
+
if body is None:
|
|
255
|
+
# No materialized body (e.g. StreamingResponse) → bypass cache.
|
|
256
|
+
return None, "streaming"
|
|
257
|
+
if len(body) > max_body_bytes:
|
|
258
|
+
return None, "size-exceeded"
|
|
259
|
+
headers = dict(response.headers.items())
|
|
260
|
+
return (
|
|
261
|
+
NeutralResponse(status=response.status_code, headers=headers, body=bytes(body)),
|
|
262
|
+
None,
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def _to_response(neutral: NeutralResponse) -> Any:
|
|
267
|
+
from starlette.responses import Response
|
|
268
|
+
|
|
269
|
+
return Response(
|
|
270
|
+
content=neutral.body,
|
|
271
|
+
status_code=neutral.status,
|
|
272
|
+
headers=dict(neutral.headers),
|
|
273
|
+
)
|