idemkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- idemkit/__init__.py +101 -0
- idemkit/_version.py +1 -0
- idemkit/adapters/__init__.py +10 -0
- idemkit/adapters/ai.py +723 -0
- idemkit/adapters/asgi.py +533 -0
- idemkit/adapters/queue.py +413 -0
- idemkit/adapters/route.py +273 -0
- idemkit/adapters/wsgi.py +393 -0
- idemkit/backends/__init__.py +30 -0
- idemkit/backends/base.py +116 -0
- idemkit/backends/dynamodb.py +489 -0
- idemkit/backends/memory.py +325 -0
- idemkit/backends/mongo.py +395 -0
- idemkit/backends/postgres.py +682 -0
- idemkit/backends/redis.py +547 -0
- idemkit/cli.py +165 -0
- idemkit/conformance/__init__.py +25 -0
- idemkit/conformance/backend.py +257 -0
- idemkit/conformance/report.py +54 -0
- idemkit/contrib/__init__.py +30 -0
- idemkit/contrib/drf.py +181 -0
- idemkit/contrib/fastapi.py +141 -0
- idemkit/contrib/kafka.py +96 -0
- idemkit/contrib/logging.py +61 -0
- idemkit/contrib/mcp.py +113 -0
- idemkit/contrib/prometheus.py +79 -0
- idemkit/contrib/pubsub.py +98 -0
- idemkit/contrib/rabbitmq.py +138 -0
- idemkit/contrib/reconciliation.py +86 -0
- idemkit/contrib/sqs.py +117 -0
- idemkit/core/__init__.py +36 -0
- idemkit/core/codecs.py +229 -0
- idemkit/core/config.py +255 -0
- idemkit/core/engine.py +331 -0
- idemkit/core/events.py +63 -0
- idemkit/core/exception_cache.py +88 -0
- idemkit/core/exceptions.py +79 -0
- idemkit/core/fingerprint.py +153 -0
- idemkit/core/policy.py +143 -0
- idemkit/core/runner.py +664 -0
- idemkit/core/state.py +95 -0
- idemkit/core/sync_bridge.py +115 -0
- idemkit/problem_details.py +119 -0
- idemkit/py.typed +0 -0
- idemkit-0.1.0.dist-info/METADATA +446 -0
- idemkit-0.1.0.dist-info/RECORD +48 -0
- idemkit-0.1.0.dist-info/WHEEL +4 -0
- idemkit-0.1.0.dist-info/entry_points.txt +2 -0
idemkit/contrib/drf.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""Django REST Framework idempotency, as a view mixin.
|
|
2
|
+
|
|
3
|
+
The WSGI middleware protects a whole Django app, but a DRF mixin lets you protect
|
|
4
|
+
*specific* views and read DRF's request (``request.user`` for ``scope``, ``request.data``
|
|
5
|
+
in the view). :func:`idempotent_view` returns a mixin class you put BEFORE the DRF
|
|
6
|
+
base class:
|
|
7
|
+
|
|
8
|
+
from rest_framework.views import APIView
|
|
9
|
+
from rest_framework.response import Response
|
|
10
|
+
from idemkit import RedisBackend, HttpConfig
|
|
11
|
+
from idemkit.contrib.drf import idempotent_view
|
|
12
|
+
|
|
13
|
+
backend = RedisBackend.from_url("redis://localhost:6379")
|
|
14
|
+
Idempotent = idempotent_view(
|
|
15
|
+
backend=backend, config=HttpConfig(scope=lambda req: str(req.user.pk))
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
class ChargeView(Idempotent, APIView):
|
|
19
|
+
def post(self, request):
|
|
20
|
+
return Response({"charged": True}, status=201)
|
|
21
|
+
|
|
22
|
+
It claims the key after DRF authentication runs (so ``request.user`` is set), replays
|
|
23
|
+
the stored response on a duplicate, and releases the claim if the view raises. Works
|
|
24
|
+
on ``APIView`` and ``ViewSet`` (both share the ``initial`` / ``handle_exception`` /
|
|
25
|
+
``finalize_response`` lifecycle). Requires Django + DRF; this module imports neither
|
|
26
|
+
until a response is built.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
from typing import Any
|
|
32
|
+
|
|
33
|
+
from idemkit.adapters.asgi import _unwrap_sf_string
|
|
34
|
+
from idemkit.backends.base import IdempotencyBackend
|
|
35
|
+
from idemkit.core.config import resolve_http_config
|
|
36
|
+
from idemkit.core.engine import (
|
|
37
|
+
EngineOutcome,
|
|
38
|
+
IdempotencyEngine,
|
|
39
|
+
NeutralRequest,
|
|
40
|
+
NeutralResponse,
|
|
41
|
+
)
|
|
42
|
+
from idemkit.core.exceptions import ConfigurationError
|
|
43
|
+
from idemkit.core.policy import HttpConfig
|
|
44
|
+
from idemkit.core.sync_bridge import register_closable, run_sync
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class _ShortCircuit(Exception):
|
|
48
|
+
"""Internal: raised from ``initial`` to return a replay/reject response without
|
|
49
|
+
running the view. Caught in the mixin's ``handle_exception``."""
|
|
50
|
+
|
|
51
|
+
def __init__(self, response: NeutralResponse) -> None:
|
|
52
|
+
self.neutral = response
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def idempotent_view(
|
|
56
|
+
*,
|
|
57
|
+
backend: IdempotencyBackend,
|
|
58
|
+
config: HttpConfig | None = None,
|
|
59
|
+
) -> type:
|
|
60
|
+
"""Build a DRF view mixin that applies idempotency.
|
|
61
|
+
|
|
62
|
+
Put the returned class BEFORE your DRF base (``class V(idempotent_view(...), APIView)``).
|
|
63
|
+
If it is placed AFTER the base, the mixin raises at class-definition time rather
|
|
64
|
+
than silently doing nothing. One engine is shared by every view using this mixin.
|
|
65
|
+
"""
|
|
66
|
+
resolved = resolve_http_config(config)
|
|
67
|
+
engine = IdempotencyEngine(backend=backend, config=resolved)
|
|
68
|
+
# The Redis/Postgres pool binds to the sync-bridge loop; close it at exit.
|
|
69
|
+
register_closable(backend)
|
|
70
|
+
|
|
71
|
+
def _extract_key(request: Any) -> str | None:
|
|
72
|
+
if resolved.key is not None:
|
|
73
|
+
try:
|
|
74
|
+
return resolved.key(request)
|
|
75
|
+
except Exception:
|
|
76
|
+
return None
|
|
77
|
+
return _unwrap_sf_string(request.headers.get("Idempotency-Key"))
|
|
78
|
+
|
|
79
|
+
def _extract_scope(request: Any) -> str | None:
|
|
80
|
+
if resolved.scope is None:
|
|
81
|
+
return None
|
|
82
|
+
try:
|
|
83
|
+
value = resolved.scope(request)
|
|
84
|
+
except Exception:
|
|
85
|
+
# A configured-but-failing extractor fails closed (engine rejects per
|
|
86
|
+
# scope_mode); it never silently shares one namespace across tenants.
|
|
87
|
+
return None
|
|
88
|
+
return value or None
|
|
89
|
+
|
|
90
|
+
def _neutral(request: Any) -> NeutralRequest:
|
|
91
|
+
try:
|
|
92
|
+
body = request.body # cached, so the view's request.data still parses
|
|
93
|
+
except Exception:
|
|
94
|
+
body = b""
|
|
95
|
+
return NeutralRequest(
|
|
96
|
+
idempotency_key=_extract_key(request),
|
|
97
|
+
scope=_extract_scope(request),
|
|
98
|
+
method=request.method,
|
|
99
|
+
path=request.path,
|
|
100
|
+
query=request.META.get("QUERY_STRING", "") if hasattr(request, "META") else "",
|
|
101
|
+
body=bytes(body) if isinstance(body, (bytes, bytearray)) else b"",
|
|
102
|
+
content_type=getattr(request, "content_type", None),
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
class IdempotencyMixin:
|
|
106
|
+
_idem_outcome: EngineOutcome | None = None
|
|
107
|
+
_idemkit_drf_mixin = True
|
|
108
|
+
|
|
109
|
+
def __init_subclass__(cls, **kwargs: Any) -> None:
|
|
110
|
+
super().__init_subclass__(**kwargs)
|
|
111
|
+
# Silent no-op guard: if a DRF base that defines `initial` precedes this
|
|
112
|
+
# mixin in the MRO, our overrides never run and idempotency does nothing.
|
|
113
|
+
# Fail at class-definition time instead of shipping unprotected.
|
|
114
|
+
mro = cls.__mro__
|
|
115
|
+
mixin_pos = next(
|
|
116
|
+
(i for i, c in enumerate(mro) if c.__dict__.get("_idemkit_drf_mixin")), None
|
|
117
|
+
)
|
|
118
|
+
base_pos = next(
|
|
119
|
+
(
|
|
120
|
+
i
|
|
121
|
+
for i, c in enumerate(mro)
|
|
122
|
+
if "initial" in c.__dict__ and not c.__dict__.get("_idemkit_drf_mixin")
|
|
123
|
+
),
|
|
124
|
+
None,
|
|
125
|
+
)
|
|
126
|
+
if mixin_pos is not None and base_pos is not None and mixin_pos > base_pos:
|
|
127
|
+
raise ConfigurationError(
|
|
128
|
+
f"idemkit: {cls.__name__} lists idempotent_view(...) AFTER its DRF base "
|
|
129
|
+
f"({mro[base_pos].__name__}), so the mixin's hooks are shadowed and "
|
|
130
|
+
"idempotency would silently do nothing. Put the mixin FIRST: "
|
|
131
|
+
"class V(idempotent_view(...), APIView)."
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
def initial(self, request: Any, *args: Any, **kwargs: Any) -> None:
|
|
135
|
+
# Run DRF auth/permissions/throttling first, so scope can read request.user.
|
|
136
|
+
super().initial(request, *args, **kwargs) # type: ignore[misc]
|
|
137
|
+
outcome = run_sync(engine.decide(_neutral(request)))
|
|
138
|
+
self._idem_outcome = outcome
|
|
139
|
+
if outcome.kind in ("replay", "reject"):
|
|
140
|
+
assert outcome.response is not None
|
|
141
|
+
raise _ShortCircuit(outcome.response)
|
|
142
|
+
|
|
143
|
+
def handle_exception(self, exc: Any) -> Any:
|
|
144
|
+
if isinstance(exc, _ShortCircuit):
|
|
145
|
+
return _to_django_response(exc.neutral)
|
|
146
|
+
# The view raised: release the claim so a retry re-runs (fail-closed).
|
|
147
|
+
outcome = getattr(self, "_idem_outcome", None)
|
|
148
|
+
if outcome is not None and outcome.kind == "pass_through" and outcome.claim_token:
|
|
149
|
+
run_sync(engine.on_error(outcome.effective_key, outcome.claim_token))
|
|
150
|
+
self._idem_outcome = None
|
|
151
|
+
return super().handle_exception(exc) # type: ignore[misc]
|
|
152
|
+
|
|
153
|
+
def finalize_response(self, request: Any, response: Any, *args: Any, **kwargs: Any) -> Any:
|
|
154
|
+
response = super().finalize_response(request, response, *args, **kwargs) # type: ignore[misc]
|
|
155
|
+
outcome = getattr(self, "_idem_outcome", None)
|
|
156
|
+
if (
|
|
157
|
+
outcome is not None
|
|
158
|
+
and outcome.kind == "pass_through"
|
|
159
|
+
and outcome.effective_key
|
|
160
|
+
and outcome.claim_token
|
|
161
|
+
):
|
|
162
|
+
response.render() # DRF renders lazily; force it to capture the bytes
|
|
163
|
+
captured = NeutralResponse(
|
|
164
|
+
status=response.status_code,
|
|
165
|
+
headers=dict(response.items()),
|
|
166
|
+
body=bytes(response.content),
|
|
167
|
+
)
|
|
168
|
+
run_sync(engine.on_complete(outcome.effective_key, outcome.claim_token, captured))
|
|
169
|
+
self._idem_outcome = None
|
|
170
|
+
return response
|
|
171
|
+
|
|
172
|
+
return IdempotencyMixin
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _to_django_response(neutral: NeutralResponse) -> Any:
|
|
176
|
+
from django.http import HttpResponse
|
|
177
|
+
|
|
178
|
+
response = HttpResponse(content=neutral.body, status=neutral.status)
|
|
179
|
+
for name, value in neutral.headers.items():
|
|
180
|
+
response[name] = value
|
|
181
|
+
return response
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""FastAPI-native idempotency via a custom route class.
|
|
2
|
+
|
|
3
|
+
The app-wide :class:`~idemkit.IdempotencyMiddleware` already works with FastAPI, but
|
|
4
|
+
a route class fits FastAPI better in two ways:
|
|
5
|
+
|
|
6
|
+
* your handler can return a ``dict`` / Pydantic model (FastAPI serializes it, then
|
|
7
|
+
idemkit captures the serialized bytes for replay), so you don't rewrite endpoints
|
|
8
|
+
to return ``JSONResponse``;
|
|
9
|
+
* your ``scope`` / ``key`` callables get the real ``Request``, so ``request.state``
|
|
10
|
+
works (the middleware only exposes a lightweight proxy).
|
|
11
|
+
|
|
12
|
+
Use it per-router or app-wide::
|
|
13
|
+
|
|
14
|
+
from fastapi import APIRouter
|
|
15
|
+
from idemkit import RedisBackend, HttpConfig
|
|
16
|
+
from idemkit.contrib.fastapi import idempotent_route
|
|
17
|
+
|
|
18
|
+
backend = RedisBackend.from_url("redis://localhost:6379")
|
|
19
|
+
router = APIRouter(
|
|
20
|
+
route_class=idempotent_route(
|
|
21
|
+
backend=backend, config=HttpConfig(scope=lambda req: req.state.user_id)
|
|
22
|
+
)
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@router.post("/charge")
|
|
27
|
+
async def charge(order: Order) -> dict:
|
|
28
|
+
return {"charged": True} # a dict is fine; idemkit captures the serialized body
|
|
29
|
+
|
|
30
|
+
For the whole app: ``app.router.route_class = idempotent_route(backend=backend, config=config)``.
|
|
31
|
+
Non-applicable methods (GET by default) and keyless requests pass straight through,
|
|
32
|
+
so the class is safe to apply broadly. Requires FastAPI.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
from typing import Any
|
|
38
|
+
|
|
39
|
+
from idemkit.backends.base import IdempotencyBackend
|
|
40
|
+
from idemkit.core.config import resolve_http_config
|
|
41
|
+
from idemkit.core.engine import IdempotencyEngine, NeutralRequest, NeutralResponse
|
|
42
|
+
from idemkit.core.policy import HttpConfig
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def idempotent_route(
|
|
46
|
+
*,
|
|
47
|
+
backend: IdempotencyBackend,
|
|
48
|
+
config: HttpConfig | None = None,
|
|
49
|
+
) -> type:
|
|
50
|
+
"""Build a FastAPI ``APIRoute`` subclass that applies idempotency.
|
|
51
|
+
|
|
52
|
+
Pass it as ``route_class=`` to an ``APIRouter`` (or assign
|
|
53
|
+
``app.router.route_class``). One engine is shared by every route on the router.
|
|
54
|
+
"""
|
|
55
|
+
try:
|
|
56
|
+
from fastapi import Request
|
|
57
|
+
from fastapi.routing import APIRoute
|
|
58
|
+
except ImportError as e: # pragma: no cover - exercised via the error message
|
|
59
|
+
raise ImportError("idemkit: install FastAPI: pip install fastapi") from e
|
|
60
|
+
|
|
61
|
+
from idemkit.adapters.asgi import _unwrap_sf_string
|
|
62
|
+
|
|
63
|
+
resolved = resolve_http_config(config)
|
|
64
|
+
engine = IdempotencyEngine(backend=backend, config=resolved)
|
|
65
|
+
|
|
66
|
+
def _extract_key(request: Request) -> str | None:
|
|
67
|
+
if resolved.key is not None:
|
|
68
|
+
try:
|
|
69
|
+
return resolved.key(request)
|
|
70
|
+
except Exception:
|
|
71
|
+
return None
|
|
72
|
+
return _unwrap_sf_string(request.headers.get("idempotency-key"))
|
|
73
|
+
|
|
74
|
+
def _extract_scope(request: Request) -> str | None:
|
|
75
|
+
if resolved.scope is None:
|
|
76
|
+
return None
|
|
77
|
+
try:
|
|
78
|
+
value = resolved.scope(request)
|
|
79
|
+
except Exception:
|
|
80
|
+
# A configured-but-failing extractor fails closed: the engine sees no
|
|
81
|
+
# identity and rejects per scope_mode (never buckets as anonymous).
|
|
82
|
+
return None
|
|
83
|
+
return value or None
|
|
84
|
+
|
|
85
|
+
class IdempotentAPIRoute(APIRoute):
|
|
86
|
+
def get_route_handler(self) -> Any:
|
|
87
|
+
original = super().get_route_handler()
|
|
88
|
+
|
|
89
|
+
async def handler(request: Request) -> Any:
|
|
90
|
+
body = await request.body() # cached, so the handler still parses it
|
|
91
|
+
neutral = NeutralRequest(
|
|
92
|
+
idempotency_key=_extract_key(request),
|
|
93
|
+
scope=_extract_scope(request),
|
|
94
|
+
method=request.method,
|
|
95
|
+
path=request.url.path,
|
|
96
|
+
query=request.url.query,
|
|
97
|
+
body=body,
|
|
98
|
+
content_type=request.headers.get("content-type"),
|
|
99
|
+
)
|
|
100
|
+
outcome = await engine.decide(neutral)
|
|
101
|
+
if outcome.kind in ("replay", "reject"):
|
|
102
|
+
assert outcome.response is not None
|
|
103
|
+
return _to_response(outcome.response)
|
|
104
|
+
if outcome.effective_key is None or outcome.claim_token is None:
|
|
105
|
+
return await original(request) # idempotency does not apply
|
|
106
|
+
try:
|
|
107
|
+
response = await original(request)
|
|
108
|
+
except Exception:
|
|
109
|
+
await engine.on_error(outcome.effective_key, outcome.claim_token)
|
|
110
|
+
raise
|
|
111
|
+
captured = _capture(response)
|
|
112
|
+
if captured is None:
|
|
113
|
+
# A streaming/file response has no buffered body to cache;
|
|
114
|
+
# release so a retry re-runs (the client already got the bytes).
|
|
115
|
+
await engine.on_error(outcome.effective_key, outcome.claim_token)
|
|
116
|
+
return response
|
|
117
|
+
# on_complete enforces max_body_bytes and cacheable_status: an
|
|
118
|
+
# oversized or non-cacheable response is released, not stored.
|
|
119
|
+
await engine.on_complete(outcome.effective_key, outcome.claim_token, captured)
|
|
120
|
+
return response
|
|
121
|
+
|
|
122
|
+
return handler
|
|
123
|
+
|
|
124
|
+
return IdempotentAPIRoute
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _to_response(neutral: NeutralResponse) -> Any:
|
|
128
|
+
from starlette.responses import Response
|
|
129
|
+
|
|
130
|
+
return Response(content=neutral.body, status_code=neutral.status, headers=neutral.headers)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _capture(response: Any) -> NeutralResponse | None:
|
|
134
|
+
body = getattr(response, "body", None)
|
|
135
|
+
if not isinstance(body, (bytes, bytearray)):
|
|
136
|
+
return None # streaming / file response: no buffered body to store
|
|
137
|
+
return NeutralResponse(
|
|
138
|
+
status=response.status_code,
|
|
139
|
+
headers=dict(response.headers),
|
|
140
|
+
body=bytes(body),
|
|
141
|
+
)
|
idemkit/contrib/kafka.py
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""Kafka glue for idemkit's queue surface.
|
|
2
|
+
|
|
3
|
+
Kafka redelivers when a consumer doesn't make progress within
|
|
4
|
+
``max.poll.interval.ms`` (the group coordinator considers it dead and reassigns
|
|
5
|
+
its partitions), and on rebalances. :func:`kafka_consumer` presets an
|
|
6
|
+
:class:`~idemkit.IdempotentConsumer` so a side effect runs once per record even
|
|
7
|
+
across those redeliveries.
|
|
8
|
+
|
|
9
|
+
Two Kafka-specific choices it gets right for you:
|
|
10
|
+
|
|
11
|
+
* **Dedup id = ``topic:partition:offset``.** The globally-stable identity of a
|
|
12
|
+
record; the message *key* is not unique enough (many records share a key).
|
|
13
|
+
* **Visibility-timeout analogue = ``max.poll.interval.ms``.** Kafka has no
|
|
14
|
+
per-message visibility timeout, but the redelivery deadline is the poll
|
|
15
|
+
interval, so the lease is derived from it (kept shorter, renewed by heartbeat).
|
|
16
|
+
|
|
17
|
+
Kafka exposes **no native receive count**, so ``max_attempts`` needs a durable
|
|
18
|
+
:class:`~idemkit.adapters.queue.AttemptStore` in a multi-consumer deployment
|
|
19
|
+
(pass ``attempt_store=``); without one, counting is in-process and idemkit warns.
|
|
20
|
+
|
|
21
|
+
``group_id`` is required and becomes the scope: two consumer groups read the same
|
|
22
|
+
offsets, and each must process every record, so their dedup records must not
|
|
23
|
+
collide.
|
|
24
|
+
|
|
25
|
+
Example (confluent-kafka)::
|
|
26
|
+
|
|
27
|
+
from confluent_kafka import Consumer
|
|
28
|
+
from idemkit import RedisBackend
|
|
29
|
+
from idemkit.contrib.kafka import kafka_consumer
|
|
30
|
+
|
|
31
|
+
consumer = kafka_consumer(
|
|
32
|
+
backend=RedisBackend.from_url("redis://localhost:6379"),
|
|
33
|
+
group_id="billing",
|
|
34
|
+
max_poll_interval_seconds=300,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@consumer.handle
|
|
39
|
+
def process(record) -> None:
|
|
40
|
+
charge_customer(record.value()) # runs once per topic:partition:offset
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
kc = Consumer({"bootstrap.servers": "...", "group.id": "billing", "enable.auto.commit": False})
|
|
44
|
+
kc.subscribe(["charges"])
|
|
45
|
+
while True:
|
|
46
|
+
record = kc.poll(1.0)
|
|
47
|
+
if record is None or record.error():
|
|
48
|
+
continue
|
|
49
|
+
result = consumer.dispatch_sync(record)
|
|
50
|
+
if result.action.value == "ack":
|
|
51
|
+
kc.commit(record) # advance the offset only when done
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
from __future__ import annotations
|
|
55
|
+
|
|
56
|
+
import dataclasses
|
|
57
|
+
from typing import Any
|
|
58
|
+
|
|
59
|
+
from idemkit.adapters.queue import IdempotentConsumer
|
|
60
|
+
from idemkit.backends.base import IdempotencyBackend
|
|
61
|
+
from idemkit.core.policy import QueueConfig
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _attr(message: Any, name: str) -> Any:
|
|
65
|
+
"""Read ``name`` from a record, whether it's a method (confluent_kafka) or a
|
|
66
|
+
plain attribute (kafka-python)."""
|
|
67
|
+
value = getattr(message, name)
|
|
68
|
+
return value() if callable(value) else value
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def kafka_dedup_id(message: Any) -> str:
|
|
72
|
+
"""``topic:partition:offset`` for a confluent_kafka or kafka-python record."""
|
|
73
|
+
return f"{_attr(message, 'topic')}:{_attr(message, 'partition')}:{_attr(message, 'offset')}"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def kafka_consumer(
|
|
77
|
+
*,
|
|
78
|
+
backend: IdempotencyBackend,
|
|
79
|
+
group_id: str,
|
|
80
|
+
max_poll_interval_seconds: float = 300.0,
|
|
81
|
+
config: QueueConfig | None = None,
|
|
82
|
+
) -> IdempotentConsumer:
|
|
83
|
+
"""Build an :class:`~idemkit.IdempotentConsumer` wired for Kafka records.
|
|
84
|
+
|
|
85
|
+
Kafka presets the dedup id (``topic:partition:offset``) and the scope
|
|
86
|
+
(``group_id``). Pass a :class:`~idemkit.QueueConfig` for behaviour.
|
|
87
|
+
"""
|
|
88
|
+
cfg = config or QueueConfig()
|
|
89
|
+
presets: dict[str, Any] = {
|
|
90
|
+
"dedup_id": kafka_dedup_id,
|
|
91
|
+
"visibility_timeout_seconds": max_poll_interval_seconds,
|
|
92
|
+
}
|
|
93
|
+
if cfg.scope is None: # group_id is the scope unless the caller set one
|
|
94
|
+
presets["scope"] = lambda _m: group_id
|
|
95
|
+
cfg = dataclasses.replace(cfg, **presets)
|
|
96
|
+
return IdempotentConsumer(backend=backend, config=cfg)
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Structured-logging exporter for idemkit's observability events.
|
|
2
|
+
|
|
3
|
+
A zero-dependency handler that logs one structured line per operation. Route
|
|
4
|
+
idemkit's :class:`~idemkit.IdempotencyEvent` to your logger when you don't (yet)
|
|
5
|
+
have a metrics pipeline::
|
|
6
|
+
|
|
7
|
+
from idemkit import MethodConfig, idempotent
|
|
8
|
+
from idemkit.contrib.logging import logging_handler
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@idempotent(
|
|
12
|
+
backend=backend,
|
|
13
|
+
config=MethodConfig(
|
|
14
|
+
key_fields=["order_id"],
|
|
15
|
+
event_handlers=(logging_handler(),),
|
|
16
|
+
),
|
|
17
|
+
)
|
|
18
|
+
async def refund(*, order_id): ...
|
|
19
|
+
|
|
20
|
+
Each record carries the decision, surface, backend, latencies, and the hashed
|
|
21
|
+
``effective_key`` (safe to log; never the raw idempotency key). Fields go in the
|
|
22
|
+
``extra`` dict so a JSON log formatter picks them up as structured fields.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import logging
|
|
28
|
+
|
|
29
|
+
from idemkit.core.events import EventHandler, IdempotencyEvent
|
|
30
|
+
|
|
31
|
+
_default_logger = logging.getLogger("idemkit.events")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def logging_handler(
|
|
35
|
+
logger: logging.Logger | None = None, *, level: int = logging.INFO
|
|
36
|
+
) -> EventHandler:
|
|
37
|
+
"""Build an :data:`~idemkit.EventHandler` that logs one line per operation.
|
|
38
|
+
|
|
39
|
+
Pass your own ``logger`` (default ``idemkit.events``) and ``level``.
|
|
40
|
+
"""
|
|
41
|
+
log = logger or _default_logger
|
|
42
|
+
|
|
43
|
+
def handle(event: IdempotencyEvent) -> None:
|
|
44
|
+
log.log(
|
|
45
|
+
level,
|
|
46
|
+
"idemkit %s surface=%s backend=%s latency=%.4fs",
|
|
47
|
+
event.decision.value,
|
|
48
|
+
event.surface,
|
|
49
|
+
event.backend_name,
|
|
50
|
+
event.latency_seconds,
|
|
51
|
+
extra={
|
|
52
|
+
"idempotency_decision": event.decision.value,
|
|
53
|
+
"idempotency_surface": event.surface,
|
|
54
|
+
"idempotency_backend": event.backend_name,
|
|
55
|
+
"idempotency_effective_key": event.effective_key,
|
|
56
|
+
"idempotency_latency_seconds": event.latency_seconds,
|
|
57
|
+
"idempotency_wait_seconds": event.wait_duration_seconds,
|
|
58
|
+
},
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
return handle
|
idemkit/contrib/mcp.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""Enforce idempotency on MCP / LLM tool calls.
|
|
2
|
+
|
|
3
|
+
The Model Context Protocol lets a tool advertise ``idempotentHint: true`` in its
|
|
4
|
+
annotations — but that is advisory metadata for the model/client, and **nothing
|
|
5
|
+
enforces it**. An agent that retries, re-plans, or runs parallel branches can
|
|
6
|
+
still call a side-effectful tool twice.
|
|
7
|
+
|
|
8
|
+
idemkit's method surface (:func:`idemkit.idempotent`) *is* the enforcement layer.
|
|
9
|
+
This module is a thin bridge:
|
|
10
|
+
|
|
11
|
+
* :func:`mcp_idempotent` — decorate a tool so duplicate calls with the same
|
|
12
|
+
arguments replay the first result instead of re-running, and stamp the
|
|
13
|
+
``idempotentHint`` so the tool truthfully advertises what it now guarantees.
|
|
14
|
+
* :func:`wrap_if_idempotent` — given a handler and its MCP annotations, enforce
|
|
15
|
+
dedup automatically *when the tool already declares* ``idempotentHint: true``.
|
|
16
|
+
* :func:`read_idempotent_hint` — read the hint off a function or an annotations
|
|
17
|
+
object/dict.
|
|
18
|
+
|
|
19
|
+
No dependency on the ``mcp`` package: these operate on plain functions and an
|
|
20
|
+
annotations mapping, so they work with the official Python SDK (FastMCP), a hand
|
|
21
|
+
-rolled server, or an Anthropic / OpenAI tool-dispatch loop.
|
|
22
|
+
|
|
23
|
+
Example — a FastMCP tool that really is safe to call twice::
|
|
24
|
+
|
|
25
|
+
from mcp.server.fastmcp import FastMCP
|
|
26
|
+
from idemkit import RedisBackend
|
|
27
|
+
from idemkit.contrib.mcp import mcp_idempotent
|
|
28
|
+
|
|
29
|
+
mcp = FastMCP("payments")
|
|
30
|
+
backend = RedisBackend.from_url("redis://localhost:6379")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@mcp.tool()
|
|
34
|
+
@mcp_idempotent(backend=backend, config=MethodConfig(key_fields=["order_id", "amount"]))
|
|
35
|
+
async def refund(order_id: str, amount: int) -> dict:
|
|
36
|
+
return await payments.refund(order_id, amount) # charged once per (order_id, amount)
|
|
37
|
+
|
|
38
|
+
The agent re-emitting ``refund("A123", 50)`` on a re-plan replays the first
|
|
39
|
+
result instead of issuing a second refund.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import inspect
|
|
45
|
+
from collections.abc import Callable, Mapping
|
|
46
|
+
from typing import Any, cast
|
|
47
|
+
|
|
48
|
+
from idemkit.adapters.ai import idempotent, idempotent_sync
|
|
49
|
+
from idemkit.backends.base import IdempotencyBackend
|
|
50
|
+
from idemkit.core.policy import MethodConfig
|
|
51
|
+
|
|
52
|
+
HINT_KEY = "idempotentHint"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def mcp_idempotent(
|
|
56
|
+
*,
|
|
57
|
+
backend: IdempotencyBackend,
|
|
58
|
+
config: MethodConfig | None = None,
|
|
59
|
+
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
|
|
60
|
+
"""Enforce idempotency on an MCP tool and mark it as idempotent.
|
|
61
|
+
|
|
62
|
+
Works on ``async def`` and plain ``def`` tools alike (it picks the matching
|
|
63
|
+
idemkit decorator). Dedup on ``config.key_fields`` — the argument fields that
|
|
64
|
+
define "the same call"; volatile per-turn ids (``tool_call_id``) are never
|
|
65
|
+
listed, so a retry with a fresh id still dedupes.
|
|
66
|
+
|
|
67
|
+
The returned function carries ``idempotent_hint = True`` so a server can surface
|
|
68
|
+
``idempotentHint`` in the tool's advertised annotations.
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
def decorate(fn: Callable[..., Any]) -> Callable[..., Any]:
|
|
72
|
+
deco = idempotent if inspect.iscoroutinefunction(fn) else idempotent_sync
|
|
73
|
+
wrapped: Any = deco(backend=backend, config=config)(fn)
|
|
74
|
+
wrapped.idempotent_hint = True
|
|
75
|
+
return cast("Callable[..., Any]", wrapped)
|
|
76
|
+
|
|
77
|
+
return decorate
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def read_idempotent_hint(source: Any) -> bool:
|
|
81
|
+
"""Return whether ``source`` declares ``idempotentHint``.
|
|
82
|
+
|
|
83
|
+
``source`` may be a function (checks the ``idempotent_hint`` attribute set by
|
|
84
|
+
:func:`mcp_idempotent`), a mapping (checks the ``idempotentHint`` key), or any
|
|
85
|
+
object exposing an ``idempotentHint`` attribute (e.g. an MCP tool annotations
|
|
86
|
+
object).
|
|
87
|
+
"""
|
|
88
|
+
if isinstance(source, Mapping):
|
|
89
|
+
return bool(source.get(HINT_KEY, False))
|
|
90
|
+
if hasattr(source, "idempotent_hint"):
|
|
91
|
+
return bool(source.idempotent_hint)
|
|
92
|
+
if hasattr(source, HINT_KEY):
|
|
93
|
+
return bool(getattr(source, HINT_KEY))
|
|
94
|
+
return False
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def wrap_if_idempotent(
|
|
98
|
+
fn: Callable[..., Any],
|
|
99
|
+
*,
|
|
100
|
+
annotations: Any,
|
|
101
|
+
backend: IdempotencyBackend,
|
|
102
|
+
config: MethodConfig | None = None,
|
|
103
|
+
) -> Callable[..., Any]:
|
|
104
|
+
"""Enforce dedup on ``fn`` only if its MCP ``annotations`` declare the hint.
|
|
105
|
+
|
|
106
|
+
Use this to auto-apply idempotency across a registry of tools: a tool that
|
|
107
|
+
already advertises ``idempotentHint: true`` gets real enforcement; a tool that
|
|
108
|
+
doesn't is returned unchanged (wrapping a pure tool only wastes storage, so
|
|
109
|
+
idemkit never guesses — the declaration is the caller's, §8.2).
|
|
110
|
+
"""
|
|
111
|
+
if not read_idempotent_hint(annotations):
|
|
112
|
+
return fn
|
|
113
|
+
return mcp_idempotent(backend=backend, config=config)(fn)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""Prometheus exporter for idemkit's observability events.
|
|
2
|
+
|
|
3
|
+
idemkit emits one :class:`~idemkit.IdempotencyEvent` per operation; you decide
|
|
4
|
+
where it goes. This is a ready-made handler that records the standard four metrics
|
|
5
|
+
so you get a dashboard without writing the plumbing yourself::
|
|
6
|
+
|
|
7
|
+
from idemkit import IdempotencyMiddleware, RedisBackend, HttpConfig
|
|
8
|
+
from idemkit.contrib.prometheus import prometheus_handler
|
|
9
|
+
|
|
10
|
+
app.add_middleware(
|
|
11
|
+
IdempotencyMiddleware,
|
|
12
|
+
backend=RedisBackend.from_url("redis://..."),
|
|
13
|
+
config=HttpConfig(event_handlers=(prometheus_handler(),)),
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
The same handler works on every surface (HTTP, queue, method), so one dashboard
|
|
17
|
+
covers all three: filter by the ``surface`` label.
|
|
18
|
+
|
|
19
|
+
Metrics (namespaced ``idemkit_`` by default):
|
|
20
|
+
|
|
21
|
+
* ``idemkit_operations_total``: Counter, labels ``decision`` / ``surface`` / ``backend``.
|
|
22
|
+
Derive replay rate, conflict rate, and storage-error rate from ``decision``.
|
|
23
|
+
* ``idemkit_latency_seconds``: Histogram, labels ``surface`` / ``backend``.
|
|
24
|
+
* ``idemkit_wait_seconds``: Histogram, labels ``surface`` / ``backend`` (only observed
|
|
25
|
+
when a duplicate waited on an in-flight run).
|
|
26
|
+
|
|
27
|
+
The hashed ``effective_key`` is deliberately NOT a label: it is high-cardinality and
|
|
28
|
+
would blow up your metrics store. Requires the ``prometheus`` extra
|
|
29
|
+
(``pip install 'idemkit[prometheus]'``).
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
from typing import Any
|
|
35
|
+
|
|
36
|
+
from idemkit.core.events import EventHandler, IdempotencyEvent
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def prometheus_handler(registry: Any = None, *, namespace: str = "idemkit") -> EventHandler:
|
|
40
|
+
"""Build an :data:`~idemkit.EventHandler` that records idemkit metrics.
|
|
41
|
+
|
|
42
|
+
Pass a custom ``registry`` (a ``prometheus_client.CollectorRegistry``) to isolate
|
|
43
|
+
the metrics, e.g. in tests; the default is the global registry. ``namespace``
|
|
44
|
+
prefixes every metric name.
|
|
45
|
+
"""
|
|
46
|
+
try:
|
|
47
|
+
from prometheus_client import Counter, Histogram
|
|
48
|
+
except ImportError as e: # pragma: no cover - exercised via the error message
|
|
49
|
+
raise ImportError(
|
|
50
|
+
"idemkit: install the `prometheus` extra: pip install 'idemkit[prometheus]'"
|
|
51
|
+
) from e
|
|
52
|
+
|
|
53
|
+
kwargs: dict[str, Any] = {"registry": registry} if registry is not None else {}
|
|
54
|
+
operations = Counter(
|
|
55
|
+
f"{namespace}_operations_total",
|
|
56
|
+
"idemkit operations by decision, surface, and backend.",
|
|
57
|
+
["decision", "surface", "backend"],
|
|
58
|
+
**kwargs,
|
|
59
|
+
)
|
|
60
|
+
latency = Histogram(
|
|
61
|
+
f"{namespace}_latency_seconds",
|
|
62
|
+
"idemkit end-to-end operation latency.",
|
|
63
|
+
["surface", "backend"],
|
|
64
|
+
**kwargs,
|
|
65
|
+
)
|
|
66
|
+
wait = Histogram(
|
|
67
|
+
f"{namespace}_wait_seconds",
|
|
68
|
+
"Time a duplicate waited on an in-flight run before replaying.",
|
|
69
|
+
["surface", "backend"],
|
|
70
|
+
**kwargs,
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
def handle(event: IdempotencyEvent) -> None:
|
|
74
|
+
operations.labels(event.decision.value, event.surface, event.backend_name).inc()
|
|
75
|
+
latency.labels(event.surface, event.backend_name).observe(event.latency_seconds)
|
|
76
|
+
if event.wait_duration_seconds:
|
|
77
|
+
wait.labels(event.surface, event.backend_name).observe(event.wait_duration_seconds)
|
|
78
|
+
|
|
79
|
+
return handle
|