rillkit 0.1.0__py3-none-any.whl

rillkit/__init__.py

@@ -0,0 +1,25 @@
+"""rillkit: simple publish and consume for Redis Streams."""
+
+from .ack import AckPolicy
+from .broker import Broker, Publisher
+from .codec import Codec, JsonCodec
+from .exceptions import (
+    AckMessage,
+    DecodeError,
+    NackMessage,
+    PublishError,
+    RejectMessage,
+    RillkitError,
+)
+from .message import Message
+from .middleware import BaseMiddleware, Idempotent, PublishCommand
+from .settings import Settings
+from .stream import Stream
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AckMessage", "AckPolicy", "BaseMiddleware", "Broker", "Codec", "DecodeError", "Idempotent",
+    "JsonCodec", "Message", "NackMessage", "PublishCommand", "PublishError", "Publisher",
+    "RejectMessage", "RillkitError", "Settings", "Stream",
+]

rillkit/ack.py

@@ -0,0 +1,18 @@
+from enum import Enum
+
+
+class AckPolicy(str, Enum):
+    """What happens to a message when its handler finishes.
+
+    NACK_ON_ERROR  Ack when the handler returns, nack when it raises. This is the default.
+    ACK_ALWAYS     Ack when the handler returns or raises. Failures are logged and dropped, never
+                   retried.
+    ACK_FIRST      Ack before the handler runs. At-most-once: a crash loses the message.
+    MANUAL         rillkit never acks. The handler must call ``msg.ack()``. If it returns without
+                   acking, or raises, the message stays pending and is retried.
+    """
+
+    NACK_ON_ERROR = "nack_on_error"
+    ACK_ALWAYS = "ack_always"
+    ACK_FIRST = "ack_first"
+    MANUAL = "manual"

rillkit/broker.py

@@ -0,0 +1,271 @@
+import logging
+import signal
+import sys
+import threading
+from collections.abc import Sequence
+from functools import partial
+from typing import Any, Callable, Optional
+
+import redis
+
+from .ack import AckPolicy
+from .consumer import HEARTBEAT_PREFIX, Subscriber, _text
+from .exceptions import PublishError
+from .middleware import BaseMiddleware, PublishCommand
+from .settings import Settings
+from .stream import Stream, StreamLike, as_stream
+
+log = logging.getLogger("rillkit")
+
+Hook = Callable[[], Any]
+
+
+class Publisher:
+    """Returned by ``broker.publisher(stream)``. Use it as a decorator to publish a handler's return
+    value, or call ``.publish(body)`` directly."""
+
+    def __init__(self, broker: "Broker", stream: Stream, maxlen: Optional[int] = None) -> None:
+        self.broker = broker
+        self.stream = stream
+        self.maxlen = maxlen
+
+    def __call__(self, func: Callable[..., Any]) -> Callable[..., Any]:
+        publishers = list(getattr(func, "_rillkit_publishers", ()))
+        publishers.append(self)
+        func._rillkit_publishers = publishers  # type: ignore[attr-defined]
+        return func
+
+    def publish(self, body: Any) -> str:
+        return self.broker.publish(self.stream, body, maxlen=self.maxlen)
+
+
+class Broker:
+    """Owns the Redis client, the subscribers and the lifecycle hooks.
+
+        broker = Broker.from_env()
+
+        @broker.subscribe(Stream("orders", group="workers"))
+        def handle(order: dict): ...
+
+        broker.publish("orders", {"id": 1})
+        broker.run()
+    """
+
+    def __init__(
+        self,
+        url: Optional[str] = None,
+        *,
+        client: Optional["redis.Redis"] = None,
+        settings: Optional[Settings] = None,
+        middlewares: Sequence[BaseMiddleware] = (),
+    ) -> None:
+        if client is None:
+            client = (settings or Settings(url=url)).client()
+        self.client = client
+        self.middlewares: list[BaseMiddleware] = []
+        for mw in middlewares:
+            self.use(mw)
+        self.subscribers: list[Subscriber] = []
+        self._hooks: dict[str, list[Hook]] = {
+            "on_startup": [], "after_startup": [], "on_shutdown": [], "after_shutdown": [],
+        }
+        self._stop = threading.Event()
+        self._stop_lock = threading.Lock()
+        self._stopped = False
+
+    @classmethod
+    def from_env(cls, prefix: str = "REDIS_", **kwargs: Any) -> "Broker":
+        return cls(settings=Settings.from_env(prefix), **kwargs)
+
+    # middleware
+
+    def use(self, middleware: BaseMiddleware) -> BaseMiddleware:
+        middleware.broker = self
+        self.middlewares.append(middleware)
+        return middleware
+
+    # subscribing and publishing
+
+    def subscribe(
+        self,
+        stream: StreamLike,
+        *,
+        group: Optional[str] = None,
+        ack_policy: AckPolicy = AckPolicy.NACK_ON_ERROR,
+        max_retries: Optional[int] = 3,
+        claim_idle_ms: int = 60_000,
+        backoff: float = 1.0,
+        max_backoff_ms: int = 15 * 60_000,
+        block_ms: int = 5000,
+        count: int = 10,
+        middlewares: Sequence[BaseMiddleware] = (),
+        consumer: Optional[str] = None,
+        heartbeat: bool = True,
+        heartbeat_ttl: int = 60,
+        group_start_id: str = "0",
+    ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+        """Register a handler for a stream. The handler is returned unchanged."""
+        resolved = as_stream(stream)
+        if group:
+            resolved = Stream(resolved.name, group, resolved.dlq, resolved.maxlen, resolved.codec)
+        own = list(middlewares)
+        for m in own:
+            m.broker = self
+
+        def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+            self.subscribers.append(Subscriber(
+                self, resolved, func, ack_policy=ack_policy, max_retries=max_retries,
+                claim_idle_ms=claim_idle_ms, backoff=backoff, max_backoff_ms=max_backoff_ms,
+                block_ms=block_ms, count=count, middlewares=own,
+                consumer=consumer, heartbeat=heartbeat, heartbeat_ttl=heartbeat_ttl,
+                group_start_id=group_start_id,
+            ))
+            return func
+
+        return decorator
+
+    def publisher(self, stream: StreamLike, *, maxlen: Optional[int] = None) -> Publisher:
+        return Publisher(self, as_stream(stream), maxlen)
+
+    def publish(
+        self,
+        stream: StreamLike,
+        body: Any,
+        *,
+        maxlen: Optional[int] = None,
+        approximate: bool = True,
+    ) -> str:
+        """Add one entry to the stream and return its id. Raises ``PublishError`` on failure."""
+        resolved = as_stream(stream)
+        cmd = PublishCommand(
+            resolved, body, maxlen if maxlen is not None else resolved.maxlen, approximate
+        )
+        call: Callable[[PublishCommand], str] = self._xadd
+        for mw in reversed(self.middlewares):
+            call = partial(mw.publish_scope, call)
+        return call(cmd)
+
+    def _xadd(self, cmd: PublishCommand) -> str:
+        try:
+            payload = cmd.stream.codec.encode(cmd.body)
+            entry_id = self.client.xadd(
+                cmd.stream.name, {"data": payload}, maxlen=cmd.maxlen, approximate=cmd.approximate
+            )
+        except (redis.RedisError, TypeError, ValueError) as e:
+            raise PublishError(f"could not publish to {cmd.stream.name}: {e}") from e
+        return _text(entry_id)
+
+    # lifecycle hooks
+
+    def _hook(self, name: str) -> Callable[[Hook], Hook]:
+        def register(func: Hook) -> Hook:
+            self._hooks[name].append(func)
+            return func
+        return register
+
+    def on_startup(self, func: Hook) -> Hook:
+        """Runs first, before the connection is checked."""
+        return self._hook("on_startup")(func)
+
+    def after_startup(self, func: Hook) -> Hook:
+        """Runs once groups exist and the consumer threads are running."""
+        return self._hook("after_startup")(func)
+
+    def on_shutdown(self, func: Hook) -> Hook:
+        """Runs after the consumers finished their current message, before the client closes."""
+        return self._hook("on_shutdown")(func)
+
+    def after_shutdown(self, func: Hook) -> Hook:
+        """Runs last, after the client is closed."""
+        return self._hook("after_shutdown")(func)
+
+    def _run_hooks(self, name: str) -> None:
+        for func in self._hooks[name]:
+            func()
+
+    # running
+
+    def start(self) -> None:
+        """Start all consumers in background threads and return."""
+        self._stop.clear()
+        self._stopped = False
+        self._run_hooks("on_startup")
+        self.client.ping()
+        for sub in self.subscribers:
+            sub.setup()
+        for sub in self.subscribers:
+            sub.start()
+        self._run_hooks("after_startup")
+
+    def stop(self, timeout: Optional[float] = None) -> None:
+        """Ask the consumers to stop, wait for the current messages to finish, then clean up.
+
+        Safe to call more than once or from another thread while ``run()`` is blocking: the shutdown
+        hooks and the client close happen only once per start."""
+        with self._stop_lock:
+            if self._stopped:
+                return
+            self._stopped = True
+            for sub in self.subscribers:
+                sub.stop()
+            for sub in self.subscribers:
+                sub.join(timeout)
+            try:
+                self._run_hooks("on_shutdown")
+            finally:
+                self.client.close()
+                self._run_hooks("after_shutdown")
+                self._stop.set()
+
+    def run(self, handle_signals: bool = True, shutdown_timeout: Optional[float] = None) -> None:
+        """Start the consumers and block until SIGINT/SIGTERM (or ``stop()``) shuts them down.
+
+        The first signal finishes the current message and exits cleanly. A second signal exits now.
+        """
+        requested = threading.Event()
+        previous = {}
+
+        def on_signal(signum: int, frame: Any) -> None:
+            if requested.is_set():
+                log.warning("second shutdown signal received, exiting now")
+                sys.exit(1)
+            log.info("shutdown requested, finishing the current message before exiting")
+            requested.set()
+
+        if handle_signals and threading.current_thread() is threading.main_thread():
+            for sig in (signal.SIGINT, signal.SIGTERM):
+                previous[sig] = signal.signal(sig, on_signal)
+        try:
+            self.start()
+            while not requested.wait(0.5):
+                if not any(s._thread and s._thread.is_alive() for s in self.subscribers):
+                    break
+            self.stop(shutdown_timeout)
+        finally:
+            for sig, handler in previous.items():
+                signal.signal(sig, handler)
+
+    # operations
+
+    def active_workers(self, stream: StreamLike, group: Optional[str] = None) -> list[str]:
+        """Consumer names with a live heartbeat. Unlike XINFO CONSUMERS this includes idle workers
+        that have not received a message yet."""
+        s = as_stream(stream)
+        prefix = f"{HEARTBEAT_PREFIX}{s.name}:{group or s.group}:"
+        return sorted(_text(k)[len(prefix):] for k in self.client.scan_iter(match=f"{prefix}*"))
+
+    def replay_dlq(self, stream: StreamLike, count: int = 100) -> int:
+        """Move up to ``count`` entries from the DLQ back onto the stream. Returns how many."""
+        s = as_stream(stream)
+        moved = 0
+        rows: Any = self.client.xrange(s.dlq_name, count=count)
+        for entry_id, raw in rows:
+            fields: dict[Any, Any] = {
+                _text(k): _text(v) for k, v in raw.items() if not _text(k).startswith("x-")
+            }
+            pipe = self.client.pipeline()
+            pipe.xadd(s.name, fields)
+            pipe.xdel(s.dlq_name, entry_id)
+            pipe.execute()
+            moved += 1
+        return moved

rillkit/codec.py

@@ -0,0 +1,28 @@
+import json
+from typing import Any, Protocol
+
+from .exceptions import DecodeError
+
+
+class Codec(Protocol):
+    def encode(self, body: Any) -> str: ...
+
+    def decode(self, raw: str) -> Any: ...
+
+
+class JsonCodec:
+    """JSON in the ``data`` field of the stream entry.
+
+    Pydantic models are dumped with ``model_dump_json``.
+    """
+
+    def encode(self, body: Any) -> str:
+        if hasattr(body, "model_dump_json"):
+            return body.model_dump_json()
+        return json.dumps(body, default=str)
+
+    def decode(self, raw: str) -> Any:
+        try:
+            return json.loads(raw)
+        except (TypeError, ValueError) as e:
+            raise DecodeError(f"body is not valid JSON: {e}") from e

rillkit/consumer.py

@@ -0,0 +1,343 @@
+import inspect
+import logging
+import os
+import socket
+import threading
+import time
+from collections.abc import Sequence
+from functools import partial
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Optional,
+    get_type_hints,
+)
+
+import redis
+
+from .ack import AckPolicy
+from .exceptions import AckMessage, DecodeError, NackMessage, RejectMessage
+from .message import Message
+from .middleware import BaseMiddleware
+from .stream import Stream
+
+if TYPE_CHECKING:
+    from .broker import Broker
+
+log = logging.getLogger("rillkit")
+
+RETRY_BACKOFF_SECONDS = 5
+HEARTBEAT_PREFIX = "rillkit:workers:"
+
+
+def _text(value: Any) -> str:
+    return value.decode() if isinstance(value, bytes) else value
+
+
+def _fields(raw: dict[Any, Any]) -> dict[str, str]:
+    return {_text(k): _text(v) for k, v in raw.items()}
+
+
+def _is_model(annotation: Any) -> bool:
+    return inspect.isclass(annotation) and hasattr(annotation, "model_validate")
+
+
+def _plan(handler: Callable[..., Any]) -> list[tuple[str, str, Any]]:
+    """Decide what each handler parameter receives: the ``Message``, the plain body, or a model."""
+    try:
+        hints = get_type_hints(handler)
+    except Exception:
+        hints = getattr(handler, "__annotations__", {})
+    plan: list[tuple[str, str, Any]] = []
+    bodies = 0
+    for name, param in inspect.signature(handler).parameters.items():
+        if param.kind in (param.VAR_POSITIONAL, param.VAR_KEYWORD):
+            continue
+        ann = hints.get(name, param.annotation)
+        if ann is Message or (ann is param.empty and name in ("msg", "message")):
+            plan.append((name, "message", None))
+        elif _is_model(ann):
+            plan.append((name, "model", ann))
+            bodies += 1
+        else:
+            plan.append((name, "body", None))
+            bodies += 1
+    if bodies > 1:
+        raise TypeError(f"{handler!r} takes more than one body parameter")
+    return plan
+
+
+class Subscriber:
+    """Runs one consumer loop for one stream and group: claim stuck messages, then read new ones."""
+
+    def __init__(
+        self,
+        broker: "Broker",
+        stream: Stream,
+        handler: Callable[..., Any],
+        *,
+        ack_policy: AckPolicy = AckPolicy.NACK_ON_ERROR,
+        max_retries: Optional[int] = 3,
+        claim_idle_ms: int = 60_000,
+        backoff: float = 1.0,
+        max_backoff_ms: int = 15 * 60_000,
+        block_ms: int = 5000,
+        count: int = 10,
+        middlewares: Sequence[BaseMiddleware] = (),
+        consumer: Optional[str] = None,
+        heartbeat: bool = True,
+        heartbeat_ttl: int = 60,
+        group_start_id: str = "0",
+    ) -> None:
+        if not stream.group:
+            raise ValueError(f"stream {stream.name!r} needs a group to be consumed")
+        self.broker = broker
+        self.stream = stream
+        self.group: str = stream.group
+        self.handler = handler
+        self.ack_policy = AckPolicy(ack_policy)
+        self.max_retries = max_retries
+        self.claim_idle_ms = claim_idle_ms
+        self.backoff = backoff
+        self.max_backoff_ms = max_backoff_ms
+        self.block_ms = block_ms
+        self.count = count
+        self.own_middlewares = list(middlewares)
+        self.consumer = consumer or f"{stream.group}-{socket.gethostname()}-{os.getpid()}"
+        self.heartbeat = heartbeat
+        self.heartbeat_ttl = heartbeat_ttl
+        self.group_start_id = group_start_id
+        self._plan = _plan(handler)
+        self._stop = threading.Event()
+        self._thread: Optional[threading.Thread] = None
+
+    @property
+    def client(self) -> "redis.Redis":
+        return self.broker.client
+
+    @property
+    def middlewares(self) -> list[BaseMiddleware]:
+        """Broker-wide middleware first (outermost), then this subscriber's own."""
+        return [*self.broker.middlewares, *self.own_middlewares]
+
+    @property
+    def heartbeat_key(self) -> str:
+        return f"{HEARTBEAT_PREFIX}{self.stream.name}:{self.group}:{self.consumer}"
+
+    # lifecycle
+
+    def setup(self) -> None:
+        try:
+            self.client.xgroup_create(
+                self.stream.name, self.group, id=self.group_start_id, mkstream=True
+            )
+            log.debug("created group %s on %s", self.group, self.stream.name)
+        except redis.exceptions.ResponseError as e:
+            if "BUSYGROUP" not in str(e):
+                raise
+
+    def start(self) -> None:
+        self._thread = threading.Thread(
+            target=self._run, name=f"rillkit-{self.stream.name}", daemon=True
+        )
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stop.set()
+
+    def join(self, timeout: Optional[float] = None) -> None:
+        if self._thread:
+            self._thread.join(timeout)
+
+    def _run(self) -> None:
+        log.info(
+            "consumer started | stream: %s | group: %s | consumer: %s | policy: %s"
+            " | max retries: %s",
+            self.stream.name, self.group, self.consumer, self.ack_policy.value,
+            self.max_retries,
+        )
+        try:
+            while not self._stop.is_set():
+                self._beat()
+                try:
+                    for msg in self._claim():
+                        self._process(msg)
+                    for msg in self._read():
+                        if self._stop.is_set():
+                            break  # unread leftovers stay pending, claimed after the idle time
+                        self._process(msg)
+                except redis.exceptions.ResponseError as e:
+                    if "NOGROUP" in str(e):
+                        self.setup()
+                    else:
+                        self._loop_error(e)
+                except Exception as e:  # redis or other loop-level error: keep the worker alive
+                    self._loop_error(e)
+        finally:
+            if self.heartbeat:
+                try:
+                    self.client.delete(self.heartbeat_key)
+                except Exception as e:
+                    log.error("could not remove heartbeat key: %s", e)
+            log.info(
+                "consumer stopped | stream: %s | consumer: %s", self.stream.name, self.consumer
+            )
+
+    def _loop_error(self, e: Exception) -> None:
+        log.error("consumer loop error, retrying in %s s: %s", RETRY_BACKOFF_SECONDS, e)
+        self._stop.wait(RETRY_BACKOFF_SECONDS)
+
+    def _beat(self) -> None:
+        if not self.heartbeat:
+            return
+        try:
+            self.client.set(self.heartbeat_key, int(time.time()), ex=self.heartbeat_ttl)
+        except Exception as e:
+            log.error("heartbeat failed: %s", e)
+
+    # reading
+
+    def _message(self, id: Any, raw: dict[Any, Any], deliveries: int) -> Message:
+        return Message(_text(id), self.stream, _fields(raw), deliveries, self.consumer, self.client)
+
+    def _read(self) -> list[Message]:
+        result: Any = self.client.xreadgroup(
+            self.group, self.consumer, {self.stream.name: ">"},
+            count=self.count, block=self.block_ms,
+        )
+        return [self._message(i, f, 1) for _, entries in result or [] for i, f in entries]
+
+    def _required_idle_ms(self, times_delivered: int) -> float:
+        wait = self.claim_idle_ms * (self.backoff ** max(times_delivered - 1, 0))
+        return min(wait, self.max_backoff_ms)
+
+    def _exhausted(self, deliveries: int) -> bool:
+        return self.max_retries is not None and deliveries > self.max_retries
+
+    def _claim(self) -> list[Message]:
+        """Take over pending messages idle long enough, honouring the retry backoff."""
+        pending: Any = self.client.xpending_range(
+            self.stream.name, self.group, "-", "+", self.count, idle=self.claim_idle_ms
+        )
+        due = {
+            _text(p["message_id"]): p["times_delivered"]
+            for p in pending
+            if p["time_since_delivered"] >= self._required_idle_ms(p["times_delivered"])
+        }
+        if not due:
+            return []
+        claimed: Any = self.client.xclaim(
+            self.stream.name, self.group, self.consumer, self.claim_idle_ms, list(due)
+        )
+        messages = []
+        for id, raw in claimed:
+            if raw is None:  # entry was trimmed from the stream while pending
+                self.client.xack(self.stream.name, self.group, id)
+                continue
+            attempts = due[_text(id)]
+            if self._exhausted(attempts):
+                # not run again, so the delivery count is the attempts it already had
+                self._dead_letter(self._message(id, raw, attempts), "retries exhausted")
+                continue
+            msg = self._message(id, raw, attempts + 1)
+            messages.append(msg)
+        return messages
+
+    # processing
+
+    def _invoke(self, msg: Message) -> Any:
+        kwargs: dict[str, Any] = {}
+        for name, kind, model in self._plan:
+            if kind == "message":
+                kwargs[name] = msg
+            elif kind == "model":
+                try:
+                    kwargs[name] = model.model_validate(msg.body)
+                except DecodeError:
+                    raise
+                except Exception as e:
+                    raise DecodeError(f"body failed validation: {e}") from e
+            else:
+                kwargs[name] = msg.body
+        result = self.handler(**kwargs)
+        publishers = getattr(self.handler, "_rillkit_publishers", ())
+        if result is not None:
+            for publisher in publishers:
+                publisher.publish(result)
+        return result
+
+    def _chain(self) -> Callable[[Message], Any]:
+        call = self._invoke
+        for mw in reversed(self.middlewares):
+            call = partial(mw.consume_scope, call)
+        return call
+
+    def _process(self, msg: Message) -> None:
+        policy = self.ack_policy
+        exc: Optional[BaseException] = None
+        if policy is AckPolicy.ACK_FIRST:
+            msg.ack()
+        try:
+            for mw in self.middlewares:
+                mw.on_receive(msg)
+            self._chain()(msg)
+        except AckMessage:
+            msg.ack()
+        except NackMessage as e:
+            msg.nack()
+            exc = e
+        except RejectMessage as e:
+            msg.reject(str(e) or "rejected")
+            exc = e
+        except DecodeError as e:
+            log.error("message %s cannot be decoded, sending to DLQ: %s", msg.id, e)
+            exc = e
+            self._dead_letter(msg, str(e))
+        except Exception as e:
+            log.error("handler failed for message %s (delivery %s): %s", msg.id, msg.deliveries, e)
+            exc = e
+            msg.error = e
+            if msg.rejected:
+                pass  # the handler already asked to skip retries
+            elif policy is AckPolicy.ACK_ALWAYS:
+                msg.ack()
+            elif policy in (AckPolicy.NACK_ON_ERROR, AckPolicy.MANUAL):
+                msg.nack()
+        else:
+            auto_ack = policy in (AckPolicy.NACK_ON_ERROR, AckPolicy.ACK_ALWAYS)
+            if auto_ack and not msg.nacked and not msg.rejected:
+                msg.ack()
+
+        if msg.rejected:
+            reason = msg.reject_reason or (str(exc) or type(exc).__name__ if exc else "rejected")
+            self._dead_letter(msg, reason)
+        elif msg.nacked and not msg.acked and self._exhausted(msg.deliveries):
+            self._dead_letter(msg, (str(exc) or type(exc).__name__) if exc else "nacked")
+        self._after(msg, exc)
+        self._beat()
+
+    def _after(self, msg: Message, exc: Optional[BaseException]) -> None:
+        for mw in self.middlewares:
+            try:
+                mw.after_processed(msg, exc)
+            except Exception as e:
+                log.error("after_processed failed in %s: %s", type(mw).__name__, e)
+
+    def _dead_letter(self, msg: Message, error: str) -> None:
+        """Copy the entry to the DLQ stream with context, then ack it so it stops being retried."""
+        log.warning("moving message %s to %s: %s", msg.id, self.stream.dlq_name, error)
+        entry: dict[Any, Any] = dict(msg.raw)
+        entry.update({
+            "x-original-id": msg.id,
+            "x-stream": self.stream.name,
+            "x-group": self.group,
+            "x-error": error[:1000],
+            "x-deliveries": str(msg.deliveries),
+        })
+        pipe = self.client.pipeline()
+        pipe.xadd(self.stream.dlq_name, entry)
+        pipe.xack(self.stream.name, self.group, msg.id)
+        pipe.execute()
+        msg.acked = True
+        msg.nacked = False

rillkit/exceptions.py

@@ -0,0 +1,28 @@
+"""Exceptions raised by rillkit, and control-flow exceptions handlers can raise."""
+
+
+class RillkitError(Exception):
+    """Base class for all rillkit errors."""
+
+
+class PublishError(RillkitError):
+    """The message could not be written to the stream."""
+
+
+class DecodeError(RillkitError):
+    """The body could not be decoded or validated. Never retried, goes straight to the DLQ."""
+
+
+class AckMessage(RillkitError):
+    """Raise from a handler to stop processing and acknowledge the message."""
+
+
+class RejectMessage(RillkitError):
+    """Raise from a handler to stop processing and send the message to the DLQ without retrying.
+
+    The text of the exception is stored as the DLQ ``x-error``.
+    """
+
+
+class NackMessage(RillkitError):
+    """Raise from a handler to stop processing and nack the message so it is retried."""

rillkit/message.py

@@ -0,0 +1,74 @@
+from typing import TYPE_CHECKING, Any, Optional
+
+from .exceptions import DecodeError
+from .stream import Stream
+
+if TYPE_CHECKING:
+    import redis
+
+_MISSING = object()
+
+
+class Message:
+    """One stream entry as seen by a handler or middleware.
+
+    ``body`` is the decoded payload (a plain dict for JSON objects). ``deliveries`` is how many
+    times Redis has delivered this entry to a consumer in the group, counting the current one.
+    """
+
+    def __init__(
+        self,
+        id: str,
+        stream: Stream,
+        raw: dict[str, str],
+        deliveries: int,
+        consumer: str,
+        client: "redis.Redis",
+    ) -> None:
+        self.id = id
+        self.stream = stream
+        self.raw = raw
+        self.deliveries = deliveries
+        self.consumer = consumer
+        self.acked = False
+        self.nacked = False
+        self.rejected = False
+        self.reject_reason: Optional[str] = None
+        self._client = client
+        self._body: Any = _MISSING
+        self.error: Optional[BaseException] = None
+
+    @property
+    def body(self) -> Any:
+        if self._body is _MISSING:
+            if "data" not in self.raw:
+                raise DecodeError("entry has no 'data' field")
+            self._body = self.stream.codec.decode(self.raw["data"])
+        return self._body
+
+    def ack(self) -> None:
+        """Acknowledge the message. It is removed from the pending list and never redelivered."""
+        if self.acked:
+            return
+        assert self.stream.group is not None  # only consumed streams have messages
+        self._client.xack(self.stream.name, self.stream.group, self.id)
+        self.acked = True
+        self.nacked = False
+
+    def nack(self) -> None:
+        """Leave the message pending so it is retried (or sent to the DLQ once retries run out)."""
+        if not self.acked:
+            self.nacked = True
+
+    def reject(self, reason: Optional[str] = None) -> None:
+        """Skip the remaining retries: the message goes to the DLQ as soon as the handler finishes.
+
+        Use it for messages that can never succeed (bad input, unknown customer). ``reason`` is
+        stored as ``x-error`` in the DLQ entry. Works under every ``AckPolicy``.
+        """
+        self.rejected = True
+        self.reject_reason = reason
+        self.nacked = False
+
+    def __repr__(self) -> str:
+        return f"Message(id={self.id!r}, stream={self.stream.name!r}, deliveries={self.deliveries})"

rillkit/middleware.py

@@ -0,0 +1,75 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Callable, Optional
+
+from .exceptions import AckMessage
+from .message import Message
+
+if TYPE_CHECKING:
+    from .broker import Broker
+    from .stream import Stream
+
+
+@dataclass
+class PublishCommand:
+    """What ``publish_scope`` receives. Mutate it to change what gets published."""
+
+    stream: "Stream"
+    body: Any
+    maxlen: Optional[int] = None
+    approximate: bool = True
+
+
+class BaseMiddleware:
+    """Subclass and override any hook. Instances are shared by all messages, so keep per-message
+    state on the ``Message``, not on ``self``. ``self.broker`` is set when the middleware is added.
+
+    Order: ``on_receive`` -> ``consume_scope`` (wraps decode + handler) -> ``after_processed``.
+    The first middleware in the list is the outermost.
+    """
+
+    broker: "Broker"
+
+    def on_receive(self, msg: Message) -> None:
+        """Called first for every delivered message, before decoding."""
+
+    def consume_scope(self, call_next: Callable[[Message], Any], msg: Message) -> Any:
+        """Wrap the handler call. Raise ``AckMessage``/``NackMessage`` to short-circuit."""
+        return call_next(msg)
+
+    def publish_scope(self, call_next: Callable[[PublishCommand], str], cmd: PublishCommand) -> str:
+        """Wrap every publish, including handler return values."""
+        return call_next(cmd)
+
+    def after_processed(self, msg: Message, exc: Optional[BaseException]) -> None:
+        """Always called last. ``exc`` is the handler error, if any."""
+
+
+class Idempotent(BaseMiddleware):
+    """Skip messages whose key was already processed successfully.
+
+    Redis delivers at least once, so a crash after the handler finished but before the ack means a
+    redelivery. The key is stored with ``ttl`` seconds once the handler returns without raising.
+
+        broker = Broker(middlewares=[Idempotent(key=lambda m: m.body["request_id"])])
+    """
+
+    def __init__(
+        self,
+        key: Callable[[Message], str],
+        ttl: int = 7 * 24 * 3600,
+        prefix: str = "rillkit:done:",
+    ):
+        self.key = key
+        self.ttl = ttl
+        self.prefix = prefix
+
+    def _redis_key(self, msg: Message) -> str:
+        return f"{self.prefix}{msg.stream.name}:{self.key(msg)}"
+
+    def consume_scope(self, call_next: Callable[[Message], Any], msg: Message) -> Any:
+        redis_key = self._redis_key(msg)
+        if self.broker.client.exists(redis_key):
+            raise AckMessage()
+        result = call_next(msg)
+        self.broker.client.set(redis_key, msg.id, ex=self.ttl)
+        return result

rillkit/settings.py

@@ -0,0 +1,39 @@
+import os
+from dataclasses import dataclass
+from typing import Optional
+
+import redis
+
+
+@dataclass
+class Settings:
+    """Connection settings.
+
+    ``Settings.from_env()`` reads REDIS_URL, or REDIS_HOST/PORT/DB/PASSWORD/SSL.
+    """
+
+    url: Optional[str] = None
+    host: str = "localhost"
+    port: int = 6379
+    db: int = 0
+    password: Optional[str] = None
+    ssl: bool = False
+
+    @classmethod
+    def from_env(cls, prefix: str = "REDIS_") -> "Settings":
+        env = os.environ
+        return cls(
+            url=env.get(f"{prefix}URL"),
+            host=env.get(f"{prefix}HOST", "localhost"),
+            port=int(env.get(f"{prefix}PORT", 6379)),
+            db=int(env.get(f"{prefix}DB", 0)),
+            password=env.get(f"{prefix}PASSWORD"),
+            ssl=env.get(f"{prefix}SSL", "").lower() in ("1", "true", "yes"),
+        )
+
+    def client(self) -> "redis.Redis":
+        if self.url:
+            return redis.Redis.from_url(self.url)
+        return redis.Redis(
+            host=self.host, port=self.port, db=self.db, password=self.password, ssl=self.ssl
+        )

rillkit/stream.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass, field
+from typing import Optional, Union
+
+from .codec import Codec, JsonCodec
+
+
+@dataclass(frozen=True)
+class Stream:
+    """A stream name plus everything that should be agreed on by both sides.
+
+    ``group`` is only needed by consumers. ``dlq`` defaults to ``<name>-dlq``.
+    """
+
+    name: str
+    group: Optional[str] = None
+    dlq: Optional[str] = None
+    maxlen: Optional[int] = None
+    codec: Codec = field(default_factory=JsonCodec, compare=False, repr=False)
+
+    @property
+    def dlq_name(self) -> str:
+        return self.dlq or f"{self.name}-dlq"
+
+
+StreamLike = Union[str, Stream]
+
+
+def as_stream(stream: StreamLike) -> Stream:
+    return stream if isinstance(stream, Stream) else Stream(stream)

rillkit-0.1.0.dist-info/METADATA

@@ -0,0 +1,81 @@
+Metadata-Version: 2.5
+Name: rillkit
+Version: 0.1.0
+Summary: Simple publish and consume for Redis Streams: consumer groups, retries, DLQ, ack policies and middleware.
+Project-URL: Homepage, https://nimzymaina.github.io/rillkit/
+Project-URL: Documentation, https://nimzymaina.github.io/rillkit/
+Project-URL: Repository, https://github.com/NimzyMaina/rillkit
+Project-URL: Issues, https://github.com/NimzyMaina/rillkit/issues
+Author: Nimrod Maina
+License: MIT
+License-File: LICENSE
+Keywords: consumer-group,messaging,queue,redis,streams,worker
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: redis>=5.0
+Provides-Extra: pydantic
+Requires-Dist: pydantic>=2.0; extra == 'pydantic'
+Description-Content-Type: text/markdown
+
+# RillKit
+
+Simple publish and consume for Redis Streams. Consumer groups, retries with backoff, a dead-letter
+queue, ack policies, middleware and lifecycle hooks, without the boilerplate.
+
+**Docs:** https://nimzymaina.github.io/rillkit/
+
+```bash
+pip install rillkit             # plain dict bodies
+pip install "rillkit[pydantic]" # optional typed bodies
+```
+
+```python
+from rillkit import Broker, Stream
+
+orders = Stream("orders", group="workers")
+broker = Broker.from_env()  # REDIS_URL, or REDIS_HOST / REDIS_PORT / REDIS_PASSWORD / REDIS_SSL
+
+
+@broker.subscribe(orders, max_retries=3)
+def handle(order: dict):
+    print("got", order)
+
+
+if __name__ == "__main__":
+    broker.publish(orders, {"id": 1})
+    broker.run()  # blocks; Ctrl-C / SIGTERM finishes the current message and exits
+```
+
+## Features
+
+- Consumer groups: group creation, `XREADGROUP`, and `XAUTOCLAIM`-style takeover of stuck messages
+- At-least-once delivery with retries (`max_retries`, optional exponential `backoff`)
+- Dead-letter queue with the original id, error and delivery count; `broker.replay_dlq()`
+- `msg.nack()` to retry, `msg.reject()` (or `RejectMessage`) to skip retries and go straight to the DLQ
+- `AckPolicy`: `NACK_ON_ERROR` (default), `ACK_ALWAYS`, `ACK_FIRST`, `MANUAL`
+- Plain dicts by default, Pydantic models when you annotate the handler with one
+- Middleware hooks (`on_receive`, `consume_scope`, `publish_scope`, `after_processed`) and an `Idempotent` middleware
+- `on_startup`, `after_startup`, `on_shutdown`, `after_shutdown` hooks and graceful SIGTERM shutdown
+- Worker heartbeats: `broker.active_workers(stream)` includes idle workers
+- Stack `@broker.publisher(...)` under `@broker.subscribe(...)` to publish a handler's return value
+- The wire format is a single `data` field holding JSON, so plain redis-py producers interoperate
+
+## Development
+
+```bash
+uv sync --group dev
+REDIS_URL=redis://localhost:6379/15 uv run pytest   # needs a Redis
+npm install && npm run docs:dev                     # docs
+```
+
+MIT licensed.

rillkit-0.1.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+rillkit/__init__.py,sha256=Nd-o6hG0u4yzPs-iZU5xA2dwfHtQhNkGmqKjRH1_Jf4,734
+rillkit/ack.py,sha256=JKm-E0ggLpY_jsK2eVFtruWu7gzBq1XNYrxJIZtuvgE,722
+rillkit/broker.py,sha256=z-d_vvLligmFpTTteFsiCAqIkVhAwARNwwkuluuGwzs,9936
+rillkit/codec.py,sha256=D5PS2fF_nBkZXGgYcGfRJOzOQM7K3c4DvBgXaJnm3LI,710
+rillkit/consumer.py,sha256=E1KH6bEWIqTe1D7eVvEFu0IbQ2qntADAcGOTyy0L4Uw,12511
+rillkit/exceptions.py,sha256=bjxioj7x438t_34GokSGlf0fC2V9hsXG19bA8TV5JQI,830
+rillkit/message.py,sha256=nwjBIBa3KlC1vOIszAWt7SfFdMU7FW2lDE8sBKgn2TI,2451
+rillkit/middleware.py,sha256=UPhoG9vaRFeh_CYYRROnl1jJiOPkzaeYLcUR55Sc7p0,2606
+rillkit/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+rillkit/settings.py,sha256=jFXD1e4MgSiArvWl3Eu_g0WWR1b2qv9ebXQ_u4nuVdk,1088
+rillkit/stream.py,sha256=YEf7P4w6VMDHibRGVdq4yBTnYcRcaYkijWnvBMcQBGg,748
+rillkit-0.1.0.dist-info/METADATA,sha256=ggADWT_yFXImp-E4kTVmuHu2Qq7vHa99CIWvfVNC0cI,3255
+rillkit-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+rillkit-0.1.0.dist-info/licenses/LICENSE,sha256=oWHoi3y0pjlKKUoN9T8Pfswaojv_Mg98vX6U1uz770k,1069
+rillkit-0.1.0.dist-info/RECORD,,

rillkit-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

rillkit-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nimrod Maina
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.