rillkit 0.1.0__tar.gz

rillkit-0.1.0/.gitignore

@@ -0,0 +1,12 @@
+.venv/
+__pycache__/
+*.egg-info/
+dist/
+.env
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+node_modules/
+docs/.vitepress/cache/
+docs/.vitepress/dist/
+.DS_Store

rillkit-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Nimrod Maina
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

rillkit-0.1.0/PKG-INFO

@@ -0,0 +1,81 @@
+Metadata-Version: 2.5
+Name: rillkit
+Version: 0.1.0
+Summary: Simple publish and consume for Redis Streams: consumer groups, retries, DLQ, ack policies and middleware.
+Project-URL: Homepage, https://nimzymaina.github.io/rillkit/
+Project-URL: Documentation, https://nimzymaina.github.io/rillkit/
+Project-URL: Repository, https://github.com/NimzyMaina/rillkit
+Project-URL: Issues, https://github.com/NimzyMaina/rillkit/issues
+Author: Nimrod Maina
+License: MIT
+License-File: LICENSE
+Keywords: consumer-group,messaging,queue,redis,streams,worker
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Typing :: Typed
+Requires-Python: >=3.9
+Requires-Dist: redis>=5.0
+Provides-Extra: pydantic
+Requires-Dist: pydantic>=2.0; extra == 'pydantic'
+Description-Content-Type: text/markdown
+
+# RillKit
+
+Simple publish and consume for Redis Streams. Consumer groups, retries with backoff, a dead-letter
+queue, ack policies, middleware and lifecycle hooks, without the boilerplate.
+
+**Docs:** https://nimzymaina.github.io/rillkit/
+
+```bash
+pip install rillkit             # plain dict bodies
+pip install "rillkit[pydantic]" # optional typed bodies
+```
+
+```python
+from rillkit import Broker, Stream
+
+orders = Stream("orders", group="workers")
+broker = Broker.from_env()  # REDIS_URL, or REDIS_HOST / REDIS_PORT / REDIS_PASSWORD / REDIS_SSL
+
+
+@broker.subscribe(orders, max_retries=3)
+def handle(order: dict):
+    print("got", order)
+
+
+if __name__ == "__main__":
+    broker.publish(orders, {"id": 1})
+    broker.run()  # blocks; Ctrl-C / SIGTERM finishes the current message and exits
+```
+
+## Features
+
+- Consumer groups: group creation, `XREADGROUP`, and `XAUTOCLAIM`-style takeover of stuck messages
+- At-least-once delivery with retries (`max_retries`, optional exponential `backoff`)
+- Dead-letter queue with the original id, error and delivery count; `broker.replay_dlq()`
+- `msg.nack()` to retry, `msg.reject()` (or `RejectMessage`) to skip retries and go straight to the DLQ
+- `AckPolicy`: `NACK_ON_ERROR` (default), `ACK_ALWAYS`, `ACK_FIRST`, `MANUAL`
+- Plain dicts by default, Pydantic models when you annotate the handler with one
+- Middleware hooks (`on_receive`, `consume_scope`, `publish_scope`, `after_processed`) and an `Idempotent` middleware
+- `on_startup`, `after_startup`, `on_shutdown`, `after_shutdown` hooks and graceful SIGTERM shutdown
+- Worker heartbeats: `broker.active_workers(stream)` includes idle workers
+- Stack `@broker.publisher(...)` under `@broker.subscribe(...)` to publish a handler's return value
+- The wire format is a single `data` field holding JSON, so plain redis-py producers interoperate
+
+## Development
+
+```bash
+uv sync --group dev
+REDIS_URL=redis://localhost:6379/15 uv run pytest   # needs a Redis
+npm install && npm run docs:dev                     # docs
+```
+
+MIT licensed.

rillkit-0.1.0/README.md

@@ -0,0 +1,52 @@
+# RillKit
+
+Simple publish and consume for Redis Streams. Consumer groups, retries with backoff, a dead-letter
+queue, ack policies, middleware and lifecycle hooks, without the boilerplate.
+
+**Docs:** https://nimzymaina.github.io/rillkit/
+
+```bash
+pip install rillkit             # plain dict bodies
+pip install "rillkit[pydantic]" # optional typed bodies
+```
+
+```python
+from rillkit import Broker, Stream
+
+orders = Stream("orders", group="workers")
+broker = Broker.from_env()  # REDIS_URL, or REDIS_HOST / REDIS_PORT / REDIS_PASSWORD / REDIS_SSL
+
+
+@broker.subscribe(orders, max_retries=3)
+def handle(order: dict):
+    print("got", order)
+
+
+if __name__ == "__main__":
+    broker.publish(orders, {"id": 1})
+    broker.run()  # blocks; Ctrl-C / SIGTERM finishes the current message and exits
+```
+
+## Features
+
+- Consumer groups: group creation, `XREADGROUP`, and `XAUTOCLAIM`-style takeover of stuck messages
+- At-least-once delivery with retries (`max_retries`, optional exponential `backoff`)
+- Dead-letter queue with the original id, error and delivery count; `broker.replay_dlq()`
+- `msg.nack()` to retry, `msg.reject()` (or `RejectMessage`) to skip retries and go straight to the DLQ
+- `AckPolicy`: `NACK_ON_ERROR` (default), `ACK_ALWAYS`, `ACK_FIRST`, `MANUAL`
+- Plain dicts by default, Pydantic models when you annotate the handler with one
+- Middleware hooks (`on_receive`, `consume_scope`, `publish_scope`, `after_processed`) and an `Idempotent` middleware
+- `on_startup`, `after_startup`, `on_shutdown`, `after_shutdown` hooks and graceful SIGTERM shutdown
+- Worker heartbeats: `broker.active_workers(stream)` includes idle workers
+- Stack `@broker.publisher(...)` under `@broker.subscribe(...)` to publish a handler's return value
+- The wire format is a single `data` field holding JSON, so plain redis-py producers interoperate
+
+## Development
+
+```bash
+uv sync --group dev
+REDIS_URL=redis://localhost:6379/15 uv run pytest   # needs a Redis
+npm install && npm run docs:dev                     # docs
+```
+
+MIT licensed.

rillkit-0.1.0/pyproject.toml

@@ -0,0 +1,57 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "rillkit"
+version = "0.1.0"
+description = "Simple publish and consume for Redis Streams: consumer groups, retries, DLQ, ack policies and middleware."
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+authors = [{ name = "Nimrod Maina" }]
+keywords = ["redis", "streams", "consumer-group", "queue", "messaging", "worker"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries",
+    "Typing :: Typed",
+]
+dependencies = ["redis>=5.0"]
+
+[project.optional-dependencies]
+pydantic = ["pydantic>=2.0"]
+
+[project.urls]
+Homepage = "https://nimzymaina.github.io/rillkit/"
+Documentation = "https://nimzymaina.github.io/rillkit/"
+Repository = "https://github.com/NimzyMaina/rillkit"
+Issues = "https://github.com/NimzyMaina/rillkit/issues"
+
+[dependency-groups]
+dev = ["pytest>=8", "pytest-timeout>=2", "pydantic>=2", "ruff>=0.6", "mypy>=1.10", "build", "twine"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/rillkit"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src", "tests", "README.md", "LICENSE", "pyproject.toml"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+timeout = 60
+
+[tool.ruff]
+line-length = 100
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
+ignore = ["UP007", "UP045"]

rillkit-0.1.0/src/rillkit/__init__.py

@@ -0,0 +1,25 @@
+"""rillkit: simple publish and consume for Redis Streams."""
+
+from .ack import AckPolicy
+from .broker import Broker, Publisher
+from .codec import Codec, JsonCodec
+from .exceptions import (
+    AckMessage,
+    DecodeError,
+    NackMessage,
+    PublishError,
+    RejectMessage,
+    RillkitError,
+)
+from .message import Message
+from .middleware import BaseMiddleware, Idempotent, PublishCommand
+from .settings import Settings
+from .stream import Stream
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "AckMessage", "AckPolicy", "BaseMiddleware", "Broker", "Codec", "DecodeError", "Idempotent",
+    "JsonCodec", "Message", "NackMessage", "PublishCommand", "PublishError", "Publisher",
+    "RejectMessage", "RillkitError", "Settings", "Stream",
+]

rillkit-0.1.0/src/rillkit/ack.py

@@ -0,0 +1,18 @@
+from enum import Enum
+
+
+class AckPolicy(str, Enum):
+    """What happens to a message when its handler finishes.
+
+    NACK_ON_ERROR  Ack when the handler returns, nack when it raises. This is the default.
+    ACK_ALWAYS     Ack when the handler returns or raises. Failures are logged and dropped, never
+                   retried.
+    ACK_FIRST      Ack before the handler runs. At-most-once: a crash loses the message.
+    MANUAL         rillkit never acks. The handler must call ``msg.ack()``. If it returns without
+                   acking, or raises, the message stays pending and is retried.
+    """
+
+    NACK_ON_ERROR = "nack_on_error"
+    ACK_ALWAYS = "ack_always"
+    ACK_FIRST = "ack_first"
+    MANUAL = "manual"

rillkit-0.1.0/src/rillkit/broker.py

@@ -0,0 +1,271 @@
+import logging
+import signal
+import sys
+import threading
+from collections.abc import Sequence
+from functools import partial
+from typing import Any, Callable, Optional
+
+import redis
+
+from .ack import AckPolicy
+from .consumer import HEARTBEAT_PREFIX, Subscriber, _text
+from .exceptions import PublishError
+from .middleware import BaseMiddleware, PublishCommand
+from .settings import Settings
+from .stream import Stream, StreamLike, as_stream
+
+log = logging.getLogger("rillkit")
+
+Hook = Callable[[], Any]
+
+
+class Publisher:
+    """Returned by ``broker.publisher(stream)``. Use it as a decorator to publish a handler's return
+    value, or call ``.publish(body)`` directly."""
+
+    def __init__(self, broker: "Broker", stream: Stream, maxlen: Optional[int] = None) -> None:
+        self.broker = broker
+        self.stream = stream
+        self.maxlen = maxlen
+
+    def __call__(self, func: Callable[..., Any]) -> Callable[..., Any]:
+        publishers = list(getattr(func, "_rillkit_publishers", ()))
+        publishers.append(self)
+        func._rillkit_publishers = publishers  # type: ignore[attr-defined]
+        return func
+
+    def publish(self, body: Any) -> str:
+        return self.broker.publish(self.stream, body, maxlen=self.maxlen)
+
+
+class Broker:
+    """Owns the Redis client, the subscribers and the lifecycle hooks.
+
+        broker = Broker.from_env()
+
+        @broker.subscribe(Stream("orders", group="workers"))
+        def handle(order: dict): ...
+
+        broker.publish("orders", {"id": 1})
+        broker.run()
+    """
+
+    def __init__(
+        self,
+        url: Optional[str] = None,
+        *,
+        client: Optional["redis.Redis"] = None,
+        settings: Optional[Settings] = None,
+        middlewares: Sequence[BaseMiddleware] = (),
+    ) -> None:
+        if client is None:
+            client = (settings or Settings(url=url)).client()
+        self.client = client
+        self.middlewares: list[BaseMiddleware] = []
+        for mw in middlewares:
+            self.use(mw)
+        self.subscribers: list[Subscriber] = []
+        self._hooks: dict[str, list[Hook]] = {
+            "on_startup": [], "after_startup": [], "on_shutdown": [], "after_shutdown": [],
+        }
+        self._stop = threading.Event()
+        self._stop_lock = threading.Lock()
+        self._stopped = False
+
+    @classmethod
+    def from_env(cls, prefix: str = "REDIS_", **kwargs: Any) -> "Broker":
+        return cls(settings=Settings.from_env(prefix), **kwargs)
+
+    # middleware
+
+    def use(self, middleware: BaseMiddleware) -> BaseMiddleware:
+        middleware.broker = self
+        self.middlewares.append(middleware)
+        return middleware
+
+    # subscribing and publishing
+
+    def subscribe(
+        self,
+        stream: StreamLike,
+        *,
+        group: Optional[str] = None,
+        ack_policy: AckPolicy = AckPolicy.NACK_ON_ERROR,
+        max_retries: Optional[int] = 3,
+        claim_idle_ms: int = 60_000,
+        backoff: float = 1.0,
+        max_backoff_ms: int = 15 * 60_000,
+        block_ms: int = 5000,
+        count: int = 10,
+        middlewares: Sequence[BaseMiddleware] = (),
+        consumer: Optional[str] = None,
+        heartbeat: bool = True,
+        heartbeat_ttl: int = 60,
+        group_start_id: str = "0",
+    ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+        """Register a handler for a stream. The handler is returned unchanged."""
+        resolved = as_stream(stream)
+        if group:
+            resolved = Stream(resolved.name, group, resolved.dlq, resolved.maxlen, resolved.codec)
+        own = list(middlewares)
+        for m in own:
+            m.broker = self
+
+        def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+            self.subscribers.append(Subscriber(
+                self, resolved, func, ack_policy=ack_policy, max_retries=max_retries,
+                claim_idle_ms=claim_idle_ms, backoff=backoff, max_backoff_ms=max_backoff_ms,
+                block_ms=block_ms, count=count, middlewares=own,
+                consumer=consumer, heartbeat=heartbeat, heartbeat_ttl=heartbeat_ttl,
+                group_start_id=group_start_id,
+            ))
+            return func
+
+        return decorator
+
+    def publisher(self, stream: StreamLike, *, maxlen: Optional[int] = None) -> Publisher:
+        return Publisher(self, as_stream(stream), maxlen)
+
+    def publish(
+        self,
+        stream: StreamLike,
+        body: Any,
+        *,
+        maxlen: Optional[int] = None,
+        approximate: bool = True,
+    ) -> str:
+        """Add one entry to the stream and return its id. Raises ``PublishError`` on failure."""
+        resolved = as_stream(stream)
+        cmd = PublishCommand(
+            resolved, body, maxlen if maxlen is not None else resolved.maxlen, approximate
+        )
+        call: Callable[[PublishCommand], str] = self._xadd
+        for mw in reversed(self.middlewares):
+            call = partial(mw.publish_scope, call)
+        return call(cmd)
+
+    def _xadd(self, cmd: PublishCommand) -> str:
+        try:
+            payload = cmd.stream.codec.encode(cmd.body)
+            entry_id = self.client.xadd(
+                cmd.stream.name, {"data": payload}, maxlen=cmd.maxlen, approximate=cmd.approximate
+            )
+        except (redis.RedisError, TypeError, ValueError) as e:
+            raise PublishError(f"could not publish to {cmd.stream.name}: {e}") from e
+        return _text(entry_id)
+
+    # lifecycle hooks
+
+    def _hook(self, name: str) -> Callable[[Hook], Hook]:
+        def register(func: Hook) -> Hook:
+            self._hooks[name].append(func)
+            return func
+        return register
+
+    def on_startup(self, func: Hook) -> Hook:
+        """Runs first, before the connection is checked."""
+        return self._hook("on_startup")(func)
+
+    def after_startup(self, func: Hook) -> Hook:
+        """Runs once groups exist and the consumer threads are running."""
+        return self._hook("after_startup")(func)
+
+    def on_shutdown(self, func: Hook) -> Hook:
+        """Runs after the consumers finished their current message, before the client closes."""
+        return self._hook("on_shutdown")(func)
+
+    def after_shutdown(self, func: Hook) -> Hook:
+        """Runs last, after the client is closed."""
+        return self._hook("after_shutdown")(func)
+
+    def _run_hooks(self, name: str) -> None:
+        for func in self._hooks[name]:
+            func()
+
+    # running
+
+    def start(self) -> None:
+        """Start all consumers in background threads and return."""
+        self._stop.clear()
+        self._stopped = False
+        self._run_hooks("on_startup")
+        self.client.ping()
+        for sub in self.subscribers:
+            sub.setup()
+        for sub in self.subscribers:
+            sub.start()
+        self._run_hooks("after_startup")
+
+    def stop(self, timeout: Optional[float] = None) -> None:
+        """Ask the consumers to stop, wait for the current messages to finish, then clean up.
+
+        Safe to call more than once or from another thread while ``run()`` is blocking: the shutdown
+        hooks and the client close happen only once per start."""
+        with self._stop_lock:
+            if self._stopped:
+                return
+            self._stopped = True
+            for sub in self.subscribers:
+                sub.stop()
+            for sub in self.subscribers:
+                sub.join(timeout)
+            try:
+                self._run_hooks("on_shutdown")
+            finally:
+                self.client.close()
+                self._run_hooks("after_shutdown")
+                self._stop.set()
+
+    def run(self, handle_signals: bool = True, shutdown_timeout: Optional[float] = None) -> None:
+        """Start the consumers and block until SIGINT/SIGTERM (or ``stop()``) shuts them down.
+
+        The first signal finishes the current message and exits cleanly. A second signal exits now.
+        """
+        requested = threading.Event()
+        previous = {}
+
+        def on_signal(signum: int, frame: Any) -> None:
+            if requested.is_set():
+                log.warning("second shutdown signal received, exiting now")
+                sys.exit(1)
+            log.info("shutdown requested, finishing the current message before exiting")
+            requested.set()
+
+        if handle_signals and threading.current_thread() is threading.main_thread():
+            for sig in (signal.SIGINT, signal.SIGTERM):
+                previous[sig] = signal.signal(sig, on_signal)
+        try:
+            self.start()
+            while not requested.wait(0.5):
+                if not any(s._thread and s._thread.is_alive() for s in self.subscribers):
+                    break
+            self.stop(shutdown_timeout)
+        finally:
+            for sig, handler in previous.items():
+                signal.signal(sig, handler)
+
+    # operations
+
+    def active_workers(self, stream: StreamLike, group: Optional[str] = None) -> list[str]:
+        """Consumer names with a live heartbeat. Unlike XINFO CONSUMERS this includes idle workers
+        that have not received a message yet."""
+        s = as_stream(stream)
+        prefix = f"{HEARTBEAT_PREFIX}{s.name}:{group or s.group}:"
+        return sorted(_text(k)[len(prefix):] for k in self.client.scan_iter(match=f"{prefix}*"))
+
+    def replay_dlq(self, stream: StreamLike, count: int = 100) -> int:
+        """Move up to ``count`` entries from the DLQ back onto the stream. Returns how many."""
+        s = as_stream(stream)
+        moved = 0
+        rows: Any = self.client.xrange(s.dlq_name, count=count)
+        for entry_id, raw in rows:
+            fields: dict[Any, Any] = {
+                _text(k): _text(v) for k, v in raw.items() if not _text(k).startswith("x-")
+            }
+            pipe = self.client.pipeline()
+            pipe.xadd(s.name, fields)
+            pipe.xdel(s.dlq_name, entry_id)
+            pipe.execute()
+            moved += 1
+        return moved

rillkit-0.1.0/src/rillkit/codec.py

@@ -0,0 +1,28 @@
+import json
+from typing import Any, Protocol
+
+from .exceptions import DecodeError
+
+
+class Codec(Protocol):
+    def encode(self, body: Any) -> str: ...
+
+    def decode(self, raw: str) -> Any: ...
+
+
+class JsonCodec:
+    """JSON in the ``data`` field of the stream entry.
+
+    Pydantic models are dumped with ``model_dump_json``.
+    """
+
+    def encode(self, body: Any) -> str:
+        if hasattr(body, "model_dump_json"):
+            return body.model_dump_json()
+        return json.dumps(body, default=str)
+
+    def decode(self, raw: str) -> Any:
+        try:
+            return json.loads(raw)
+        except (TypeError, ValueError) as e:
+            raise DecodeError(f"body is not valid JSON: {e}") from e