rillkit 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rillkit-0.1.1 → rillkit-0.2.0}/.gitignore +1 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/PKG-INFO +17 -4
- {rillkit-0.1.1 → rillkit-0.2.0}/README.md +14 -3
- {rillkit-0.1.1 → rillkit-0.2.0}/pyproject.toml +16 -2
- {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/__init__.py +8 -5
- {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/broker.py +40 -17
- rillkit-0.2.0/src/rillkit/clients.py +294 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/consumer.py +39 -37
- {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/exceptions.py +4 -0
- rillkit-0.2.0/src/rillkit/keys.py +127 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/message.py +7 -6
- rillkit-0.2.0/src/rillkit/settings.py +142 -0
- rillkit-0.2.0/tests/conftest.py +99 -0
- rillkit-0.2.0/tests/helpers.py +100 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_blocking_timeout.py +9 -11
- rillkit-0.2.0/tests/test_cluster.py +180 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_consume.py +19 -19
- rillkit-0.2.0/tests/test_keys.py +60 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_lifecycle.py +61 -6
- {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_middleware.py +8 -8
- {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_publish.py +9 -9
- {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_publisher_decorator.py +6 -6
- {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_retries.py +38 -38
- rillkit-0.2.0/tests/test_settings.py +363 -0
- rillkit-0.1.1/src/rillkit/settings.py +0 -57
- rillkit-0.1.1/tests/conftest.py +0 -56
- rillkit-0.1.1/tests/helpers.py +0 -74
- rillkit-0.1.1/tests/test_settings.py +0 -79
- {rillkit-0.1.1 → rillkit-0.2.0}/LICENSE +0 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/ack.py +0 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/codec.py +0 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/middleware.py +0 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/py.typed +0 -0
- {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/stream.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: rillkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Simple publish and consume for Redis Streams: consumer groups, retries, DLQ, ack policies and middleware.
|
|
5
5
|
Project-URL: Homepage, https://nimzymaina.github.io/rillkit/
|
|
6
6
|
Project-URL: Documentation, https://nimzymaina.github.io/rillkit/
|
|
@@ -25,18 +25,22 @@ Requires-Python: >=3.9
|
|
|
25
25
|
Requires-Dist: redis>=5.0
|
|
26
26
|
Provides-Extra: pydantic
|
|
27
27
|
Requires-Dist: pydantic>=2.0; extra == 'pydantic'
|
|
28
|
+
Provides-Extra: valkey
|
|
29
|
+
Requires-Dist: valkey<6.3,>=6.1.1; extra == 'valkey'
|
|
28
30
|
Description-Content-Type: text/markdown
|
|
29
31
|
|
|
30
32
|
# RillKit
|
|
31
33
|
|
|
32
|
-
Simple publish and consume for Redis
|
|
33
|
-
queue, ack policies, middleware and lifecycle hooks, without the
|
|
34
|
+
Simple publish and consume for Redis and Valkey streams, standalone or cluster. Consumer groups,
|
|
35
|
+
retries with backoff, a dead-letter queue, ack policies, middleware and lifecycle hooks, without the
|
|
36
|
+
boilerplate.
|
|
34
37
|
|
|
35
38
|
**Docs:** https://nimzymaina.github.io/rillkit/
|
|
36
39
|
|
|
37
40
|
```bash
|
|
38
41
|
pip install rillkit # plain dict bodies
|
|
39
42
|
pip install "rillkit[pydantic]" # optional typed bodies
|
|
43
|
+
pip install "rillkit[valkey]" # optional valkey-py client
|
|
40
44
|
```
|
|
41
45
|
|
|
42
46
|
```python
|
|
@@ -69,13 +73,22 @@ if __name__ == "__main__":
|
|
|
69
73
|
- Worker heartbeats: `broker.active_workers(stream)` includes idle workers
|
|
70
74
|
- Stack `@broker.publisher(...)` under `@broker.subscribe(...)` to publish a handler's return value
|
|
71
75
|
- The wire format is a single `data` field holding JSON, so plain redis-py producers interoperate
|
|
76
|
+
- Redis or Valkey (`RILLKIT_BACKEND=valkey`), standalone or Cluster (`RILLKIT_MODE=cluster`), with
|
|
77
|
+
the stream and its DLQ in one hash slot so the move to the DLQ stays a transaction
|
|
72
78
|
|
|
73
79
|
## Development
|
|
74
80
|
|
|
75
81
|
```bash
|
|
76
82
|
uv sync --group dev
|
|
77
|
-
REDIS_URL=redis://localhost:6379/15 uv run pytest # needs a Redis
|
|
83
|
+
REDIS_URL=redis://localhost:6379/15 uv run pytest # needs a Redis; only db 15 is flushed
|
|
84
|
+
RILLKIT_TEST_BACKEND=valkey REDIS_URL=redis://localhost:6380/15 uv run pytest # valkey-py
|
|
85
|
+
|
|
86
|
+
scripts/test-cluster.sh valkey 7101 # a six-node cluster in Docker
|
|
87
|
+
RILLKIT_TEST_BACKEND=valkey RILLKIT_TEST_MODE=cluster \
|
|
88
|
+
RILLKIT_TEST_CLUSTER_NODES=127.0.0.1:7101 uv run pytest
|
|
89
|
+
|
|
78
90
|
npm install && npm run docs:dev # docs
|
|
91
|
+
npm run docs:build:versions # latest plus a snapshot per release tag
|
|
79
92
|
```
|
|
80
93
|
|
|
81
94
|
MIT licensed.
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
# RillKit
|
|
2
2
|
|
|
3
|
-
Simple publish and consume for Redis
|
|
4
|
-
queue, ack policies, middleware and lifecycle hooks, without the
|
|
3
|
+
Simple publish and consume for Redis and Valkey streams, standalone or cluster. Consumer groups,
|
|
4
|
+
retries with backoff, a dead-letter queue, ack policies, middleware and lifecycle hooks, without the
|
|
5
|
+
boilerplate.
|
|
5
6
|
|
|
6
7
|
**Docs:** https://nimzymaina.github.io/rillkit/
|
|
7
8
|
|
|
8
9
|
```bash
|
|
9
10
|
pip install rillkit # plain dict bodies
|
|
10
11
|
pip install "rillkit[pydantic]" # optional typed bodies
|
|
12
|
+
pip install "rillkit[valkey]" # optional valkey-py client
|
|
11
13
|
```
|
|
12
14
|
|
|
13
15
|
```python
|
|
@@ -40,13 +42,22 @@ if __name__ == "__main__":
|
|
|
40
42
|
- Worker heartbeats: `broker.active_workers(stream)` includes idle workers
|
|
41
43
|
- Stack `@broker.publisher(...)` under `@broker.subscribe(...)` to publish a handler's return value
|
|
42
44
|
- The wire format is a single `data` field holding JSON, so plain redis-py producers interoperate
|
|
45
|
+
- Redis or Valkey (`RILLKIT_BACKEND=valkey`), standalone or Cluster (`RILLKIT_MODE=cluster`), with
|
|
46
|
+
the stream and its DLQ in one hash slot so the move to the DLQ stays a transaction
|
|
43
47
|
|
|
44
48
|
## Development
|
|
45
49
|
|
|
46
50
|
```bash
|
|
47
51
|
uv sync --group dev
|
|
48
|
-
REDIS_URL=redis://localhost:6379/15 uv run pytest # needs a Redis
|
|
52
|
+
REDIS_URL=redis://localhost:6379/15 uv run pytest # needs a Redis; only db 15 is flushed
|
|
53
|
+
RILLKIT_TEST_BACKEND=valkey REDIS_URL=redis://localhost:6380/15 uv run pytest # valkey-py
|
|
54
|
+
|
|
55
|
+
scripts/test-cluster.sh valkey 7101 # a six-node cluster in Docker
|
|
56
|
+
RILLKIT_TEST_BACKEND=valkey RILLKIT_TEST_MODE=cluster \
|
|
57
|
+
RILLKIT_TEST_CLUSTER_NODES=127.0.0.1:7101 uv run pytest
|
|
58
|
+
|
|
49
59
|
npm install && npm run docs:dev # docs
|
|
60
|
+
npm run docs:build:versions # latest plus a snapshot per release tag
|
|
50
61
|
```
|
|
51
62
|
|
|
52
63
|
MIT licensed.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "rillkit"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.2.0"
|
|
8
8
|
description = "Simple publish and consume for Redis Streams: consumer groups, retries, DLQ, ack policies and middleware."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -28,6 +28,7 @@ dependencies = ["redis>=5.0"]
|
|
|
28
28
|
|
|
29
29
|
[project.optional-dependencies]
|
|
30
30
|
pydantic = ["pydantic>=2.0"]
|
|
31
|
+
valkey = ["valkey>=6.1.1,<6.3"]
|
|
31
32
|
|
|
32
33
|
[project.urls]
|
|
33
34
|
Homepage = "https://nimzymaina.github.io/rillkit/"
|
|
@@ -36,7 +37,16 @@ Repository = "https://github.com/NimzyMaina/rillkit"
|
|
|
36
37
|
Issues = "https://github.com/NimzyMaina/rillkit/issues"
|
|
37
38
|
|
|
38
39
|
[dependency-groups]
|
|
39
|
-
dev = [
|
|
40
|
+
dev = [
|
|
41
|
+
"pytest>=8",
|
|
42
|
+
"pytest-timeout>=2",
|
|
43
|
+
"pydantic>=2",
|
|
44
|
+
"ruff>=0.6",
|
|
45
|
+
"mypy>=1.10",
|
|
46
|
+
"build",
|
|
47
|
+
"twine",
|
|
48
|
+
"valkey>=6.1.1,<6.3",
|
|
49
|
+
]
|
|
40
50
|
|
|
41
51
|
[tool.hatch.build.targets.wheel]
|
|
42
52
|
packages = ["src/rillkit"]
|
|
@@ -47,6 +57,10 @@ include = ["src", "tests", "README.md", "LICENSE", "pyproject.toml"]
|
|
|
47
57
|
[tool.pytest.ini_options]
|
|
48
58
|
testpaths = ["tests"]
|
|
49
59
|
timeout = 60
|
|
60
|
+
markers = [
|
|
61
|
+
"standalone_only: needs a standalone server (skipped when RILLKIT_TEST_MODE=cluster)",
|
|
62
|
+
"cluster_only: needs a cluster (RILLKIT_TEST_MODE=cluster)",
|
|
63
|
+
]
|
|
50
64
|
|
|
51
65
|
[tool.ruff]
|
|
52
66
|
line-length = 100
|
|
@@ -1,25 +1,28 @@
|
|
|
1
|
-
"""rillkit: simple publish and consume for Redis
|
|
1
|
+
"""rillkit: simple publish and consume for Redis and Valkey streams."""
|
|
2
2
|
|
|
3
3
|
from .ack import AckPolicy
|
|
4
4
|
from .broker import Broker, Publisher
|
|
5
5
|
from .codec import Codec, JsonCodec
|
|
6
6
|
from .exceptions import (
|
|
7
7
|
AckMessage,
|
|
8
|
+
ConfigurationError,
|
|
8
9
|
DecodeError,
|
|
9
10
|
NackMessage,
|
|
10
11
|
PublishError,
|
|
11
12
|
RejectMessage,
|
|
12
13
|
RillkitError,
|
|
13
14
|
)
|
|
15
|
+
from .keys import HashTagKeys, KeyBuilder, PlainKeys
|
|
14
16
|
from .message import Message
|
|
15
17
|
from .middleware import BaseMiddleware, Idempotent, PublishCommand
|
|
16
18
|
from .settings import Settings
|
|
17
19
|
from .stream import Stream
|
|
18
20
|
|
|
19
|
-
__version__ = "0.
|
|
21
|
+
__version__ = "0.2.0"
|
|
20
22
|
|
|
21
23
|
__all__ = [
|
|
22
|
-
"AckMessage", "AckPolicy", "BaseMiddleware", "Broker", "Codec", "
|
|
23
|
-
"
|
|
24
|
-
"
|
|
24
|
+
"AckMessage", "AckPolicy", "BaseMiddleware", "Broker", "Codec", "ConfigurationError",
|
|
25
|
+
"DecodeError", "HashTagKeys", "Idempotent", "JsonCodec", "KeyBuilder", "Message",
|
|
26
|
+
"NackMessage", "PlainKeys", "PublishCommand", "PublishError", "Publisher", "RejectMessage",
|
|
27
|
+
"RillkitError", "Settings", "Stream",
|
|
25
28
|
]
|
|
@@ -6,11 +6,11 @@ from collections.abc import Sequence
|
|
|
6
6
|
from functools import partial
|
|
7
7
|
from typing import Any, Callable, Optional
|
|
8
8
|
|
|
9
|
-
import redis
|
|
10
|
-
|
|
11
9
|
from .ack import AckPolicy
|
|
12
|
-
from .
|
|
10
|
+
from .clients import CLIENT_ERRORS, Client, is_cluster, run_together
|
|
11
|
+
from .consumer import Subscriber, _text
|
|
13
12
|
from .exceptions import PublishError
|
|
13
|
+
from .keys import HashTagKeys, KeyBuilder, PlainKeys, glob_escape
|
|
14
14
|
from .middleware import BaseMiddleware, PublishCommand
|
|
15
15
|
from .settings import Settings
|
|
16
16
|
from .stream import Stream, StreamLike, as_stream
|
|
@@ -40,7 +40,7 @@ class Publisher:
|
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
class Broker:
|
|
43
|
-
"""Owns the
|
|
43
|
+
"""Owns the client, the subscribers and the lifecycle hooks.
|
|
44
44
|
|
|
45
45
|
broker = Broker.from_env()
|
|
46
46
|
|
|
@@ -49,19 +49,32 @@ class Broker:
|
|
|
49
49
|
|
|
50
50
|
broker.publish("orders", {"id": 1})
|
|
51
51
|
broker.run()
|
|
52
|
+
|
|
53
|
+
``client`` can be any redis-py or valkey-py client, standalone or cluster. It is used as is;
|
|
54
|
+
``stop()`` closes it. ``keys`` picks the physical key layout; by default it comes from
|
|
55
|
+
``settings.key_layout``, else plain keys on a standalone client and hash-tagged keys on a
|
|
56
|
+
cluster client.
|
|
52
57
|
"""
|
|
53
58
|
|
|
54
59
|
def __init__(
|
|
55
60
|
self,
|
|
56
61
|
url: Optional[str] = None,
|
|
57
62
|
*,
|
|
58
|
-
client: Optional[
|
|
63
|
+
client: Optional[Client] = None,
|
|
59
64
|
settings: Optional[Settings] = None,
|
|
60
65
|
middlewares: Sequence[BaseMiddleware] = (),
|
|
66
|
+
keys: Optional[KeyBuilder] = None,
|
|
61
67
|
) -> None:
|
|
62
68
|
if client is None:
|
|
63
|
-
|
|
69
|
+
settings = settings or Settings(url=url)
|
|
70
|
+
client = settings.client()
|
|
64
71
|
self.client = client
|
|
72
|
+
if keys is None:
|
|
73
|
+
if settings is not None and settings.key_layout is not None:
|
|
74
|
+
keys = settings.key_builder()
|
|
75
|
+
else:
|
|
76
|
+
keys = HashTagKeys() if is_cluster(client) else PlainKeys()
|
|
77
|
+
self.keys: KeyBuilder = keys
|
|
65
78
|
self.middlewares: list[BaseMiddleware] = []
|
|
66
79
|
for mw in middlewares:
|
|
67
80
|
self.use(mw)
|
|
@@ -149,9 +162,10 @@ class Broker:
|
|
|
149
162
|
try:
|
|
150
163
|
payload = cmd.stream.codec.encode(cmd.body)
|
|
151
164
|
entry_id = self.client.xadd(
|
|
152
|
-
cmd.stream
|
|
165
|
+
self.keys.stream(cmd.stream), {"data": payload}, maxlen=cmd.maxlen,
|
|
166
|
+
approximate=cmd.approximate,
|
|
153
167
|
)
|
|
154
|
-
except (
|
|
168
|
+
except CLIENT_ERRORS + (TypeError, ValueError) as e:
|
|
155
169
|
raise PublishError(f"could not publish to {cmd.stream.name}: {e}") from e
|
|
156
170
|
return _text(entry_id)
|
|
157
171
|
|
|
@@ -249,23 +263,32 @@ class Broker:
|
|
|
249
263
|
|
|
250
264
|
def active_workers(self, stream: StreamLike, group: Optional[str] = None) -> list[str]:
|
|
251
265
|
"""Consumer names with a live heartbeat. Unlike XINFO CONSUMERS this includes idle workers
|
|
252
|
-
that have not received a message yet.
|
|
266
|
+
that have not received a message yet.
|
|
267
|
+
|
|
268
|
+
On a cluster every primary is scanned, and a key seen twice (during a slot migration) is
|
|
269
|
+
counted once. If a node cannot be reached the client error is raised rather than
|
|
270
|
+
returning a partial list."""
|
|
253
271
|
s = as_stream(stream)
|
|
254
|
-
prefix =
|
|
255
|
-
|
|
272
|
+
prefix = self.keys.heartbeat_prefix(s, str(group or s.group))
|
|
273
|
+
found = {_text(k) for k in self.client.scan_iter(match=f"{glob_escape(prefix)}*")}
|
|
274
|
+
return sorted(k[len(prefix):] for k in found)
|
|
256
275
|
|
|
257
276
|
def replay_dlq(self, stream: StreamLike, count: int = 100) -> int:
|
|
258
|
-
"""Move up to ``count`` entries from the DLQ back onto the stream. Returns how many.
|
|
277
|
+
"""Move up to ``count`` entries from the DLQ back onto the stream. Returns how many.
|
|
278
|
+
|
|
279
|
+
Each move is one transaction when the stream and DLQ keys share a slot (always outside a
|
|
280
|
+
cluster). Otherwise the entry is added to the stream first and then deleted from the DLQ,
|
|
281
|
+
so a failure in between can replay it twice but never drops it."""
|
|
259
282
|
s = as_stream(stream)
|
|
283
|
+
stream_key, dlq_key = self.keys.stream(s), self.keys.dlq(s)
|
|
260
284
|
moved = 0
|
|
261
|
-
rows: Any = self.client.xrange(
|
|
285
|
+
rows: Any = self.client.xrange(dlq_key, count=count)
|
|
262
286
|
for entry_id, raw in rows:
|
|
263
287
|
fields: dict[Any, Any] = {
|
|
264
288
|
_text(k): _text(v) for k, v in raw.items() if not _text(k).startswith("x-")
|
|
265
289
|
}
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
pipe.execute()
|
|
290
|
+
run_together(
|
|
291
|
+
self.client, [("xadd", (stream_key, fields)), ("xdel", (dlq_key, entry_id))]
|
|
292
|
+
)
|
|
270
293
|
moved += 1
|
|
271
294
|
return moved
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
"""Building Redis or Valkey clients, and the small differences between the two libraries.
|
|
2
|
+
|
|
3
|
+
rillkit talks to whichever client it is given through the commands both libraries share. This
|
|
4
|
+
module holds the parts that are not shared: the client classes, the options each constructor
|
|
5
|
+
accepts, the exception classes, and how to reach a single node of a cluster.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import importlib
|
|
9
|
+
import inspect
|
|
10
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from typing import TYPE_CHECKING, Any, Optional
|
|
13
|
+
from urllib.parse import parse_qsl, unquote, urlparse
|
|
14
|
+
|
|
15
|
+
import redis
|
|
16
|
+
from redis.crc import key_slot
|
|
17
|
+
|
|
18
|
+
from .exceptions import ConfigurationError
|
|
19
|
+
|
|
20
|
+
if TYPE_CHECKING:
|
|
21
|
+
from .settings import Settings
|
|
22
|
+
|
|
23
|
+
BACKENDS = ("redis", "valkey")
|
|
24
|
+
MODES = ("standalone", "cluster")
|
|
25
|
+
|
|
26
|
+
# Options that pick the server or the credentials. They have their own Settings fields, so they are
|
|
27
|
+
# refused in client_kwargs instead of quietly fighting with those fields.
|
|
28
|
+
RESERVED_OPTIONS = {
|
|
29
|
+
"host": "host", "port": "port", "db": "db", "url": "url", "startup_nodes": "cluster_nodes",
|
|
30
|
+
"username": "username", "password": "password", "ssl": "ssl",
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
# A client instance. Typed loosely because it is one of four classes from two libraries.
|
|
34
|
+
Client = Any
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class Backend:
|
|
39
|
+
"""The classes and option lists of one client library."""
|
|
40
|
+
|
|
41
|
+
name: str
|
|
42
|
+
standalone: type
|
|
43
|
+
cluster: type
|
|
44
|
+
cluster_node: type
|
|
45
|
+
standalone_options: frozenset[str]
|
|
46
|
+
cluster_options: frozenset[str]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _constructor_options(cls: type) -> set[str]:
|
|
50
|
+
params = inspect.signature(cls.__init__).parameters.values() # type: ignore[misc]
|
|
51
|
+
return {p.name for p in params if p.kind not in (p.VAR_KEYWORD, p.VAR_POSITIONAL)} - {"self"}
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _load(name: str) -> Backend:
|
|
55
|
+
try:
|
|
56
|
+
module = importlib.import_module(name)
|
|
57
|
+
cluster_module = importlib.import_module(f"{name}.cluster")
|
|
58
|
+
except ImportError as e:
|
|
59
|
+
raise ConfigurationError(
|
|
60
|
+
f'backend "{name}" needs the {name} package: pip install "rillkit[{name}]"'
|
|
61
|
+
) from e
|
|
62
|
+
standalone = module.Redis if name == "redis" else module.Valkey
|
|
63
|
+
cluster = module.RedisCluster if name == "redis" else module.ValkeyCluster
|
|
64
|
+
# Cluster clients forward only the keys in this list to each node and silently drop the rest,
|
|
65
|
+
# so it is the real list of what works, on top of the cluster constructor's own arguments.
|
|
66
|
+
list_name = "REDIS_ALLOWED_KEYS" if name == "redis" else "VALKEY_ALLOWED_KEYS"
|
|
67
|
+
forwarded: Iterable[str] = getattr(cluster_module, list_name, ())
|
|
68
|
+
disabled = set(getattr(cluster_module, "KWARGS_DISABLED_KEYS", ()))
|
|
69
|
+
cluster_options = (_constructor_options(cluster) | (set(forwarded) - disabled)) - {"db"}
|
|
70
|
+
return Backend(
|
|
71
|
+
name=name,
|
|
72
|
+
standalone=standalone,
|
|
73
|
+
cluster=cluster,
|
|
74
|
+
cluster_node=cluster_module.ClusterNode,
|
|
75
|
+
standalone_options=frozenset(_constructor_options(standalone) - set(RESERVED_OPTIONS)),
|
|
76
|
+
cluster_options=frozenset(cluster_options - set(RESERVED_OPTIONS)),
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
_backends: dict[str, Backend] = {}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def backend(name: str) -> Backend:
|
|
84
|
+
"""The client library for ``name`` ("redis" or "valkey"), imported on first use."""
|
|
85
|
+
if name not in BACKENDS:
|
|
86
|
+
raise ConfigurationError(f"unknown backend {name!r}, expected one of {BACKENDS}")
|
|
87
|
+
if name not in _backends:
|
|
88
|
+
_backends[name] = _load(name)
|
|
89
|
+
return _backends[name]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# exceptions
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _exception_classes(*names: str) -> tuple[type[Exception], ...]:
|
|
96
|
+
"""The named exception classes from redis-py, plus valkey-py's when it is installed."""
|
|
97
|
+
found: list[type[Exception]] = []
|
|
98
|
+
modules = [redis.exceptions]
|
|
99
|
+
try:
|
|
100
|
+
modules.append(importlib.import_module("valkey.exceptions"))
|
|
101
|
+
except ImportError:
|
|
102
|
+
pass
|
|
103
|
+
for module in modules:
|
|
104
|
+
for name in names:
|
|
105
|
+
cls = getattr(module, name, None)
|
|
106
|
+
if cls is not None and cls not in found:
|
|
107
|
+
found.append(cls)
|
|
108
|
+
return tuple(found)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# Any error raised by either client, including cluster errors, which do not derive from the
|
|
112
|
+
# library's base error class.
|
|
113
|
+
CLIENT_ERRORS = _exception_classes(
|
|
114
|
+
"RedisError", "RedisClusterException", "ValkeyError", "ValkeyClusterException"
|
|
115
|
+
)
|
|
116
|
+
# The server answered with an error (BUSYGROUP, NOGROUP, WRONGTYPE, MOVED, ...).
|
|
117
|
+
RESPONSE_ERRORS = _exception_classes("ResponseError")
|
|
118
|
+
# The server could not be reached or did not answer in time.
|
|
119
|
+
CONNECTION_ERRORS = _exception_classes("ConnectionError", "TimeoutError")
|
|
120
|
+
# The cluster topology changed under a command, so it may succeed once the slots are reloaded.
|
|
121
|
+
CLUSTER_REDIRECT_ERRORS = _exception_classes(
|
|
122
|
+
"MovedError", "AskError", "TryAgainError", "ClusterDownError"
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# building clients
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _url_credentials(url: str) -> bool:
|
|
130
|
+
"""Whether the url carries a username or password, in the userinfo or the query string."""
|
|
131
|
+
parsed = urlparse(url)
|
|
132
|
+
query = {k.lower() for k, _ in parse_qsl(parsed.query, True)}
|
|
133
|
+
return bool(parsed.username or parsed.password or query & {"username", "password"})
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _url_db(url: str) -> int:
|
|
137
|
+
path = urlparse(url).path.strip("/")
|
|
138
|
+
if not path:
|
|
139
|
+
return 0
|
|
140
|
+
try:
|
|
141
|
+
return int(unquote(path))
|
|
142
|
+
except ValueError as e:
|
|
143
|
+
raise ConfigurationError("the database in the url path must be a number") from e
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _check_options(options: Mapping[str, Any], allowed: Iterable[str], where: str) -> None:
|
|
147
|
+
allowed = set(allowed)
|
|
148
|
+
for key in options:
|
|
149
|
+
if key in RESERVED_OPTIONS:
|
|
150
|
+
raise ConfigurationError(
|
|
151
|
+
f"client_kwargs[{key!r}] is not allowed: set Settings.{RESERVED_OPTIONS[key]}"
|
|
152
|
+
)
|
|
153
|
+
if key not in allowed:
|
|
154
|
+
raise ConfigurationError(f"client_kwargs[{key!r}] is not an option of {where}")
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def parse_node(node: Any) -> tuple[str, int]:
|
|
158
|
+
"""``"host:port"``, ``("host", port)`` or ``"host"`` (port 6379) as a (host, port) pair."""
|
|
159
|
+
if isinstance(node, str):
|
|
160
|
+
host, sep, port = node.strip().rpartition(":")
|
|
161
|
+
if not sep:
|
|
162
|
+
host, port = port, "6379"
|
|
163
|
+
host = host.strip("[]") # [::1]:7000
|
|
164
|
+
elif isinstance(node, Sequence) and len(node) == 2:
|
|
165
|
+
host, port = node
|
|
166
|
+
else:
|
|
167
|
+
raise ConfigurationError(f"cluster node {node!r} is not 'host:port' or (host, port)")
|
|
168
|
+
try:
|
|
169
|
+
return str(host), int(port)
|
|
170
|
+
except ValueError as e:
|
|
171
|
+
raise ConfigurationError(f"cluster node {node!r} has an invalid port") from e
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def create_client(settings: "Settings") -> Client:
|
|
175
|
+
"""Build the client ``settings`` describe. Raises ``ConfigurationError`` for bad settings."""
|
|
176
|
+
lib = backend(settings.backend)
|
|
177
|
+
options: dict[str, Any] = {"socket_timeout": settings.socket_timeout}
|
|
178
|
+
options.update(settings.client_kwargs)
|
|
179
|
+
url = settings.url
|
|
180
|
+
if url is None or not _url_credentials(url):
|
|
181
|
+
# Credentials embedded in the url win; the separate fields fill in when it has none.
|
|
182
|
+
if settings.username is not None:
|
|
183
|
+
options["username"] = settings.username
|
|
184
|
+
if settings.password is not None:
|
|
185
|
+
options["password"] = settings.password
|
|
186
|
+
if settings.mode == "cluster":
|
|
187
|
+
return _cluster_client(lib, settings, options)
|
|
188
|
+
return _standalone_client(lib, settings, options)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _standalone_client(lib: Backend, settings: "Settings", options: dict[str, Any]) -> Client:
|
|
192
|
+
_check_options(settings.client_kwargs, lib.standalone_options, lib.standalone.__name__)
|
|
193
|
+
if settings.cluster_nodes:
|
|
194
|
+
raise ConfigurationError("cluster_nodes is set but mode is 'standalone'")
|
|
195
|
+
cls: Any = lib.standalone
|
|
196
|
+
if settings.url:
|
|
197
|
+
client = cls.from_url(settings.url, **options)
|
|
198
|
+
else:
|
|
199
|
+
client = cls(
|
|
200
|
+
host=settings.host, port=settings.port, db=settings.db, ssl=settings.ssl, **options
|
|
201
|
+
)
|
|
202
|
+
pool = client.connection_pool
|
|
203
|
+
try:
|
|
204
|
+
# Build (without connecting) one connection now, so an option the connection class does
|
|
205
|
+
# not accept fails here rather than on the first command.
|
|
206
|
+
pool.connection_class(**pool.connection_kwargs)
|
|
207
|
+
except TypeError as e:
|
|
208
|
+
client.close()
|
|
209
|
+
raise ConfigurationError(f"invalid client option: {e}") from e
|
|
210
|
+
return client
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _cluster_client(lib: Backend, settings: "Settings", options: dict[str, Any]) -> Client:
|
|
214
|
+
_check_options(settings.client_kwargs, lib.cluster_options, lib.cluster.__name__)
|
|
215
|
+
if settings.db != 0 or (settings.url and _url_db(settings.url) != 0):
|
|
216
|
+
raise ConfigurationError("cluster mode only has database 0")
|
|
217
|
+
node_cls: Any = lib.cluster_node
|
|
218
|
+
cls: Any = lib.cluster
|
|
219
|
+
nodes = [node_cls(*parse_node(n)) for n in settings.cluster_nodes]
|
|
220
|
+
if settings.url:
|
|
221
|
+
# The url is one seed node plus credentials and TLS; cluster_nodes adds more seeds.
|
|
222
|
+
return cls.from_url(settings.url, startup_nodes=nodes or None, **options)
|
|
223
|
+
if not nodes:
|
|
224
|
+
nodes = [node_cls(settings.host, settings.port)]
|
|
225
|
+
return cls(startup_nodes=nodes, ssl=settings.ssl, **options)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
# working with clients of either library and mode
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def is_cluster(client: Client) -> bool:
|
|
232
|
+
return hasattr(client, "nodes_manager") and hasattr(client, "get_node_from_key")
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _node_client(node: Any) -> Client:
|
|
236
|
+
conn = getattr(node, "redis_connection", None) or getattr(node, "valkey_connection", None)
|
|
237
|
+
if conn is None:
|
|
238
|
+
raise ConfigurationError(f"cluster node {node!r} has no client")
|
|
239
|
+
return conn
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def socket_timeout(client: Client) -> Optional[float]:
|
|
243
|
+
"""The read timeout a connection from this client will use, or None for no timeout."""
|
|
244
|
+
try:
|
|
245
|
+
if is_cluster(client):
|
|
246
|
+
client = _node_client(client.get_primaries()[0])
|
|
247
|
+
pool = client.connection_pool
|
|
248
|
+
conn = pool.connection_class(**pool.connection_kwargs) # built, never connected
|
|
249
|
+
except Exception:
|
|
250
|
+
return None
|
|
251
|
+
return getattr(conn, "socket_timeout", None) or getattr(conn, "_socket_timeout", None)
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def same_slot(*keys: str) -> bool:
|
|
255
|
+
return len({key_slot(k.encode()) for k in keys}) == 1
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
Command = tuple[str, tuple[Any, ...]]
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def run_together(client: Client, commands: Sequence[Command]) -> list[Any]:
|
|
262
|
+
"""Run ``commands`` (method name, args) atomically where the keys allow it, else in order.
|
|
263
|
+
|
|
264
|
+
The first argument of every command is its key. Standalone clients use MULTI/EXEC. On a
|
|
265
|
+
cluster, keys in one hash slot also use MULTI/EXEC, sent to the node that owns the slot.
|
|
266
|
+
Keys in different slots cannot be in one transaction, so the commands run one after another
|
|
267
|
+
and a failure leaves the earlier ones applied: callers order them so that is safe to retry.
|
|
268
|
+
"""
|
|
269
|
+
if not is_cluster(client):
|
|
270
|
+
pipe = client.pipeline(transaction=True)
|
|
271
|
+
for name, args in commands:
|
|
272
|
+
getattr(pipe, name)(*args)
|
|
273
|
+
return list(pipe.execute())
|
|
274
|
+
keys = [args[0] for _, args in commands]
|
|
275
|
+
if not same_slot(*keys):
|
|
276
|
+
return [getattr(client, name)(*args) for name, args in commands]
|
|
277
|
+
return _node_transaction(client, keys[0], commands)
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _node_transaction(client: Client, key: str, commands: Sequence[Command]) -> list[Any]:
|
|
281
|
+
# Not every cluster client supports transactional pipelines (valkey-py 6.1 refuses them), so
|
|
282
|
+
# run MULTI/EXEC on the owning node directly. If the slot moved, the queued commands fail
|
|
283
|
+
# with MOVED, EXEC applies nothing, and one retry after reloading the slot map follows it.
|
|
284
|
+
for attempt in (1, 2):
|
|
285
|
+
pipe = _node_client(client.get_node_from_key(key)).pipeline(transaction=True)
|
|
286
|
+
for name, args in commands:
|
|
287
|
+
getattr(pipe, name)(*args)
|
|
288
|
+
try:
|
|
289
|
+
return list(pipe.execute())
|
|
290
|
+
except CLUSTER_REDIRECT_ERRORS + CONNECTION_ERRORS:
|
|
291
|
+
if attempt == 2:
|
|
292
|
+
raise
|
|
293
|
+
client.nodes_manager.initialize()
|
|
294
|
+
raise AssertionError("unreachable")
|