rillkit 0.1.1__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. {rillkit-0.1.1 → rillkit-0.2.0}/.gitignore +1 -0
  2. {rillkit-0.1.1 → rillkit-0.2.0}/PKG-INFO +17 -4
  3. {rillkit-0.1.1 → rillkit-0.2.0}/README.md +14 -3
  4. {rillkit-0.1.1 → rillkit-0.2.0}/pyproject.toml +16 -2
  5. {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/__init__.py +8 -5
  6. {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/broker.py +40 -17
  7. rillkit-0.2.0/src/rillkit/clients.py +294 -0
  8. {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/consumer.py +39 -37
  9. {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/exceptions.py +4 -0
  10. rillkit-0.2.0/src/rillkit/keys.py +127 -0
  11. {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/message.py +7 -6
  12. rillkit-0.2.0/src/rillkit/settings.py +142 -0
  13. rillkit-0.2.0/tests/conftest.py +99 -0
  14. rillkit-0.2.0/tests/helpers.py +100 -0
  15. {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_blocking_timeout.py +9 -11
  16. rillkit-0.2.0/tests/test_cluster.py +180 -0
  17. {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_consume.py +19 -19
  18. rillkit-0.2.0/tests/test_keys.py +60 -0
  19. {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_lifecycle.py +61 -6
  20. {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_middleware.py +8 -8
  21. {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_publish.py +9 -9
  22. {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_publisher_decorator.py +6 -6
  23. {rillkit-0.1.1 → rillkit-0.2.0}/tests/test_retries.py +38 -38
  24. rillkit-0.2.0/tests/test_settings.py +363 -0
  25. rillkit-0.1.1/src/rillkit/settings.py +0 -57
  26. rillkit-0.1.1/tests/conftest.py +0 -56
  27. rillkit-0.1.1/tests/helpers.py +0 -74
  28. rillkit-0.1.1/tests/test_settings.py +0 -79
  29. {rillkit-0.1.1 → rillkit-0.2.0}/LICENSE +0 -0
  30. {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/ack.py +0 -0
  31. {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/codec.py +0 -0
  32. {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/middleware.py +0 -0
  33. {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/py.typed +0 -0
  34. {rillkit-0.1.1 → rillkit-0.2.0}/src/rillkit/stream.py +0 -0
@@ -9,4 +9,5 @@ dist/
9
9
  node_modules/
10
10
  docs/.vitepress/cache/
11
11
  docs/.vitepress/dist/
12
+ .docs-build/
12
13
  .DS_Store
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: rillkit
3
- Version: 0.1.1
3
+ Version: 0.2.0
4
4
  Summary: Simple publish and consume for Redis Streams: consumer groups, retries, DLQ, ack policies and middleware.
5
5
  Project-URL: Homepage, https://nimzymaina.github.io/rillkit/
6
6
  Project-URL: Documentation, https://nimzymaina.github.io/rillkit/
@@ -25,18 +25,22 @@ Requires-Python: >=3.9
25
25
  Requires-Dist: redis>=5.0
26
26
  Provides-Extra: pydantic
27
27
  Requires-Dist: pydantic>=2.0; extra == 'pydantic'
28
+ Provides-Extra: valkey
29
+ Requires-Dist: valkey<6.3,>=6.1.1; extra == 'valkey'
28
30
  Description-Content-Type: text/markdown
29
31
 
30
32
  # RillKit
31
33
 
32
- Simple publish and consume for Redis Streams. Consumer groups, retries with backoff, a dead-letter
33
- queue, ack policies, middleware and lifecycle hooks, without the boilerplate.
34
+ Simple publish and consume for Redis and Valkey streams, standalone or cluster. Consumer groups,
35
+ retries with backoff, a dead-letter queue, ack policies, middleware and lifecycle hooks, without the
36
+ boilerplate.
34
37
 
35
38
  **Docs:** https://nimzymaina.github.io/rillkit/
36
39
 
37
40
  ```bash
38
41
  pip install rillkit # plain dict bodies
39
42
  pip install "rillkit[pydantic]" # optional typed bodies
43
+ pip install "rillkit[valkey]" # optional valkey-py client
40
44
  ```
41
45
 
42
46
  ```python
@@ -69,13 +73,22 @@ if __name__ == "__main__":
69
73
  - Worker heartbeats: `broker.active_workers(stream)` includes idle workers
70
74
  - Stack `@broker.publisher(...)` under `@broker.subscribe(...)` to publish a handler's return value
71
75
  - The wire format is a single `data` field holding JSON, so plain redis-py producers interoperate
76
+ - Redis or Valkey (`RILLKIT_BACKEND=valkey`), standalone or Cluster (`RILLKIT_MODE=cluster`), with
77
+ the stream and its DLQ in one hash slot so the move to the DLQ stays a transaction
72
78
 
73
79
  ## Development
74
80
 
75
81
  ```bash
76
82
  uv sync --group dev
77
- REDIS_URL=redis://localhost:6379/15 uv run pytest # needs a Redis
83
+ REDIS_URL=redis://localhost:6379/15 uv run pytest # needs a Redis; only db 15 is flushed
84
+ RILLKIT_TEST_BACKEND=valkey REDIS_URL=redis://localhost:6380/15 uv run pytest # valkey-py
85
+
86
+ scripts/test-cluster.sh valkey 7101 # a six-node cluster in Docker
87
+ RILLKIT_TEST_BACKEND=valkey RILLKIT_TEST_MODE=cluster \
88
+ RILLKIT_TEST_CLUSTER_NODES=127.0.0.1:7101 uv run pytest
89
+
78
90
  npm install && npm run docs:dev # docs
91
+ npm run docs:build:versions # latest plus a snapshot per release tag
79
92
  ```
80
93
 
81
94
  MIT licensed.
@@ -1,13 +1,15 @@
1
1
  # RillKit
2
2
 
3
- Simple publish and consume for Redis Streams. Consumer groups, retries with backoff, a dead-letter
4
- queue, ack policies, middleware and lifecycle hooks, without the boilerplate.
3
+ Simple publish and consume for Redis and Valkey streams, standalone or cluster. Consumer groups,
4
+ retries with backoff, a dead-letter queue, ack policies, middleware and lifecycle hooks, without the
5
+ boilerplate.
5
6
 
6
7
  **Docs:** https://nimzymaina.github.io/rillkit/
7
8
 
8
9
  ```bash
9
10
  pip install rillkit # plain dict bodies
10
11
  pip install "rillkit[pydantic]" # optional typed bodies
12
+ pip install "rillkit[valkey]" # optional valkey-py client
11
13
  ```
12
14
 
13
15
  ```python
@@ -40,13 +42,22 @@ if __name__ == "__main__":
40
42
  - Worker heartbeats: `broker.active_workers(stream)` includes idle workers
41
43
  - Stack `@broker.publisher(...)` under `@broker.subscribe(...)` to publish a handler's return value
42
44
  - The wire format is a single `data` field holding JSON, so plain redis-py producers interoperate
45
+ - Redis or Valkey (`RILLKIT_BACKEND=valkey`), standalone or Cluster (`RILLKIT_MODE=cluster`), with
46
+ the stream and its DLQ in one hash slot so the move to the DLQ stays a transaction
43
47
 
44
48
  ## Development
45
49
 
46
50
  ```bash
47
51
  uv sync --group dev
48
- REDIS_URL=redis://localhost:6379/15 uv run pytest # needs a Redis
52
+ REDIS_URL=redis://localhost:6379/15 uv run pytest # needs a Redis; only db 15 is flushed
53
+ RILLKIT_TEST_BACKEND=valkey REDIS_URL=redis://localhost:6380/15 uv run pytest # valkey-py
54
+
55
+ scripts/test-cluster.sh valkey 7101 # a six-node cluster in Docker
56
+ RILLKIT_TEST_BACKEND=valkey RILLKIT_TEST_MODE=cluster \
57
+ RILLKIT_TEST_CLUSTER_NODES=127.0.0.1:7101 uv run pytest
58
+
49
59
  npm install && npm run docs:dev # docs
60
+ npm run docs:build:versions # latest plus a snapshot per release tag
50
61
  ```
51
62
 
52
63
  MIT licensed.
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "rillkit"
7
- version = "0.1.1"
7
+ version = "0.2.0"
8
8
  description = "Simple publish and consume for Redis Streams: consumer groups, retries, DLQ, ack policies and middleware."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -28,6 +28,7 @@ dependencies = ["redis>=5.0"]
28
28
 
29
29
  [project.optional-dependencies]
30
30
  pydantic = ["pydantic>=2.0"]
31
+ valkey = ["valkey>=6.1.1,<6.3"]
31
32
 
32
33
  [project.urls]
33
34
  Homepage = "https://nimzymaina.github.io/rillkit/"
@@ -36,7 +37,16 @@ Repository = "https://github.com/NimzyMaina/rillkit"
36
37
  Issues = "https://github.com/NimzyMaina/rillkit/issues"
37
38
 
38
39
  [dependency-groups]
39
- dev = ["pytest>=8", "pytest-timeout>=2", "pydantic>=2", "ruff>=0.6", "mypy>=1.10", "build", "twine"]
40
+ dev = [
41
+ "pytest>=8",
42
+ "pytest-timeout>=2",
43
+ "pydantic>=2",
44
+ "ruff>=0.6",
45
+ "mypy>=1.10",
46
+ "build",
47
+ "twine",
48
+ "valkey>=6.1.1,<6.3",
49
+ ]
40
50
 
41
51
  [tool.hatch.build.targets.wheel]
42
52
  packages = ["src/rillkit"]
@@ -47,6 +57,10 @@ include = ["src", "tests", "README.md", "LICENSE", "pyproject.toml"]
47
57
  [tool.pytest.ini_options]
48
58
  testpaths = ["tests"]
49
59
  timeout = 60
60
+ markers = [
61
+ "standalone_only: needs a standalone server (skipped when RILLKIT_TEST_MODE=cluster)",
62
+ "cluster_only: needs a cluster (RILLKIT_TEST_MODE=cluster)",
63
+ ]
50
64
 
51
65
  [tool.ruff]
52
66
  line-length = 100
@@ -1,25 +1,28 @@
1
- """rillkit: simple publish and consume for Redis Streams."""
1
+ """rillkit: simple publish and consume for Redis and Valkey streams."""
2
2
 
3
3
  from .ack import AckPolicy
4
4
  from .broker import Broker, Publisher
5
5
  from .codec import Codec, JsonCodec
6
6
  from .exceptions import (
7
7
  AckMessage,
8
+ ConfigurationError,
8
9
  DecodeError,
9
10
  NackMessage,
10
11
  PublishError,
11
12
  RejectMessage,
12
13
  RillkitError,
13
14
  )
15
+ from .keys import HashTagKeys, KeyBuilder, PlainKeys
14
16
  from .message import Message
15
17
  from .middleware import BaseMiddleware, Idempotent, PublishCommand
16
18
  from .settings import Settings
17
19
  from .stream import Stream
18
20
 
19
- __version__ = "0.1.1"
21
+ __version__ = "0.2.0"
20
22
 
21
23
  __all__ = [
22
- "AckMessage", "AckPolicy", "BaseMiddleware", "Broker", "Codec", "DecodeError", "Idempotent",
23
- "JsonCodec", "Message", "NackMessage", "PublishCommand", "PublishError", "Publisher",
24
- "RejectMessage", "RillkitError", "Settings", "Stream",
24
+ "AckMessage", "AckPolicy", "BaseMiddleware", "Broker", "Codec", "ConfigurationError",
25
+ "DecodeError", "HashTagKeys", "Idempotent", "JsonCodec", "KeyBuilder", "Message",
26
+ "NackMessage", "PlainKeys", "PublishCommand", "PublishError", "Publisher", "RejectMessage",
27
+ "RillkitError", "Settings", "Stream",
25
28
  ]
@@ -6,11 +6,11 @@ from collections.abc import Sequence
6
6
  from functools import partial
7
7
  from typing import Any, Callable, Optional
8
8
 
9
- import redis
10
-
11
9
  from .ack import AckPolicy
12
- from .consumer import HEARTBEAT_PREFIX, Subscriber, _text
10
+ from .clients import CLIENT_ERRORS, Client, is_cluster, run_together
11
+ from .consumer import Subscriber, _text
13
12
  from .exceptions import PublishError
13
+ from .keys import HashTagKeys, KeyBuilder, PlainKeys, glob_escape
14
14
  from .middleware import BaseMiddleware, PublishCommand
15
15
  from .settings import Settings
16
16
  from .stream import Stream, StreamLike, as_stream
@@ -40,7 +40,7 @@ class Publisher:
40
40
 
41
41
 
42
42
  class Broker:
43
- """Owns the Redis client, the subscribers and the lifecycle hooks.
43
+ """Owns the client, the subscribers and the lifecycle hooks.
44
44
 
45
45
  broker = Broker.from_env()
46
46
 
@@ -49,19 +49,32 @@ class Broker:
49
49
 
50
50
  broker.publish("orders", {"id": 1})
51
51
  broker.run()
52
+
53
+ ``client`` can be any redis-py or valkey-py client, standalone or cluster. It is used as is;
54
+ ``stop()`` closes it. ``keys`` picks the physical key layout; by default it comes from
55
+ ``settings.key_layout``, else plain keys on a standalone client and hash-tagged keys on a
56
+ cluster client.
52
57
  """
53
58
 
54
59
  def __init__(
55
60
  self,
56
61
  url: Optional[str] = None,
57
62
  *,
58
- client: Optional["redis.Redis"] = None,
63
+ client: Optional[Client] = None,
59
64
  settings: Optional[Settings] = None,
60
65
  middlewares: Sequence[BaseMiddleware] = (),
66
+ keys: Optional[KeyBuilder] = None,
61
67
  ) -> None:
62
68
  if client is None:
63
- client = (settings or Settings(url=url)).client()
69
+ settings = settings or Settings(url=url)
70
+ client = settings.client()
64
71
  self.client = client
72
+ if keys is None:
73
+ if settings is not None and settings.key_layout is not None:
74
+ keys = settings.key_builder()
75
+ else:
76
+ keys = HashTagKeys() if is_cluster(client) else PlainKeys()
77
+ self.keys: KeyBuilder = keys
65
78
  self.middlewares: list[BaseMiddleware] = []
66
79
  for mw in middlewares:
67
80
  self.use(mw)
@@ -149,9 +162,10 @@ class Broker:
149
162
  try:
150
163
  payload = cmd.stream.codec.encode(cmd.body)
151
164
  entry_id = self.client.xadd(
152
- cmd.stream.name, {"data": payload}, maxlen=cmd.maxlen, approximate=cmd.approximate
165
+ self.keys.stream(cmd.stream), {"data": payload}, maxlen=cmd.maxlen,
166
+ approximate=cmd.approximate,
153
167
  )
154
- except (redis.RedisError, TypeError, ValueError) as e:
168
+ except CLIENT_ERRORS + (TypeError, ValueError) as e:
155
169
  raise PublishError(f"could not publish to {cmd.stream.name}: {e}") from e
156
170
  return _text(entry_id)
157
171
 
@@ -249,23 +263,32 @@ class Broker:
249
263
 
250
264
  def active_workers(self, stream: StreamLike, group: Optional[str] = None) -> list[str]:
251
265
  """Consumer names with a live heartbeat. Unlike XINFO CONSUMERS this includes idle workers
252
- that have not received a message yet."""
266
+ that have not received a message yet.
267
+
268
+ On a cluster every primary is scanned, and a key seen twice (during a slot migration) is
269
+ counted once. If a node cannot be reached the client error is raised rather than
270
+ returning a partial list."""
253
271
  s = as_stream(stream)
254
- prefix = f"{HEARTBEAT_PREFIX}{s.name}:{group or s.group}:"
255
- return sorted(_text(k)[len(prefix):] for k in self.client.scan_iter(match=f"{prefix}*"))
272
+ prefix = self.keys.heartbeat_prefix(s, str(group or s.group))
273
+ found = {_text(k) for k in self.client.scan_iter(match=f"{glob_escape(prefix)}*")}
274
+ return sorted(k[len(prefix):] for k in found)
256
275
 
257
276
  def replay_dlq(self, stream: StreamLike, count: int = 100) -> int:
258
- """Move up to ``count`` entries from the DLQ back onto the stream. Returns how many."""
277
+ """Move up to ``count`` entries from the DLQ back onto the stream. Returns how many.
278
+
279
+ Each move is one transaction when the stream and DLQ keys share a slot (always outside a
280
+ cluster). Otherwise the entry is added to the stream first and then deleted from the DLQ,
281
+ so a failure in between can replay it twice but never drops it."""
259
282
  s = as_stream(stream)
283
+ stream_key, dlq_key = self.keys.stream(s), self.keys.dlq(s)
260
284
  moved = 0
261
- rows: Any = self.client.xrange(s.dlq_name, count=count)
285
+ rows: Any = self.client.xrange(dlq_key, count=count)
262
286
  for entry_id, raw in rows:
263
287
  fields: dict[Any, Any] = {
264
288
  _text(k): _text(v) for k, v in raw.items() if not _text(k).startswith("x-")
265
289
  }
266
- pipe = self.client.pipeline()
267
- pipe.xadd(s.name, fields)
268
- pipe.xdel(s.dlq_name, entry_id)
269
- pipe.execute()
290
+ run_together(
291
+ self.client, [("xadd", (stream_key, fields)), ("xdel", (dlq_key, entry_id))]
292
+ )
270
293
  moved += 1
271
294
  return moved
@@ -0,0 +1,294 @@
1
+ """Building Redis or Valkey clients, and the small differences between the two libraries.
2
+
3
+ rillkit talks to whichever client it is given through the commands both libraries share. This
4
+ module holds the parts that are not shared: the client classes, the options each constructor
5
+ accepts, the exception classes, and how to reach a single node of a cluster.
6
+ """
7
+
8
+ import importlib
9
+ import inspect
10
+ from collections.abc import Iterable, Mapping, Sequence
11
+ from dataclasses import dataclass
12
+ from typing import TYPE_CHECKING, Any, Optional
13
+ from urllib.parse import parse_qsl, unquote, urlparse
14
+
15
+ import redis
16
+ from redis.crc import key_slot
17
+
18
+ from .exceptions import ConfigurationError
19
+
20
+ if TYPE_CHECKING:
21
+ from .settings import Settings
22
+
23
+ BACKENDS = ("redis", "valkey")
24
+ MODES = ("standalone", "cluster")
25
+
26
+ # Options that pick the server or the credentials. They have their own Settings fields, so they are
27
+ # refused in client_kwargs instead of quietly fighting with those fields.
28
+ RESERVED_OPTIONS = {
29
+ "host": "host", "port": "port", "db": "db", "url": "url", "startup_nodes": "cluster_nodes",
30
+ "username": "username", "password": "password", "ssl": "ssl",
31
+ }
32
+
33
+ # A client instance. Typed loosely because it is one of four classes from two libraries.
34
+ Client = Any
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class Backend:
39
+ """The classes and option lists of one client library."""
40
+
41
+ name: str
42
+ standalone: type
43
+ cluster: type
44
+ cluster_node: type
45
+ standalone_options: frozenset[str]
46
+ cluster_options: frozenset[str]
47
+
48
+
49
+ def _constructor_options(cls: type) -> set[str]:
50
+ params = inspect.signature(cls.__init__).parameters.values() # type: ignore[misc]
51
+ return {p.name for p in params if p.kind not in (p.VAR_KEYWORD, p.VAR_POSITIONAL)} - {"self"}
52
+
53
+
54
+ def _load(name: str) -> Backend:
55
+ try:
56
+ module = importlib.import_module(name)
57
+ cluster_module = importlib.import_module(f"{name}.cluster")
58
+ except ImportError as e:
59
+ raise ConfigurationError(
60
+ f'backend "{name}" needs the {name} package: pip install "rillkit[{name}]"'
61
+ ) from e
62
+ standalone = module.Redis if name == "redis" else module.Valkey
63
+ cluster = module.RedisCluster if name == "redis" else module.ValkeyCluster
64
+ # Cluster clients forward only the keys in this list to each node and silently drop the rest,
65
+ # so it is the real list of what works, on top of the cluster constructor's own arguments.
66
+ list_name = "REDIS_ALLOWED_KEYS" if name == "redis" else "VALKEY_ALLOWED_KEYS"
67
+ forwarded: Iterable[str] = getattr(cluster_module, list_name, ())
68
+ disabled = set(getattr(cluster_module, "KWARGS_DISABLED_KEYS", ()))
69
+ cluster_options = (_constructor_options(cluster) | (set(forwarded) - disabled)) - {"db"}
70
+ return Backend(
71
+ name=name,
72
+ standalone=standalone,
73
+ cluster=cluster,
74
+ cluster_node=cluster_module.ClusterNode,
75
+ standalone_options=frozenset(_constructor_options(standalone) - set(RESERVED_OPTIONS)),
76
+ cluster_options=frozenset(cluster_options - set(RESERVED_OPTIONS)),
77
+ )
78
+
79
+
80
+ _backends: dict[str, Backend] = {}
81
+
82
+
83
+ def backend(name: str) -> Backend:
84
+ """The client library for ``name`` ("redis" or "valkey"), imported on first use."""
85
+ if name not in BACKENDS:
86
+ raise ConfigurationError(f"unknown backend {name!r}, expected one of {BACKENDS}")
87
+ if name not in _backends:
88
+ _backends[name] = _load(name)
89
+ return _backends[name]
90
+
91
+
92
+ # exceptions
93
+
94
+
95
+ def _exception_classes(*names: str) -> tuple[type[Exception], ...]:
96
+ """The named exception classes from redis-py, plus valkey-py's when it is installed."""
97
+ found: list[type[Exception]] = []
98
+ modules = [redis.exceptions]
99
+ try:
100
+ modules.append(importlib.import_module("valkey.exceptions"))
101
+ except ImportError:
102
+ pass
103
+ for module in modules:
104
+ for name in names:
105
+ cls = getattr(module, name, None)
106
+ if cls is not None and cls not in found:
107
+ found.append(cls)
108
+ return tuple(found)
109
+
110
+
111
+ # Any error raised by either client, including cluster errors, which do not derive from the
112
+ # library's base error class.
113
+ CLIENT_ERRORS = _exception_classes(
114
+ "RedisError", "RedisClusterException", "ValkeyError", "ValkeyClusterException"
115
+ )
116
+ # The server answered with an error (BUSYGROUP, NOGROUP, WRONGTYPE, MOVED, ...).
117
+ RESPONSE_ERRORS = _exception_classes("ResponseError")
118
+ # The server could not be reached or did not answer in time.
119
+ CONNECTION_ERRORS = _exception_classes("ConnectionError", "TimeoutError")
120
+ # The cluster topology changed under a command, so it may succeed once the slots are reloaded.
121
+ CLUSTER_REDIRECT_ERRORS = _exception_classes(
122
+ "MovedError", "AskError", "TryAgainError", "ClusterDownError"
123
+ )
124
+
125
+
126
+ # building clients
127
+
128
+
129
+ def _url_credentials(url: str) -> bool:
130
+ """Whether the url carries a username or password, in the userinfo or the query string."""
131
+ parsed = urlparse(url)
132
+ query = {k.lower() for k, _ in parse_qsl(parsed.query, True)}
133
+ return bool(parsed.username or parsed.password or query & {"username", "password"})
134
+
135
+
136
+ def _url_db(url: str) -> int:
137
+ path = urlparse(url).path.strip("/")
138
+ if not path:
139
+ return 0
140
+ try:
141
+ return int(unquote(path))
142
+ except ValueError as e:
143
+ raise ConfigurationError("the database in the url path must be a number") from e
144
+
145
+
146
+ def _check_options(options: Mapping[str, Any], allowed: Iterable[str], where: str) -> None:
147
+ allowed = set(allowed)
148
+ for key in options:
149
+ if key in RESERVED_OPTIONS:
150
+ raise ConfigurationError(
151
+ f"client_kwargs[{key!r}] is not allowed: set Settings.{RESERVED_OPTIONS[key]}"
152
+ )
153
+ if key not in allowed:
154
+ raise ConfigurationError(f"client_kwargs[{key!r}] is not an option of {where}")
155
+
156
+
157
+ def parse_node(node: Any) -> tuple[str, int]:
158
+ """``"host:port"``, ``("host", port)`` or ``"host"`` (port 6379) as a (host, port) pair."""
159
+ if isinstance(node, str):
160
+ host, sep, port = node.strip().rpartition(":")
161
+ if not sep:
162
+ host, port = port, "6379"
163
+ host = host.strip("[]") # [::1]:7000
164
+ elif isinstance(node, Sequence) and len(node) == 2:
165
+ host, port = node
166
+ else:
167
+ raise ConfigurationError(f"cluster node {node!r} is not 'host:port' or (host, port)")
168
+ try:
169
+ return str(host), int(port)
170
+ except ValueError as e:
171
+ raise ConfigurationError(f"cluster node {node!r} has an invalid port") from e
172
+
173
+
174
+ def create_client(settings: "Settings") -> Client:
175
+ """Build the client ``settings`` describe. Raises ``ConfigurationError`` for bad settings."""
176
+ lib = backend(settings.backend)
177
+ options: dict[str, Any] = {"socket_timeout": settings.socket_timeout}
178
+ options.update(settings.client_kwargs)
179
+ url = settings.url
180
+ if url is None or not _url_credentials(url):
181
+ # Credentials embedded in the url win; the separate fields fill in when it has none.
182
+ if settings.username is not None:
183
+ options["username"] = settings.username
184
+ if settings.password is not None:
185
+ options["password"] = settings.password
186
+ if settings.mode == "cluster":
187
+ return _cluster_client(lib, settings, options)
188
+ return _standalone_client(lib, settings, options)
189
+
190
+
191
+ def _standalone_client(lib: Backend, settings: "Settings", options: dict[str, Any]) -> Client:
192
+ _check_options(settings.client_kwargs, lib.standalone_options, lib.standalone.__name__)
193
+ if settings.cluster_nodes:
194
+ raise ConfigurationError("cluster_nodes is set but mode is 'standalone'")
195
+ cls: Any = lib.standalone
196
+ if settings.url:
197
+ client = cls.from_url(settings.url, **options)
198
+ else:
199
+ client = cls(
200
+ host=settings.host, port=settings.port, db=settings.db, ssl=settings.ssl, **options
201
+ )
202
+ pool = client.connection_pool
203
+ try:
204
+ # Build (without connecting) one connection now, so an option the connection class does
205
+ # not accept fails here rather than on the first command.
206
+ pool.connection_class(**pool.connection_kwargs)
207
+ except TypeError as e:
208
+ client.close()
209
+ raise ConfigurationError(f"invalid client option: {e}") from e
210
+ return client
211
+
212
+
213
+ def _cluster_client(lib: Backend, settings: "Settings", options: dict[str, Any]) -> Client:
214
+ _check_options(settings.client_kwargs, lib.cluster_options, lib.cluster.__name__)
215
+ if settings.db != 0 or (settings.url and _url_db(settings.url) != 0):
216
+ raise ConfigurationError("cluster mode only has database 0")
217
+ node_cls: Any = lib.cluster_node
218
+ cls: Any = lib.cluster
219
+ nodes = [node_cls(*parse_node(n)) for n in settings.cluster_nodes]
220
+ if settings.url:
221
+ # The url is one seed node plus credentials and TLS; cluster_nodes adds more seeds.
222
+ return cls.from_url(settings.url, startup_nodes=nodes or None, **options)
223
+ if not nodes:
224
+ nodes = [node_cls(settings.host, settings.port)]
225
+ return cls(startup_nodes=nodes, ssl=settings.ssl, **options)
226
+
227
+
228
+ # working with clients of either library and mode
229
+
230
+
231
+ def is_cluster(client: Client) -> bool:
232
+ return hasattr(client, "nodes_manager") and hasattr(client, "get_node_from_key")
233
+
234
+
235
+ def _node_client(node: Any) -> Client:
236
+ conn = getattr(node, "redis_connection", None) or getattr(node, "valkey_connection", None)
237
+ if conn is None:
238
+ raise ConfigurationError(f"cluster node {node!r} has no client")
239
+ return conn
240
+
241
+
242
+ def socket_timeout(client: Client) -> Optional[float]:
243
+ """The read timeout a connection from this client will use, or None for no timeout."""
244
+ try:
245
+ if is_cluster(client):
246
+ client = _node_client(client.get_primaries()[0])
247
+ pool = client.connection_pool
248
+ conn = pool.connection_class(**pool.connection_kwargs) # built, never connected
249
+ except Exception:
250
+ return None
251
+ return getattr(conn, "socket_timeout", None) or getattr(conn, "_socket_timeout", None)
252
+
253
+
254
+ def same_slot(*keys: str) -> bool:
255
+ return len({key_slot(k.encode()) for k in keys}) == 1
256
+
257
+
258
+ Command = tuple[str, tuple[Any, ...]]
259
+
260
+
261
+ def run_together(client: Client, commands: Sequence[Command]) -> list[Any]:
262
+ """Run ``commands`` (method name, args) atomically where the keys allow it, else in order.
263
+
264
+ The first argument of every command is its key. Standalone clients use MULTI/EXEC. On a
265
+ cluster, keys in one hash slot also use MULTI/EXEC, sent to the node that owns the slot.
266
+ Keys in different slots cannot be in one transaction, so the commands run one after another
267
+ and a failure leaves the earlier ones applied: callers order them so that is safe to retry.
268
+ """
269
+ if not is_cluster(client):
270
+ pipe = client.pipeline(transaction=True)
271
+ for name, args in commands:
272
+ getattr(pipe, name)(*args)
273
+ return list(pipe.execute())
274
+ keys = [args[0] for _, args in commands]
275
+ if not same_slot(*keys):
276
+ return [getattr(client, name)(*args) for name, args in commands]
277
+ return _node_transaction(client, keys[0], commands)
278
+
279
+
280
+ def _node_transaction(client: Client, key: str, commands: Sequence[Command]) -> list[Any]:
281
+ # Not every cluster client supports transactional pipelines (valkey-py 6.1 refuses them), so
282
+ # run MULTI/EXEC on the owning node directly. If the slot moved, the queued commands fail
283
+ # with MOVED, EXEC applies nothing, and one retry after reloading the slot map follows it.
284
+ for attempt in (1, 2):
285
+ pipe = _node_client(client.get_node_from_key(key)).pipeline(transaction=True)
286
+ for name, args in commands:
287
+ getattr(pipe, name)(*args)
288
+ try:
289
+ return list(pipe.execute())
290
+ except CLUSTER_REDIRECT_ERRORS + CONNECTION_ERRORS:
291
+ if attempt == 2:
292
+ raise
293
+ client.nodes_manager.initialize()
294
+ raise AssertionError("unreachable")