coredis 5.0.0rc1__py3-none-any.whl → 5.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of coredis might be problematic. Click here for more details.
- coredis/_protocols.py +0 -23
- coredis/_utils.py +8 -4
- coredis/_version.py +3 -3
- coredis/commands/core.py +50 -8
- coredis/commands/script.py +19 -20
- coredis/commands/sentinel.py +9 -7
- coredis/pipeline.py +124 -137
- coredis/response/_callbacks/sentinel.py +28 -25
- coredis/response/_callbacks/vector_sets.py +44 -11
- coredis/retry.py +12 -13
- coredis/sentinel.py +18 -5
- coredis/tokens.py +5 -0
- coredis/typing.py +1 -3
- {coredis-5.0.0rc1.dist-info → coredis-5.0.1.dist-info}/METADATA +1 -5
- {coredis-5.0.0rc1.dist-info → coredis-5.0.1.dist-info}/RECORD +18 -18
- {coredis-5.0.0rc1.dist-info → coredis-5.0.1.dist-info}/WHEEL +0 -0
- {coredis-5.0.0rc1.dist-info → coredis-5.0.1.dist-info}/licenses/LICENSE +0 -0
- {coredis-5.0.0rc1.dist-info → coredis-5.0.1.dist-info}/top_level.txt +0 -0
coredis/pipeline.py
CHANGED
|
@@ -18,6 +18,7 @@ from coredis.client import Client, RedisCluster
|
|
|
18
18
|
from coredis.commands import CommandRequest, CommandResponseT
|
|
19
19
|
from coredis.commands._key_spec import KeySpec
|
|
20
20
|
from coredis.commands.constants import CommandName, NodeFlag
|
|
21
|
+
from coredis.commands.request import TransformedResponse
|
|
21
22
|
from coredis.commands.script import Script
|
|
22
23
|
from coredis.connection import BaseConnection, ClusterConnection, CommandInvocation, Connection
|
|
23
24
|
from coredis.exceptions import (
|
|
@@ -88,16 +89,14 @@ def wrap_pipeline_method(
|
|
|
88
89
|
def wrapper(*args: P.args, **kwargs: P.kwargs) -> Awaitable[R]:
|
|
89
90
|
return func(*args, **kwargs)
|
|
90
91
|
|
|
91
|
-
wrapper.__annotations__ = wrapper.__annotations__.copy()
|
|
92
|
-
wrapper.__annotations__["return"] = kls
|
|
93
92
|
wrapper.__doc__ = textwrap.dedent(wrapper.__doc__ or "")
|
|
94
93
|
wrapper.__doc__ = f"""
|
|
95
|
-
Pipeline variant of :meth:`coredis.Redis.{func.__name__}` that does not execute
|
|
96
|
-
immediately and instead pushes the command into a stack for batch send.
|
|
94
|
+
.. note:: Pipeline variant of :meth:`coredis.Redis.{func.__name__}` that does not execute
|
|
95
|
+
immediately and instead pushes the command into a stack for batch send.
|
|
97
96
|
|
|
98
|
-
The return value can be retrieved either as part of the tuple returned by
|
|
99
|
-
:meth:`~{kls.__name__}.execute` or by awaiting the :class:`~coredis.commands.CommandRequest`
|
|
100
|
-
instance after calling :meth:`~{kls.__name__}.execute`
|
|
97
|
+
The return value can be retrieved either as part of the tuple returned by
|
|
98
|
+
:meth:`~{kls.__name__}.execute` or by awaiting the :class:`~coredis.commands.CommandRequest`
|
|
99
|
+
instance after calling :meth:`~{kls.__name__}.execute`
|
|
101
100
|
|
|
102
101
|
{wrapper.__doc__}
|
|
103
102
|
"""
|
|
@@ -105,6 +104,11 @@ instance after calling :meth:`~{kls.__name__}.execute`
|
|
|
105
104
|
|
|
106
105
|
|
|
107
106
|
class PipelineCommandRequest(CommandRequest[CommandResponseT]):
|
|
107
|
+
"""
|
|
108
|
+
Command request used within a pipeline. Handles immediate execution for WATCH or
|
|
109
|
+
watched commands outside explicit transactions, otherwise queues the command.
|
|
110
|
+
"""
|
|
111
|
+
|
|
108
112
|
client: Pipeline[Any] | ClusterPipeline[Any]
|
|
109
113
|
queued_response: Awaitable[bytes | str]
|
|
110
114
|
|
|
@@ -115,23 +119,58 @@ class PipelineCommandRequest(CommandRequest[CommandResponseT]):
|
|
|
115
119
|
*arguments: ValueT,
|
|
116
120
|
callback: Callable[..., CommandResponseT],
|
|
117
121
|
execution_parameters: ExecutionParameters | None = None,
|
|
122
|
+
parent: CommandRequest[Any] | None = None,
|
|
118
123
|
) -> None:
|
|
119
124
|
super().__init__(
|
|
120
|
-
client,
|
|
125
|
+
client,
|
|
126
|
+
name,
|
|
127
|
+
*arguments,
|
|
128
|
+
callback=callback,
|
|
129
|
+
execution_parameters=execution_parameters,
|
|
130
|
+
)
|
|
131
|
+
if not parent:
|
|
132
|
+
if (client.watching or name == CommandName.WATCH) and not client.explicit_transaction:
|
|
133
|
+
self.response = client.immediate_execute_command(
|
|
134
|
+
self, callback=callback, **self.execution_parameters
|
|
135
|
+
)
|
|
136
|
+
else:
|
|
137
|
+
client.pipeline_execute_command(self) # type: ignore[arg-type]
|
|
138
|
+
self.parent = parent
|
|
139
|
+
|
|
140
|
+
def transform(
|
|
141
|
+
self, transformer: type[TransformedResponse]
|
|
142
|
+
) -> CommandRequest[TransformedResponse]:
|
|
143
|
+
transform_func = functools.partial(
|
|
144
|
+
self.type_adapter.deserialize,
|
|
145
|
+
return_type=transformer,
|
|
146
|
+
)
|
|
147
|
+
return cast(type[PipelineCommandRequest[TransformedResponse]], self.__class__)(
|
|
148
|
+
self.client,
|
|
149
|
+
self.name,
|
|
150
|
+
*self.arguments,
|
|
151
|
+
callback=lambda resp, **k: transform_func(resp),
|
|
152
|
+
execution_parameters=self.execution_parameters,
|
|
153
|
+
parent=self,
|
|
121
154
|
)
|
|
122
|
-
if (client.watching or name == CommandName.WATCH) and not client.explicit_transaction:
|
|
123
|
-
self.response = client.immediate_execute_command(
|
|
124
|
-
self, callback=callback, **self.execution_parameters
|
|
125
|
-
)
|
|
126
|
-
else:
|
|
127
|
-
client.pipeline_execute_command(self) # type: ignore[arg-type]
|
|
128
155
|
|
|
129
156
|
async def __backward_compatibility_return(self) -> Pipeline[Any] | ClusterPipeline[Any]:
|
|
157
|
+
"""
|
|
158
|
+
For backward compatibility: returns the pipeline instance when awaited before execute().
|
|
159
|
+
"""
|
|
130
160
|
return self.client
|
|
131
161
|
|
|
132
162
|
def __await__(self) -> Generator[None, None, CommandResponseT]:
|
|
133
163
|
if hasattr(self, "response"):
|
|
134
164
|
return self.response.__await__()
|
|
165
|
+
elif self.parent:
|
|
166
|
+
|
|
167
|
+
async def _transformed() -> CommandResponseT:
|
|
168
|
+
if (r := await self.parent) == self.client: # type: ignore
|
|
169
|
+
return r # type: ignore
|
|
170
|
+
else:
|
|
171
|
+
return self.callback(r)
|
|
172
|
+
|
|
173
|
+
return _transformed().__await__()
|
|
135
174
|
else:
|
|
136
175
|
warnings.warn(
|
|
137
176
|
"""
|
|
@@ -140,12 +179,17 @@ has no effect and returns the pipeline instance itself for backward compatibilit
|
|
|
140
179
|
|
|
141
180
|
To add commands to a pipeline simply call the methods synchronously. The awaitable response
|
|
142
181
|
can be awaited after calling `execute()` to retrieve a statically typed response if required.
|
|
143
|
-
"""
|
|
182
|
+
""",
|
|
183
|
+
stacklevel=2,
|
|
144
184
|
)
|
|
145
185
|
return self.__backward_compatibility_return().__await__() # type: ignore[return-value]
|
|
146
186
|
|
|
147
187
|
|
|
148
188
|
class ClusterPipelineCommandRequest(PipelineCommandRequest[CommandResponseT]):
|
|
189
|
+
"""
|
|
190
|
+
Command request for cluster pipelines, tracks position and result for cluster routing.
|
|
191
|
+
"""
|
|
192
|
+
|
|
149
193
|
def __init__(
|
|
150
194
|
self,
|
|
151
195
|
client: ClusterPipeline[Any],
|
|
@@ -153,16 +197,26 @@ class ClusterPipelineCommandRequest(PipelineCommandRequest[CommandResponseT]):
|
|
|
153
197
|
*arguments: ValueT,
|
|
154
198
|
callback: Callable[..., CommandResponseT],
|
|
155
199
|
execution_parameters: ExecutionParameters | None = None,
|
|
200
|
+
parent: CommandRequest[Any] | None = None,
|
|
156
201
|
) -> None:
|
|
157
202
|
self.position: int = 0
|
|
158
203
|
self.result: Any | None = None
|
|
159
204
|
self.asking: bool = False
|
|
160
205
|
super().__init__(
|
|
161
|
-
client,
|
|
206
|
+
client,
|
|
207
|
+
name,
|
|
208
|
+
*arguments,
|
|
209
|
+
callback=callback,
|
|
210
|
+
execution_parameters=execution_parameters,
|
|
211
|
+
parent=parent,
|
|
162
212
|
)
|
|
163
213
|
|
|
164
214
|
|
|
165
215
|
class NodeCommands:
|
|
216
|
+
"""
|
|
217
|
+
Helper for grouping and executing commands on a single cluster node, handling transactions if needed.
|
|
218
|
+
"""
|
|
219
|
+
|
|
166
220
|
def __init__(
|
|
167
221
|
self,
|
|
168
222
|
client: RedisCluster[AnyStr],
|
|
@@ -188,14 +242,11 @@ class NodeCommands:
|
|
|
188
242
|
connection = self.connection
|
|
189
243
|
commands = self.commands
|
|
190
244
|
|
|
191
|
-
#
|
|
192
|
-
# and ensure that nothing is sitting there from a previous run.
|
|
193
|
-
|
|
245
|
+
# Reset results for all commands before writing.
|
|
194
246
|
for c in commands:
|
|
195
247
|
c.result = None
|
|
196
248
|
|
|
197
|
-
#
|
|
198
|
-
# send all the commands and catch connection and timeout errors.
|
|
249
|
+
# Batch all commands into a single request for efficiency.
|
|
199
250
|
try:
|
|
200
251
|
if self.in_transaction:
|
|
201
252
|
self.multi_cmd = await connection.create_request(
|
|
@@ -307,20 +358,16 @@ class ClusterPipelineMeta(PipelineMeta):
|
|
|
307
358
|
|
|
308
359
|
|
|
309
360
|
class Pipeline(Client[AnyStr], metaclass=PipelineMeta):
|
|
310
|
-
"""Pipeline for the Redis class"""
|
|
311
|
-
|
|
312
361
|
"""
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
saving all the values in a list to Redis.
|
|
362
|
+
Pipeline for batching multiple commands to a Redis server.
|
|
363
|
+
Supports transactions and command stacking.
|
|
316
364
|
|
|
317
365
|
All commands executed within a pipeline are wrapped with MULTI and EXEC
|
|
318
|
-
calls
|
|
319
|
-
executed atomically.
|
|
366
|
+
calls when :paramref:`transaction` is ``True``.
|
|
320
367
|
|
|
321
368
|
Any command raising an exception does *not* halt the execution of
|
|
322
369
|
subsequent commands in the pipeline. Instead, the exception is caught
|
|
323
|
-
and its instance is placed into the response list returned by
|
|
370
|
+
and its instance is placed into the response list returned by :meth:`execute`
|
|
324
371
|
"""
|
|
325
372
|
|
|
326
373
|
command_stack: list[PipelineCommandRequest[Any]]
|
|
@@ -340,7 +387,7 @@ class Pipeline(Client[AnyStr], metaclass=PipelineMeta):
|
|
|
340
387
|
self.watching = False
|
|
341
388
|
self.watches: Parameters[KeyT] | None = watches or None
|
|
342
389
|
self.command_stack = []
|
|
343
|
-
self.cache = None
|
|
390
|
+
self.cache = None
|
|
344
391
|
self.explicit_transaction = False
|
|
345
392
|
self.scripts: set[Script[AnyStr]] = set()
|
|
346
393
|
self.timeout = timeout
|
|
@@ -385,30 +432,21 @@ class Pipeline(Client[AnyStr], metaclass=PipelineMeta):
|
|
|
385
432
|
|
|
386
433
|
async def clear(self) -> None:
|
|
387
434
|
"""
|
|
388
|
-
|
|
389
|
-
back to the pool
|
|
435
|
+
Clear the pipeline, reset state, and release the connection back to the pool.
|
|
390
436
|
"""
|
|
391
437
|
self.command_stack.clear()
|
|
392
438
|
self.scripts = set()
|
|
393
|
-
#
|
|
394
|
-
# watching something
|
|
395
|
-
|
|
439
|
+
# Reset connection state if we were watching something.
|
|
396
440
|
if self.watching and self.connection:
|
|
397
441
|
try:
|
|
398
|
-
# call this manually since our unwatch or
|
|
399
|
-
# immediate_execute_command methods can call clear()
|
|
400
442
|
request = await self.connection.create_request(CommandName.UNWATCH, decode=False)
|
|
401
443
|
await request
|
|
402
444
|
except ConnectionError:
|
|
403
|
-
# disconnect will also remove any previous WATCHes
|
|
404
445
|
self.connection.disconnect()
|
|
405
|
-
#
|
|
446
|
+
# Reset pipeline state and release connection if needed.
|
|
406
447
|
self.watching = False
|
|
407
448
|
self.watches = []
|
|
408
449
|
self.explicit_transaction = False
|
|
409
|
-
# we can safely return the connection to the pool here since we're
|
|
410
|
-
# sure we're no longer WATCHing anything
|
|
411
|
-
|
|
412
450
|
if self.connection:
|
|
413
451
|
self.connection_pool.release(self.connection)
|
|
414
452
|
self.connection = None
|
|
@@ -422,19 +460,14 @@ class Pipeline(Client[AnyStr], metaclass=PipelineMeta):
|
|
|
422
460
|
)
|
|
423
461
|
def reset(self) -> CommandRequest[None]:
|
|
424
462
|
"""
|
|
425
|
-
|
|
426
|
-
back to the pool
|
|
427
|
-
|
|
428
|
-
:meta private:
|
|
463
|
+
Deprecated. Use :meth:`clear` instead.
|
|
429
464
|
"""
|
|
430
465
|
return self.clear() # type: ignore
|
|
431
466
|
|
|
432
467
|
def multi(self) -> None:
|
|
433
468
|
"""
|
|
434
|
-
|
|
435
|
-
are issued. End the transactional block with `execute`.
|
|
469
|
+
Start a transactional block after WATCH commands. End with `execute()`.
|
|
436
470
|
"""
|
|
437
|
-
|
|
438
471
|
if self.explicit_transaction:
|
|
439
472
|
raise RedisError("Cannot issue nested calls to MULTI")
|
|
440
473
|
|
|
@@ -504,15 +537,7 @@ class Pipeline(Client[AnyStr], metaclass=PipelineMeta):
|
|
|
504
537
|
command: PipelineCommandRequest[R],
|
|
505
538
|
) -> None:
|
|
506
539
|
"""
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
Returns the current Pipeline object back so commands can be
|
|
510
|
-
chained together, such as:
|
|
511
|
-
|
|
512
|
-
pipe = pipe.set('foo', 'bar').incr('baz').decr('bang')
|
|
513
|
-
|
|
514
|
-
At some other point, you can then run: pipe.execute(),
|
|
515
|
-
which will execute all commands queued in the pipe.
|
|
540
|
+
Queue a command for execution on the next `execute()` call.
|
|
516
541
|
|
|
517
542
|
:meta private:
|
|
518
543
|
"""
|
|
@@ -704,7 +729,9 @@ class Pipeline(Client[AnyStr], metaclass=PipelineMeta):
|
|
|
704
729
|
)
|
|
705
730
|
|
|
706
731
|
async def execute(self, raise_on_error: bool = True) -> tuple[Any, ...]:
|
|
707
|
-
"""
|
|
732
|
+
"""
|
|
733
|
+
Execute all queued commands in the pipeline. Returns a tuple of results.
|
|
734
|
+
"""
|
|
708
735
|
stack = self.command_stack
|
|
709
736
|
|
|
710
737
|
if not stack:
|
|
@@ -748,9 +775,8 @@ class Pipeline(Client[AnyStr], metaclass=PipelineMeta):
|
|
|
748
775
|
|
|
749
776
|
def watch(self, *keys: KeyT) -> CommandRequest[bool]:
|
|
750
777
|
"""
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
pipeline buffering mode, call :meth:`multi`.
|
|
778
|
+
Watch the given keys for changes. Switches to immediate execution mode
|
|
779
|
+
until :meth:`multi` is called.
|
|
754
780
|
"""
|
|
755
781
|
if self.explicit_transaction:
|
|
756
782
|
raise RedisError("Cannot issue a WATCH after a MULTI")
|
|
@@ -759,13 +785,21 @@ class Pipeline(Client[AnyStr], metaclass=PipelineMeta):
|
|
|
759
785
|
|
|
760
786
|
def unwatch(self) -> CommandRequest[bool]:
|
|
761
787
|
"""
|
|
762
|
-
|
|
763
|
-
to buffered mode.
|
|
788
|
+
Remove all key watches and return to buffered mode.
|
|
764
789
|
"""
|
|
765
790
|
return self.create_request(CommandName.UNWATCH, callback=SimpleStringCallback())
|
|
766
791
|
|
|
767
792
|
|
|
768
793
|
class ClusterPipeline(Client[AnyStr], metaclass=ClusterPipelineMeta):
|
|
794
|
+
"""
|
|
795
|
+
Pipeline for batching commands to a Redis Cluster.
|
|
796
|
+
Handles routing, transactions, and error management across nodes.
|
|
797
|
+
|
|
798
|
+
.. warning:: Unlike :class:`Pipeline`, :paramref:`transaction` is ``False`` by
|
|
799
|
+
default as there is limited support for transactions in redis cluster
|
|
800
|
+
(only keys in the same slot can be part of a transaction).
|
|
801
|
+
"""
|
|
802
|
+
|
|
769
803
|
client: RedisCluster[AnyStr]
|
|
770
804
|
connection_pool: ClusterConnectionPool
|
|
771
805
|
command_stack: list[ClusterPipelineCommandRequest[Any]]
|
|
@@ -791,7 +825,7 @@ class ClusterPipeline(Client[AnyStr], metaclass=ClusterPipelineMeta):
|
|
|
791
825
|
self.watches: Parameters[KeyT] | None = watches or None
|
|
792
826
|
self.watching = False
|
|
793
827
|
self.explicit_transaction = False
|
|
794
|
-
self.cache = None
|
|
828
|
+
self.cache = None
|
|
795
829
|
self.timeout = timeout
|
|
796
830
|
self.type_adapter = client.type_adapter
|
|
797
831
|
|
|
@@ -811,9 +845,8 @@ class ClusterPipeline(Client[AnyStr], metaclass=ClusterPipelineMeta):
|
|
|
811
845
|
|
|
812
846
|
def watch(self, *keys: KeyT) -> CommandRequest[bool]:
|
|
813
847
|
"""
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
pipeline buffering mode, call :meth:`multi`.
|
|
848
|
+
Watch the given keys for changes. Switches to immediate execution mode
|
|
849
|
+
until :meth:`multi` is called.
|
|
817
850
|
"""
|
|
818
851
|
if self.explicit_transaction:
|
|
819
852
|
raise RedisError("Cannot issue a WATCH after a MULTI")
|
|
@@ -822,8 +855,7 @@ class ClusterPipeline(Client[AnyStr], metaclass=ClusterPipelineMeta):
|
|
|
822
855
|
|
|
823
856
|
async def unwatch(self) -> bool:
|
|
824
857
|
"""
|
|
825
|
-
|
|
826
|
-
to buffered mode.
|
|
858
|
+
Remove all key watches and return to buffered mode.
|
|
827
859
|
"""
|
|
828
860
|
if self._watched_connection:
|
|
829
861
|
try:
|
|
@@ -898,7 +930,9 @@ class ClusterPipeline(Client[AnyStr], metaclass=ClusterPipelineMeta):
|
|
|
898
930
|
exception.args = (msg,) + exception.args[1:]
|
|
899
931
|
|
|
900
932
|
async def execute(self, raise_on_error: bool = True) -> tuple[object, ...]:
|
|
901
|
-
"""
|
|
933
|
+
"""
|
|
934
|
+
Execute all queued commands in the cluster pipeline. Returns a tuple of results.
|
|
935
|
+
"""
|
|
902
936
|
await self.connection_pool.initialize()
|
|
903
937
|
|
|
904
938
|
if not self.command_stack:
|
|
@@ -915,8 +949,7 @@ class ClusterPipeline(Client[AnyStr], metaclass=ClusterPipelineMeta):
|
|
|
915
949
|
|
|
916
950
|
async def clear(self) -> None:
|
|
917
951
|
"""
|
|
918
|
-
|
|
919
|
-
back to the pool
|
|
952
|
+
Clear the pipeline, reset state, and release any held connections.
|
|
920
953
|
"""
|
|
921
954
|
self.command_stack = []
|
|
922
955
|
|
|
@@ -981,9 +1014,7 @@ class ClusterPipeline(Client[AnyStr], metaclass=ClusterPipelineMeta):
|
|
|
981
1014
|
if self.explicit_transaction:
|
|
982
1015
|
request = await conn.create_request(CommandName.DISCARD)
|
|
983
1016
|
await request
|
|
984
|
-
# If at least one watched key is modified before the EXEC
|
|
985
|
-
# the whole transaction aborts,
|
|
986
|
-
# and EXEC returns a Null reply to notify that the transaction failed.
|
|
1017
|
+
# If at least one watched key is modified before EXEC, the transaction aborts and EXEC returns null.
|
|
987
1018
|
|
|
988
1019
|
if node_commands.exec_cmd and await node_commands.exec_cmd is None:
|
|
989
1020
|
raise WatchError
|
|
@@ -1006,104 +1037,57 @@ class ClusterPipeline(Client[AnyStr], metaclass=ClusterPipelineMeta):
|
|
|
1006
1037
|
self, raise_on_error: bool = True, allow_redirections: bool = True
|
|
1007
1038
|
) -> tuple[object, ...]:
|
|
1008
1039
|
"""
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
`allow_redirections` If the pipeline should follow `ASK` & `MOVED` responses
|
|
1012
|
-
automatically. If set to false it will raise RedisClusterException.
|
|
1040
|
+
Execute all queued commands in the cluster pipeline, handling redirections
|
|
1041
|
+
and retries as needed.
|
|
1013
1042
|
|
|
1014
1043
|
:meta private:
|
|
1015
1044
|
"""
|
|
1016
|
-
#
|
|
1017
|
-
# if we have to run through it again, we only retry the commands that failed.
|
|
1045
|
+
# On first send, queue all commands. On retry, only failed ones.
|
|
1018
1046
|
attempt = sorted(self.command_stack, key=lambda x: x.position)
|
|
1019
1047
|
|
|
1020
|
-
|
|
1021
|
-
# build a list of node objects based on node names we need to
|
|
1048
|
+
# Group commands by node for efficient network usage.
|
|
1022
1049
|
nodes: dict[str, NodeCommands] = {}
|
|
1023
|
-
# as we move through each command that still needs to be processed,
|
|
1024
|
-
# we figure out the slot number that command maps to, then from the slot determine the node.
|
|
1025
1050
|
for c in attempt:
|
|
1026
|
-
# refer to our internal node -> slot table that tells us where a given
|
|
1027
|
-
# command should route to.
|
|
1028
1051
|
slot = self._determine_slot(c.name, *c.arguments)
|
|
1029
1052
|
node = self.connection_pool.get_node_by_slot(slot)
|
|
1030
|
-
|
|
1031
1053
|
if node.name not in nodes:
|
|
1032
1054
|
nodes[node.name] = NodeCommands(
|
|
1033
1055
|
self.client,
|
|
1034
1056
|
await self.connection_pool.get_connection_by_node(node),
|
|
1035
1057
|
timeout=self.timeout,
|
|
1036
1058
|
)
|
|
1037
|
-
|
|
1038
1059
|
nodes[node.name].append(c)
|
|
1039
1060
|
|
|
1040
|
-
#
|
|
1041
|
-
# we write to all the open sockets for each node first, before reading anything
|
|
1042
|
-
# this allows us to flush all the requests out across the network essentially in parallel
|
|
1043
|
-
# so that we can read them all in parallel as they come back.
|
|
1044
|
-
# we dont' multiplex on the sockets as they come available, but that shouldn't make
|
|
1045
|
-
# too much difference.
|
|
1061
|
+
# Write to all nodes, then read from all nodes in sequence.
|
|
1046
1062
|
node_commands = nodes.values()
|
|
1047
|
-
|
|
1048
1063
|
for n in node_commands:
|
|
1049
1064
|
await n.write()
|
|
1050
|
-
|
|
1051
1065
|
for n in node_commands:
|
|
1052
1066
|
await n.read()
|
|
1053
1067
|
|
|
1054
|
-
#
|
|
1055
|
-
#
|
|
1056
|
-
# release connections back into the pool if for some reason the socket has data still left
|
|
1057
|
-
# in it from a previous operation. The write and read operations already have try/catch
|
|
1058
|
-
# around them for all known types of errors including connection and socket level errors.
|
|
1059
|
-
# So if we hit an exception, something really bad happened and putting any of
|
|
1060
|
-
# these connections back into the pool is a very bad idea.
|
|
1061
|
-
# the socket might have unread buffer still sitting in it, and then the
|
|
1062
|
-
# next time we read from it we pass the buffered result back from a previous
|
|
1063
|
-
# command and every single request after to that connection will always get
|
|
1064
|
-
# a mismatched result. (not just theoretical, I saw this happen on production x.x).
|
|
1068
|
+
# Release all connections back to the pool only if safe (no unread buffer).
|
|
1069
|
+
# If an error occurred, do not release to avoid buffer mismatches.
|
|
1065
1070
|
for n in nodes.values():
|
|
1066
1071
|
protocol_version = n.connection.protocol_version
|
|
1067
1072
|
self.connection_pool.release(n.connection)
|
|
1068
|
-
|
|
1069
|
-
#
|
|
1070
|
-
# if we have more commands to attempt, we've run into problems.
|
|
1071
|
-
# collect all the commands we are allowed to retry.
|
|
1072
|
-
# (MOVED, ASK, or connection errors or timeout errors)
|
|
1073
|
+
|
|
1074
|
+
# Retry MOVED/ASK/connection errors one by one if allowed.
|
|
1073
1075
|
attempt = sorted(
|
|
1074
1076
|
(c for c in attempt if isinstance(c.result, ERRORS_ALLOW_RETRY)),
|
|
1075
1077
|
key=lambda x: x.position,
|
|
1076
1078
|
)
|
|
1077
1079
|
|
|
1078
1080
|
if attempt and allow_redirections:
|
|
1079
|
-
# RETRY MAGIC HAPPENS HERE!
|
|
1080
|
-
# send these remaing comamnds one at a time using `execute_command`
|
|
1081
|
-
# in the main client. This keeps our retry logic in one place mostly,
|
|
1082
|
-
# and allows us to be more confident in correctness of behavior.
|
|
1083
|
-
# at this point any speed gains from pipelining have been lost
|
|
1084
|
-
# anyway, so we might as well make the best attempt to get the correct
|
|
1085
|
-
# behavior.
|
|
1086
|
-
#
|
|
1087
|
-
# The client command will handle retries for each individual command
|
|
1088
|
-
# sequentially as we pass each one into `execute_command`. Any exceptions
|
|
1089
|
-
# that bubble out should only appear once all retries have been exhausted.
|
|
1090
|
-
#
|
|
1091
|
-
# If a lot of commands have failed, we'll be setting the
|
|
1092
|
-
# flag to rebuild the slots table from scratch. So MOVED errors should
|
|
1093
|
-
# correct .commandsthemselves fairly quickly.
|
|
1094
1081
|
await self.connection_pool.nodes.increment_reinitialize_counter(len(attempt))
|
|
1095
|
-
|
|
1096
1082
|
for c in attempt:
|
|
1097
1083
|
try:
|
|
1098
|
-
# send each command individually like we do in the main client.
|
|
1099
1084
|
c.result = await self.client.execute_command(
|
|
1100
1085
|
RedisCommand(c.name, c.arguments), **c.execution_parameters
|
|
1101
1086
|
)
|
|
1102
1087
|
except RedisError as e:
|
|
1103
1088
|
c.result = e
|
|
1104
1089
|
|
|
1105
|
-
#
|
|
1106
|
-
# to the sequence of commands issued in the stack in pipeline.execute()
|
|
1090
|
+
# Flatten results to match the original command order.
|
|
1107
1091
|
response = []
|
|
1108
1092
|
for c in sorted(self.command_stack, key=lambda x: x.position):
|
|
1109
1093
|
r = c.result
|
|
@@ -1121,8 +1105,9 @@ class ClusterPipeline(Client[AnyStr], metaclass=ClusterPipelineMeta):
|
|
|
1121
1105
|
def _determine_slot(
|
|
1122
1106
|
self, command: bytes, *args: ValueT, **options: Unpack[ExecutionParameters]
|
|
1123
1107
|
) -> int:
|
|
1124
|
-
"""
|
|
1125
|
-
|
|
1108
|
+
"""
|
|
1109
|
+
Determine the hash slot for the given command and arguments.
|
|
1110
|
+
"""
|
|
1126
1111
|
keys: tuple[RedisValueT, ...] = cast(
|
|
1127
1112
|
tuple[RedisValueT, ...], options.get("keys")
|
|
1128
1113
|
) or KeySpec.extract_keys(command, *args) # type: ignore
|
|
@@ -1138,13 +1123,15 @@ class ClusterPipeline(Client[AnyStr], metaclass=ClusterPipelineMeta):
|
|
|
1138
1123
|
return slots.pop()
|
|
1139
1124
|
|
|
1140
1125
|
def _fail_on_redirect(self, allow_redirections: bool) -> None:
|
|
1126
|
+
"""
|
|
1127
|
+
Raise if redirections are not allowed in the pipeline.
|
|
1128
|
+
"""
|
|
1141
1129
|
if not allow_redirections:
|
|
1142
1130
|
raise RedisClusterException("ASK & MOVED redirection not allowed in this pipeline")
|
|
1143
1131
|
|
|
1144
1132
|
def multi(self) -> None:
|
|
1145
1133
|
"""
|
|
1146
|
-
|
|
1147
|
-
are issued. End the transactional block with `execute`.
|
|
1134
|
+
Start a transactional block after WATCH commands. End with `execute()`.
|
|
1148
1135
|
"""
|
|
1149
1136
|
if self.explicit_transaction:
|
|
1150
1137
|
raise RedisError("Cannot issue nested calls to MULTI")
|
|
@@ -7,7 +7,6 @@ from coredis.response._callbacks import ResponseCallback
|
|
|
7
7
|
from coredis.response._callbacks.server import InfoCallback
|
|
8
8
|
from coredis.typing import (
|
|
9
9
|
AnyStr,
|
|
10
|
-
MutableMapping,
|
|
11
10
|
ResponsePrimitive,
|
|
12
11
|
ResponseType,
|
|
13
12
|
)
|
|
@@ -41,9 +40,9 @@ SENTINEL_STATE_INT_FIELDS = {
|
|
|
41
40
|
|
|
42
41
|
def sentinel_state_typed(
|
|
43
42
|
response: list[str],
|
|
44
|
-
) ->
|
|
43
|
+
) -> EncodingInsensitiveDict[str, str | int | bool]:
|
|
45
44
|
it = iter(response)
|
|
46
|
-
result:
|
|
45
|
+
result: EncodingInsensitiveDict[str, str | int | bool] = EncodingInsensitiveDict()
|
|
47
46
|
|
|
48
47
|
for key, value in zip(it, it):
|
|
49
48
|
if key in SENTINEL_STATE_INT_FIELDS:
|
|
@@ -54,8 +53,8 @@ def sentinel_state_typed(
|
|
|
54
53
|
|
|
55
54
|
|
|
56
55
|
def add_flags(
|
|
57
|
-
result:
|
|
58
|
-
) ->
|
|
56
|
+
result: EncodingInsensitiveDict[str, int | str | bool],
|
|
57
|
+
) -> EncodingInsensitiveDict[str, int | str | bool]:
|
|
59
58
|
flags = set(nativestr(result["flags"]).split(","))
|
|
60
59
|
for name, flag in (
|
|
61
60
|
("is_master", "master"),
|
|
@@ -72,7 +71,7 @@ def add_flags(
|
|
|
72
71
|
|
|
73
72
|
def parse_sentinel_state(
|
|
74
73
|
item: list[ResponsePrimitive],
|
|
75
|
-
) ->
|
|
74
|
+
) -> EncodingInsensitiveDict[str, int | str | bool]:
|
|
76
75
|
result = sentinel_state_typed([nativestr(k) for k in item])
|
|
77
76
|
result = add_flags(result)
|
|
78
77
|
return result
|
|
@@ -82,34 +81,34 @@ class PrimaryCallback(
|
|
|
82
81
|
ResponseCallback[
|
|
83
82
|
ResponseType,
|
|
84
83
|
dict[ResponsePrimitive, ResponsePrimitive],
|
|
85
|
-
dict[str,
|
|
84
|
+
dict[str, ResponsePrimitive],
|
|
86
85
|
]
|
|
87
86
|
):
|
|
88
87
|
def transform(
|
|
89
88
|
self,
|
|
90
89
|
response: ResponseType,
|
|
91
|
-
) -> dict[str,
|
|
92
|
-
return
|
|
90
|
+
) -> dict[str, ResponsePrimitive]:
|
|
91
|
+
return parse_sentinel_state(cast(list[ResponsePrimitive], response)).stringify_keys()
|
|
93
92
|
|
|
94
93
|
def transform_3(
|
|
95
94
|
self,
|
|
96
95
|
response: dict[ResponsePrimitive, ResponsePrimitive],
|
|
97
|
-
) -> dict[str,
|
|
98
|
-
return
|
|
96
|
+
) -> dict[str, ResponsePrimitive]:
|
|
97
|
+
return add_flags(EncodingInsensitiveDict(response)).stringify_keys()
|
|
99
98
|
|
|
100
99
|
|
|
101
100
|
class PrimariesCallback(
|
|
102
101
|
ResponseCallback[
|
|
103
102
|
list[ResponseType],
|
|
104
103
|
list[ResponseType],
|
|
105
|
-
dict[str, dict[str,
|
|
104
|
+
dict[str, dict[str, ResponsePrimitive]],
|
|
106
105
|
]
|
|
107
106
|
):
|
|
108
107
|
def transform(
|
|
109
108
|
self,
|
|
110
109
|
response: list[ResponseType] | dict[ResponsePrimitive, ResponsePrimitive],
|
|
111
|
-
) -> dict[str, dict[str,
|
|
112
|
-
result: dict[str, dict[str,
|
|
110
|
+
) -> dict[str, dict[str, ResponsePrimitive]]:
|
|
111
|
+
result: dict[str, dict[str, ResponseType]] = {}
|
|
113
112
|
|
|
114
113
|
for item in response:
|
|
115
114
|
state = PrimaryCallback()(item)
|
|
@@ -120,11 +119,11 @@ class PrimariesCallback(
|
|
|
120
119
|
def transform_3(
|
|
121
120
|
self,
|
|
122
121
|
response: list[ResponseType],
|
|
123
|
-
) -> dict[str, dict[str,
|
|
124
|
-
states: dict[str, dict[str,
|
|
122
|
+
) -> dict[str, dict[str, ResponsePrimitive]]:
|
|
123
|
+
states: dict[str, dict[str, ResponsePrimitive]] = {}
|
|
125
124
|
for state in response:
|
|
126
|
-
|
|
127
|
-
states[nativestr(
|
|
125
|
+
state = add_flags(EncodingInsensitiveDict(state)).stringify_keys()
|
|
126
|
+
states[nativestr(state["name"])] = state
|
|
128
127
|
return states
|
|
129
128
|
|
|
130
129
|
|
|
@@ -132,20 +131,24 @@ class SentinelsStateCallback(
|
|
|
132
131
|
ResponseCallback[
|
|
133
132
|
list[ResponseType],
|
|
134
133
|
list[ResponseType],
|
|
135
|
-
tuple[dict[str,
|
|
134
|
+
tuple[dict[str, ResponsePrimitive], ...],
|
|
136
135
|
]
|
|
137
136
|
):
|
|
138
137
|
def transform(
|
|
139
138
|
self,
|
|
140
139
|
response: list[ResponseType],
|
|
141
|
-
) -> tuple[dict[str,
|
|
142
|
-
return tuple(
|
|
140
|
+
) -> tuple[dict[str, ResponsePrimitive], ...]:
|
|
141
|
+
return tuple(
|
|
142
|
+
parse_sentinel_state([nativestr(i) for i in item]).stringify_keys() for item in response
|
|
143
|
+
)
|
|
143
144
|
|
|
144
145
|
def transform_3(
|
|
145
146
|
self,
|
|
146
147
|
response: list[ResponseType],
|
|
147
|
-
) -> tuple[dict[str,
|
|
148
|
-
return tuple(
|
|
148
|
+
) -> tuple[dict[str, ResponsePrimitive], ...]:
|
|
149
|
+
return tuple(
|
|
150
|
+
add_flags(EncodingInsensitiveDict(state)).stringify_keys() for state in response
|
|
151
|
+
)
|
|
149
152
|
|
|
150
153
|
|
|
151
154
|
class GetPrimaryCallback(
|
|
@@ -166,11 +169,11 @@ class SentinelInfoCallback(
|
|
|
166
169
|
ResponseCallback[
|
|
167
170
|
list[ResponseType],
|
|
168
171
|
list[ResponseType],
|
|
169
|
-
dict[AnyStr, dict[int, dict[str,
|
|
172
|
+
dict[AnyStr, dict[int, dict[str, ResponsePrimitive]]],
|
|
170
173
|
]
|
|
171
174
|
):
|
|
172
175
|
def transform(
|
|
173
176
|
self,
|
|
174
177
|
response: list[ResponseType],
|
|
175
|
-
) -> dict[AnyStr, dict[int, dict[str,
|
|
178
|
+
) -> dict[AnyStr, dict[int, dict[str, ResponsePrimitive]]]:
|
|
176
179
|
return {response[0]: {r[0]: InfoCallback()(r[1]) for r in response[1]}}
|