patchwork-client-redis 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- patchwork/client/redis/__init__.py +2 -0
- patchwork/client/redis/common.py +24 -0
- patchwork/client/redis/publisher.py +129 -0
- patchwork/client/redis/subscriber.py +398 -0
- patchwork_client_redis-1.0.0.dist-info/METADATA +50 -0
- patchwork_client_redis-1.0.0.dist-info/RECORD +9 -0
- patchwork_client_redis-1.0.0.dist-info/WHEEL +5 -0
- patchwork_client_redis-1.0.0.dist-info/licenses/LICENSE +21 -0
- patchwork_client_redis-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from datetime import UTC, datetime
|
|
2
|
+
|
|
3
|
+
from redis.asyncio import Redis
|
|
4
|
+
|
|
5
|
+
#: stream entry field holding encoded message, the source of truth
|
|
6
|
+
DATA_FIELD = b'data'
|
|
7
|
+
|
|
8
|
+
#: stream entry fields copied from the message, for filtering and debugging without decoding
|
|
9
|
+
HEADER_FIELDS = (b'id', b'type', b'queue', b'not_before', b'expires')
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def encode_timestamp(value: datetime) -> bytes:
|
|
13
|
+
return str(int(value.timestamp() * 1000)).encode()
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def decode_timestamp(value: bytes) -> datetime:
|
|
17
|
+
return datetime.fromtimestamp(int(value) / 1000, tz=UTC)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def create_client(url: str, **options) -> Redis:
|
|
21
|
+
"""
|
|
22
|
+
Creates Redis client from URL, eg. `redis://localhost:6379/0`. Responses are not decoded (bytes).
|
|
23
|
+
"""
|
|
24
|
+
return Redis.from_url(url, decode_responses=False, **options)
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
|
|
3
|
+
from patchwork.core import AsyncPublisher, Capabilities, Component, Message
|
|
4
|
+
from patchwork.core.client.publisher import DEFAULT_MAX_MSG_SIZE
|
|
5
|
+
from patchwork.core.serializers import Serializer
|
|
6
|
+
|
|
7
|
+
from redis.asyncio import Redis
|
|
8
|
+
from redis.exceptions import ConnectionError as RedisConnectionError
|
|
9
|
+
from redis.exceptions import TimeoutError as RedisTimeoutError
|
|
10
|
+
|
|
11
|
+
from .common import DATA_FIELD, create_client, encode_timestamp
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def message_fields(data: bytes, message: Message) -> dict[bytes, bytes]:
|
|
15
|
+
"""
|
|
16
|
+
Builds stream entry fields: encoded message and a copy of selected message fields.
|
|
17
|
+
"""
|
|
18
|
+
fields = {
|
|
19
|
+
DATA_FIELD: data,
|
|
20
|
+
b'id': str(message.id).encode(),
|
|
21
|
+
b'type': message.type.encode(),
|
|
22
|
+
b'queue': message.queue.encode(),
|
|
23
|
+
}
|
|
24
|
+
if message.not_before is not None:
|
|
25
|
+
fields[b'not_before'] = encode_timestamp(message.not_before)
|
|
26
|
+
if message.expires is not None:
|
|
27
|
+
fields[b'expires'] = encode_timestamp(message.expires)
|
|
28
|
+
return fields
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class AsyncRedisPublisher(AsyncPublisher):
|
|
32
|
+
"""
|
|
33
|
+
Redis Streams publisher, each queue is a stream. Messages are appended with `XADD`.
|
|
34
|
+
|
|
35
|
+
Redis streams don't support delayed delivery, configure `postpone_queue` and run patchwork-postponer to support
|
|
36
|
+
messages with `not_before` in the future.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
capabilities = Capabilities(delayed_delivery=False, native_headers=True, redelivery_count=False)
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
*,
|
|
44
|
+
url: str | None = None,
|
|
45
|
+
client: Redis | None = None,
|
|
46
|
+
parent: Component | None = None,
|
|
47
|
+
enabled: bool = True,
|
|
48
|
+
serializer: Serializer | None = None,
|
|
49
|
+
max_message_size: int = DEFAULT_MAX_MSG_SIZE,
|
|
50
|
+
postpone_queue: str | None = None,
|
|
51
|
+
maxlen: int | None = None,
|
|
52
|
+
stream_prefix: str = '',
|
|
53
|
+
send_timeout: float = 10.0,
|
|
54
|
+
retry_attempts: int = 3,
|
|
55
|
+
retry_time_s: float = 1.0,
|
|
56
|
+
):
|
|
57
|
+
"""
|
|
58
|
+
:param url: Redis URL, eg. `redis://localhost:6379/0`, required if `client` is not given
|
|
59
|
+
:param client: Redis client (`redis.asyncio.Redis` with `decode_responses=False`), if given it's not closed
|
|
60
|
+
by the publisher
|
|
61
|
+
:param postpone_queue: queue for messages with `not_before` in the future
|
|
62
|
+
:param maxlen: approximate max length of streams (`XADD MAXLEN ~`), None means no trimming.
|
|
63
|
+
|
|
64
|
+
!!! danger
|
|
65
|
+
Trimming removes entries regardless of consumer groups, messages not consumed yet are lost.
|
|
66
|
+
|
|
67
|
+
:param stream_prefix: prefix of stream keys, stream key is `prefix + queue name`
|
|
68
|
+
:param send_timeout: default send timeout in seconds
|
|
69
|
+
:param retry_attempts: number of retries on connection errors
|
|
70
|
+
:param retry_time_s: time between retries in seconds
|
|
71
|
+
"""
|
|
72
|
+
super().__init__(
|
|
73
|
+
parent=parent,
|
|
74
|
+
enabled=enabled,
|
|
75
|
+
serializer=serializer,
|
|
76
|
+
max_message_size=max_message_size,
|
|
77
|
+
postpone_queue=postpone_queue,
|
|
78
|
+
)
|
|
79
|
+
if url is None and client is None:
|
|
80
|
+
raise ValueError("either `url` or `client` must be given")
|
|
81
|
+
|
|
82
|
+
self._url = url
|
|
83
|
+
self._client = client
|
|
84
|
+
self._owns_client = client is None
|
|
85
|
+
self._maxlen = maxlen
|
|
86
|
+
self._stream_prefix = stream_prefix
|
|
87
|
+
self._send_timeout = send_timeout
|
|
88
|
+
self._retry_attempts = retry_attempts
|
|
89
|
+
self._retry_time_s = retry_time_s
|
|
90
|
+
|
|
91
|
+
def __repr__(self):
|
|
92
|
+
res = super().__repr__()
|
|
93
|
+
return f"<{res[1:-1]} [{self._url or self._client}]>"
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def client(self) -> Redis:
|
|
97
|
+
return self._client
|
|
98
|
+
|
|
99
|
+
def stream_key(self, queue: str) -> str:
|
|
100
|
+
return f'{self._stream_prefix}{queue}'
|
|
101
|
+
|
|
102
|
+
async def _start(self):
|
|
103
|
+
if self._client is None:
|
|
104
|
+
self._client = create_client(self._url)
|
|
105
|
+
await self._client.ping()
|
|
106
|
+
|
|
107
|
+
async def _stop(self):
|
|
108
|
+
if self._owns_client and self._client is not None:
|
|
109
|
+
await self._client.aclose()
|
|
110
|
+
self._client = None
|
|
111
|
+
|
|
112
|
+
async def _send(self, data: bytes, message: Message, *, queue: str, timeout: float | None = None):
|
|
113
|
+
fields = message_fields(data, message)
|
|
114
|
+
timeout = self._send_timeout if timeout is None else timeout
|
|
115
|
+
kwargs = {'maxlen': self._maxlen, 'approximate': True} if self._maxlen else {}
|
|
116
|
+
|
|
117
|
+
for attempt in range(self._retry_attempts + 1):
|
|
118
|
+
try:
|
|
119
|
+
entry_id = await asyncio.wait_for(
|
|
120
|
+
self._client.xadd(self.stream_key(queue), fields, **kwargs), timeout
|
|
121
|
+
)
|
|
122
|
+
except (RedisConnectionError, RedisTimeoutError) as exc:
|
|
123
|
+
self.logger.warning(f"can't send message to {queue}: {exc.__class__.__name__}({exc})")
|
|
124
|
+
if attempt == self._retry_attempts:
|
|
125
|
+
raise TimeoutError(f"can't send message to {queue}") from exc
|
|
126
|
+
await asyncio.sleep(self._retry_time_s)
|
|
127
|
+
else:
|
|
128
|
+
self.logger.debug(f"message {message.id} appended to {queue} as {entry_id!r}")
|
|
129
|
+
return
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import os
|
|
3
|
+
import socket
|
|
4
|
+
import time
|
|
5
|
+
import uuid
|
|
6
|
+
from collections import deque
|
|
7
|
+
from collections.abc import Awaitable, Callable, Iterable
|
|
8
|
+
from typing import Any, NamedTuple, TypeVar
|
|
9
|
+
|
|
10
|
+
from patchwork.core import (
|
|
11
|
+
AsyncSubscriber,
|
|
12
|
+
Capabilities,
|
|
13
|
+
CommitLost,
|
|
14
|
+
Component,
|
|
15
|
+
Delivery,
|
|
16
|
+
SubscriberClosed,
|
|
17
|
+
)
|
|
18
|
+
from patchwork.core.serializers import Serializer
|
|
19
|
+
|
|
20
|
+
from redis.asyncio import Redis
|
|
21
|
+
from redis.exceptions import ConnectionError as RedisConnectionError
|
|
22
|
+
from redis.exceptions import ResponseError
|
|
23
|
+
from redis.exceptions import TimeoutError as RedisTimeoutError
|
|
24
|
+
|
|
25
|
+
from .common import DATA_FIELD, create_client, decode_timestamp
|
|
26
|
+
|
|
27
|
+
T = TypeVar('T')
|
|
28
|
+
|
|
29
|
+
_RETRYABLE = (RedisConnectionError, RedisTimeoutError, OSError)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class RedisReceipt(NamedTuple):
|
|
33
|
+
#: stream key
|
|
34
|
+
stream: str
|
|
35
|
+
#: stream entry ID
|
|
36
|
+
entry_id: bytes
|
|
37
|
+
#: number of deliveries of the entry including this one
|
|
38
|
+
deliveries: int
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class _Entry(NamedTuple):
|
|
42
|
+
stream: str
|
|
43
|
+
entry_id: bytes
|
|
44
|
+
fields: dict[bytes, bytes]
|
|
45
|
+
deliveries: int
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def default_consumer_name() -> str:
|
|
49
|
+
return f'{socket.gethostname()}-{os.getpid()}-{uuid.uuid4().hex[:8]}'
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class AsyncRedisSubscriber(AsyncSubscriber):
|
|
53
|
+
"""
|
|
54
|
+
Redis Streams subscriber working in a consumer group.
|
|
55
|
+
|
|
56
|
+
- messages are read with `XREADGROUP`, each message is pending (in the group PEL) until committed (`XACK`),
|
|
57
|
+
commits may come in any order
|
|
58
|
+
- rollback claims the message back (`XCLAIM`), it's delivered again by the next `get()` call and broker
|
|
59
|
+
delivery counter is increased (`Delivery.redelivered`)
|
|
60
|
+
- messages pending on consumers which went away are claimed (`XAUTOCLAIM`) after `claim_idle_ms`,
|
|
61
|
+
so `claim_idle_ms` works like a visibility timeout: processing of a message must not take longer
|
|
62
|
+
- with a stable `consumer` name (eg. pod name), messages pending on the consumer are delivered again
|
|
63
|
+
immediately after restart
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
capabilities = Capabilities(delayed_delivery=False, native_headers=True, redelivery_count=True)
|
|
67
|
+
|
|
68
|
+
def __init__(
|
|
69
|
+
self,
|
|
70
|
+
*,
|
|
71
|
+
queues: Iterable[str],
|
|
72
|
+
group: str,
|
|
73
|
+
url: str | None = None,
|
|
74
|
+
client: Redis | None = None,
|
|
75
|
+
consumer: str | None = None,
|
|
76
|
+
parent: Component | None = None,
|
|
77
|
+
enabled: bool = True,
|
|
78
|
+
serializer: Serializer | None = None,
|
|
79
|
+
stream_prefix: str = '',
|
|
80
|
+
start_id: str = '0',
|
|
81
|
+
create_group: bool = True,
|
|
82
|
+
block_ms: int = 1000,
|
|
83
|
+
claim_idle_ms: int | None = 60_000,
|
|
84
|
+
claim_interval_s: float | None = None,
|
|
85
|
+
claim_batch: int = 10,
|
|
86
|
+
retry_attempts: int = 5,
|
|
87
|
+
retry_time_s: float = 1.0,
|
|
88
|
+
):
|
|
89
|
+
"""
|
|
90
|
+
:param queues: queues (streams) to read
|
|
91
|
+
:param group: consumer group name, each group receives all messages
|
|
92
|
+
:param url: Redis URL, eg. `redis://localhost:6379/0`, required if `client` is not given
|
|
93
|
+
:param client: Redis client (`redis.asyncio.Redis` with `decode_responses=False`), if given it's not closed
|
|
94
|
+
by the subscriber. Subscriber blocks one connection while waiting for messages.
|
|
95
|
+
:param consumer: consumer name, unique random name by default. Use a stable name (eg. pod name) to get
|
|
96
|
+
messages pending on the consumer immediately after restart.
|
|
97
|
+
:param stream_prefix: prefix of stream keys, stream key is `prefix + queue name`
|
|
98
|
+
:param start_id: where a new consumer group starts reading the stream: `0` (all messages) or `$`
|
|
99
|
+
(only new messages)
|
|
100
|
+
:param create_group: create missing consumer groups (and streams)
|
|
101
|
+
:param block_ms: max time of a single blocking read in milliseconds
|
|
102
|
+
:param claim_idle_ms: messages pending longer than this on other consumers are claimed and delivered again,
|
|
103
|
+
None disables claiming (messages of dead consumers are never delivered again!)
|
|
104
|
+
:param claim_interval_s: how often to check for messages to claim, half of `claim_idle_ms` by default
|
|
105
|
+
:param claim_batch: max number of messages claimed at once
|
|
106
|
+
:param retry_attempts: number of retries on connection errors, then subscriber fails
|
|
107
|
+
:param retry_time_s: time between retries in seconds
|
|
108
|
+
"""
|
|
109
|
+
super().__init__(parent=parent, enabled=enabled, serializer=serializer)
|
|
110
|
+
if url is None and client is None:
|
|
111
|
+
raise ValueError("either `url` or `client` must be given")
|
|
112
|
+
|
|
113
|
+
self._queues = list(queues)
|
|
114
|
+
if not self._queues:
|
|
115
|
+
raise ValueError("no queues to listen on")
|
|
116
|
+
|
|
117
|
+
self._group = group
|
|
118
|
+
self._url = url
|
|
119
|
+
self._client = client
|
|
120
|
+
self._owns_client = client is None
|
|
121
|
+
self._consumer = consumer or default_consumer_name()
|
|
122
|
+
self._stream_prefix = stream_prefix
|
|
123
|
+
self._start_id = start_id
|
|
124
|
+
self._create_group = create_group
|
|
125
|
+
self._block_ms = block_ms
|
|
126
|
+
self._claim_idle_ms = claim_idle_ms
|
|
127
|
+
if claim_interval_s is None and claim_idle_ms is not None:
|
|
128
|
+
claim_interval_s = claim_idle_ms / 1000 / 2
|
|
129
|
+
self._claim_interval_s = claim_interval_s
|
|
130
|
+
self._claim_batch = claim_batch
|
|
131
|
+
self._retry_attempts = retry_attempts
|
|
132
|
+
self._retry_time_s = retry_time_s
|
|
133
|
+
|
|
134
|
+
self._ready: deque[_Entry] = deque()
|
|
135
|
+
self._next_claim = 0.0
|
|
136
|
+
self._claim_cursors: dict[str, bytes] = {}
|
|
137
|
+
self._recover_own = True
|
|
138
|
+
|
|
139
|
+
def __repr__(self):
|
|
140
|
+
res = super().__repr__()
|
|
141
|
+
return f"<{res[1:-1]} [{self._group}/{self._consumer}]>"
|
|
142
|
+
|
|
143
|
+
@property
|
|
144
|
+
def queues(self) -> list[str]:
|
|
145
|
+
return list(self._queues)
|
|
146
|
+
|
|
147
|
+
@property
|
|
148
|
+
def group(self) -> str:
|
|
149
|
+
return self._group
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def consumer(self) -> str:
|
|
153
|
+
return self._consumer
|
|
154
|
+
|
|
155
|
+
@property
|
|
156
|
+
def client(self) -> Redis:
|
|
157
|
+
return self._client
|
|
158
|
+
|
|
159
|
+
def stream_key(self, queue: str) -> str:
|
|
160
|
+
return f'{self._stream_prefix}{queue}'
|
|
161
|
+
|
|
162
|
+
def _queue_name(self, stream: str) -> str:
|
|
163
|
+
return stream[len(self._stream_prefix):]
|
|
164
|
+
|
|
165
|
+
@property
|
|
166
|
+
def _streams(self) -> list[str]:
|
|
167
|
+
return [self.stream_key(q) for q in self._queues]
|
|
168
|
+
|
|
169
|
+
# --- lifecycle ---
|
|
170
|
+
|
|
171
|
+
async def _start(self):
|
|
172
|
+
if self._client is None:
|
|
173
|
+
self._client = create_client(self._url)
|
|
174
|
+
|
|
175
|
+
try:
|
|
176
|
+
await self._client.ping()
|
|
177
|
+
if self._create_group:
|
|
178
|
+
for stream in self._streams:
|
|
179
|
+
await self._ensure_group(stream)
|
|
180
|
+
except BaseException:
|
|
181
|
+
await self._close_client()
|
|
182
|
+
raise
|
|
183
|
+
|
|
184
|
+
self._ready.clear()
|
|
185
|
+
self._claim_cursors.clear()
|
|
186
|
+
self._next_claim = 0.0
|
|
187
|
+
self._recover_own = True
|
|
188
|
+
|
|
189
|
+
async def _stop(self):
|
|
190
|
+
# messages pending on this consumer stay in the group PEL, they are claimed by other consumers after
|
|
191
|
+
# `claim_idle_ms` or delivered again when consumer with the same name starts
|
|
192
|
+
self._ready.clear()
|
|
193
|
+
await self._close_client()
|
|
194
|
+
|
|
195
|
+
async def _close_client(self):
|
|
196
|
+
if self._owns_client and self._client is not None:
|
|
197
|
+
await self._client.aclose()
|
|
198
|
+
self._client = None
|
|
199
|
+
|
|
200
|
+
async def _ensure_group(self, stream: str):
|
|
201
|
+
try:
|
|
202
|
+
await self._client.xgroup_create(stream, self._group, id=self._start_id, mkstream=True)
|
|
203
|
+
self.logger.info(f"consumer group {self._group} created on {stream}")
|
|
204
|
+
except ResponseError as exc:
|
|
205
|
+
if 'BUSYGROUP' not in str(exc):
|
|
206
|
+
raise
|
|
207
|
+
|
|
208
|
+
# --- redis calls ---
|
|
209
|
+
|
|
210
|
+
async def _call(self, fn: Callable[[], Awaitable[T]]) -> T:
|
|
211
|
+
"""
|
|
212
|
+
Calls Redis, retries on connection errors and missing consumer group. If retries are exhausted subscriber
|
|
213
|
+
fails.
|
|
214
|
+
"""
|
|
215
|
+
for attempt in range(self._retry_attempts + 1):
|
|
216
|
+
try:
|
|
217
|
+
return await fn()
|
|
218
|
+
except ResponseError as exc:
|
|
219
|
+
if 'NOGROUP' in str(exc) and self._create_group and attempt < self._retry_attempts:
|
|
220
|
+
# stream or group has been deleted, create it again
|
|
221
|
+
self.logger.warning(f"consumer group missing, recreating: {exc}")
|
|
222
|
+
for stream in self._streams:
|
|
223
|
+
await self._ensure_group(stream)
|
|
224
|
+
continue
|
|
225
|
+
self._fail(exc)
|
|
226
|
+
raise SubscriberClosed(f"{self!r} failed") from exc
|
|
227
|
+
except _RETRYABLE as exc:
|
|
228
|
+
if attempt == self._retry_attempts:
|
|
229
|
+
self._fail(exc)
|
|
230
|
+
raise SubscriberClosed(f"{self!r} failed") from exc
|
|
231
|
+
self.logger.warning(f"redis call failed ({exc.__class__.__name__}: {exc}), retrying")
|
|
232
|
+
await asyncio.sleep(self._retry_time_s)
|
|
233
|
+
|
|
234
|
+
raise AssertionError('unreachable')
|
|
235
|
+
|
|
236
|
+
# --- fetching ---
|
|
237
|
+
|
|
238
|
+
@staticmethod
|
|
239
|
+
def _parse_read(response: Any) -> dict[str, list]:
|
|
240
|
+
"""
|
|
241
|
+
Converts `XREADGROUP` response to `{stream: [(entry_id, fields), ...]}`.
|
|
242
|
+
"""
|
|
243
|
+
if not response:
|
|
244
|
+
return {}
|
|
245
|
+
items = response.items() if isinstance(response, dict) else response
|
|
246
|
+
result: dict[str, list] = {}
|
|
247
|
+
for stream, entries in items:
|
|
248
|
+
stream = stream.decode() if isinstance(stream, bytes) else stream
|
|
249
|
+
result.setdefault(stream, []).extend(entries)
|
|
250
|
+
return result
|
|
251
|
+
|
|
252
|
+
async def _delivery_counts(self, stream: str, ids: list[bytes]) -> dict[bytes, int]:
|
|
253
|
+
pending = await self._call(lambda: self._client.xpending_range(
|
|
254
|
+
stream, self._group, min=ids[0], max=ids[-1], count=len(ids) * 2 + 10, consumername=self._consumer
|
|
255
|
+
))
|
|
256
|
+
return {p['message_id']: p['times_delivered'] for p in pending}
|
|
257
|
+
|
|
258
|
+
async def _with_counts(self, stream: str, items: list) -> list[_Entry]:
|
|
259
|
+
entries = [(entry_id, fields) for entry_id, fields in items if entry_id is not None]
|
|
260
|
+
if not entries:
|
|
261
|
+
return []
|
|
262
|
+
ids = sorted((e[0] for e in entries), key=_entry_sort_key)
|
|
263
|
+
counts = await self._delivery_counts(stream, ids)
|
|
264
|
+
return [_Entry(stream, entry_id, fields or {}, counts.get(entry_id, 1)) for entry_id, fields in entries]
|
|
265
|
+
|
|
266
|
+
async def _recover_own_pending(self):
|
|
267
|
+
"""
|
|
268
|
+
Reads messages pending on this consumer (eg. after restart with the same consumer name).
|
|
269
|
+
"""
|
|
270
|
+
self._recover_own = False
|
|
271
|
+
for stream in self._streams:
|
|
272
|
+
response = await self._call(lambda s=stream: self._client.xreadgroup(
|
|
273
|
+
self._group, self._consumer, {s: '0'}, count=self._claim_batch * 10
|
|
274
|
+
))
|
|
275
|
+
for _, items in self._parse_read(response).items():
|
|
276
|
+
# reading consumer history increments delivery counters too
|
|
277
|
+
self._ready.extend(await self._with_counts(stream, items))
|
|
278
|
+
if self._ready:
|
|
279
|
+
self.logger.info(f"{len(self._ready)} messages pending on consumer {self._consumer} recovered")
|
|
280
|
+
|
|
281
|
+
async def _claim(self):
|
|
282
|
+
"""
|
|
283
|
+
Claims messages pending too long on other consumers.
|
|
284
|
+
"""
|
|
285
|
+
if self._claim_idle_ms is None or time.monotonic() < self._next_claim:
|
|
286
|
+
return
|
|
287
|
+
self._next_claim = time.monotonic() + self._claim_interval_s
|
|
288
|
+
|
|
289
|
+
for stream in self._streams:
|
|
290
|
+
cursor = self._claim_cursors.get(stream, b'0-0')
|
|
291
|
+
response = await self._call(lambda s=stream, c=cursor: self._client.xautoclaim(
|
|
292
|
+
s, self._group, self._consumer, min_idle_time=self._claim_idle_ms, start_id=c,
|
|
293
|
+
count=self._claim_batch,
|
|
294
|
+
))
|
|
295
|
+
next_cursor, items = response[0], response[1]
|
|
296
|
+
self._claim_cursors[stream] = next_cursor
|
|
297
|
+
claimed = await self._with_counts(stream, items)
|
|
298
|
+
if claimed:
|
|
299
|
+
self.logger.info(f"{len(claimed)} messages claimed from {stream}")
|
|
300
|
+
self._ready.extend(claimed)
|
|
301
|
+
|
|
302
|
+
async def _next_entry(self, timeout: float | None) -> _Entry:
|
|
303
|
+
deadline = None if timeout is None else time.monotonic() + timeout
|
|
304
|
+
|
|
305
|
+
while True:
|
|
306
|
+
if self._recover_own:
|
|
307
|
+
await self._recover_own_pending()
|
|
308
|
+
await self._claim()
|
|
309
|
+
|
|
310
|
+
while self._ready:
|
|
311
|
+
entry = self._ready.popleft()
|
|
312
|
+
if DATA_FIELD in entry.fields:
|
|
313
|
+
return entry
|
|
314
|
+
# entry has been deleted from the stream (eg. trimmed), nothing to deliver
|
|
315
|
+
self.logger.warning(f"message {entry.entry_id!r} deleted from {entry.stream}, skipped")
|
|
316
|
+
await self._call(lambda e=entry: self._client.xack(e.stream, self._group, e.entry_id))
|
|
317
|
+
|
|
318
|
+
if deadline is None:
|
|
319
|
+
block = self._block_ms
|
|
320
|
+
else:
|
|
321
|
+
remaining_ms = int((deadline - time.monotonic()) * 1000)
|
|
322
|
+
if remaining_ms <= 0:
|
|
323
|
+
block = None
|
|
324
|
+
else:
|
|
325
|
+
block = min(self._block_ms, remaining_ms)
|
|
326
|
+
if self._claim_idle_ms is not None:
|
|
327
|
+
# wake up for the next claim check
|
|
328
|
+
until_claim_ms = int((self._next_claim - time.monotonic()) * 1000)
|
|
329
|
+
block = max(1, min(block, until_claim_ms)) if block is not None else block
|
|
330
|
+
|
|
331
|
+
started = time.monotonic()
|
|
332
|
+
response = await self._call(lambda: self._client.xreadgroup(
|
|
333
|
+
self._group, self._consumer, {s: '>' for s in self._streams}, count=1, block=block
|
|
334
|
+
))
|
|
335
|
+
if not response and block and time.monotonic() - started < block / 1000:
|
|
336
|
+
# server returned earlier than requested (eg. BLOCK not supported by a proxy),
|
|
337
|
+
# don't spin in a busy loop
|
|
338
|
+
await asyncio.sleep(min(0.01, block / 1000))
|
|
339
|
+
for stream, items in self._parse_read(response).items():
|
|
340
|
+
self._ready.extend(_Entry(stream, entry_id, fields or {}, 1) for entry_id, fields in items)
|
|
341
|
+
|
|
342
|
+
if not self._ready and deadline is not None and time.monotonic() >= deadline:
|
|
343
|
+
raise TimeoutError('no messages')
|
|
344
|
+
|
|
345
|
+
async def _fetch(self, *, timeout: float | None = None) -> Delivery:
|
|
346
|
+
entry = await self._next_entry(timeout)
|
|
347
|
+
fields = entry.fields
|
|
348
|
+
headers = {k.decode(errors='replace'): v for k, v in fields.items() if k != DATA_FIELD}
|
|
349
|
+
|
|
350
|
+
return self._make_delivery(
|
|
351
|
+
fields[DATA_FIELD],
|
|
352
|
+
queue=self._queue_name(entry.stream),
|
|
353
|
+
receipt=RedisReceipt(stream=entry.stream, entry_id=entry.entry_id, deliveries=entry.deliveries),
|
|
354
|
+
headers=headers,
|
|
355
|
+
hints=self._hints(headers),
|
|
356
|
+
redelivered=max(0, entry.deliveries - 1),
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
@staticmethod
|
|
360
|
+
def _hints(headers: dict[str, bytes]) -> dict[str, Any]:
|
|
361
|
+
hints: dict[str, Any] = {}
|
|
362
|
+
try:
|
|
363
|
+
if 'type' in headers:
|
|
364
|
+
hints['type'] = headers['type'].decode()
|
|
365
|
+
for name in ('not_before', 'expires'):
|
|
366
|
+
if name in headers:
|
|
367
|
+
hints[name] = decode_timestamp(headers[name])
|
|
368
|
+
except (UnicodeDecodeError, ValueError, OverflowError):
|
|
369
|
+
# invalid headers are not critical, message is the source of truth
|
|
370
|
+
return {}
|
|
371
|
+
return hints
|
|
372
|
+
|
|
373
|
+
# --- settlement ---
|
|
374
|
+
|
|
375
|
+
async def _commit(self, delivery: Delivery, *, timeout: float | None = None):
|
|
376
|
+
receipt: RedisReceipt = delivery.receipt
|
|
377
|
+
acked = await asyncio.wait_for(
|
|
378
|
+
self._call(lambda: self._client.xack(receipt.stream, self._group, receipt.entry_id)), timeout
|
|
379
|
+
)
|
|
380
|
+
if not acked:
|
|
381
|
+
raise CommitLost(f"message {receipt.entry_id!r} is not pending anymore")
|
|
382
|
+
|
|
383
|
+
async def _rollback(self, delivery: Delivery, *, timeout: float | None = None):
|
|
384
|
+
receipt: RedisReceipt = delivery.receipt
|
|
385
|
+
# claim the message back: resets idle time and increments delivery counter
|
|
386
|
+
claimed = await asyncio.wait_for(self._call(lambda: self._client.xclaim(
|
|
387
|
+
receipt.stream, self._group, self._consumer, min_idle_time=0, message_ids=[receipt.entry_id]
|
|
388
|
+
)), timeout)
|
|
389
|
+
if not claimed:
|
|
390
|
+
raise CommitLost(f"message {receipt.entry_id!r} is not pending anymore")
|
|
391
|
+
|
|
392
|
+
entry_id, fields = claimed[0]
|
|
393
|
+
self._ready.append(_Entry(receipt.stream, entry_id, fields or {}, receipt.deliveries + 1))
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _entry_sort_key(entry_id: bytes) -> tuple[int, int]:
|
|
397
|
+
ms, _, seq = entry_id.partition(b'-')
|
|
398
|
+
return int(ms), int(seq or 0)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: patchwork-client-redis
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Redis Streams client for Patchwork - The distributed asynchronous microframework
|
|
5
|
+
Author: Paweł Pecio
|
|
6
|
+
License: MIT
|
|
7
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
|
|
15
|
+
Requires-Python: >=3.11.0
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: patchwork-core<3,>=2.0.0
|
|
19
|
+
Requires-Dist: redis>=5.0
|
|
20
|
+
Provides-Extra: docs
|
|
21
|
+
Requires-Dist: mkdocs<2,==1.6.1; extra == "docs"
|
|
22
|
+
Requires-Dist: mkdocs-material; extra == "docs"
|
|
23
|
+
Requires-Dist: mkdocstrings[python]; extra == "docs"
|
|
24
|
+
Dynamic: license-file
|
|
25
|
+
|
|
26
|
+
# Patchwork Client Redis
|
|
27
|
+
|
|
28
|
+
Redis Streams client for Patchwork - an asynchronous distributed microframework.
|
|
29
|
+
|
|
30
|
+
## Requirements
|
|
31
|
+
|
|
32
|
+
* Python 3.11+
|
|
33
|
+
* Redis 7.0+
|
|
34
|
+
* `patchwork-core` 2.0+
|
|
35
|
+
|
|
36
|
+
## Installation
|
|
37
|
+
|
|
38
|
+
``pip install patchwork-client-redis``
|
|
39
|
+
|
|
40
|
+
## Development
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
uv sync --all-extras --dev
|
|
44
|
+
uv run pytest tests
|
|
45
|
+
REDIS_URL=redis://127.0.0.1:6379/0 uv run pytest int
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Documentation
|
|
49
|
+
|
|
50
|
+
https://patchwork-framework.gitlab.io/redis-client
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
patchwork/client/redis/__init__.py,sha256=IMkmFtvMcmixeMoKFZafPuhknTHaK9-4R7tZdyu5Czo,102
|
|
2
|
+
patchwork/client/redis/common.py,sha256=ibqlMuWqzbYHaBp2GPFkm7H5XhuZeeviDqNiPa5w9lI,772
|
|
3
|
+
patchwork/client/redis/publisher.py,sha256=K5_2GVaeTNfYB-OvBtG4o3NRfBUuSDGy3OBbNwpqKZY,5112
|
|
4
|
+
patchwork/client/redis/subscriber.py,sha256=9LuwF-bVUeCBC4LogWtYLGE5uVHY5-XVwXxorroUy3s,16449
|
|
5
|
+
patchwork_client_redis-1.0.0.dist-info/licenses/LICENSE,sha256=2j7OLS2vXIEshgbSkiCU9RgYJY1M4OBBtO6ECbo7OB8,1056
|
|
6
|
+
patchwork_client_redis-1.0.0.dist-info/METADATA,sha256=qqcKZdaqmO2TY7VKyK-J5HRLl5vhc2c_dO-47HLWIOo,1431
|
|
7
|
+
patchwork_client_redis-1.0.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
8
|
+
patchwork_client_redis-1.0.0.dist-info/top_level.txt,sha256=aLF4R1vFf2HfqmKmZHlBkLny_-8Ye5nT-Pi8pKL2vE8,10
|
|
9
|
+
patchwork_client_redis-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2019
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
patchwork
|