DefStream-Redis 0.1.5__13-py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of DefStream-Redis might be problematic. Click here for more details.

@@ -0,0 +1,191 @@
1
+ from logging import getLogger
2
+ from typing import (
3
+ Any,
4
+ Optional,
5
+ Union,
6
+ )
7
+
8
+ from redis import Redis
9
+ from redis.cluster import RedisCluster
10
+
11
+ from DefStream.Broker import Event as BaseEvent
12
+
13
+ __all__ = (
14
+ 'Event',
15
+ )
16
+
17
+
18
+ class Event(BaseEvent):
19
+ logger = getLogger('DefStream.Broker.Implements.Redis.Event')
20
+
21
+ def __init__(
22
+ self,
23
+ connection: Union[Redis, RedisCluster],
24
+ queue: str,
25
+ id: str,
26
+ body: Any,
27
+ group: Optional[str] = None,
28
+ consumer: Optional[str] = None,
29
+ ):
30
+ self.connection = connection
31
+ self.queue = queue
32
+ self.id = id
33
+ self.payload = body
34
+ self.group = group
35
+ self.consumer = consumer
36
+ self.settled = False
37
+
38
+ @property
39
+ def body(self) -> Any:
40
+ return self.payload
41
+
42
+ @body.setter
43
+ def body(
44
+ self,
45
+ body: Any,
46
+ ) -> None:
47
+ self.payload = body
48
+
49
+ @property
50
+ def src(self) -> str:
51
+ return self.queue
52
+
53
+ def ack(self) -> None:
54
+ if self.group is None:
55
+ self.logger.warning(
56
+ 'Rejected ACK for history Event without a consumer group: queue=%s id=%s',
57
+ self.queue,
58
+ self.id,
59
+ )
60
+ raise RuntimeError('Events from read() have no consumer group to acknowledge')
61
+ if not self.settled:
62
+ self.logger.debug(
63
+ 'Attempting Event ACK: queue=%s group=%s consumer=%s id=%s',
64
+ self.queue,
65
+ self.group,
66
+ self.consumer,
67
+ self.id,
68
+ )
69
+ try:
70
+ result = self.connection.xack(
71
+ self.queue,
72
+ self.group,
73
+ self.id,
74
+ )
75
+ except Exception as error:
76
+ self.logger.error(
77
+ 'Event ACK failed: queue=%s group=%s consumer=%s id=%s error_type=%s',
78
+ self.queue,
79
+ self.group,
80
+ self.consumer,
81
+ self.id,
82
+ type(error).__name__,
83
+ )
84
+ raise
85
+ self.settled = True
86
+ if isinstance(result, int) and result > 0:
87
+ self.logger.debug(
88
+ 'Event ACK completed: queue=%s group=%s consumer=%s id=%s count=%s',
89
+ self.queue,
90
+ self.group,
91
+ self.consumer,
92
+ self.id,
93
+ result,
94
+ )
95
+ else:
96
+ self.logger.debug(
97
+ 'No Event ACK target: queue=%s group=%s consumer=%s id=%s count=%s',
98
+ self.queue,
99
+ self.group,
100
+ self.consumer,
101
+ self.id,
102
+ result,
103
+ )
104
+ else:
105
+ self.logger.debug(
106
+ 'Skipping ACK for settled Event: queue=%s group=%s consumer=%s id=%s',
107
+ self.queue,
108
+ self.group,
109
+ self.consumer,
110
+ self.id,
111
+ )
112
+
113
+ def nack(self) -> None:
114
+ if self.group is None:
115
+ self.logger.warning(
116
+ 'Rejected NACK for history Event without a consumer group: queue=%s id=%s',
117
+ self.queue,
118
+ self.id,
119
+ )
120
+ raise RuntimeError('Events from read() have no consumer group to requeue')
121
+ if not self.settled:
122
+ self.logger.debug(
123
+ 'Attempting Event requeue: queue=%s group=%s consumer=%s id=%s',
124
+ self.queue,
125
+ self.group,
126
+ self.consumer,
127
+ self.id,
128
+ )
129
+ # Requeue and acknowledge atomically on one stream (also valid in Cluster).
130
+ try:
131
+ result = self.connection.eval(
132
+ """
133
+ local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
134
+ if #pending == 0 then return 0 end
135
+ local entries = redis.call('XRANGE', KEYS[1], ARGV[2], ARGV[2])
136
+ if #entries == 0 then return redis.error_reply('Pending entry was deleted') end
137
+ redis.call('XADD', KEYS[1], '*', unpack(entries[1][2]))
138
+ return redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
139
+ """,
140
+ 1,
141
+ self.queue,
142
+ self.group,
143
+ self.id,
144
+ )
145
+ except Exception as error:
146
+ self.logger.error(
147
+ 'Event atomic requeue failed: queue=%s group=%s consumer=%s id=%s error_type=%s',
148
+ self.queue,
149
+ self.group,
150
+ self.consumer,
151
+ self.id,
152
+ type(error).__name__,
153
+ )
154
+ raise
155
+ self.settled = True
156
+ if isinstance(result, int) and result > 0:
157
+ self.logger.debug(
158
+ 'Event requeue and ACK completed: queue=%s group=%s consumer=%s id=%s count=%s',
159
+ self.queue,
160
+ self.group,
161
+ self.consumer,
162
+ self.id,
163
+ result,
164
+ )
165
+ else:
166
+ self.logger.debug(
167
+ 'No Event requeue target: queue=%s group=%s consumer=%s id=%s count=%s',
168
+ self.queue,
169
+ self.group,
170
+ self.consumer,
171
+ self.id,
172
+ result,
173
+ )
174
+ else:
175
+ self.logger.debug(
176
+ 'Skipping NACK for settled Event: queue=%s group=%s consumer=%s id=%s',
177
+ self.queue,
178
+ self.group,
179
+ self.consumer,
180
+ self.id,
181
+ )
182
+
183
+ def reject(self) -> None:
184
+ self.logger.debug(
185
+ 'Event reject requested (currently implemented as ACK): queue=%s group=%s consumer=%s id=%s',
186
+ self.queue,
187
+ self.group,
188
+ self.consumer,
189
+ self.id,
190
+ )
191
+ self.ack()
@@ -0,0 +1,284 @@
1
+ from logging import getLogger
2
+ from time import (
3
+ monotonic,
4
+ sleep,
5
+ )
6
+ from typing import (
7
+ Any,
8
+ Iterable,
9
+ Optional,
10
+ Union,
11
+ )
12
+ from uuid import uuid4
13
+
14
+ from redis import Redis
15
+ from redis.cluster import RedisCluster
16
+ from redis.exceptions import ResponseError
17
+
18
+ from DefStream.Broker import (
19
+ Poppable,
20
+ Readable,
21
+ Sendable,
22
+ )
23
+ from .Event import Event
24
+ from .Serializer import (
25
+ Decoder,
26
+ Encoder,
27
+ )
28
+ from .Subscription import Subscription
29
+
30
+ __all__ = (
31
+ 'Queue',
32
+ )
33
+
34
+
35
+ class Queue(
36
+ Sendable,
37
+ Poppable,
38
+ Readable,
39
+ ):
40
+ logger = getLogger('DefStream.Broker.Implements.Redis.Queue')
41
+
42
+ def __init__(
43
+ self,
44
+ connection: Union[Redis, RedisCluster],
45
+ encode: Encoder,
46
+ decode: Decoder,
47
+ name: str,
48
+ group: str = 'DefStream',
49
+ ):
50
+ self.subscription = Subscription(
51
+ name,
52
+ group,
53
+ )
54
+ self.connection = connection
55
+ self.encode = encode
56
+ self.decode = decode
57
+ self.queue = name
58
+ self.group = group
59
+ self.consumer = uuid4().hex
60
+
61
+ def __str__(self) -> str:
62
+ return self.queue
63
+
64
+ def create_group(self) -> None:
65
+ self.logger.debug(
66
+ 'Creating Consumer Group: queue=%s group=%s',
67
+ self.queue,
68
+ self.group,
69
+ )
70
+ try:
71
+ self.connection.xgroup_create(
72
+ self.queue,
73
+ self.group,
74
+ id='0-0',
75
+ mkstream=True,
76
+ )
77
+ except ResponseError as error:
78
+ if not str(error).startswith('BUSYGROUP '):
79
+ self.logger.error(
80
+ 'Consumer Group creation failed: queue=%s group=%s error_type=%s',
81
+ self.queue,
82
+ self.group,
83
+ type(error).__name__,
84
+ )
85
+ raise
86
+ self.logger.debug(
87
+ 'Reusing existing Consumer Group: queue=%s group=%s',
88
+ self.queue,
89
+ self.group,
90
+ )
91
+ return
92
+ except Exception as error:
93
+ self.logger.error(
94
+ 'Consumer Group creation failed: queue=%s group=%s error_type=%s',
95
+ self.queue,
96
+ self.group,
97
+ type(error).__name__,
98
+ )
99
+ raise
100
+ self.logger.info(
101
+ 'Consumer Group creation completed: queue=%s group=%s',
102
+ self.queue,
103
+ self.group,
104
+ )
105
+
106
+ def send(
107
+ self,
108
+ body: Any,
109
+ ) -> str:
110
+ self.logger.debug(
111
+ 'Publishing Redis Stream message: queue=%s',
112
+ self.queue,
113
+ )
114
+ try:
115
+ id = self.connection.xadd(
116
+ self.queue,
117
+ {'body': self.encode(body)},
118
+ )
119
+ message_id = id.decode('ascii')
120
+ except Exception as error:
121
+ self.logger.error(
122
+ 'Redis Stream publication or serialization failed: queue=%s error_type=%s',
123
+ self.queue,
124
+ type(error).__name__,
125
+ )
126
+ raise
127
+ self.logger.debug(
128
+ 'Redis Stream message published: queue=%s id=%s',
129
+ self.queue,
130
+ message_id,
131
+ )
132
+ return message_id
133
+
134
+ def event(
135
+ self,
136
+ id: bytes,
137
+ fields: dict,
138
+ group: Optional[str] = None,
139
+ consumer: Optional[str] = None,
140
+ ) -> Event:
141
+ try:
142
+ message_id = id.decode('ascii')
143
+ body = self.decode(fields[b'body'])
144
+ except Exception as error:
145
+ self.logger.error(
146
+ 'Redis message deserialization or Event conversion failed: queue=%s group=%s id=%s error_type=%s',
147
+ self.queue,
148
+ group,
149
+ id,
150
+ type(error).__name__,
151
+ )
152
+ raise
153
+ self.logger.debug(
154
+ 'Converting message to Event: queue=%s group=%s consumer=%s id=%s',
155
+ self.queue,
156
+ group,
157
+ consumer,
158
+ message_id,
159
+ )
160
+ return Event(
161
+ self.connection,
162
+ self.queue,
163
+ message_id,
164
+ body,
165
+ group,
166
+ consumer,
167
+ )
168
+
169
+ def pop(
170
+ self,
171
+ timeout: Optional[int] = None,
172
+ ) -> Optional[Event]:
173
+ if timeout is not None and timeout < 0:
174
+ raise ValueError('Redis pop timeout must be non-negative milliseconds')
175
+ self.logger.debug(
176
+ 'Starting group-based single-message reception: queue=%s group=%s consumer=%s timeout=%sms',
177
+ self.queue,
178
+ self.group,
179
+ self.consumer,
180
+ timeout,
181
+ )
182
+ self.create_group()
183
+ deadline = monotonic() + timeout / 1000 if timeout else None
184
+ while True:
185
+ try:
186
+ messages = self.connection.xreadgroup(
187
+ self.group,
188
+ self.consumer,
189
+ {self.queue: '>'},
190
+ count=1,
191
+ )
192
+ except Exception as error:
193
+ self.logger.error(
194
+ 'Group-based single-message reception failed: queue=%s group=%s error_type=%s',
195
+ self.queue,
196
+ self.group,
197
+ type(error).__name__,
198
+ )
199
+ raise
200
+ for _, entries in messages:
201
+ for id, fields in entries:
202
+ event = self.event(
203
+ id,
204
+ fields,
205
+ self.group,
206
+ self.consumer,
207
+ )
208
+ self.logger.debug(
209
+ 'Group message received: queue=%s group=%s id=%s',
210
+ self.queue,
211
+ self.group,
212
+ event.id,
213
+ )
214
+ return event
215
+ if deadline is not None and monotonic() >= deadline:
216
+ self.logger.debug(
217
+ 'Group-based single-message wait timed out: queue=%s group=%s timeout=%sms',
218
+ self.queue,
219
+ self.group,
220
+ timeout,
221
+ )
222
+ return None
223
+ sleep(0.01)
224
+
225
+ def read(
226
+ self,
227
+ offset: Union[int, str] = 0,
228
+ timeout: Optional[int] = None,
229
+ ) -> Iterable[Event]:
230
+ if timeout is not None and timeout < 0:
231
+ raise ValueError('Redis read timeout must be non-negative milliseconds')
232
+ cursor = str(offset)
233
+ deadline = monotonic() + timeout / 1000 if timeout else None
234
+ self.logger.debug(
235
+ 'Starting history read without a consumer group: queue=%s offset=%s timeout=%sms',
236
+ self.queue,
237
+ cursor,
238
+ timeout,
239
+ )
240
+ try:
241
+ while True:
242
+ if deadline is not None and monotonic() >= deadline:
243
+ self.logger.debug(
244
+ 'History read without a consumer group timed out: queue=%s cursor=%s timeout=%sms',
245
+ self.queue,
246
+ cursor,
247
+ timeout,
248
+ )
249
+ return
250
+ try:
251
+ messages = self.connection.xread(
252
+ {self.queue: cursor},
253
+ count=1,
254
+ )
255
+ except Exception as error:
256
+ self.logger.error(
257
+ 'History read without a consumer group failed: queue=%s cursor=%s error_type=%s',
258
+ self.queue,
259
+ cursor,
260
+ type(error).__name__,
261
+ )
262
+ raise
263
+ if not messages:
264
+ sleep(0.01)
265
+ continue
266
+ for _, entries in messages:
267
+ for id, fields in entries:
268
+ event = self.event(
269
+ id,
270
+ fields,
271
+ )
272
+ cursor = event.id
273
+ self.logger.debug(
274
+ 'History message received without a consumer group: queue=%s id=%s',
275
+ self.queue,
276
+ event.id,
277
+ )
278
+ yield event
279
+ finally:
280
+ self.logger.debug(
281
+ 'History read without a consumer group ended: queue=%s cursor=%s',
282
+ self.queue,
283
+ cursor,
284
+ )
@@ -0,0 +1,48 @@
1
+ from json import (
2
+ dumps,
3
+ loads,
4
+ )
5
+ from typing import (
6
+ Any,
7
+ Protocol,
8
+ )
9
+
10
+ __all__ = (
11
+ 'Encoder',
12
+ 'Decoder',
13
+ 'JavaScriptObjectNotationEncoder',
14
+ 'JavaScriptObjectNotationDecoder',
15
+ )
16
+
17
+
18
+ class Encoder(Protocol):
19
+ def __call__(
20
+ self,
21
+ body: Any,
22
+ ) -> bytes: ...
23
+
24
+
25
+ class Decoder(Protocol):
26
+ def __call__(
27
+ self,
28
+ body: bytes,
29
+ ) -> Any: ...
30
+
31
+
32
+ class JavaScriptObjectNotationEncoder(Encoder):
33
+ def __call__(
34
+ self,
35
+ body: Any,
36
+ ) -> bytes:
37
+ return dumps(
38
+ body,
39
+ ensure_ascii=False,
40
+ ).encode('utf-8')
41
+
42
+
43
+ class JavaScriptObjectNotationDecoder(Decoder):
44
+ def __call__(
45
+ self,
46
+ body: bytes,
47
+ ) -> Any:
48
+ return loads(body)
@@ -0,0 +1,46 @@
1
+ from collections.abc import Mapping
2
+ from dataclasses import dataclass
3
+ from typing import Any
4
+
5
+ __all__ = (
6
+ 'Subscription',
7
+ )
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class Subscription:
12
+ queue: str
13
+ group: str = 'DefStream'
14
+
15
+ def __post_init__(self) -> None:
16
+ for value in (
17
+ self.queue,
18
+ self.group,
19
+ ):
20
+ if not isinstance(
21
+ value,
22
+ str,
23
+ ) or not value:
24
+ raise ValueError('Redis queue and group must be non-empty strings')
25
+
26
+ @classmethod
27
+ def parse(
28
+ cls,
29
+ parameters: Any,
30
+ ) -> 'Subscription':
31
+ if isinstance(
32
+ parameters,
33
+ cls,
34
+ ):
35
+ return parameters
36
+ if isinstance(
37
+ parameters,
38
+ str,
39
+ ):
40
+ return cls(parameters)
41
+ if isinstance(
42
+ parameters,
43
+ Mapping,
44
+ ):
45
+ return cls(**dict(parameters))
46
+ raise TypeError('Redis subscriptions require a queue name, Subscription or mapping')
@@ -0,0 +1,29 @@
1
+ from DefStream.Broker import (
2
+ EventHandler,
3
+ Reject,
4
+ Requeue,
5
+ )
6
+ from .Configuration import (
7
+ Configuration,
8
+ ConnectionType,
9
+ )
10
+ from .Connection import (
11
+ Connection,
12
+ )
13
+ from .Consumer import Consumer
14
+ from .Event import Event
15
+ from .Queue import Queue
16
+ from .Subscription import Subscription
17
+
18
+ __all__ = (
19
+ 'Connection',
20
+ 'Configuration',
21
+ 'ConnectionType',
22
+ 'Consumer',
23
+ 'Event',
24
+ 'EventHandler',
25
+ 'Queue',
26
+ 'Subscription',
27
+ 'Reject',
28
+ 'Requeue',
29
+ )