DefStream-Redis 0.1.23__106-py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,59 @@
1
+ from enum import Enum
2
+ from typing import Optional
3
+
4
+ from DefStream.Broker import Configuration as BaseConfiguration
5
+ from .Serializer import (
6
+ Decoder,
7
+ Encoder,
8
+ JavaScriptObjectNotationDecoder,
9
+ JavaScriptObjectNotationEncoder,
10
+ )
11
+
12
+ __all__ = (
13
+ 'Configuration',
14
+ 'ConnectionType',
15
+ )
16
+
17
+
18
+ class ConnectionType(Enum):
19
+ Pool = 'pool'
20
+ Cluster = 'cluster'
21
+
22
+
23
+ class Configuration(BaseConfiguration):
24
+ def __init__(
25
+ self,
26
+ host: str = '127.0.0.1',
27
+ port: int = 6379,
28
+ username: Optional[str] = None,
29
+ password: Optional[str] = None,
30
+ secure: bool = False,
31
+ type: ConnectionType = ConnectionType.Pool,
32
+ max: int = 10,
33
+ database: int = 0,
34
+ timeout: float = 5,
35
+ encode: Optional[Encoder] = None,
36
+ decode: Optional[Decoder] = None,
37
+ ):
38
+ if not isinstance(
39
+ type,
40
+ ConnectionType,
41
+ ):
42
+ raise ValueError('Redis connection type must be a ConnectionType')
43
+ if max < 1:
44
+ raise ValueError('Redis max connections must be positive')
45
+ if database < 0 or (type == ConnectionType.Cluster and database != 0):
46
+ raise ValueError('Redis database must be non-negative; Cluster requires database 0')
47
+ if timeout <= 0:
48
+ raise ValueError('Redis connection timeout must be positive')
49
+ self.host = host
50
+ self.port = port
51
+ self.username = username
52
+ self.password = password
53
+ self.secure = secure
54
+ self.type = type
55
+ self.max = max
56
+ self.database = database
57
+ self.timeout = timeout
58
+ self.encode = encode if encode is not None else JavaScriptObjectNotationEncoder()
59
+ self.decode = decode if decode is not None else JavaScriptObjectNotationDecoder()
@@ -0,0 +1,265 @@
1
+ from logging import getLogger
2
+ from typing import (
3
+ Any,
4
+ Optional,
5
+ Union,
6
+ )
7
+
8
+ from redis import Redis
9
+ from redis.cluster import RedisCluster
10
+
11
+ from DefStream.Broker import (
12
+ Connection as BaseConnection,
13
+ GetQueue,
14
+ GetStream,
15
+ GetConsumer,
16
+ Declarable,
17
+ Emittable,
18
+ EventHandler,
19
+ )
20
+ from .Configuration import (
21
+ Configuration,
22
+ ConnectionType,
23
+ )
24
+ from .Consumer import Consumer
25
+ from .Queue import Queue
26
+ from .Subscription import Subscription
27
+
28
+ __all__ = (
29
+ 'Connection',
30
+ )
31
+
32
+
33
+ class Connection(
34
+ BaseConnection,
35
+ GetQueue,
36
+ GetStream,
37
+ GetConsumer,
38
+ Declarable,
39
+ Emittable,
40
+ ):
41
+ logger = getLogger('Broker.Redis.Connection')
42
+
43
+ def __init__(
44
+ self,
45
+ conf: Configuration,
46
+ ):
47
+ self.conf = conf
48
+ self.connection: Optional[Union[Redis, RedisCluster]] = None
49
+
50
+ def connect(self) -> None:
51
+ if self.connection is not None:
52
+ self.logger.debug('Reusing existing Redis connection')
53
+ return
54
+ connection_type = self.conf.type.value
55
+ self.logger.info(
56
+ 'Connecting to Redis: type=%s',
57
+ connection_type,
58
+ )
59
+ self.logger.debug(
60
+ 'Redis connection settings: host=%s port=%s database=%s TLS=%s max=%s timeout=%ss',
61
+ self.conf.host,
62
+ self.conf.port,
63
+ self.conf.database,
64
+ self.conf.secure,
65
+ self.conf.max,
66
+ self.conf.timeout,
67
+ )
68
+ parameters = {
69
+ 'host': self.conf.host,
70
+ 'port': self.conf.port,
71
+ 'username': self.conf.username,
72
+ 'password': self.conf.password,
73
+ 'ssl': self.conf.secure,
74
+ 'max_connections': self.conf.max,
75
+ 'socket_connect_timeout': self.conf.timeout,
76
+ 'socket_timeout': self.conf.timeout,
77
+ 'decode_responses': False,
78
+ }
79
+ connection = None
80
+ try:
81
+ if self.conf.type == ConnectionType.Cluster:
82
+ connection = RedisCluster(**parameters)
83
+ else:
84
+ connection = Redis(
85
+ db=self.conf.database,
86
+ **parameters,
87
+ )
88
+ connection.ping()
89
+ except Exception as error:
90
+ self.logger.error(
91
+ 'Redis connection or PING failed: type=%s error_type=%s',
92
+ connection_type,
93
+ type(error).__name__,
94
+ )
95
+ if connection is not None:
96
+ connection.close()
97
+ raise
98
+ self.connection = connection
99
+ self.logger.info(
100
+ 'Redis connection verified with PING: type=%s',
101
+ connection_type,
102
+ )
103
+
104
+ def connected(self) -> Union[Redis, RedisCluster]:
105
+ if self.connection is None:
106
+ raise RuntimeError('Connect the Redis broker before use')
107
+ return self.connection
108
+
109
+ def declare(
110
+ self,
111
+ parameters: Any,
112
+ ) -> None:
113
+ subscription = Subscription.parse(parameters)
114
+ self.logger.debug(
115
+ 'Delegating stream declaration: queue=%s group=%s',
116
+ subscription.queue,
117
+ subscription.group,
118
+ )
119
+ self.create_queue(
120
+ subscription.queue,
121
+ group=subscription.group,
122
+ )
123
+
124
+ def emit(
125
+ self,
126
+ body: Any,
127
+ parameters: Any,
128
+ ) -> str:
129
+ subscription = Subscription.parse(parameters)
130
+ self.logger.debug(
131
+ 'Delegating message publication: queue=%s group=%s',
132
+ subscription.queue,
133
+ subscription.group,
134
+ )
135
+ return self.queue(
136
+ subscription.queue,
137
+ group=subscription.group,
138
+ ).send(body)
139
+
140
+ def queue(
141
+ self,
142
+ name: str,
143
+ group: str = 'DefStream',
144
+ ) -> Queue:
145
+ self.logger.debug(
146
+ 'Creating Queue: queue=%s group=%s',
147
+ name,
148
+ group,
149
+ )
150
+ return Queue(
151
+ self.connected(),
152
+ self.conf.encode,
153
+ self.conf.decode,
154
+ name,
155
+ group,
156
+ )
157
+
158
+ def stream(
159
+ self,
160
+ name: str,
161
+ group: str = 'DefStream',
162
+ ) -> Queue:
163
+ return self.queue(
164
+ name,
165
+ group,
166
+ )
167
+
168
+ def consumer(
169
+ self,
170
+ handler: Optional[EventHandler] = None,
171
+ qos: int = 1,
172
+ reclaim_idle: Optional[int] = 60000,
173
+ ) -> Consumer:
174
+ self.logger.debug(
175
+ 'Creating Consumer: qos=%s reclaim_idle=%sms',
176
+ qos,
177
+ reclaim_idle,
178
+ )
179
+ return Consumer(
180
+ self.connected(),
181
+ self.conf.encode,
182
+ self.conf.decode,
183
+ handler,
184
+ qos,
185
+ reclaim_idle,
186
+ )
187
+
188
+ def create_queue(
189
+ self,
190
+ name: str,
191
+ group: str = 'DefStream',
192
+ ) -> None:
193
+ self.logger.debug(
194
+ 'Delegating Consumer Group creation: queue=%s group=%s',
195
+ name,
196
+ group,
197
+ )
198
+ self.queue(
199
+ name,
200
+ group,
201
+ ).create_group()
202
+
203
+ def delete_queue(
204
+ self,
205
+ name: Union[str, Queue],
206
+ ) -> int:
207
+ queue = str(name)
208
+ self.logger.debug(
209
+ 'Deleting Redis stream: queue=%s',
210
+ queue,
211
+ )
212
+ try:
213
+ deleted = self.connected().delete(queue)
214
+ except Exception as error:
215
+ self.logger.error(
216
+ 'Redis stream deletion failed: queue=%s error_type=%s',
217
+ queue,
218
+ type(error).__name__,
219
+ )
220
+ raise
221
+ if isinstance(deleted, int) and deleted > 0:
222
+ self.logger.info(
223
+ 'Redis stream deletion completed: queue=%s deleted=%s',
224
+ queue,
225
+ deleted,
226
+ )
227
+ else:
228
+ self.logger.info(
229
+ 'No Redis stream to delete: queue=%s deleted=%s',
230
+ queue,
231
+ deleted,
232
+ )
233
+ return deleted
234
+
235
+ def create_stream(
236
+ self,
237
+ name: str,
238
+ group: str = 'DefStream',
239
+ ) -> None:
240
+ self.create_queue(
241
+ name,
242
+ group,
243
+ )
244
+
245
+ def delete_stream(
246
+ self,
247
+ name: Union[str, Queue],
248
+ ) -> int:
249
+ return self.delete_queue(name)
250
+
251
+ def close(self) -> None:
252
+ if self.connection is not None:
253
+ self.logger.info('Closing Redis connection')
254
+ try:
255
+ self.connection.close()
256
+ except Exception as error:
257
+ self.logger.error(
258
+ 'Redis connection close failed: error_type=%s',
259
+ type(error).__name__,
260
+ )
261
+ raise
262
+ self.connection = None
263
+ self.logger.info('Redis connection closed')
264
+ else:
265
+ self.logger.debug('No Redis connection to close')
@@ -0,0 +1,368 @@
1
+ from collections import deque
2
+ from logging import getLogger
3
+ from queue import (
4
+ Empty,
5
+ SimpleQueue,
6
+ )
7
+ from threading import Event as Wakeup
8
+ from typing import (
9
+ Any,
10
+ Callable,
11
+ Dict,
12
+ Optional,
13
+ Tuple,
14
+ Union,
15
+ )
16
+ from uuid import uuid4
17
+
18
+ from redis import Redis
19
+ from redis.cluster import RedisCluster
20
+
21
+ from DefStream.Broker import (
22
+ Consumer as BaseConsumer,
23
+ Event as BaseEvent,
24
+ EventHandler,
25
+ )
26
+ from .Event import (
27
+ Event,
28
+ Requeue,
29
+ )
30
+ from .Queue import Queue
31
+ from .Serializer import (
32
+ Decoder,
33
+ Encoder,
34
+ )
35
+ from .Subscription import Subscription
36
+ from DefStream.Broker.Event import EventErrorPolicy
37
+
38
+ __all__ = (
39
+ 'Consumer',
40
+ )
41
+
42
+
43
+ class Consumer(BaseConsumer):
44
+ logger = getLogger('Broker.Redis.Consumer')
45
+
46
+ def __init__(
47
+ self,
48
+ connection: Union[Redis, RedisCluster],
49
+ encode: Encoder,
50
+ decode: Decoder,
51
+ handler: Optional[EventHandler] = None,
52
+ qos: int = 1,
53
+ reclaim_idle: Optional[int] = 60000,
54
+ ):
55
+ if isinstance(
56
+ qos,
57
+ bool,
58
+ ) or not isinstance(
59
+ qos,
60
+ int,
61
+ ) or qos < 1:
62
+ raise ValueError('Redis qos must be a positive integer')
63
+ if reclaim_idle is not None and reclaim_idle < 1:
64
+ raise ValueError('Redis reclaim_idle must be positive milliseconds or None')
65
+ self.connection = connection
66
+ self.encode = encode
67
+ self.decode = decode
68
+ self.handler = handler
69
+ self.qos = qos
70
+ self.reclaim_idle = reclaim_idle
71
+ self.name = uuid4().hex
72
+ self.subscriptions: Dict[str, Tuple[Queue, EventHandler]] = {}
73
+ self.rotation = deque()
74
+ self.pending: Dict[Tuple[str, str], Event] = {}
75
+ self.cursors: Dict[str, str] = {}
76
+ self.callbacks: SimpleQueue[Callable[[], None]] = SimpleQueue()
77
+ self.wakeup = Wakeup()
78
+ self.stopped = False
79
+ self.closed = False
80
+ self.logger.debug(
81
+ 'Consumer created: consumer=%s qos=%s reclaim_idle=%sms',
82
+ self.name,
83
+ self.qos,
84
+ self.reclaim_idle,
85
+ )
86
+
87
+ def subs(
88
+ self,
89
+ queue: Any,
90
+ handler: Optional[EventHandler] = None,
91
+ ) -> None:
92
+ if self.closed:
93
+ raise RuntimeError('Redis consumer is closed')
94
+ subscription = Subscription.parse(queue)
95
+ handler = handler if handler is not None else self.handler
96
+ if handler is None:
97
+ raise ValueError('subs requires a handler or a default handler')
98
+ if subscription.queue in self.subscriptions:
99
+ raise ValueError('A Run is already subscribed to queue {}'.format(subscription.queue))
100
+ self.logger.debug(
101
+ 'Preparing Consumer subscription: queue=%s group=%s consumer=%s',
102
+ subscription.queue,
103
+ subscription.group,
104
+ self.name,
105
+ )
106
+ stream = Queue(
107
+ self.connection,
108
+ self.encode,
109
+ self.decode,
110
+ subscription.queue,
111
+ subscription.group,
112
+ )
113
+ stream.create_group()
114
+ self.subscriptions[subscription.queue] = (
115
+ stream,
116
+ handler,
117
+ )
118
+ self.cursors[subscription.queue] = '0-0'
119
+ self.rotation.append(subscription.queue)
120
+ self.logger.info(
121
+ 'Consumer subscribed: queue=%s group=%s consumer=%s qos=%s reclaim_idle=%sms',
122
+ subscription.queue,
123
+ subscription.group,
124
+ self.name,
125
+ self.qos,
126
+ self.reclaim_idle,
127
+ )
128
+
129
+ def call(
130
+ self,
131
+ callback: Callable[[], None],
132
+ ) -> None:
133
+ if self.closed:
134
+ raise RuntimeError('Redis consumer is closed')
135
+ self.callbacks.put(callback)
136
+ self.wakeup.set()
137
+ self.logger.debug(
138
+ 'Consumer callback scheduled: consumer=%s',
139
+ self.name,
140
+ )
141
+
142
+ def fail(
143
+ self,
144
+ event: BaseEvent,
145
+ policy: Optional[EventErrorPolicy] = None,
146
+ ) -> None:
147
+ self.logger.debug(
148
+ 'Applying message failure policy: consumer=%s policy=%s',
149
+ self.name,
150
+ type(policy).__name__ if policy is not None else 'Requeue',
151
+ )
152
+ (policy if policy is not None else Requeue())(event)
153
+
154
+ def drain(self) -> None:
155
+ while True:
156
+ try:
157
+ callback = self.callbacks.get_nowait()
158
+ except Empty:
159
+ break
160
+ self.logger.debug(
161
+ 'Executing Consumer callback: consumer=%s',
162
+ self.name,
163
+ )
164
+ try:
165
+ callback()
166
+ except Exception as error:
167
+ self.logger.error(
168
+ 'Consumer callback failed: consumer=%s error_type=%s',
169
+ self.name,
170
+ type(error).__name__,
171
+ )
172
+ raise
173
+ pending_count = len(self.pending)
174
+ self.pending = {
175
+ key: event
176
+ for key, event in self.pending.items()
177
+ if not event.settled
178
+ }
179
+ settled_count = pending_count - len(self.pending)
180
+ if settled_count:
181
+ self.logger.debug(
182
+ 'Settled Events removed: consumer=%s removed=%s pending=%s',
183
+ self.name,
184
+ settled_count,
185
+ len(self.pending),
186
+ )
187
+
188
+ def deliver(
189
+ self,
190
+ stream: Queue,
191
+ handler: EventHandler,
192
+ entries: list,
193
+ ) -> None:
194
+ for id, fields in entries:
195
+ key = (
196
+ stream.queue,
197
+ id.decode('ascii'),
198
+ )
199
+ if key in self.pending:
200
+ self.logger.debug(
201
+ 'Skipping delivery of already pending message: queue=%s group=%s id=%s',
202
+ stream.queue,
203
+ stream.group,
204
+ key[1],
205
+ )
206
+ continue
207
+ event = stream.event(
208
+ id,
209
+ fields,
210
+ stream.group,
211
+ self.name,
212
+ )
213
+ self.pending[key] = event
214
+ self.logger.debug(
215
+ 'Delivering message: queue=%s group=%s consumer=%s id=%s pending=%s',
216
+ stream.queue,
217
+ stream.group,
218
+ self.name,
219
+ event.id,
220
+ len(self.pending),
221
+ )
222
+ try:
223
+ handler(event)
224
+ except Exception as error:
225
+ self.logger.error(
226
+ 'Message handler dispatch failed: queue=%s group=%s consumer=%s id=%s error_type=%s',
227
+ stream.queue,
228
+ stream.group,
229
+ self.name,
230
+ event.id,
231
+ type(error).__name__,
232
+ )
233
+ raise
234
+ self.drain()
235
+
236
+ def run(self) -> None:
237
+ if self.closed:
238
+ raise RuntimeError('Redis consumer is closed')
239
+ if not self.subscriptions:
240
+ raise RuntimeError('Subscribe before running the Redis consumer')
241
+ self.logger.info(
242
+ 'Starting Consumer consumption loop: consumer=%s subscriptions=%s qos=%s reclaim_idle=%sms',
243
+ self.name,
244
+ len(self.subscriptions),
245
+ self.qos,
246
+ self.reclaim_idle,
247
+ )
248
+ qos_limited = False
249
+ while not self.stopped:
250
+ self.wakeup.clear()
251
+ self.drain()
252
+ for _ in range(len(self.rotation)):
253
+ available = self.qos - len(self.pending)
254
+ if available <= 0:
255
+ if not qos_limited:
256
+ self.logger.debug(
257
+ 'Deferring reads at QoS limit: consumer=%s qos=%s pending=%s',
258
+ self.name,
259
+ self.qos,
260
+ len(self.pending),
261
+ )
262
+ qos_limited = True
263
+ break
264
+ qos_limited = False
265
+ if self.stopped:
266
+ break
267
+ name = self.rotation.popleft()
268
+ self.rotation.append(name)
269
+ stream, handler = self.subscriptions[name]
270
+ if self.reclaim_idle is not None:
271
+ old_cursor = self.cursors[stream.queue]
272
+ try:
273
+ claimed = self.connection.xautoclaim(
274
+ stream.queue,
275
+ stream.group,
276
+ self.name,
277
+ self.reclaim_idle,
278
+ start_id=old_cursor,
279
+ count=available,
280
+ )
281
+ except Exception as error:
282
+ self.logger.error(
283
+ 'Pending message reclaim failed: queue=%s group=%s consumer=%s error_type=%s',
284
+ stream.queue,
285
+ stream.group,
286
+ self.name,
287
+ type(error).__name__,
288
+ )
289
+ raise
290
+ self.cursors[stream.queue] = claimed[0].decode('ascii')
291
+ if claimed[1] or self.cursors[stream.queue] != old_cursor:
292
+ self.logger.debug(
293
+ 'Pending message reclaim result: queue=%s group=%s consumer=%s count=%s cursor=%s',
294
+ stream.queue,
295
+ stream.group,
296
+ self.name,
297
+ len(claimed[1]),
298
+ self.cursors[stream.queue],
299
+ )
300
+ self.deliver(
301
+ stream,
302
+ handler,
303
+ claimed[1],
304
+ )
305
+ available = self.qos - len(self.pending)
306
+ if available <= 0:
307
+ if not qos_limited:
308
+ self.logger.debug(
309
+ 'Deferring reads at QoS limit: consumer=%s qos=%s pending=%s',
310
+ self.name,
311
+ self.qos,
312
+ len(self.pending),
313
+ )
314
+ qos_limited = True
315
+ continue
316
+ qos_limited = False
317
+ if self.stopped:
318
+ continue
319
+ try:
320
+ messages = self.connection.xreadgroup(
321
+ stream.group,
322
+ self.name,
323
+ {stream.queue: '>'},
324
+ count=available,
325
+ )
326
+ except Exception as error:
327
+ self.logger.error(
328
+ 'New message read failed: queue=%s group=%s consumer=%s error_type=%s',
329
+ stream.queue,
330
+ stream.group,
331
+ self.name,
332
+ type(error).__name__,
333
+ )
334
+ raise
335
+ for _, entries in messages:
336
+ if not isinstance(entries, list):
337
+ raise TypeError('Redis Stream read returned invalid entries')
338
+ self.deliver(
339
+ stream,
340
+ handler,
341
+ entries,
342
+ )
343
+ self.wakeup.wait(0.05)
344
+ self.logger.info(
345
+ 'Consumer consumption loop ended: consumer=%s pending=%s',
346
+ self.name,
347
+ len(self.pending),
348
+ )
349
+
350
+ def stop(self) -> None:
351
+ if not self.stopped:
352
+ self.logger.info(
353
+ 'Consumer stop requested: consumer=%s pending=%s',
354
+ self.name,
355
+ len(self.pending),
356
+ )
357
+ self.stopped = True
358
+ self.wakeup.set()
359
+
360
+ def close(self) -> None:
361
+ if not self.closed:
362
+ self.logger.info(
363
+ 'Consumer closed: consumer=%s pending=%s',
364
+ self.name,
365
+ len(self.pending),
366
+ )
367
+ self.stop()
368
+ self.closed = True