DefStream-Redis 0.1.5__13-py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of DefStream-Redis might be problematic. Click here for more details.

@@ -0,0 +1,59 @@
1
+ from enum import Enum
2
+ from typing import Optional
3
+
4
+ from DefStream.Broker import Configuration as BaseConfiguration
5
+ from .Serializer import (
6
+ Decoder,
7
+ Encoder,
8
+ JavaScriptObjectNotationDecoder,
9
+ JavaScriptObjectNotationEncoder,
10
+ )
11
+
12
+ __all__ = (
13
+ 'Configuration',
14
+ 'ConnectionType',
15
+ )
16
+
17
+
18
+ class ConnectionType(Enum):
19
+ Pool = 'pool'
20
+ Cluster = 'cluster'
21
+
22
+
23
+ class Configuration(BaseConfiguration):
24
+ def __init__(
25
+ self,
26
+ host: str = '127.0.0.1',
27
+ port: int = 6379,
28
+ username: Optional[str] = None,
29
+ password: Optional[str] = None,
30
+ secure: bool = False,
31
+ type: ConnectionType = ConnectionType.Pool,
32
+ max: int = 10,
33
+ database: int = 0,
34
+ timeout: float = 5,
35
+ encode: Optional[Encoder] = None,
36
+ decode: Optional[Decoder] = None,
37
+ ):
38
+ if not isinstance(
39
+ type,
40
+ ConnectionType,
41
+ ):
42
+ raise ValueError('Redis connection type must be a ConnectionType')
43
+ if max < 1:
44
+ raise ValueError('Redis max connections must be positive')
45
+ if database < 0 or (type == ConnectionType.Cluster and database != 0):
46
+ raise ValueError('Redis database must be non-negative; Cluster requires database 0')
47
+ if timeout <= 0:
48
+ raise ValueError('Redis connection timeout must be positive')
49
+ self.host = host
50
+ self.port = port
51
+ self.username = username
52
+ self.password = password
53
+ self.secure = secure
54
+ self.type = type
55
+ self.max = max
56
+ self.database = database
57
+ self.timeout = timeout
58
+ self.encode = encode if encode is not None else JavaScriptObjectNotationEncoder()
59
+ self.decode = decode if decode is not None else JavaScriptObjectNotationDecoder()
@@ -0,0 +1,265 @@
1
+ from logging import getLogger
2
+ from typing import (
3
+ Any,
4
+ Optional,
5
+ Union,
6
+ )
7
+
8
+ from redis import Redis
9
+ from redis.cluster import RedisCluster
10
+
11
+ from DefStream.Broker import (
12
+ Connection as BaseConnection,
13
+ GetQueue,
14
+ GetStream,
15
+ GetConsumer,
16
+ Declarable,
17
+ Emittable,
18
+ EventHandler,
19
+ )
20
+ from .Configuration import (
21
+ Configuration,
22
+ ConnectionType,
23
+ )
24
+ from .Consumer import Consumer
25
+ from .Queue import Queue
26
+ from .Subscription import Subscription
27
+
28
+ __all__ = (
29
+ 'Connection',
30
+ )
31
+
32
+
33
+ class Connection(
34
+ BaseConnection,
35
+ GetQueue,
36
+ GetStream,
37
+ GetConsumer,
38
+ Declarable,
39
+ Emittable,
40
+ ):
41
+ logger = getLogger('DefStream.Broker.Implements.Redis.Connection')
42
+
43
+ def __init__(
44
+ self,
45
+ conf: Configuration,
46
+ ):
47
+ self.conf = conf
48
+ self.connection: Optional[Union[Redis, RedisCluster]] = None
49
+
50
+ def connect(self) -> None:
51
+ if self.connection is not None:
52
+ self.logger.debug('Reusing existing Redis connection')
53
+ return
54
+ connection_type = self.conf.type.value
55
+ self.logger.info(
56
+ 'Connecting to Redis: type=%s',
57
+ connection_type,
58
+ )
59
+ self.logger.debug(
60
+ 'Redis connection settings: host=%s port=%s database=%s TLS=%s max=%s timeout=%ss',
61
+ self.conf.host,
62
+ self.conf.port,
63
+ self.conf.database,
64
+ self.conf.secure,
65
+ self.conf.max,
66
+ self.conf.timeout,
67
+ )
68
+ parameters = {
69
+ 'host': self.conf.host,
70
+ 'port': self.conf.port,
71
+ 'username': self.conf.username,
72
+ 'password': self.conf.password,
73
+ 'ssl': self.conf.secure,
74
+ 'max_connections': self.conf.max,
75
+ 'socket_connect_timeout': self.conf.timeout,
76
+ 'socket_timeout': self.conf.timeout,
77
+ 'decode_responses': False,
78
+ }
79
+ connection = None
80
+ try:
81
+ if self.conf.type == ConnectionType.Cluster:
82
+ connection = RedisCluster(**parameters)
83
+ else:
84
+ connection = Redis(
85
+ db=self.conf.database,
86
+ **parameters,
87
+ )
88
+ connection.ping()
89
+ except Exception as error:
90
+ self.logger.error(
91
+ 'Redis connection or PING failed: type=%s error_type=%s',
92
+ connection_type,
93
+ type(error).__name__,
94
+ )
95
+ if connection is not None:
96
+ connection.close()
97
+ raise
98
+ self.connection = connection
99
+ self.logger.info(
100
+ 'Redis connection verified with PING: type=%s',
101
+ connection_type,
102
+ )
103
+
104
+ def connected(self) -> Union[Redis, RedisCluster]:
105
+ if self.connection is None:
106
+ raise RuntimeError('Connect the Redis broker before use')
107
+ return self.connection
108
+
109
+ def declare(
110
+ self,
111
+ parameters: Any,
112
+ ) -> None:
113
+ subscription = Subscription.parse(parameters)
114
+ self.logger.debug(
115
+ 'Delegating stream declaration: queue=%s group=%s',
116
+ subscription.queue,
117
+ subscription.group,
118
+ )
119
+ self.create_queue(
120
+ subscription.queue,
121
+ group=subscription.group,
122
+ )
123
+
124
+ def emit(
125
+ self,
126
+ body: Any,
127
+ parameters: Any,
128
+ ) -> str:
129
+ subscription = Subscription.parse(parameters)
130
+ self.logger.debug(
131
+ 'Delegating message publication: queue=%s group=%s',
132
+ subscription.queue,
133
+ subscription.group,
134
+ )
135
+ return self.queue(
136
+ subscription.queue,
137
+ group=subscription.group,
138
+ ).send(body)
139
+
140
+ def queue(
141
+ self,
142
+ name: str,
143
+ group: str = 'DefStream',
144
+ ) -> Queue:
145
+ self.logger.debug(
146
+ 'Creating Queue: queue=%s group=%s',
147
+ name,
148
+ group,
149
+ )
150
+ return Queue(
151
+ self.connected(),
152
+ self.conf.encode,
153
+ self.conf.decode,
154
+ name,
155
+ group,
156
+ )
157
+
158
+ def stream(
159
+ self,
160
+ name: str,
161
+ group: str = 'DefStream',
162
+ ) -> Queue:
163
+ return self.queue(
164
+ name,
165
+ group,
166
+ )
167
+
168
+ def consumer(
169
+ self,
170
+ handler: Optional[EventHandler] = None,
171
+ qos: int = 1,
172
+ reclaim_idle: Optional[int] = 60000,
173
+ ) -> Consumer:
174
+ self.logger.debug(
175
+ 'Creating Consumer: qos=%s reclaim_idle=%sms',
176
+ qos,
177
+ reclaim_idle,
178
+ )
179
+ return Consumer(
180
+ self.connected(),
181
+ self.conf.encode,
182
+ self.conf.decode,
183
+ handler,
184
+ qos,
185
+ reclaim_idle,
186
+ )
187
+
188
+ def create_queue(
189
+ self,
190
+ name: str,
191
+ group: str = 'DefStream',
192
+ ) -> None:
193
+ self.logger.debug(
194
+ 'Delegating Consumer Group creation: queue=%s group=%s',
195
+ name,
196
+ group,
197
+ )
198
+ self.queue(
199
+ name,
200
+ group,
201
+ ).create_group()
202
+
203
+ def delete_queue(
204
+ self,
205
+ name: Union[str, Queue],
206
+ ) -> int:
207
+ queue = str(name)
208
+ self.logger.debug(
209
+ 'Deleting Redis stream: queue=%s',
210
+ queue,
211
+ )
212
+ try:
213
+ deleted = self.connected().delete(queue)
214
+ except Exception as error:
215
+ self.logger.error(
216
+ 'Redis stream deletion failed: queue=%s error_type=%s',
217
+ queue,
218
+ type(error).__name__,
219
+ )
220
+ raise
221
+ if isinstance(deleted, int) and deleted > 0:
222
+ self.logger.info(
223
+ 'Redis stream deletion completed: queue=%s deleted=%s',
224
+ queue,
225
+ deleted,
226
+ )
227
+ else:
228
+ self.logger.info(
229
+ 'No Redis stream to delete: queue=%s deleted=%s',
230
+ queue,
231
+ deleted,
232
+ )
233
+ return deleted
234
+
235
+ def create_stream(
236
+ self,
237
+ name: str,
238
+ group: str = 'DefStream',
239
+ ) -> None:
240
+ self.create_queue(
241
+ name,
242
+ group,
243
+ )
244
+
245
+ def delete_stream(
246
+ self,
247
+ name: Union[str, Queue],
248
+ ) -> int:
249
+ return self.delete_queue(name)
250
+
251
+ def close(self) -> None:
252
+ if self.connection is not None:
253
+ self.logger.info('Closing Redis connection')
254
+ try:
255
+ self.connection.close()
256
+ except Exception as error:
257
+ self.logger.error(
258
+ 'Redis connection close failed: error_type=%s',
259
+ type(error).__name__,
260
+ )
261
+ raise
262
+ self.connection = None
263
+ self.logger.info('Redis connection closed')
264
+ else:
265
+ self.logger.debug('No Redis connection to close')
@@ -0,0 +1,364 @@
1
+ from collections import deque
2
+ from logging import getLogger
3
+ from queue import (
4
+ Empty,
5
+ SimpleQueue,
6
+ )
7
+ from threading import Event as Wakeup
8
+ from typing import (
9
+ Any,
10
+ Callable,
11
+ Dict,
12
+ Optional,
13
+ Tuple,
14
+ Union,
15
+ )
16
+ from uuid import uuid4
17
+
18
+ from redis import Redis
19
+ from redis.cluster import RedisCluster
20
+
21
+ from DefStream.Broker import (
22
+ Consumer as BaseConsumer,
23
+ Event as BaseEvent,
24
+ EventError,
25
+ EventHandler,
26
+ Requeue,
27
+ )
28
+ from .Event import Event
29
+ from .Queue import Queue
30
+ from .Serializer import (
31
+ Decoder,
32
+ Encoder,
33
+ )
34
+ from .Subscription import Subscription
35
+
36
+ __all__ = (
37
+ 'Consumer',
38
+ )
39
+
40
+
41
+ class Consumer(BaseConsumer):
42
+ logger = getLogger('DefStream.Broker.Implements.Redis.Consumer')
43
+
44
+ def __init__(
45
+ self,
46
+ connection: Union[Redis, RedisCluster],
47
+ encode: Encoder,
48
+ decode: Decoder,
49
+ handler: Optional[EventHandler] = None,
50
+ qos: int = 1,
51
+ reclaim_idle: Optional[int] = 60000,
52
+ ):
53
+ if isinstance(
54
+ qos,
55
+ bool,
56
+ ) or not isinstance(
57
+ qos,
58
+ int,
59
+ ) or qos < 1:
60
+ raise ValueError('Redis qos must be a positive integer')
61
+ if reclaim_idle is not None and reclaim_idle < 1:
62
+ raise ValueError('Redis reclaim_idle must be positive milliseconds or None')
63
+ self.connection = connection
64
+ self.encode = encode
65
+ self.decode = decode
66
+ self.handler = handler
67
+ self.qos = qos
68
+ self.reclaim_idle = reclaim_idle
69
+ self.name = uuid4().hex
70
+ self.subscriptions: Dict[str, Tuple[Queue, EventHandler]] = {}
71
+ self.rotation = deque()
72
+ self.pending: Dict[Tuple[str, str], Event] = {}
73
+ self.cursors: Dict[str, str] = {}
74
+ self.callbacks: SimpleQueue[Callable[[], None]] = SimpleQueue()
75
+ self.wakeup = Wakeup()
76
+ self.stopped = False
77
+ self.closed = False
78
+ self.logger.debug(
79
+ 'Consumer created: consumer=%s qos=%s reclaim_idle=%sms',
80
+ self.name,
81
+ self.qos,
82
+ self.reclaim_idle,
83
+ )
84
+
85
+ def subs(
86
+ self,
87
+ queue: Any,
88
+ handler: Optional[EventHandler] = None,
89
+ ) -> None:
90
+ if self.closed:
91
+ raise RuntimeError('Redis consumer is closed')
92
+ subscription = Subscription.parse(queue)
93
+ handler = handler if handler is not None else self.handler
94
+ if handler is None:
95
+ raise ValueError('subs requires a handler or a default handler')
96
+ if subscription.queue in self.subscriptions:
97
+ raise ValueError('A Run is already subscribed to queue {}'.format(subscription.queue))
98
+ self.logger.debug(
99
+ 'Preparing Consumer subscription: queue=%s group=%s consumer=%s',
100
+ subscription.queue,
101
+ subscription.group,
102
+ self.name,
103
+ )
104
+ stream = Queue(
105
+ self.connection,
106
+ self.encode,
107
+ self.decode,
108
+ subscription.queue,
109
+ subscription.group,
110
+ )
111
+ stream.create_group()
112
+ self.subscriptions[subscription.queue] = (
113
+ stream,
114
+ handler,
115
+ )
116
+ self.cursors[subscription.queue] = '0-0'
117
+ self.rotation.append(subscription.queue)
118
+ self.logger.info(
119
+ 'Consumer subscribed: queue=%s group=%s consumer=%s qos=%s reclaim_idle=%sms',
120
+ subscription.queue,
121
+ subscription.group,
122
+ self.name,
123
+ self.qos,
124
+ self.reclaim_idle,
125
+ )
126
+
127
+ def call(
128
+ self,
129
+ callback: Callable[[], None],
130
+ ) -> None:
131
+ if self.closed:
132
+ raise RuntimeError('Redis consumer is closed')
133
+ self.callbacks.put(callback)
134
+ self.wakeup.set()
135
+ self.logger.debug(
136
+ 'Consumer callback scheduled: consumer=%s',
137
+ self.name,
138
+ )
139
+
140
+ def fail(
141
+ self,
142
+ event: BaseEvent,
143
+ policy: Optional[EventError] = None,
144
+ ) -> None:
145
+ self.logger.debug(
146
+ 'Applying message failure policy: consumer=%s policy=%s',
147
+ self.name,
148
+ type(policy).__name__ if policy is not None else 'Requeue',
149
+ )
150
+ (policy if policy is not None else Requeue())(event)
151
+
152
+ def drain(self) -> None:
153
+ while True:
154
+ try:
155
+ callback = self.callbacks.get_nowait()
156
+ except Empty:
157
+ break
158
+ self.logger.debug(
159
+ 'Executing Consumer callback: consumer=%s',
160
+ self.name,
161
+ )
162
+ try:
163
+ callback()
164
+ except Exception as error:
165
+ self.logger.error(
166
+ 'Consumer callback failed: consumer=%s error_type=%s',
167
+ self.name,
168
+ type(error).__name__,
169
+ )
170
+ raise
171
+ pending_count = len(self.pending)
172
+ self.pending = {
173
+ key: event
174
+ for key, event in self.pending.items()
175
+ if not event.settled
176
+ }
177
+ settled_count = pending_count - len(self.pending)
178
+ if settled_count:
179
+ self.logger.debug(
180
+ 'Settled Events removed: consumer=%s removed=%s pending=%s',
181
+ self.name,
182
+ settled_count,
183
+ len(self.pending),
184
+ )
185
+
186
+ def deliver(
187
+ self,
188
+ stream: Queue,
189
+ handler: EventHandler,
190
+ entries: list,
191
+ ) -> None:
192
+ for id, fields in entries:
193
+ key = (
194
+ stream.queue,
195
+ id.decode('ascii'),
196
+ )
197
+ if key in self.pending:
198
+ self.logger.debug(
199
+ 'Skipping delivery of already pending message: queue=%s group=%s id=%s',
200
+ stream.queue,
201
+ stream.group,
202
+ key[1],
203
+ )
204
+ continue
205
+ event = stream.event(
206
+ id,
207
+ fields,
208
+ stream.group,
209
+ self.name,
210
+ )
211
+ self.pending[key] = event
212
+ self.logger.debug(
213
+ 'Delivering message: queue=%s group=%s consumer=%s id=%s pending=%s',
214
+ stream.queue,
215
+ stream.group,
216
+ self.name,
217
+ event.id,
218
+ len(self.pending),
219
+ )
220
+ try:
221
+ handler(event)
222
+ except Exception as error:
223
+ self.logger.error(
224
+ 'Message handler dispatch failed: queue=%s group=%s consumer=%s id=%s error_type=%s',
225
+ stream.queue,
226
+ stream.group,
227
+ self.name,
228
+ event.id,
229
+ type(error).__name__,
230
+ )
231
+ raise
232
+ self.drain()
233
+
234
+ def run(self) -> None:
235
+ if self.closed:
236
+ raise RuntimeError('Redis consumer is closed')
237
+ if not self.subscriptions:
238
+ raise RuntimeError('Subscribe before running the Redis consumer')
239
+ self.logger.info(
240
+ 'Starting Consumer consumption loop: consumer=%s subscriptions=%s qos=%s reclaim_idle=%sms',
241
+ self.name,
242
+ len(self.subscriptions),
243
+ self.qos,
244
+ self.reclaim_idle,
245
+ )
246
+ qos_limited = False
247
+ while not self.stopped:
248
+ self.wakeup.clear()
249
+ self.drain()
250
+ for _ in range(len(self.rotation)):
251
+ available = self.qos - len(self.pending)
252
+ if available <= 0:
253
+ if not qos_limited:
254
+ self.logger.debug(
255
+ 'Deferring reads at QoS limit: consumer=%s qos=%s pending=%s',
256
+ self.name,
257
+ self.qos,
258
+ len(self.pending),
259
+ )
260
+ qos_limited = True
261
+ break
262
+ qos_limited = False
263
+ if self.stopped:
264
+ break
265
+ name = self.rotation.popleft()
266
+ self.rotation.append(name)
267
+ stream, handler = self.subscriptions[name]
268
+ if self.reclaim_idle is not None:
269
+ old_cursor = self.cursors[stream.queue]
270
+ try:
271
+ claimed = self.connection.xautoclaim(
272
+ stream.queue,
273
+ stream.group,
274
+ self.name,
275
+ self.reclaim_idle,
276
+ start_id=old_cursor,
277
+ count=available,
278
+ )
279
+ except Exception as error:
280
+ self.logger.error(
281
+ 'Pending message reclaim failed: queue=%s group=%s consumer=%s error_type=%s',
282
+ stream.queue,
283
+ stream.group,
284
+ self.name,
285
+ type(error).__name__,
286
+ )
287
+ raise
288
+ self.cursors[stream.queue] = claimed[0].decode('ascii')
289
+ if claimed[1] or self.cursors[stream.queue] != old_cursor:
290
+ self.logger.debug(
291
+ 'Pending message reclaim result: queue=%s group=%s consumer=%s count=%s cursor=%s',
292
+ stream.queue,
293
+ stream.group,
294
+ self.name,
295
+ len(claimed[1]),
296
+ self.cursors[stream.queue],
297
+ )
298
+ self.deliver(
299
+ stream,
300
+ handler,
301
+ claimed[1],
302
+ )
303
+ available = self.qos - len(self.pending)
304
+ if available <= 0:
305
+ if not qos_limited:
306
+ self.logger.debug(
307
+ 'Deferring reads at QoS limit: consumer=%s qos=%s pending=%s',
308
+ self.name,
309
+ self.qos,
310
+ len(self.pending),
311
+ )
312
+ qos_limited = True
313
+ continue
314
+ qos_limited = False
315
+ if self.stopped:
316
+ continue
317
+ try:
318
+ messages = self.connection.xreadgroup(
319
+ stream.group,
320
+ self.name,
321
+ {stream.queue: '>'},
322
+ count=available,
323
+ )
324
+ except Exception as error:
325
+ self.logger.error(
326
+ 'New message read failed: queue=%s group=%s consumer=%s error_type=%s',
327
+ stream.queue,
328
+ stream.group,
329
+ self.name,
330
+ type(error).__name__,
331
+ )
332
+ raise
333
+ for _, entries in messages:
334
+ self.deliver(
335
+ stream,
336
+ handler,
337
+ entries,
338
+ )
339
+ self.wakeup.wait(0.05)
340
+ self.logger.info(
341
+ 'Consumer consumption loop ended: consumer=%s pending=%s',
342
+ self.name,
343
+ len(self.pending),
344
+ )
345
+
346
+ def stop(self) -> None:
347
+ if not self.stopped:
348
+ self.logger.info(
349
+ 'Consumer stop requested: consumer=%s pending=%s',
350
+ self.name,
351
+ len(self.pending),
352
+ )
353
+ self.stopped = True
354
+ self.wakeup.set()
355
+
356
+ def close(self) -> None:
357
+ if not self.closed:
358
+ self.logger.info(
359
+ 'Consumer closed: consumer=%s pending=%s',
360
+ self.name,
361
+ len(self.pending),
362
+ )
363
+ self.stop()
364
+ self.closed = True