DefStream-Redis 0.1.23__106-py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- DefStream/Broker/Implements/Redis/Configuration.py +59 -0
- DefStream/Broker/Implements/Redis/Connection.py +265 -0
- DefStream/Broker/Implements/Redis/Consumer.py +368 -0
- DefStream/Broker/Implements/Redis/Event.py +212 -0
- DefStream/Broker/Implements/Redis/Queue.py +284 -0
- DefStream/Broker/Implements/Redis/Serializer.py +48 -0
- DefStream/Broker/Implements/Redis/Subscription.py +46 -0
- DefStream/Broker/Implements/Redis/__init__.py +29 -0
- defstream_redis-0.1.23.dist-info/METADATA +92 -0
- defstream_redis-0.1.23.dist-info/RECORD +13 -0
- defstream_redis-0.1.23.dist-info/WHEEL +6 -0
- defstream_redis-0.1.23.dist-info/licenses/LICENSE +674 -0
- defstream_redis-0.1.23.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
from enum import Enum
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from DefStream.Broker import Configuration as BaseConfiguration
|
|
5
|
+
from .Serializer import (
|
|
6
|
+
Decoder,
|
|
7
|
+
Encoder,
|
|
8
|
+
JavaScriptObjectNotationDecoder,
|
|
9
|
+
JavaScriptObjectNotationEncoder,
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
__all__ = (
|
|
13
|
+
'Configuration',
|
|
14
|
+
'ConnectionType',
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ConnectionType(Enum):
|
|
19
|
+
Pool = 'pool'
|
|
20
|
+
Cluster = 'cluster'
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Configuration(BaseConfiguration):
|
|
24
|
+
def __init__(
|
|
25
|
+
self,
|
|
26
|
+
host: str = '127.0.0.1',
|
|
27
|
+
port: int = 6379,
|
|
28
|
+
username: Optional[str] = None,
|
|
29
|
+
password: Optional[str] = None,
|
|
30
|
+
secure: bool = False,
|
|
31
|
+
type: ConnectionType = ConnectionType.Pool,
|
|
32
|
+
max: int = 10,
|
|
33
|
+
database: int = 0,
|
|
34
|
+
timeout: float = 5,
|
|
35
|
+
encode: Optional[Encoder] = None,
|
|
36
|
+
decode: Optional[Decoder] = None,
|
|
37
|
+
):
|
|
38
|
+
if not isinstance(
|
|
39
|
+
type,
|
|
40
|
+
ConnectionType,
|
|
41
|
+
):
|
|
42
|
+
raise ValueError('Redis connection type must be a ConnectionType')
|
|
43
|
+
if max < 1:
|
|
44
|
+
raise ValueError('Redis max connections must be positive')
|
|
45
|
+
if database < 0 or (type == ConnectionType.Cluster and database != 0):
|
|
46
|
+
raise ValueError('Redis database must be non-negative; Cluster requires database 0')
|
|
47
|
+
if timeout <= 0:
|
|
48
|
+
raise ValueError('Redis connection timeout must be positive')
|
|
49
|
+
self.host = host
|
|
50
|
+
self.port = port
|
|
51
|
+
self.username = username
|
|
52
|
+
self.password = password
|
|
53
|
+
self.secure = secure
|
|
54
|
+
self.type = type
|
|
55
|
+
self.max = max
|
|
56
|
+
self.database = database
|
|
57
|
+
self.timeout = timeout
|
|
58
|
+
self.encode = encode if encode is not None else JavaScriptObjectNotationEncoder()
|
|
59
|
+
self.decode = decode if decode is not None else JavaScriptObjectNotationDecoder()
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
from logging import getLogger
|
|
2
|
+
from typing import (
|
|
3
|
+
Any,
|
|
4
|
+
Optional,
|
|
5
|
+
Union,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
from redis import Redis
|
|
9
|
+
from redis.cluster import RedisCluster
|
|
10
|
+
|
|
11
|
+
from DefStream.Broker import (
|
|
12
|
+
Connection as BaseConnection,
|
|
13
|
+
GetQueue,
|
|
14
|
+
GetStream,
|
|
15
|
+
GetConsumer,
|
|
16
|
+
Declarable,
|
|
17
|
+
Emittable,
|
|
18
|
+
EventHandler,
|
|
19
|
+
)
|
|
20
|
+
from .Configuration import (
|
|
21
|
+
Configuration,
|
|
22
|
+
ConnectionType,
|
|
23
|
+
)
|
|
24
|
+
from .Consumer import Consumer
|
|
25
|
+
from .Queue import Queue
|
|
26
|
+
from .Subscription import Subscription
|
|
27
|
+
|
|
28
|
+
__all__ = (
|
|
29
|
+
'Connection',
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Connection(
|
|
34
|
+
BaseConnection,
|
|
35
|
+
GetQueue,
|
|
36
|
+
GetStream,
|
|
37
|
+
GetConsumer,
|
|
38
|
+
Declarable,
|
|
39
|
+
Emittable,
|
|
40
|
+
):
|
|
41
|
+
logger = getLogger('Broker.Redis.Connection')
|
|
42
|
+
|
|
43
|
+
def __init__(
|
|
44
|
+
self,
|
|
45
|
+
conf: Configuration,
|
|
46
|
+
):
|
|
47
|
+
self.conf = conf
|
|
48
|
+
self.connection: Optional[Union[Redis, RedisCluster]] = None
|
|
49
|
+
|
|
50
|
+
def connect(self) -> None:
|
|
51
|
+
if self.connection is not None:
|
|
52
|
+
self.logger.debug('Reusing existing Redis connection')
|
|
53
|
+
return
|
|
54
|
+
connection_type = self.conf.type.value
|
|
55
|
+
self.logger.info(
|
|
56
|
+
'Connecting to Redis: type=%s',
|
|
57
|
+
connection_type,
|
|
58
|
+
)
|
|
59
|
+
self.logger.debug(
|
|
60
|
+
'Redis connection settings: host=%s port=%s database=%s TLS=%s max=%s timeout=%ss',
|
|
61
|
+
self.conf.host,
|
|
62
|
+
self.conf.port,
|
|
63
|
+
self.conf.database,
|
|
64
|
+
self.conf.secure,
|
|
65
|
+
self.conf.max,
|
|
66
|
+
self.conf.timeout,
|
|
67
|
+
)
|
|
68
|
+
parameters = {
|
|
69
|
+
'host': self.conf.host,
|
|
70
|
+
'port': self.conf.port,
|
|
71
|
+
'username': self.conf.username,
|
|
72
|
+
'password': self.conf.password,
|
|
73
|
+
'ssl': self.conf.secure,
|
|
74
|
+
'max_connections': self.conf.max,
|
|
75
|
+
'socket_connect_timeout': self.conf.timeout,
|
|
76
|
+
'socket_timeout': self.conf.timeout,
|
|
77
|
+
'decode_responses': False,
|
|
78
|
+
}
|
|
79
|
+
connection = None
|
|
80
|
+
try:
|
|
81
|
+
if self.conf.type == ConnectionType.Cluster:
|
|
82
|
+
connection = RedisCluster(**parameters)
|
|
83
|
+
else:
|
|
84
|
+
connection = Redis(
|
|
85
|
+
db=self.conf.database,
|
|
86
|
+
**parameters,
|
|
87
|
+
)
|
|
88
|
+
connection.ping()
|
|
89
|
+
except Exception as error:
|
|
90
|
+
self.logger.error(
|
|
91
|
+
'Redis connection or PING failed: type=%s error_type=%s',
|
|
92
|
+
connection_type,
|
|
93
|
+
type(error).__name__,
|
|
94
|
+
)
|
|
95
|
+
if connection is not None:
|
|
96
|
+
connection.close()
|
|
97
|
+
raise
|
|
98
|
+
self.connection = connection
|
|
99
|
+
self.logger.info(
|
|
100
|
+
'Redis connection verified with PING: type=%s',
|
|
101
|
+
connection_type,
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
def connected(self) -> Union[Redis, RedisCluster]:
|
|
105
|
+
if self.connection is None:
|
|
106
|
+
raise RuntimeError('Connect the Redis broker before use')
|
|
107
|
+
return self.connection
|
|
108
|
+
|
|
109
|
+
def declare(
|
|
110
|
+
self,
|
|
111
|
+
parameters: Any,
|
|
112
|
+
) -> None:
|
|
113
|
+
subscription = Subscription.parse(parameters)
|
|
114
|
+
self.logger.debug(
|
|
115
|
+
'Delegating stream declaration: queue=%s group=%s',
|
|
116
|
+
subscription.queue,
|
|
117
|
+
subscription.group,
|
|
118
|
+
)
|
|
119
|
+
self.create_queue(
|
|
120
|
+
subscription.queue,
|
|
121
|
+
group=subscription.group,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
def emit(
|
|
125
|
+
self,
|
|
126
|
+
body: Any,
|
|
127
|
+
parameters: Any,
|
|
128
|
+
) -> str:
|
|
129
|
+
subscription = Subscription.parse(parameters)
|
|
130
|
+
self.logger.debug(
|
|
131
|
+
'Delegating message publication: queue=%s group=%s',
|
|
132
|
+
subscription.queue,
|
|
133
|
+
subscription.group,
|
|
134
|
+
)
|
|
135
|
+
return self.queue(
|
|
136
|
+
subscription.queue,
|
|
137
|
+
group=subscription.group,
|
|
138
|
+
).send(body)
|
|
139
|
+
|
|
140
|
+
def queue(
|
|
141
|
+
self,
|
|
142
|
+
name: str,
|
|
143
|
+
group: str = 'DefStream',
|
|
144
|
+
) -> Queue:
|
|
145
|
+
self.logger.debug(
|
|
146
|
+
'Creating Queue: queue=%s group=%s',
|
|
147
|
+
name,
|
|
148
|
+
group,
|
|
149
|
+
)
|
|
150
|
+
return Queue(
|
|
151
|
+
self.connected(),
|
|
152
|
+
self.conf.encode,
|
|
153
|
+
self.conf.decode,
|
|
154
|
+
name,
|
|
155
|
+
group,
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
def stream(
|
|
159
|
+
self,
|
|
160
|
+
name: str,
|
|
161
|
+
group: str = 'DefStream',
|
|
162
|
+
) -> Queue:
|
|
163
|
+
return self.queue(
|
|
164
|
+
name,
|
|
165
|
+
group,
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
def consumer(
|
|
169
|
+
self,
|
|
170
|
+
handler: Optional[EventHandler] = None,
|
|
171
|
+
qos: int = 1,
|
|
172
|
+
reclaim_idle: Optional[int] = 60000,
|
|
173
|
+
) -> Consumer:
|
|
174
|
+
self.logger.debug(
|
|
175
|
+
'Creating Consumer: qos=%s reclaim_idle=%sms',
|
|
176
|
+
qos,
|
|
177
|
+
reclaim_idle,
|
|
178
|
+
)
|
|
179
|
+
return Consumer(
|
|
180
|
+
self.connected(),
|
|
181
|
+
self.conf.encode,
|
|
182
|
+
self.conf.decode,
|
|
183
|
+
handler,
|
|
184
|
+
qos,
|
|
185
|
+
reclaim_idle,
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
def create_queue(
|
|
189
|
+
self,
|
|
190
|
+
name: str,
|
|
191
|
+
group: str = 'DefStream',
|
|
192
|
+
) -> None:
|
|
193
|
+
self.logger.debug(
|
|
194
|
+
'Delegating Consumer Group creation: queue=%s group=%s',
|
|
195
|
+
name,
|
|
196
|
+
group,
|
|
197
|
+
)
|
|
198
|
+
self.queue(
|
|
199
|
+
name,
|
|
200
|
+
group,
|
|
201
|
+
).create_group()
|
|
202
|
+
|
|
203
|
+
def delete_queue(
|
|
204
|
+
self,
|
|
205
|
+
name: Union[str, Queue],
|
|
206
|
+
) -> int:
|
|
207
|
+
queue = str(name)
|
|
208
|
+
self.logger.debug(
|
|
209
|
+
'Deleting Redis stream: queue=%s',
|
|
210
|
+
queue,
|
|
211
|
+
)
|
|
212
|
+
try:
|
|
213
|
+
deleted = self.connected().delete(queue)
|
|
214
|
+
except Exception as error:
|
|
215
|
+
self.logger.error(
|
|
216
|
+
'Redis stream deletion failed: queue=%s error_type=%s',
|
|
217
|
+
queue,
|
|
218
|
+
type(error).__name__,
|
|
219
|
+
)
|
|
220
|
+
raise
|
|
221
|
+
if isinstance(deleted, int) and deleted > 0:
|
|
222
|
+
self.logger.info(
|
|
223
|
+
'Redis stream deletion completed: queue=%s deleted=%s',
|
|
224
|
+
queue,
|
|
225
|
+
deleted,
|
|
226
|
+
)
|
|
227
|
+
else:
|
|
228
|
+
self.logger.info(
|
|
229
|
+
'No Redis stream to delete: queue=%s deleted=%s',
|
|
230
|
+
queue,
|
|
231
|
+
deleted,
|
|
232
|
+
)
|
|
233
|
+
return deleted
|
|
234
|
+
|
|
235
|
+
def create_stream(
|
|
236
|
+
self,
|
|
237
|
+
name: str,
|
|
238
|
+
group: str = 'DefStream',
|
|
239
|
+
) -> None:
|
|
240
|
+
self.create_queue(
|
|
241
|
+
name,
|
|
242
|
+
group,
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
def delete_stream(
|
|
246
|
+
self,
|
|
247
|
+
name: Union[str, Queue],
|
|
248
|
+
) -> int:
|
|
249
|
+
return self.delete_queue(name)
|
|
250
|
+
|
|
251
|
+
def close(self) -> None:
|
|
252
|
+
if self.connection is not None:
|
|
253
|
+
self.logger.info('Closing Redis connection')
|
|
254
|
+
try:
|
|
255
|
+
self.connection.close()
|
|
256
|
+
except Exception as error:
|
|
257
|
+
self.logger.error(
|
|
258
|
+
'Redis connection close failed: error_type=%s',
|
|
259
|
+
type(error).__name__,
|
|
260
|
+
)
|
|
261
|
+
raise
|
|
262
|
+
self.connection = None
|
|
263
|
+
self.logger.info('Redis connection closed')
|
|
264
|
+
else:
|
|
265
|
+
self.logger.debug('No Redis connection to close')
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
from collections import deque
|
|
2
|
+
from logging import getLogger
|
|
3
|
+
from queue import (
|
|
4
|
+
Empty,
|
|
5
|
+
SimpleQueue,
|
|
6
|
+
)
|
|
7
|
+
from threading import Event as Wakeup
|
|
8
|
+
from typing import (
|
|
9
|
+
Any,
|
|
10
|
+
Callable,
|
|
11
|
+
Dict,
|
|
12
|
+
Optional,
|
|
13
|
+
Tuple,
|
|
14
|
+
Union,
|
|
15
|
+
)
|
|
16
|
+
from uuid import uuid4
|
|
17
|
+
|
|
18
|
+
from redis import Redis
|
|
19
|
+
from redis.cluster import RedisCluster
|
|
20
|
+
|
|
21
|
+
from DefStream.Broker import (
|
|
22
|
+
Consumer as BaseConsumer,
|
|
23
|
+
Event as BaseEvent,
|
|
24
|
+
EventHandler,
|
|
25
|
+
)
|
|
26
|
+
from .Event import (
|
|
27
|
+
Event,
|
|
28
|
+
Requeue,
|
|
29
|
+
)
|
|
30
|
+
from .Queue import Queue
|
|
31
|
+
from .Serializer import (
|
|
32
|
+
Decoder,
|
|
33
|
+
Encoder,
|
|
34
|
+
)
|
|
35
|
+
from .Subscription import Subscription
|
|
36
|
+
from DefStream.Broker.Event import EventErrorPolicy
|
|
37
|
+
|
|
38
|
+
__all__ = (
|
|
39
|
+
'Consumer',
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class Consumer(BaseConsumer):
|
|
44
|
+
logger = getLogger('Broker.Redis.Consumer')
|
|
45
|
+
|
|
46
|
+
def __init__(
|
|
47
|
+
self,
|
|
48
|
+
connection: Union[Redis, RedisCluster],
|
|
49
|
+
encode: Encoder,
|
|
50
|
+
decode: Decoder,
|
|
51
|
+
handler: Optional[EventHandler] = None,
|
|
52
|
+
qos: int = 1,
|
|
53
|
+
reclaim_idle: Optional[int] = 60000,
|
|
54
|
+
):
|
|
55
|
+
if isinstance(
|
|
56
|
+
qos,
|
|
57
|
+
bool,
|
|
58
|
+
) or not isinstance(
|
|
59
|
+
qos,
|
|
60
|
+
int,
|
|
61
|
+
) or qos < 1:
|
|
62
|
+
raise ValueError('Redis qos must be a positive integer')
|
|
63
|
+
if reclaim_idle is not None and reclaim_idle < 1:
|
|
64
|
+
raise ValueError('Redis reclaim_idle must be positive milliseconds or None')
|
|
65
|
+
self.connection = connection
|
|
66
|
+
self.encode = encode
|
|
67
|
+
self.decode = decode
|
|
68
|
+
self.handler = handler
|
|
69
|
+
self.qos = qos
|
|
70
|
+
self.reclaim_idle = reclaim_idle
|
|
71
|
+
self.name = uuid4().hex
|
|
72
|
+
self.subscriptions: Dict[str, Tuple[Queue, EventHandler]] = {}
|
|
73
|
+
self.rotation = deque()
|
|
74
|
+
self.pending: Dict[Tuple[str, str], Event] = {}
|
|
75
|
+
self.cursors: Dict[str, str] = {}
|
|
76
|
+
self.callbacks: SimpleQueue[Callable[[], None]] = SimpleQueue()
|
|
77
|
+
self.wakeup = Wakeup()
|
|
78
|
+
self.stopped = False
|
|
79
|
+
self.closed = False
|
|
80
|
+
self.logger.debug(
|
|
81
|
+
'Consumer created: consumer=%s qos=%s reclaim_idle=%sms',
|
|
82
|
+
self.name,
|
|
83
|
+
self.qos,
|
|
84
|
+
self.reclaim_idle,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
def subs(
|
|
88
|
+
self,
|
|
89
|
+
queue: Any,
|
|
90
|
+
handler: Optional[EventHandler] = None,
|
|
91
|
+
) -> None:
|
|
92
|
+
if self.closed:
|
|
93
|
+
raise RuntimeError('Redis consumer is closed')
|
|
94
|
+
subscription = Subscription.parse(queue)
|
|
95
|
+
handler = handler if handler is not None else self.handler
|
|
96
|
+
if handler is None:
|
|
97
|
+
raise ValueError('subs requires a handler or a default handler')
|
|
98
|
+
if subscription.queue in self.subscriptions:
|
|
99
|
+
raise ValueError('A Run is already subscribed to queue {}'.format(subscription.queue))
|
|
100
|
+
self.logger.debug(
|
|
101
|
+
'Preparing Consumer subscription: queue=%s group=%s consumer=%s',
|
|
102
|
+
subscription.queue,
|
|
103
|
+
subscription.group,
|
|
104
|
+
self.name,
|
|
105
|
+
)
|
|
106
|
+
stream = Queue(
|
|
107
|
+
self.connection,
|
|
108
|
+
self.encode,
|
|
109
|
+
self.decode,
|
|
110
|
+
subscription.queue,
|
|
111
|
+
subscription.group,
|
|
112
|
+
)
|
|
113
|
+
stream.create_group()
|
|
114
|
+
self.subscriptions[subscription.queue] = (
|
|
115
|
+
stream,
|
|
116
|
+
handler,
|
|
117
|
+
)
|
|
118
|
+
self.cursors[subscription.queue] = '0-0'
|
|
119
|
+
self.rotation.append(subscription.queue)
|
|
120
|
+
self.logger.info(
|
|
121
|
+
'Consumer subscribed: queue=%s group=%s consumer=%s qos=%s reclaim_idle=%sms',
|
|
122
|
+
subscription.queue,
|
|
123
|
+
subscription.group,
|
|
124
|
+
self.name,
|
|
125
|
+
self.qos,
|
|
126
|
+
self.reclaim_idle,
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
def call(
|
|
130
|
+
self,
|
|
131
|
+
callback: Callable[[], None],
|
|
132
|
+
) -> None:
|
|
133
|
+
if self.closed:
|
|
134
|
+
raise RuntimeError('Redis consumer is closed')
|
|
135
|
+
self.callbacks.put(callback)
|
|
136
|
+
self.wakeup.set()
|
|
137
|
+
self.logger.debug(
|
|
138
|
+
'Consumer callback scheduled: consumer=%s',
|
|
139
|
+
self.name,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
def fail(
|
|
143
|
+
self,
|
|
144
|
+
event: BaseEvent,
|
|
145
|
+
policy: Optional[EventErrorPolicy] = None,
|
|
146
|
+
) -> None:
|
|
147
|
+
self.logger.debug(
|
|
148
|
+
'Applying message failure policy: consumer=%s policy=%s',
|
|
149
|
+
self.name,
|
|
150
|
+
type(policy).__name__ if policy is not None else 'Requeue',
|
|
151
|
+
)
|
|
152
|
+
(policy if policy is not None else Requeue())(event)
|
|
153
|
+
|
|
154
|
+
def drain(self) -> None:
|
|
155
|
+
while True:
|
|
156
|
+
try:
|
|
157
|
+
callback = self.callbacks.get_nowait()
|
|
158
|
+
except Empty:
|
|
159
|
+
break
|
|
160
|
+
self.logger.debug(
|
|
161
|
+
'Executing Consumer callback: consumer=%s',
|
|
162
|
+
self.name,
|
|
163
|
+
)
|
|
164
|
+
try:
|
|
165
|
+
callback()
|
|
166
|
+
except Exception as error:
|
|
167
|
+
self.logger.error(
|
|
168
|
+
'Consumer callback failed: consumer=%s error_type=%s',
|
|
169
|
+
self.name,
|
|
170
|
+
type(error).__name__,
|
|
171
|
+
)
|
|
172
|
+
raise
|
|
173
|
+
pending_count = len(self.pending)
|
|
174
|
+
self.pending = {
|
|
175
|
+
key: event
|
|
176
|
+
for key, event in self.pending.items()
|
|
177
|
+
if not event.settled
|
|
178
|
+
}
|
|
179
|
+
settled_count = pending_count - len(self.pending)
|
|
180
|
+
if settled_count:
|
|
181
|
+
self.logger.debug(
|
|
182
|
+
'Settled Events removed: consumer=%s removed=%s pending=%s',
|
|
183
|
+
self.name,
|
|
184
|
+
settled_count,
|
|
185
|
+
len(self.pending),
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
def deliver(
|
|
189
|
+
self,
|
|
190
|
+
stream: Queue,
|
|
191
|
+
handler: EventHandler,
|
|
192
|
+
entries: list,
|
|
193
|
+
) -> None:
|
|
194
|
+
for id, fields in entries:
|
|
195
|
+
key = (
|
|
196
|
+
stream.queue,
|
|
197
|
+
id.decode('ascii'),
|
|
198
|
+
)
|
|
199
|
+
if key in self.pending:
|
|
200
|
+
self.logger.debug(
|
|
201
|
+
'Skipping delivery of already pending message: queue=%s group=%s id=%s',
|
|
202
|
+
stream.queue,
|
|
203
|
+
stream.group,
|
|
204
|
+
key[1],
|
|
205
|
+
)
|
|
206
|
+
continue
|
|
207
|
+
event = stream.event(
|
|
208
|
+
id,
|
|
209
|
+
fields,
|
|
210
|
+
stream.group,
|
|
211
|
+
self.name,
|
|
212
|
+
)
|
|
213
|
+
self.pending[key] = event
|
|
214
|
+
self.logger.debug(
|
|
215
|
+
'Delivering message: queue=%s group=%s consumer=%s id=%s pending=%s',
|
|
216
|
+
stream.queue,
|
|
217
|
+
stream.group,
|
|
218
|
+
self.name,
|
|
219
|
+
event.id,
|
|
220
|
+
len(self.pending),
|
|
221
|
+
)
|
|
222
|
+
try:
|
|
223
|
+
handler(event)
|
|
224
|
+
except Exception as error:
|
|
225
|
+
self.logger.error(
|
|
226
|
+
'Message handler dispatch failed: queue=%s group=%s consumer=%s id=%s error_type=%s',
|
|
227
|
+
stream.queue,
|
|
228
|
+
stream.group,
|
|
229
|
+
self.name,
|
|
230
|
+
event.id,
|
|
231
|
+
type(error).__name__,
|
|
232
|
+
)
|
|
233
|
+
raise
|
|
234
|
+
self.drain()
|
|
235
|
+
|
|
236
|
+
def run(self) -> None:
|
|
237
|
+
if self.closed:
|
|
238
|
+
raise RuntimeError('Redis consumer is closed')
|
|
239
|
+
if not self.subscriptions:
|
|
240
|
+
raise RuntimeError('Subscribe before running the Redis consumer')
|
|
241
|
+
self.logger.info(
|
|
242
|
+
'Starting Consumer consumption loop: consumer=%s subscriptions=%s qos=%s reclaim_idle=%sms',
|
|
243
|
+
self.name,
|
|
244
|
+
len(self.subscriptions),
|
|
245
|
+
self.qos,
|
|
246
|
+
self.reclaim_idle,
|
|
247
|
+
)
|
|
248
|
+
qos_limited = False
|
|
249
|
+
while not self.stopped:
|
|
250
|
+
self.wakeup.clear()
|
|
251
|
+
self.drain()
|
|
252
|
+
for _ in range(len(self.rotation)):
|
|
253
|
+
available = self.qos - len(self.pending)
|
|
254
|
+
if available <= 0:
|
|
255
|
+
if not qos_limited:
|
|
256
|
+
self.logger.debug(
|
|
257
|
+
'Deferring reads at QoS limit: consumer=%s qos=%s pending=%s',
|
|
258
|
+
self.name,
|
|
259
|
+
self.qos,
|
|
260
|
+
len(self.pending),
|
|
261
|
+
)
|
|
262
|
+
qos_limited = True
|
|
263
|
+
break
|
|
264
|
+
qos_limited = False
|
|
265
|
+
if self.stopped:
|
|
266
|
+
break
|
|
267
|
+
name = self.rotation.popleft()
|
|
268
|
+
self.rotation.append(name)
|
|
269
|
+
stream, handler = self.subscriptions[name]
|
|
270
|
+
if self.reclaim_idle is not None:
|
|
271
|
+
old_cursor = self.cursors[stream.queue]
|
|
272
|
+
try:
|
|
273
|
+
claimed = self.connection.xautoclaim(
|
|
274
|
+
stream.queue,
|
|
275
|
+
stream.group,
|
|
276
|
+
self.name,
|
|
277
|
+
self.reclaim_idle,
|
|
278
|
+
start_id=old_cursor,
|
|
279
|
+
count=available,
|
|
280
|
+
)
|
|
281
|
+
except Exception as error:
|
|
282
|
+
self.logger.error(
|
|
283
|
+
'Pending message reclaim failed: queue=%s group=%s consumer=%s error_type=%s',
|
|
284
|
+
stream.queue,
|
|
285
|
+
stream.group,
|
|
286
|
+
self.name,
|
|
287
|
+
type(error).__name__,
|
|
288
|
+
)
|
|
289
|
+
raise
|
|
290
|
+
self.cursors[stream.queue] = claimed[0].decode('ascii')
|
|
291
|
+
if claimed[1] or self.cursors[stream.queue] != old_cursor:
|
|
292
|
+
self.logger.debug(
|
|
293
|
+
'Pending message reclaim result: queue=%s group=%s consumer=%s count=%s cursor=%s',
|
|
294
|
+
stream.queue,
|
|
295
|
+
stream.group,
|
|
296
|
+
self.name,
|
|
297
|
+
len(claimed[1]),
|
|
298
|
+
self.cursors[stream.queue],
|
|
299
|
+
)
|
|
300
|
+
self.deliver(
|
|
301
|
+
stream,
|
|
302
|
+
handler,
|
|
303
|
+
claimed[1],
|
|
304
|
+
)
|
|
305
|
+
available = self.qos - len(self.pending)
|
|
306
|
+
if available <= 0:
|
|
307
|
+
if not qos_limited:
|
|
308
|
+
self.logger.debug(
|
|
309
|
+
'Deferring reads at QoS limit: consumer=%s qos=%s pending=%s',
|
|
310
|
+
self.name,
|
|
311
|
+
self.qos,
|
|
312
|
+
len(self.pending),
|
|
313
|
+
)
|
|
314
|
+
qos_limited = True
|
|
315
|
+
continue
|
|
316
|
+
qos_limited = False
|
|
317
|
+
if self.stopped:
|
|
318
|
+
continue
|
|
319
|
+
try:
|
|
320
|
+
messages = self.connection.xreadgroup(
|
|
321
|
+
stream.group,
|
|
322
|
+
self.name,
|
|
323
|
+
{stream.queue: '>'},
|
|
324
|
+
count=available,
|
|
325
|
+
)
|
|
326
|
+
except Exception as error:
|
|
327
|
+
self.logger.error(
|
|
328
|
+
'New message read failed: queue=%s group=%s consumer=%s error_type=%s',
|
|
329
|
+
stream.queue,
|
|
330
|
+
stream.group,
|
|
331
|
+
self.name,
|
|
332
|
+
type(error).__name__,
|
|
333
|
+
)
|
|
334
|
+
raise
|
|
335
|
+
for _, entries in messages:
|
|
336
|
+
if not isinstance(entries, list):
|
|
337
|
+
raise TypeError('Redis Stream read returned invalid entries')
|
|
338
|
+
self.deliver(
|
|
339
|
+
stream,
|
|
340
|
+
handler,
|
|
341
|
+
entries,
|
|
342
|
+
)
|
|
343
|
+
self.wakeup.wait(0.05)
|
|
344
|
+
self.logger.info(
|
|
345
|
+
'Consumer consumption loop ended: consumer=%s pending=%s',
|
|
346
|
+
self.name,
|
|
347
|
+
len(self.pending),
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
def stop(self) -> None:
|
|
351
|
+
if not self.stopped:
|
|
352
|
+
self.logger.info(
|
|
353
|
+
'Consumer stop requested: consumer=%s pending=%s',
|
|
354
|
+
self.name,
|
|
355
|
+
len(self.pending),
|
|
356
|
+
)
|
|
357
|
+
self.stopped = True
|
|
358
|
+
self.wakeup.set()
|
|
359
|
+
|
|
360
|
+
def close(self) -> None:
|
|
361
|
+
if not self.closed:
|
|
362
|
+
self.logger.info(
|
|
363
|
+
'Consumer closed: consumer=%s pending=%s',
|
|
364
|
+
self.name,
|
|
365
|
+
len(self.pending),
|
|
366
|
+
)
|
|
367
|
+
self.stop()
|
|
368
|
+
self.closed = True
|