DefStream-Redis 0.1.24__110-py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- DefStream/Broker/Implements/Redis/Configuration.py +59 -0
- DefStream/Broker/Implements/Redis/Connection.py +265 -0
- DefStream/Broker/Implements/Redis/Consumer.py +368 -0
- DefStream/Broker/Implements/Redis/Event.py +212 -0
- DefStream/Broker/Implements/Redis/Queue.py +284 -0
- DefStream/Broker/Implements/Redis/Serializer.py +48 -0
- DefStream/Broker/Implements/Redis/Subscription.py +46 -0
- DefStream/Broker/Implements/Redis/__init__.py +29 -0
- defstream_redis-0.1.24.dist-info/METADATA +92 -0
- defstream_redis-0.1.24.dist-info/RECORD +13 -0
- defstream_redis-0.1.24.dist-info/WHEEL +6 -0
- defstream_redis-0.1.24.dist-info/licenses/LICENSE +674 -0
- defstream_redis-0.1.24.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
from logging import getLogger
|
|
2
|
+
from typing import (
|
|
3
|
+
Any,
|
|
4
|
+
Optional,
|
|
5
|
+
Union,
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
from redis import Redis
|
|
9
|
+
from redis.cluster import RedisCluster
|
|
10
|
+
|
|
11
|
+
from DefStream.Broker.Event import (
|
|
12
|
+
Event as BaseEvent,
|
|
13
|
+
EventErrorPolicy,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
__all__ = (
|
|
17
|
+
'Event',
|
|
18
|
+
'Requeue',
|
|
19
|
+
'Reject',
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Event(BaseEvent):
|
|
24
|
+
logger = getLogger('Broker.Redis.Event')
|
|
25
|
+
|
|
26
|
+
def __init__(
|
|
27
|
+
self,
|
|
28
|
+
connection: Union[Redis, RedisCluster],
|
|
29
|
+
queue: str,
|
|
30
|
+
id: str,
|
|
31
|
+
body: Any,
|
|
32
|
+
group: Optional[str] = None,
|
|
33
|
+
consumer: Optional[str] = None,
|
|
34
|
+
):
|
|
35
|
+
self.connection = connection
|
|
36
|
+
self.queue = queue
|
|
37
|
+
self.id = id
|
|
38
|
+
self.payload = body
|
|
39
|
+
self.group = group
|
|
40
|
+
self.consumer = consumer
|
|
41
|
+
self.settled = False
|
|
42
|
+
|
|
43
|
+
@property
|
|
44
|
+
def body(self) -> Any:
|
|
45
|
+
return self.payload
|
|
46
|
+
|
|
47
|
+
@body.setter
|
|
48
|
+
def body(
|
|
49
|
+
self,
|
|
50
|
+
body: Any,
|
|
51
|
+
) -> None:
|
|
52
|
+
self.payload = body
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def src(self) -> str:
|
|
56
|
+
return self.queue
|
|
57
|
+
|
|
58
|
+
def ack(self) -> None:
|
|
59
|
+
if self.group is None:
|
|
60
|
+
self.logger.warning(
|
|
61
|
+
'Rejected ACK for history Event without a consumer group: queue=%s id=%s',
|
|
62
|
+
self.queue,
|
|
63
|
+
self.id,
|
|
64
|
+
)
|
|
65
|
+
raise RuntimeError('Events from read() have no consumer group to acknowledge')
|
|
66
|
+
if not self.settled:
|
|
67
|
+
self.logger.debug(
|
|
68
|
+
'Attempting Event ACK: queue=%s group=%s consumer=%s id=%s',
|
|
69
|
+
self.queue,
|
|
70
|
+
self.group,
|
|
71
|
+
self.consumer,
|
|
72
|
+
self.id,
|
|
73
|
+
)
|
|
74
|
+
try:
|
|
75
|
+
result = self.connection.xack(
|
|
76
|
+
self.queue,
|
|
77
|
+
self.group,
|
|
78
|
+
self.id,
|
|
79
|
+
)
|
|
80
|
+
except Exception as error:
|
|
81
|
+
self.logger.error(
|
|
82
|
+
'Event ACK failed: queue=%s group=%s consumer=%s id=%s error_type=%s',
|
|
83
|
+
self.queue,
|
|
84
|
+
self.group,
|
|
85
|
+
self.consumer,
|
|
86
|
+
self.id,
|
|
87
|
+
type(error).__name__,
|
|
88
|
+
)
|
|
89
|
+
raise
|
|
90
|
+
self.settled = True
|
|
91
|
+
if isinstance(result, int) and result > 0:
|
|
92
|
+
self.logger.debug(
|
|
93
|
+
'Event ACK completed: queue=%s group=%s consumer=%s id=%s count=%s',
|
|
94
|
+
self.queue,
|
|
95
|
+
self.group,
|
|
96
|
+
self.consumer,
|
|
97
|
+
self.id,
|
|
98
|
+
result,
|
|
99
|
+
)
|
|
100
|
+
else:
|
|
101
|
+
self.logger.debug(
|
|
102
|
+
'No Event ACK target: queue=%s group=%s consumer=%s id=%s count=%s',
|
|
103
|
+
self.queue,
|
|
104
|
+
self.group,
|
|
105
|
+
self.consumer,
|
|
106
|
+
self.id,
|
|
107
|
+
result,
|
|
108
|
+
)
|
|
109
|
+
else:
|
|
110
|
+
self.logger.debug(
|
|
111
|
+
'Skipping ACK for settled Event: queue=%s group=%s consumer=%s id=%s',
|
|
112
|
+
self.queue,
|
|
113
|
+
self.group,
|
|
114
|
+
self.consumer,
|
|
115
|
+
self.id,
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
def nack(self) -> None:
|
|
119
|
+
if self.group is None:
|
|
120
|
+
self.logger.warning(
|
|
121
|
+
'Rejected NACK for history Event without a consumer group: queue=%s id=%s',
|
|
122
|
+
self.queue,
|
|
123
|
+
self.id,
|
|
124
|
+
)
|
|
125
|
+
raise RuntimeError('Events from read() have no consumer group to requeue')
|
|
126
|
+
if not self.settled:
|
|
127
|
+
self.logger.debug(
|
|
128
|
+
'Attempting Event requeue: queue=%s group=%s consumer=%s id=%s',
|
|
129
|
+
self.queue,
|
|
130
|
+
self.group,
|
|
131
|
+
self.consumer,
|
|
132
|
+
self.id,
|
|
133
|
+
)
|
|
134
|
+
# Requeue and acknowledge atomically on one stream (also valid in Cluster).
|
|
135
|
+
try:
|
|
136
|
+
result = self.connection.eval(
|
|
137
|
+
"""
|
|
138
|
+
local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
|
|
139
|
+
if #pending == 0 then return 0 end
|
|
140
|
+
local entries = redis.call('XRANGE', KEYS[1], ARGV[2], ARGV[2])
|
|
141
|
+
if #entries == 0 then return redis.error_reply('Pending entry was deleted') end
|
|
142
|
+
redis.call('XADD', KEYS[1], '*', unpack(entries[1][2]))
|
|
143
|
+
return redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
|
|
144
|
+
""",
|
|
145
|
+
1,
|
|
146
|
+
self.queue,
|
|
147
|
+
self.group,
|
|
148
|
+
self.id,
|
|
149
|
+
)
|
|
150
|
+
except Exception as error:
|
|
151
|
+
self.logger.error(
|
|
152
|
+
'Event atomic requeue failed: queue=%s group=%s consumer=%s id=%s error_type=%s',
|
|
153
|
+
self.queue,
|
|
154
|
+
self.group,
|
|
155
|
+
self.consumer,
|
|
156
|
+
self.id,
|
|
157
|
+
type(error).__name__,
|
|
158
|
+
)
|
|
159
|
+
raise
|
|
160
|
+
self.settled = True
|
|
161
|
+
if isinstance(result, int) and result > 0:
|
|
162
|
+
self.logger.debug(
|
|
163
|
+
'Event requeue and ACK completed: queue=%s group=%s consumer=%s id=%s count=%s',
|
|
164
|
+
self.queue,
|
|
165
|
+
self.group,
|
|
166
|
+
self.consumer,
|
|
167
|
+
self.id,
|
|
168
|
+
result,
|
|
169
|
+
)
|
|
170
|
+
else:
|
|
171
|
+
self.logger.debug(
|
|
172
|
+
'No Event requeue target: queue=%s group=%s consumer=%s id=%s count=%s',
|
|
173
|
+
self.queue,
|
|
174
|
+
self.group,
|
|
175
|
+
self.consumer,
|
|
176
|
+
self.id,
|
|
177
|
+
result,
|
|
178
|
+
)
|
|
179
|
+
else:
|
|
180
|
+
self.logger.debug(
|
|
181
|
+
'Skipping NACK for settled Event: queue=%s group=%s consumer=%s id=%s',
|
|
182
|
+
self.queue,
|
|
183
|
+
self.group,
|
|
184
|
+
self.consumer,
|
|
185
|
+
self.id,
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
def reject(self) -> None:
|
|
189
|
+
self.logger.debug(
|
|
190
|
+
'Event reject requested (currently implemented as ACK): queue=%s group=%s consumer=%s id=%s',
|
|
191
|
+
self.queue,
|
|
192
|
+
self.group,
|
|
193
|
+
self.consumer,
|
|
194
|
+
self.id,
|
|
195
|
+
)
|
|
196
|
+
self.ack()
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class Requeue(EventErrorPolicy):
|
|
200
|
+
def __call__(
|
|
201
|
+
self,
|
|
202
|
+
event: BaseEvent,
|
|
203
|
+
) -> None:
|
|
204
|
+
event.nack()
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
class Reject(EventErrorPolicy):
|
|
208
|
+
def __call__(
|
|
209
|
+
self,
|
|
210
|
+
event: BaseEvent,
|
|
211
|
+
) -> None:
|
|
212
|
+
event.reject()
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
from logging import getLogger
|
|
2
|
+
from time import (
|
|
3
|
+
monotonic,
|
|
4
|
+
sleep,
|
|
5
|
+
)
|
|
6
|
+
from typing import (
|
|
7
|
+
Any,
|
|
8
|
+
Generator,
|
|
9
|
+
Optional,
|
|
10
|
+
Union,
|
|
11
|
+
)
|
|
12
|
+
from uuid import uuid4
|
|
13
|
+
|
|
14
|
+
from redis import Redis
|
|
15
|
+
from redis.cluster import RedisCluster
|
|
16
|
+
from redis.exceptions import ResponseError
|
|
17
|
+
|
|
18
|
+
from DefStream.Broker import (
|
|
19
|
+
Poppable,
|
|
20
|
+
Readable,
|
|
21
|
+
Sendable,
|
|
22
|
+
)
|
|
23
|
+
from .Event import Event
|
|
24
|
+
from .Serializer import (
|
|
25
|
+
Decoder,
|
|
26
|
+
Encoder,
|
|
27
|
+
)
|
|
28
|
+
from .Subscription import Subscription
|
|
29
|
+
|
|
30
|
+
__all__ = (
|
|
31
|
+
'Queue',
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Queue(
|
|
36
|
+
Sendable,
|
|
37
|
+
Poppable,
|
|
38
|
+
Readable,
|
|
39
|
+
):
|
|
40
|
+
logger = getLogger('Broker.Redis.Queue')
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
connection: Union[Redis, RedisCluster],
|
|
45
|
+
encode: Encoder,
|
|
46
|
+
decode: Decoder,
|
|
47
|
+
name: str,
|
|
48
|
+
group: str = 'DefStream',
|
|
49
|
+
):
|
|
50
|
+
self.subscription = Subscription(
|
|
51
|
+
name,
|
|
52
|
+
group,
|
|
53
|
+
)
|
|
54
|
+
self.connection = connection
|
|
55
|
+
self.encode = encode
|
|
56
|
+
self.decode = decode
|
|
57
|
+
self.queue = name
|
|
58
|
+
self.group = group
|
|
59
|
+
self.consumer = uuid4().hex
|
|
60
|
+
|
|
61
|
+
def __str__(self) -> str:
|
|
62
|
+
return self.queue
|
|
63
|
+
|
|
64
|
+
def create_group(self) -> None:
|
|
65
|
+
self.logger.debug(
|
|
66
|
+
'Creating Consumer Group: queue=%s group=%s',
|
|
67
|
+
self.queue,
|
|
68
|
+
self.group,
|
|
69
|
+
)
|
|
70
|
+
try:
|
|
71
|
+
self.connection.xgroup_create(
|
|
72
|
+
self.queue,
|
|
73
|
+
self.group,
|
|
74
|
+
id='0-0',
|
|
75
|
+
mkstream=True,
|
|
76
|
+
)
|
|
77
|
+
except ResponseError as error:
|
|
78
|
+
if not str(error).startswith('BUSYGROUP '):
|
|
79
|
+
self.logger.error(
|
|
80
|
+
'Consumer Group creation failed: queue=%s group=%s error_type=%s',
|
|
81
|
+
self.queue,
|
|
82
|
+
self.group,
|
|
83
|
+
type(error).__name__,
|
|
84
|
+
)
|
|
85
|
+
raise
|
|
86
|
+
self.logger.debug(
|
|
87
|
+
'Reusing existing Consumer Group: queue=%s group=%s',
|
|
88
|
+
self.queue,
|
|
89
|
+
self.group,
|
|
90
|
+
)
|
|
91
|
+
return
|
|
92
|
+
except Exception as error:
|
|
93
|
+
self.logger.error(
|
|
94
|
+
'Consumer Group creation failed: queue=%s group=%s error_type=%s',
|
|
95
|
+
self.queue,
|
|
96
|
+
self.group,
|
|
97
|
+
type(error).__name__,
|
|
98
|
+
)
|
|
99
|
+
raise
|
|
100
|
+
self.logger.info(
|
|
101
|
+
'Consumer Group creation completed: queue=%s group=%s',
|
|
102
|
+
self.queue,
|
|
103
|
+
self.group,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
def send(
|
|
107
|
+
self,
|
|
108
|
+
body: Any,
|
|
109
|
+
) -> str:
|
|
110
|
+
self.logger.debug(
|
|
111
|
+
'Publishing Redis Stream message: queue=%s',
|
|
112
|
+
self.queue,
|
|
113
|
+
)
|
|
114
|
+
try:
|
|
115
|
+
id = self.connection.xadd(
|
|
116
|
+
self.queue,
|
|
117
|
+
{'body': self.encode(body)},
|
|
118
|
+
)
|
|
119
|
+
message_id = id.decode('ascii') if isinstance(id, bytes) else id
|
|
120
|
+
except Exception as error:
|
|
121
|
+
self.logger.error(
|
|
122
|
+
'Redis Stream publication or serialization failed: queue=%s error_type=%s',
|
|
123
|
+
self.queue,
|
|
124
|
+
type(error).__name__,
|
|
125
|
+
)
|
|
126
|
+
raise
|
|
127
|
+
self.logger.debug(
|
|
128
|
+
'Redis Stream message published: queue=%s id=%s',
|
|
129
|
+
self.queue,
|
|
130
|
+
message_id,
|
|
131
|
+
)
|
|
132
|
+
return message_id
|
|
133
|
+
|
|
134
|
+
def event(
|
|
135
|
+
self,
|
|
136
|
+
id: bytes,
|
|
137
|
+
fields: dict,
|
|
138
|
+
group: Optional[str] = None,
|
|
139
|
+
consumer: Optional[str] = None,
|
|
140
|
+
) -> Event:
|
|
141
|
+
try:
|
|
142
|
+
message_id = id.decode('ascii')
|
|
143
|
+
body = self.decode(fields[b'body'])
|
|
144
|
+
except Exception as error:
|
|
145
|
+
self.logger.error(
|
|
146
|
+
'Redis message deserialization or Event conversion failed: queue=%s group=%s id=%s error_type=%s',
|
|
147
|
+
self.queue,
|
|
148
|
+
group,
|
|
149
|
+
id,
|
|
150
|
+
type(error).__name__,
|
|
151
|
+
)
|
|
152
|
+
raise
|
|
153
|
+
self.logger.debug(
|
|
154
|
+
'Converting message to Event: queue=%s group=%s consumer=%s id=%s',
|
|
155
|
+
self.queue,
|
|
156
|
+
group,
|
|
157
|
+
consumer,
|
|
158
|
+
message_id,
|
|
159
|
+
)
|
|
160
|
+
return Event(
|
|
161
|
+
self.connection,
|
|
162
|
+
self.queue,
|
|
163
|
+
message_id,
|
|
164
|
+
body,
|
|
165
|
+
group,
|
|
166
|
+
consumer,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
def pop(
|
|
170
|
+
self,
|
|
171
|
+
timeout: Optional[int] = None,
|
|
172
|
+
) -> Optional[Event]:
|
|
173
|
+
if timeout is not None and timeout < 0:
|
|
174
|
+
raise ValueError('Redis pop timeout must be non-negative milliseconds')
|
|
175
|
+
self.logger.debug(
|
|
176
|
+
'Starting group-based single-message reception: queue=%s group=%s consumer=%s timeout=%sms',
|
|
177
|
+
self.queue,
|
|
178
|
+
self.group,
|
|
179
|
+
self.consumer,
|
|
180
|
+
timeout,
|
|
181
|
+
)
|
|
182
|
+
self.create_group()
|
|
183
|
+
deadline = monotonic() + timeout / 1000 if timeout else None
|
|
184
|
+
while True:
|
|
185
|
+
try:
|
|
186
|
+
messages = self.connection.xreadgroup(
|
|
187
|
+
self.group,
|
|
188
|
+
self.consumer,
|
|
189
|
+
{self.queue: '>'},
|
|
190
|
+
count=1,
|
|
191
|
+
)
|
|
192
|
+
except Exception as error:
|
|
193
|
+
self.logger.error(
|
|
194
|
+
'Group-based single-message reception failed: queue=%s group=%s error_type=%s',
|
|
195
|
+
self.queue,
|
|
196
|
+
self.group,
|
|
197
|
+
type(error).__name__,
|
|
198
|
+
)
|
|
199
|
+
raise
|
|
200
|
+
for _, entries in messages:
|
|
201
|
+
for id, fields in entries:
|
|
202
|
+
event = self.event(
|
|
203
|
+
id,
|
|
204
|
+
fields,
|
|
205
|
+
self.group,
|
|
206
|
+
self.consumer,
|
|
207
|
+
)
|
|
208
|
+
self.logger.debug(
|
|
209
|
+
'Group message received: queue=%s group=%s id=%s',
|
|
210
|
+
self.queue,
|
|
211
|
+
self.group,
|
|
212
|
+
event.id,
|
|
213
|
+
)
|
|
214
|
+
return event
|
|
215
|
+
if deadline is not None and monotonic() >= deadline:
|
|
216
|
+
self.logger.debug(
|
|
217
|
+
'Group-based single-message wait timed out: queue=%s group=%s timeout=%sms',
|
|
218
|
+
self.queue,
|
|
219
|
+
self.group,
|
|
220
|
+
timeout,
|
|
221
|
+
)
|
|
222
|
+
return None
|
|
223
|
+
sleep(0.01)
|
|
224
|
+
|
|
225
|
+
def read(
|
|
226
|
+
self,
|
|
227
|
+
offset: Union[int, str] = 0,
|
|
228
|
+
timeout: Optional[int] = None,
|
|
229
|
+
) -> Generator[Event, None, None]:
|
|
230
|
+
if timeout is not None and timeout < 0:
|
|
231
|
+
raise ValueError('Redis read timeout must be non-negative milliseconds')
|
|
232
|
+
cursor = str(offset)
|
|
233
|
+
deadline = monotonic() + timeout / 1000 if timeout else None
|
|
234
|
+
self.logger.debug(
|
|
235
|
+
'Starting history read without a consumer group: queue=%s offset=%s timeout=%sms',
|
|
236
|
+
self.queue,
|
|
237
|
+
cursor,
|
|
238
|
+
timeout,
|
|
239
|
+
)
|
|
240
|
+
try:
|
|
241
|
+
while True:
|
|
242
|
+
if deadline is not None and monotonic() >= deadline:
|
|
243
|
+
self.logger.debug(
|
|
244
|
+
'History read without a consumer group timed out: queue=%s cursor=%s timeout=%sms',
|
|
245
|
+
self.queue,
|
|
246
|
+
cursor,
|
|
247
|
+
timeout,
|
|
248
|
+
)
|
|
249
|
+
return
|
|
250
|
+
try:
|
|
251
|
+
messages = self.connection.xread(
|
|
252
|
+
{self.queue: cursor},
|
|
253
|
+
count=1,
|
|
254
|
+
)
|
|
255
|
+
except Exception as error:
|
|
256
|
+
self.logger.error(
|
|
257
|
+
'History read without a consumer group failed: queue=%s cursor=%s error_type=%s',
|
|
258
|
+
self.queue,
|
|
259
|
+
cursor,
|
|
260
|
+
type(error).__name__,
|
|
261
|
+
)
|
|
262
|
+
raise
|
|
263
|
+
if not messages:
|
|
264
|
+
sleep(0.01)
|
|
265
|
+
continue
|
|
266
|
+
for _, entries in messages:
|
|
267
|
+
for id, fields in entries:
|
|
268
|
+
event = self.event(
|
|
269
|
+
id,
|
|
270
|
+
fields,
|
|
271
|
+
)
|
|
272
|
+
cursor = event.id
|
|
273
|
+
self.logger.debug(
|
|
274
|
+
'History message received without a consumer group: queue=%s id=%s',
|
|
275
|
+
self.queue,
|
|
276
|
+
event.id,
|
|
277
|
+
)
|
|
278
|
+
yield event
|
|
279
|
+
finally:
|
|
280
|
+
self.logger.debug(
|
|
281
|
+
'History read without a consumer group ended: queue=%s cursor=%s',
|
|
282
|
+
self.queue,
|
|
283
|
+
cursor,
|
|
284
|
+
)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
from json import (
|
|
2
|
+
dumps,
|
|
3
|
+
loads,
|
|
4
|
+
)
|
|
5
|
+
from typing import (
|
|
6
|
+
Any,
|
|
7
|
+
Protocol,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
__all__ = (
|
|
11
|
+
'Encoder',
|
|
12
|
+
'Decoder',
|
|
13
|
+
'JavaScriptObjectNotationEncoder',
|
|
14
|
+
'JavaScriptObjectNotationDecoder',
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Encoder(Protocol):
|
|
19
|
+
def __call__(
|
|
20
|
+
self,
|
|
21
|
+
body: Any,
|
|
22
|
+
) -> bytes: ...
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Decoder(Protocol):
|
|
26
|
+
def __call__(
|
|
27
|
+
self,
|
|
28
|
+
body: bytes,
|
|
29
|
+
) -> Any: ...
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class JavaScriptObjectNotationEncoder(Encoder):
|
|
33
|
+
def __call__(
|
|
34
|
+
self,
|
|
35
|
+
body: Any,
|
|
36
|
+
) -> bytes:
|
|
37
|
+
return dumps(
|
|
38
|
+
body,
|
|
39
|
+
ensure_ascii=False,
|
|
40
|
+
).encode('utf-8')
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class JavaScriptObjectNotationDecoder(Decoder):
|
|
44
|
+
def __call__(
|
|
45
|
+
self,
|
|
46
|
+
body: bytes,
|
|
47
|
+
) -> Any:
|
|
48
|
+
return loads(body)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
from collections.abc import Mapping
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
__all__ = (
|
|
6
|
+
'Subscription',
|
|
7
|
+
)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class Subscription:
|
|
12
|
+
queue: str
|
|
13
|
+
group: str = 'stream'
|
|
14
|
+
|
|
15
|
+
def __post_init__(self) -> None:
|
|
16
|
+
for value in (
|
|
17
|
+
self.queue,
|
|
18
|
+
self.group,
|
|
19
|
+
):
|
|
20
|
+
if not isinstance(
|
|
21
|
+
value,
|
|
22
|
+
str,
|
|
23
|
+
) or not value:
|
|
24
|
+
raise ValueError('Redis queue and group must be non-empty strings')
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def parse(
|
|
28
|
+
cls,
|
|
29
|
+
parameters: Any,
|
|
30
|
+
) -> 'Subscription':
|
|
31
|
+
if isinstance(
|
|
32
|
+
parameters,
|
|
33
|
+
cls,
|
|
34
|
+
):
|
|
35
|
+
return parameters
|
|
36
|
+
if isinstance(
|
|
37
|
+
parameters,
|
|
38
|
+
str,
|
|
39
|
+
):
|
|
40
|
+
return cls(parameters)
|
|
41
|
+
if isinstance(
|
|
42
|
+
parameters,
|
|
43
|
+
Mapping,
|
|
44
|
+
):
|
|
45
|
+
return cls(**dict(parameters))
|
|
46
|
+
raise TypeError('Redis subscriptions require a queue name, Subscription or mapping')
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from DefStream.Broker import EventHandler
|
|
2
|
+
from .Configuration import (
|
|
3
|
+
Configuration,
|
|
4
|
+
ConnectionType,
|
|
5
|
+
)
|
|
6
|
+
from .Connection import (
|
|
7
|
+
Connection,
|
|
8
|
+
)
|
|
9
|
+
from .Consumer import Consumer
|
|
10
|
+
from .Event import (
|
|
11
|
+
Event,
|
|
12
|
+
Reject,
|
|
13
|
+
Requeue,
|
|
14
|
+
)
|
|
15
|
+
from .Queue import Queue
|
|
16
|
+
from .Subscription import Subscription
|
|
17
|
+
|
|
18
|
+
__all__ = (
|
|
19
|
+
'Connection',
|
|
20
|
+
'Configuration',
|
|
21
|
+
'ConnectionType',
|
|
22
|
+
'Consumer',
|
|
23
|
+
'Event',
|
|
24
|
+
'EventHandler',
|
|
25
|
+
'Queue',
|
|
26
|
+
'Subscription',
|
|
27
|
+
'Reject',
|
|
28
|
+
'Requeue',
|
|
29
|
+
)
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: DefStream-Redis
|
|
3
|
+
Version: 0.1.24
|
|
4
|
+
Summary: Synchronous message processing with pluggable broker adapters
|
|
5
|
+
Home-page: https://github.com/yong5eon/DefStream
|
|
6
|
+
Author: Heo Yongseon
|
|
7
|
+
Project-URL: Source, https://github.com/yong5eon/DefStream
|
|
8
|
+
Project-URL: Issues, https://github.com/yong5eon/DefStream/issues
|
|
9
|
+
Project-URL: Releases, https://github.com/yong5eon/DefStream/releases
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Requires-Python: >=3.9
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Requires-Dist: DefStream==0.1.24
|
|
20
|
+
Requires-Dist: redis>=5.1.1
|
|
21
|
+
Dynamic: author
|
|
22
|
+
Dynamic: classifier
|
|
23
|
+
Dynamic: description
|
|
24
|
+
Dynamic: description-content-type
|
|
25
|
+
Dynamic: home-page
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
Dynamic: project-url
|
|
28
|
+
Dynamic: requires-dist
|
|
29
|
+
Dynamic: requires-python
|
|
30
|
+
Dynamic: summary
|
|
31
|
+
|
|
32
|
+
# DefStream
|
|
33
|
+
|
|
34
|
+
Synchronous event/message processing with pluggable broker adapters
|
|
35
|
+
|
|
36
|
+
DefStream processes broker messages with ordinary Python task methods and can run them in thread or process pools. It requires Python 3.9 or later.
|
|
37
|
+
|
|
38
|
+
## Installation
|
|
39
|
+
|
|
40
|
+
Install the client extra for the broker you use. A broker server must be available separately.
|
|
41
|
+
|
|
42
|
+
```shell
|
|
43
|
+
python -m pip install "DefStream[RabbitMQ]"
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
For Redis Streams, install `DefStream[Redis]`. To install both adapters and clients, use `DefStream[RabbitMQ,Redis]`. The plain `DefStream` installation includes neither broker implementation nor client. Each extra installs a separate, same-version adapter distribution, preserving existing import paths.
|
|
47
|
+
|
|
48
|
+
Before migrating an older integrated installation or historical standalone adapters, uninstall DefStream, DefStream-RabbitMQ and DefStream-Redis, then reinstall with the desired extras. Omitting an extra later does not remove an already installed adapter.
|
|
49
|
+
|
|
50
|
+
## Quick start
|
|
51
|
+
|
|
52
|
+
The following worker listens to a local RabbitMQ server at `127.0.0.1:5672`. The example credentials are for local development only.
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from signal import SIGINT, signal
|
|
56
|
+
|
|
57
|
+
from DefStream.Broker.Implements.RabbitMQ import Configuration, Connection, Reject, Subscription
|
|
58
|
+
from DefStream.Worker import Run, ThreadPool, Worker, WorkerSetup, declare
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Add(
|
|
62
|
+
Run,
|
|
63
|
+
parameters=Subscription(queue='example.add', error='example.add.error'),
|
|
64
|
+
on_error=Reject(),
|
|
65
|
+
):
|
|
66
|
+
def run(self, a: int, b: int):
|
|
67
|
+
print(a + b)
|
|
68
|
+
return a + b
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Prepare(WorkerSetup):
|
|
72
|
+
def __call__(self, worker: Worker) -> None:
|
|
73
|
+
for task in worker.tasks:
|
|
74
|
+
declare(task=task, broker=worker.broker)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == '__main__':
|
|
78
|
+
worker = Worker(
|
|
79
|
+
Connection(Configuration(host='127.0.0.1', port=5672, username='guest', password='guest')),
|
|
80
|
+
ThreadPool(4),
|
|
81
|
+
setups=Prepare(),
|
|
82
|
+
)
|
|
83
|
+
worker.add(Add)
|
|
84
|
+
signal(SIGINT, lambda *_: worker.stop())
|
|
85
|
+
worker.run()
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Run the script after starting RabbitMQ. The worker declares its task queue and waits for messages. Store production credentials outside source code.
|
|
89
|
+
|
|
90
|
+
## License
|
|
91
|
+
|
|
92
|
+
See the `LICENSE` file included with the source distribution for the license terms.
|