DefStream-Redis 0.1.23__106-py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,212 @@
1
+ from logging import getLogger
2
+ from typing import (
3
+ Any,
4
+ Optional,
5
+ Union,
6
+ )
7
+
8
+ from redis import Redis
9
+ from redis.cluster import RedisCluster
10
+
11
+ from DefStream.Broker.Event import (
12
+ Event as BaseEvent,
13
+ EventErrorPolicy,
14
+ )
15
+
16
+ __all__ = (
17
+ 'Event',
18
+ 'Requeue',
19
+ 'Reject',
20
+ )
21
+
22
+
23
+ class Event(BaseEvent):
24
+ logger = getLogger('Broker.Redis.Event')
25
+
26
+ def __init__(
27
+ self,
28
+ connection: Union[Redis, RedisCluster],
29
+ queue: str,
30
+ id: str,
31
+ body: Any,
32
+ group: Optional[str] = None,
33
+ consumer: Optional[str] = None,
34
+ ):
35
+ self.connection = connection
36
+ self.queue = queue
37
+ self.id = id
38
+ self.payload = body
39
+ self.group = group
40
+ self.consumer = consumer
41
+ self.settled = False
42
+
43
+ @property
44
+ def body(self) -> Any:
45
+ return self.payload
46
+
47
+ @body.setter
48
+ def body(
49
+ self,
50
+ body: Any,
51
+ ) -> None:
52
+ self.payload = body
53
+
54
+ @property
55
+ def src(self) -> str:
56
+ return self.queue
57
+
58
+ def ack(self) -> None:
59
+ if self.group is None:
60
+ self.logger.warning(
61
+ 'Rejected ACK for history Event without a consumer group: queue=%s id=%s',
62
+ self.queue,
63
+ self.id,
64
+ )
65
+ raise RuntimeError('Events from read() have no consumer group to acknowledge')
66
+ if not self.settled:
67
+ self.logger.debug(
68
+ 'Attempting Event ACK: queue=%s group=%s consumer=%s id=%s',
69
+ self.queue,
70
+ self.group,
71
+ self.consumer,
72
+ self.id,
73
+ )
74
+ try:
75
+ result = self.connection.xack(
76
+ self.queue,
77
+ self.group,
78
+ self.id,
79
+ )
80
+ except Exception as error:
81
+ self.logger.error(
82
+ 'Event ACK failed: queue=%s group=%s consumer=%s id=%s error_type=%s',
83
+ self.queue,
84
+ self.group,
85
+ self.consumer,
86
+ self.id,
87
+ type(error).__name__,
88
+ )
89
+ raise
90
+ self.settled = True
91
+ if isinstance(result, int) and result > 0:
92
+ self.logger.debug(
93
+ 'Event ACK completed: queue=%s group=%s consumer=%s id=%s count=%s',
94
+ self.queue,
95
+ self.group,
96
+ self.consumer,
97
+ self.id,
98
+ result,
99
+ )
100
+ else:
101
+ self.logger.debug(
102
+ 'No Event ACK target: queue=%s group=%s consumer=%s id=%s count=%s',
103
+ self.queue,
104
+ self.group,
105
+ self.consumer,
106
+ self.id,
107
+ result,
108
+ )
109
+ else:
110
+ self.logger.debug(
111
+ 'Skipping ACK for settled Event: queue=%s group=%s consumer=%s id=%s',
112
+ self.queue,
113
+ self.group,
114
+ self.consumer,
115
+ self.id,
116
+ )
117
+
118
+ def nack(self) -> None:
119
+ if self.group is None:
120
+ self.logger.warning(
121
+ 'Rejected NACK for history Event without a consumer group: queue=%s id=%s',
122
+ self.queue,
123
+ self.id,
124
+ )
125
+ raise RuntimeError('Events from read() have no consumer group to requeue')
126
+ if not self.settled:
127
+ self.logger.debug(
128
+ 'Attempting Event requeue: queue=%s group=%s consumer=%s id=%s',
129
+ self.queue,
130
+ self.group,
131
+ self.consumer,
132
+ self.id,
133
+ )
134
+ # Requeue and acknowledge atomically on one stream (also valid in Cluster).
135
+ try:
136
+ result = self.connection.eval(
137
+ """
138
+ local pending = redis.call('XPENDING', KEYS[1], ARGV[1], ARGV[2], ARGV[2], 1)
139
+ if #pending == 0 then return 0 end
140
+ local entries = redis.call('XRANGE', KEYS[1], ARGV[2], ARGV[2])
141
+ if #entries == 0 then return redis.error_reply('Pending entry was deleted') end
142
+ redis.call('XADD', KEYS[1], '*', unpack(entries[1][2]))
143
+ return redis.call('XACK', KEYS[1], ARGV[1], ARGV[2])
144
+ """,
145
+ 1,
146
+ self.queue,
147
+ self.group,
148
+ self.id,
149
+ )
150
+ except Exception as error:
151
+ self.logger.error(
152
+ 'Event atomic requeue failed: queue=%s group=%s consumer=%s id=%s error_type=%s',
153
+ self.queue,
154
+ self.group,
155
+ self.consumer,
156
+ self.id,
157
+ type(error).__name__,
158
+ )
159
+ raise
160
+ self.settled = True
161
+ if isinstance(result, int) and result > 0:
162
+ self.logger.debug(
163
+ 'Event requeue and ACK completed: queue=%s group=%s consumer=%s id=%s count=%s',
164
+ self.queue,
165
+ self.group,
166
+ self.consumer,
167
+ self.id,
168
+ result,
169
+ )
170
+ else:
171
+ self.logger.debug(
172
+ 'No Event requeue target: queue=%s group=%s consumer=%s id=%s count=%s',
173
+ self.queue,
174
+ self.group,
175
+ self.consumer,
176
+ self.id,
177
+ result,
178
+ )
179
+ else:
180
+ self.logger.debug(
181
+ 'Skipping NACK for settled Event: queue=%s group=%s consumer=%s id=%s',
182
+ self.queue,
183
+ self.group,
184
+ self.consumer,
185
+ self.id,
186
+ )
187
+
188
+ def reject(self) -> None:
189
+ self.logger.debug(
190
+ 'Event reject requested (currently implemented as ACK): queue=%s group=%s consumer=%s id=%s',
191
+ self.queue,
192
+ self.group,
193
+ self.consumer,
194
+ self.id,
195
+ )
196
+ self.ack()
197
+
198
+
199
+ class Requeue(EventErrorPolicy):
200
+ def __call__(
201
+ self,
202
+ event: BaseEvent,
203
+ ) -> None:
204
+ event.nack()
205
+
206
+
207
+ class Reject(EventErrorPolicy):
208
+ def __call__(
209
+ self,
210
+ event: BaseEvent,
211
+ ) -> None:
212
+ event.reject()
@@ -0,0 +1,284 @@
1
+ from logging import getLogger
2
+ from time import (
3
+ monotonic,
4
+ sleep,
5
+ )
6
+ from typing import (
7
+ Any,
8
+ Generator,
9
+ Optional,
10
+ Union,
11
+ )
12
+ from uuid import uuid4
13
+
14
+ from redis import Redis
15
+ from redis.cluster import RedisCluster
16
+ from redis.exceptions import ResponseError
17
+
18
+ from DefStream.Broker import (
19
+ Poppable,
20
+ Readable,
21
+ Sendable,
22
+ )
23
+ from .Event import Event
24
+ from .Serializer import (
25
+ Decoder,
26
+ Encoder,
27
+ )
28
+ from .Subscription import Subscription
29
+
30
+ __all__ = (
31
+ 'Queue',
32
+ )
33
+
34
+
35
+ class Queue(
36
+ Sendable,
37
+ Poppable,
38
+ Readable,
39
+ ):
40
+ logger = getLogger('Broker.Redis.Queue')
41
+
42
+ def __init__(
43
+ self,
44
+ connection: Union[Redis, RedisCluster],
45
+ encode: Encoder,
46
+ decode: Decoder,
47
+ name: str,
48
+ group: str = 'DefStream',
49
+ ):
50
+ self.subscription = Subscription(
51
+ name,
52
+ group,
53
+ )
54
+ self.connection = connection
55
+ self.encode = encode
56
+ self.decode = decode
57
+ self.queue = name
58
+ self.group = group
59
+ self.consumer = uuid4().hex
60
+
61
+ def __str__(self) -> str:
62
+ return self.queue
63
+
64
+ def create_group(self) -> None:
65
+ self.logger.debug(
66
+ 'Creating Consumer Group: queue=%s group=%s',
67
+ self.queue,
68
+ self.group,
69
+ )
70
+ try:
71
+ self.connection.xgroup_create(
72
+ self.queue,
73
+ self.group,
74
+ id='0-0',
75
+ mkstream=True,
76
+ )
77
+ except ResponseError as error:
78
+ if not str(error).startswith('BUSYGROUP '):
79
+ self.logger.error(
80
+ 'Consumer Group creation failed: queue=%s group=%s error_type=%s',
81
+ self.queue,
82
+ self.group,
83
+ type(error).__name__,
84
+ )
85
+ raise
86
+ self.logger.debug(
87
+ 'Reusing existing Consumer Group: queue=%s group=%s',
88
+ self.queue,
89
+ self.group,
90
+ )
91
+ return
92
+ except Exception as error:
93
+ self.logger.error(
94
+ 'Consumer Group creation failed: queue=%s group=%s error_type=%s',
95
+ self.queue,
96
+ self.group,
97
+ type(error).__name__,
98
+ )
99
+ raise
100
+ self.logger.info(
101
+ 'Consumer Group creation completed: queue=%s group=%s',
102
+ self.queue,
103
+ self.group,
104
+ )
105
+
106
+ def send(
107
+ self,
108
+ body: Any,
109
+ ) -> str:
110
+ self.logger.debug(
111
+ 'Publishing Redis Stream message: queue=%s',
112
+ self.queue,
113
+ )
114
+ try:
115
+ id = self.connection.xadd(
116
+ self.queue,
117
+ {'body': self.encode(body)},
118
+ )
119
+ message_id = id.decode('ascii') if isinstance(id, bytes) else id
120
+ except Exception as error:
121
+ self.logger.error(
122
+ 'Redis Stream publication or serialization failed: queue=%s error_type=%s',
123
+ self.queue,
124
+ type(error).__name__,
125
+ )
126
+ raise
127
+ self.logger.debug(
128
+ 'Redis Stream message published: queue=%s id=%s',
129
+ self.queue,
130
+ message_id,
131
+ )
132
+ return message_id
133
+
134
+ def event(
135
+ self,
136
+ id: bytes,
137
+ fields: dict,
138
+ group: Optional[str] = None,
139
+ consumer: Optional[str] = None,
140
+ ) -> Event:
141
+ try:
142
+ message_id = id.decode('ascii')
143
+ body = self.decode(fields[b'body'])
144
+ except Exception as error:
145
+ self.logger.error(
146
+ 'Redis message deserialization or Event conversion failed: queue=%s group=%s id=%s error_type=%s',
147
+ self.queue,
148
+ group,
149
+ id,
150
+ type(error).__name__,
151
+ )
152
+ raise
153
+ self.logger.debug(
154
+ 'Converting message to Event: queue=%s group=%s consumer=%s id=%s',
155
+ self.queue,
156
+ group,
157
+ consumer,
158
+ message_id,
159
+ )
160
+ return Event(
161
+ self.connection,
162
+ self.queue,
163
+ message_id,
164
+ body,
165
+ group,
166
+ consumer,
167
+ )
168
+
169
+ def pop(
170
+ self,
171
+ timeout: Optional[int] = None,
172
+ ) -> Optional[Event]:
173
+ if timeout is not None and timeout < 0:
174
+ raise ValueError('Redis pop timeout must be non-negative milliseconds')
175
+ self.logger.debug(
176
+ 'Starting group-based single-message reception: queue=%s group=%s consumer=%s timeout=%sms',
177
+ self.queue,
178
+ self.group,
179
+ self.consumer,
180
+ timeout,
181
+ )
182
+ self.create_group()
183
+ deadline = monotonic() + timeout / 1000 if timeout else None
184
+ while True:
185
+ try:
186
+ messages = self.connection.xreadgroup(
187
+ self.group,
188
+ self.consumer,
189
+ {self.queue: '>'},
190
+ count=1,
191
+ )
192
+ except Exception as error:
193
+ self.logger.error(
194
+ 'Group-based single-message reception failed: queue=%s group=%s error_type=%s',
195
+ self.queue,
196
+ self.group,
197
+ type(error).__name__,
198
+ )
199
+ raise
200
+ for _, entries in messages:
201
+ for id, fields in entries:
202
+ event = self.event(
203
+ id,
204
+ fields,
205
+ self.group,
206
+ self.consumer,
207
+ )
208
+ self.logger.debug(
209
+ 'Group message received: queue=%s group=%s id=%s',
210
+ self.queue,
211
+ self.group,
212
+ event.id,
213
+ )
214
+ return event
215
+ if deadline is not None and monotonic() >= deadline:
216
+ self.logger.debug(
217
+ 'Group-based single-message wait timed out: queue=%s group=%s timeout=%sms',
218
+ self.queue,
219
+ self.group,
220
+ timeout,
221
+ )
222
+ return None
223
+ sleep(0.01)
224
+
225
+ def read(
226
+ self,
227
+ offset: Union[int, str] = 0,
228
+ timeout: Optional[int] = None,
229
+ ) -> Generator[Event, None, None]:
230
+ if timeout is not None and timeout < 0:
231
+ raise ValueError('Redis read timeout must be non-negative milliseconds')
232
+ cursor = str(offset)
233
+ deadline = monotonic() + timeout / 1000 if timeout else None
234
+ self.logger.debug(
235
+ 'Starting history read without a consumer group: queue=%s offset=%s timeout=%sms',
236
+ self.queue,
237
+ cursor,
238
+ timeout,
239
+ )
240
+ try:
241
+ while True:
242
+ if deadline is not None and monotonic() >= deadline:
243
+ self.logger.debug(
244
+ 'History read without a consumer group timed out: queue=%s cursor=%s timeout=%sms',
245
+ self.queue,
246
+ cursor,
247
+ timeout,
248
+ )
249
+ return
250
+ try:
251
+ messages = self.connection.xread(
252
+ {self.queue: cursor},
253
+ count=1,
254
+ )
255
+ except Exception as error:
256
+ self.logger.error(
257
+ 'History read without a consumer group failed: queue=%s cursor=%s error_type=%s',
258
+ self.queue,
259
+ cursor,
260
+ type(error).__name__,
261
+ )
262
+ raise
263
+ if not messages:
264
+ sleep(0.01)
265
+ continue
266
+ for _, entries in messages:
267
+ for id, fields in entries:
268
+ event = self.event(
269
+ id,
270
+ fields,
271
+ )
272
+ cursor = event.id
273
+ self.logger.debug(
274
+ 'History message received without a consumer group: queue=%s id=%s',
275
+ self.queue,
276
+ event.id,
277
+ )
278
+ yield event
279
+ finally:
280
+ self.logger.debug(
281
+ 'History read without a consumer group ended: queue=%s cursor=%s',
282
+ self.queue,
283
+ cursor,
284
+ )
@@ -0,0 +1,48 @@
1
+ from json import (
2
+ dumps,
3
+ loads,
4
+ )
5
+ from typing import (
6
+ Any,
7
+ Protocol,
8
+ )
9
+
10
+ __all__ = (
11
+ 'Encoder',
12
+ 'Decoder',
13
+ 'JavaScriptObjectNotationEncoder',
14
+ 'JavaScriptObjectNotationDecoder',
15
+ )
16
+
17
+
18
+ class Encoder(Protocol):
19
+ def __call__(
20
+ self,
21
+ body: Any,
22
+ ) -> bytes: ...
23
+
24
+
25
+ class Decoder(Protocol):
26
+ def __call__(
27
+ self,
28
+ body: bytes,
29
+ ) -> Any: ...
30
+
31
+
32
+ class JavaScriptObjectNotationEncoder(Encoder):
33
+ def __call__(
34
+ self,
35
+ body: Any,
36
+ ) -> bytes:
37
+ return dumps(
38
+ body,
39
+ ensure_ascii=False,
40
+ ).encode('utf-8')
41
+
42
+
43
+ class JavaScriptObjectNotationDecoder(Decoder):
44
+ def __call__(
45
+ self,
46
+ body: bytes,
47
+ ) -> Any:
48
+ return loads(body)
@@ -0,0 +1,46 @@
1
+ from collections.abc import Mapping
2
+ from dataclasses import dataclass
3
+ from typing import Any
4
+
5
+ __all__ = (
6
+ 'Subscription',
7
+ )
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class Subscription:
12
+ queue: str
13
+ group: str = 'stream'
14
+
15
+ def __post_init__(self) -> None:
16
+ for value in (
17
+ self.queue,
18
+ self.group,
19
+ ):
20
+ if not isinstance(
21
+ value,
22
+ str,
23
+ ) or not value:
24
+ raise ValueError('Redis queue and group must be non-empty strings')
25
+
26
+ @classmethod
27
+ def parse(
28
+ cls,
29
+ parameters: Any,
30
+ ) -> 'Subscription':
31
+ if isinstance(
32
+ parameters,
33
+ cls,
34
+ ):
35
+ return parameters
36
+ if isinstance(
37
+ parameters,
38
+ str,
39
+ ):
40
+ return cls(parameters)
41
+ if isinstance(
42
+ parameters,
43
+ Mapping,
44
+ ):
45
+ return cls(**dict(parameters))
46
+ raise TypeError('Redis subscriptions require a queue name, Subscription or mapping')
@@ -0,0 +1,29 @@
1
+ from DefStream.Broker import EventHandler
2
+ from .Configuration import (
3
+ Configuration,
4
+ ConnectionType,
5
+ )
6
+ from .Connection import (
7
+ Connection,
8
+ )
9
+ from .Consumer import Consumer
10
+ from .Event import (
11
+ Event,
12
+ Reject,
13
+ Requeue,
14
+ )
15
+ from .Queue import Queue
16
+ from .Subscription import Subscription
17
+
18
+ __all__ = (
19
+ 'Connection',
20
+ 'Configuration',
21
+ 'ConnectionType',
22
+ 'Consumer',
23
+ 'Event',
24
+ 'EventHandler',
25
+ 'Queue',
26
+ 'Subscription',
27
+ 'Reject',
28
+ 'Requeue',
29
+ )
@@ -0,0 +1,92 @@
1
+ Metadata-Version: 2.4
2
+ Name: DefStream-Redis
3
+ Version: 0.1.23
4
+ Summary: Synchronous message processing with pluggable broker adapters
5
+ Home-page: https://github.com/yong5eon/DefStream
6
+ Author: Heo Yongseon
7
+ Project-URL: Source, https://github.com/yong5eon/DefStream
8
+ Project-URL: Issues, https://github.com/yong5eon/DefStream/issues
9
+ Project-URL: Releases, https://github.com/yong5eon/DefStream/releases
10
+ Classifier: Programming Language :: Python :: 3.9
11
+ Classifier: Programming Language :: Python :: 3.10
12
+ Classifier: Programming Language :: Python :: 3.11
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Programming Language :: Python :: 3.13
15
+ Classifier: Programming Language :: Python :: 3.14
16
+ Requires-Python: >=3.9
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Requires-Dist: DefStream==0.1.23
20
+ Requires-Dist: redis>=5.1.1
21
+ Dynamic: author
22
+ Dynamic: classifier
23
+ Dynamic: description
24
+ Dynamic: description-content-type
25
+ Dynamic: home-page
26
+ Dynamic: license-file
27
+ Dynamic: project-url
28
+ Dynamic: requires-dist
29
+ Dynamic: requires-python
30
+ Dynamic: summary
31
+
32
+ # DefStream
33
+
34
+ Synchronous event/message processing with pluggable broker adapters
35
+
36
+ DefStream processes broker messages with ordinary Python task methods and can run them in thread or process pools. It requires Python 3.9 or later.
37
+
38
+ ## Installation
39
+
40
+ Install the client extra for the broker you use. A broker server must be available separately.
41
+
42
+ ```shell
43
+ python -m pip install "DefStream[RabbitMQ]"
44
+ ```
45
+
46
+ For Redis Streams, install `DefStream[Redis]`. To install both adapters and clients, use `DefStream[RabbitMQ,Redis]`. The plain `DefStream` installation includes neither broker implementation nor client. Each extra installs a separate, same-version adapter distribution, preserving existing import paths.
47
+
48
+ Before migrating an older integrated installation or historical standalone adapters, uninstall DefStream, DefStream-RabbitMQ and DefStream-Redis, then reinstall with the desired extras. Omitting an extra later does not remove an already installed adapter.
49
+
50
+ ## Quick start
51
+
52
+ The following worker listens to a local RabbitMQ server at `127.0.0.1:5672`. The example credentials are for local development only.
53
+
54
+ ```python
55
+ from signal import SIGINT, signal
56
+
57
+ from DefStream.Broker.Implements.RabbitMQ import Configuration, Connection, Reject, Subscription
58
+ from DefStream.Worker import Run, ThreadPool, Worker, WorkerSetup, declare
59
+
60
+
61
+ class Add(
62
+ Run,
63
+ parameters=Subscription(queue='example.add', error='example.add.error'),
64
+ on_error=Reject(),
65
+ ):
66
+ def run(self, a: int, b: int):
67
+ print(a + b)
68
+ return a + b
69
+
70
+
71
+ class Prepare(WorkerSetup):
72
+ def __call__(self, worker: Worker) -> None:
73
+ for task in worker.tasks:
74
+ declare(task=task, broker=worker.broker)
75
+
76
+
77
+ if __name__ == '__main__':
78
+ worker = Worker(
79
+ Connection(Configuration(host='127.0.0.1', port=5672, username='guest', password='guest')),
80
+ ThreadPool(4),
81
+ setups=Prepare(),
82
+ )
83
+ worker.add(Add)
84
+ signal(SIGINT, lambda *_: worker.stop())
85
+ worker.run()
86
+ ```
87
+
88
+ Run the script after starting RabbitMQ. The worker declares its task queue and waits for messages. Store production credentials outside source code.
89
+
90
+ ## License
91
+
92
+ See the `LICENSE` file included with the source distribution for the license terms.