DefStream 0.1.19__79-py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- DefStream/Broker/Configuration.py +17 -0
- DefStream/Broker/Connection.py +141 -0
- DefStream/Broker/Consumer.py +59 -0
- DefStream/Broker/Errors.py +14 -0
- DefStream/Broker/Event.py +35 -0
- DefStream/Broker/Implements/RabbitMQ/Configuration.py +49 -0
- DefStream/Broker/Implements/RabbitMQ/Connection.py +547 -0
- DefStream/Broker/Implements/RabbitMQ/Consumer.py +360 -0
- DefStream/Broker/Implements/RabbitMQ/Event.py +146 -0
- DefStream/Broker/Implements/RabbitMQ/Exchange.py +277 -0
- DefStream/Broker/Implements/RabbitMQ/Queue.py +321 -0
- DefStream/Broker/Implements/RabbitMQ/Serializer.py +75 -0
- DefStream/Broker/Implements/RabbitMQ/Stream.py +181 -0
- DefStream/Broker/Implements/RabbitMQ/Subscription.py +74 -0
- DefStream/Broker/Implements/RabbitMQ/__init__.py +45 -0
- DefStream/Broker/Implements/Redis/Configuration.py +59 -0
- DefStream/Broker/Implements/Redis/Connection.py +265 -0
- DefStream/Broker/Implements/Redis/Consumer.py +368 -0
- DefStream/Broker/Implements/Redis/Event.py +212 -0
- DefStream/Broker/Implements/Redis/Queue.py +284 -0
- DefStream/Broker/Implements/Redis/Serializer.py +48 -0
- DefStream/Broker/Implements/Redis/Subscription.py +46 -0
- DefStream/Broker/Implements/Redis/__init__.py +29 -0
- DefStream/Broker/Implements/__init__.py +1 -0
- DefStream/Broker/Priority.py +26 -0
- DefStream/Broker/Protocol.py +45 -0
- DefStream/Broker/__init__.py +53 -0
- DefStream/Descriptor/DataObject.py +409 -0
- DefStream/Descriptor/Types.py +248 -0
- DefStream/Descriptor/Value.py +188 -0
- DefStream/Descriptor/__init__.py +172 -0
- DefStream/Inspector/Extension.py +69 -0
- DefStream/Inspector/Extensions/Metrics.py +316 -0
- DefStream/Inspector/Extensions/Ping.py +107 -0
- DefStream/Inspector/Extensions/Play.html +373 -0
- DefStream/Inspector/Extensions/Play.py +415 -0
- DefStream/Inspector/Extensions/Specifications.py +176 -0
- DefStream/Inspector/Extensions/Status.html +206 -0
- DefStream/Inspector/Extensions/Status.py +124 -0
- DefStream/Inspector/Extensions/__init__.py +13 -0
- DefStream/Inspector/Inspector.py +354 -0
- DefStream/Inspector/Response.py +14 -0
- DefStream/Inspector/__init__.py +9 -0
- DefStream/Validator/Pattern.py +36 -0
- DefStream/Validator/Patterns/Array.py +30 -0
- DefStream/Validator/Patterns/Common.py +92 -0
- DefStream/Validator/Patterns/Compare.py +180 -0
- DefStream/Validator/Patterns/Condition.py +60 -0
- DefStream/Validator/Patterns/DataObject.py +93 -0
- DefStream/Validator/Patterns/DateTime.py +111 -0
- DefStream/Validator/Patterns/Number.py +43 -0
- DefStream/Validator/Patterns/Object.py +158 -0
- DefStream/Validator/Patterns/Size.py +110 -0
- DefStream/Validator/Patterns/String.py +92 -0
- DefStream/Validator/Patterns/Type.py +355 -0
- DefStream/Validator/Patterns/__init__.py +137 -0
- DefStream/Validator/Validate.py +29 -0
- DefStream/Validator/Validator.py +30 -0
- DefStream/Validator/__init__.py +11 -0
- DefStream/Worker/Executor.py +371 -0
- DefStream/Worker/Message.py +180 -0
- DefStream/Worker/Pool.py +245 -0
- DefStream/Worker/Run.py +357 -0
- DefStream/Worker/Specification.py +87 -0
- DefStream/Worker/Startup.py +174 -0
- DefStream/Worker/Worker.py +421 -0
- DefStream/Worker/__init__.py +47 -0
- DefStream/__init__.py +0 -0
- defstream-0.1.19.dist-info/METADATA +92 -0
- defstream-0.1.19.dist-info/RECORD +73 -0
- defstream-0.1.19.dist-info/WHEEL +6 -0
- defstream-0.1.19.dist-info/licenses/LICENSE +674 -0
- defstream-0.1.19.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,421 @@
|
|
|
1
|
+
from abc import (
|
|
2
|
+
ABC,
|
|
3
|
+
abstractmethod,
|
|
4
|
+
)
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from logging import getLogger
|
|
7
|
+
from threading import Event as PyEvent
|
|
8
|
+
from typing import (
|
|
9
|
+
Any,
|
|
10
|
+
List,
|
|
11
|
+
Callable,
|
|
12
|
+
Optional,
|
|
13
|
+
Sequence,
|
|
14
|
+
Type,
|
|
15
|
+
Tuple,
|
|
16
|
+
Union,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
from ..Broker import (
|
|
20
|
+
Connection,
|
|
21
|
+
GetConsumer,
|
|
22
|
+
Consumer,
|
|
23
|
+
EventHandler,
|
|
24
|
+
)
|
|
25
|
+
from ..Broker.Event import Event
|
|
26
|
+
from .Message import task_parameters
|
|
27
|
+
from ..Inspector import Inspector
|
|
28
|
+
from .Pool import (
|
|
29
|
+
Pool,
|
|
30
|
+
PoolFull,
|
|
31
|
+
)
|
|
32
|
+
from .Run import Run
|
|
33
|
+
from .Executor import run_cleanups
|
|
34
|
+
from .Startup import startup
|
|
35
|
+
|
|
36
|
+
__all__ = (
|
|
37
|
+
'Worker',
|
|
38
|
+
'WorkerSetup',
|
|
39
|
+
'WorkerCleanup',
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
class WorkerSetup(ABC):
|
|
43
|
+
@abstractmethod
|
|
44
|
+
def __call__(
|
|
45
|
+
self,
|
|
46
|
+
worker: 'Worker',
|
|
47
|
+
) -> None:
|
|
48
|
+
raise NotImplementedError
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class WorkerCleanup(ABC):
|
|
52
|
+
@abstractmethod
|
|
53
|
+
def __call__(
|
|
54
|
+
self,
|
|
55
|
+
worker: 'Worker',
|
|
56
|
+
) -> None:
|
|
57
|
+
raise NotImplementedError
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def worker_hooks(
|
|
61
|
+
hooks: Optional[Union[Callable[['Worker'], None], Sequence[Callable[['Worker'], None]]]],
|
|
62
|
+
) -> Tuple[Callable[['Worker'], None], ...]:
|
|
63
|
+
result = () if hooks is None else tuple(hooks) if isinstance(
|
|
64
|
+
hooks,
|
|
65
|
+
Sequence,
|
|
66
|
+
) else (hooks,)
|
|
67
|
+
if not all(callable(hook) for hook in result):
|
|
68
|
+
raise TypeError('Worker hooks must be callable')
|
|
69
|
+
return result
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class Worker:
|
|
73
|
+
logger = getLogger('Worker')
|
|
74
|
+
|
|
75
|
+
def __init__(
|
|
76
|
+
self,
|
|
77
|
+
broker: Connection,
|
|
78
|
+
pool: Pool,
|
|
79
|
+
setups: Optional[Union[WorkerSetup, Sequence[WorkerSetup]]] = None,
|
|
80
|
+
cleanups: Optional[Union[WorkerCleanup, Sequence[WorkerCleanup]]] = None,
|
|
81
|
+
inspector: Optional[Inspector] = None,
|
|
82
|
+
):
|
|
83
|
+
self.broker = broker
|
|
84
|
+
self.pool = pool
|
|
85
|
+
self.tasks: List[Type[Run]] = []
|
|
86
|
+
self.consumer: Optional[Consumer] = None
|
|
87
|
+
self.started = False
|
|
88
|
+
self.running = PyEvent()
|
|
89
|
+
self.stopping = PyEvent()
|
|
90
|
+
self.finished = PyEvent()
|
|
91
|
+
self.pending = 0
|
|
92
|
+
self.setups = worker_hooks(setups)
|
|
93
|
+
self.cleanups = worker_hooks(cleanups)
|
|
94
|
+
self.inspector = inspector
|
|
95
|
+
|
|
96
|
+
def add(
|
|
97
|
+
self,
|
|
98
|
+
run: Type[Run],
|
|
99
|
+
) -> None:
|
|
100
|
+
if self.started:
|
|
101
|
+
raise RuntimeError('Cannot register runs after starting the worker')
|
|
102
|
+
task_parameters(run)
|
|
103
|
+
if run in self.tasks:
|
|
104
|
+
raise ValueError('Run is already registered')
|
|
105
|
+
if any(
|
|
106
|
+
registered.name == run.name
|
|
107
|
+
for registered in self.tasks
|
|
108
|
+
):
|
|
109
|
+
raise ValueError('Duplicate Run name: ' + run.name)
|
|
110
|
+
self.tasks.append(run)
|
|
111
|
+
self.logger.debug(
|
|
112
|
+
'Task registered: task=%s.%s',
|
|
113
|
+
run.__module__,
|
|
114
|
+
run.__qualname__,
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
def run(self) -> None:
|
|
118
|
+
if self.started:
|
|
119
|
+
raise RuntimeError('Worker can only be started once')
|
|
120
|
+
if not self.tasks:
|
|
121
|
+
raise RuntimeError('Register at least one Run before starting')
|
|
122
|
+
self.started = True
|
|
123
|
+
self.logger.info(
|
|
124
|
+
'Starting Worker: tasks=%s',
|
|
125
|
+
len(self.tasks),
|
|
126
|
+
)
|
|
127
|
+
try:
|
|
128
|
+
startup(self)
|
|
129
|
+
if self.inspector is not None:
|
|
130
|
+
self.inspector.start(self)
|
|
131
|
+
self.logger.debug('Connecting to broker')
|
|
132
|
+
self.broker.connect()
|
|
133
|
+
self.logger.debug('Broker connected')
|
|
134
|
+
for setup in self.setups:
|
|
135
|
+
self.logger.debug(
|
|
136
|
+
'Starting Worker setup: hook=%s',
|
|
137
|
+
type(setup).__qualname__,
|
|
138
|
+
)
|
|
139
|
+
setup(self)
|
|
140
|
+
self.logger.debug(
|
|
141
|
+
'Worker setup completed: hook=%s',
|
|
142
|
+
type(setup).__qualname__,
|
|
143
|
+
)
|
|
144
|
+
if not isinstance(self.broker, GetConsumer):
|
|
145
|
+
raise TypeError('Worker broker must support consumer creation')
|
|
146
|
+
self.consumer = self.broker.consumer()
|
|
147
|
+
self.logger.debug('Consumer created')
|
|
148
|
+
for run in self.tasks:
|
|
149
|
+
self.logger.debug(
|
|
150
|
+
'Subscribing task: task=%s.%s',
|
|
151
|
+
run.__module__,
|
|
152
|
+
run.__qualname__,
|
|
153
|
+
)
|
|
154
|
+
self.consumer.subs(
|
|
155
|
+
run.parameters,
|
|
156
|
+
Handler(
|
|
157
|
+
self,
|
|
158
|
+
run,
|
|
159
|
+
),
|
|
160
|
+
)
|
|
161
|
+
self.logger.debug(
|
|
162
|
+
'Task subscribed: task=%s.%s',
|
|
163
|
+
run.__module__,
|
|
164
|
+
run.__qualname__,
|
|
165
|
+
)
|
|
166
|
+
if not self.stopping.is_set():
|
|
167
|
+
self.running.set()
|
|
168
|
+
self.logger.info('Worker ready to consume')
|
|
169
|
+
self.consumer.run()
|
|
170
|
+
finally:
|
|
171
|
+
self.running.clear()
|
|
172
|
+
self.finished.set()
|
|
173
|
+
if self.pending:
|
|
174
|
+
self.logger.error(
|
|
175
|
+
'Consumer exited with unconfirmed deliveries: pending=%s',
|
|
176
|
+
self.pending,
|
|
177
|
+
)
|
|
178
|
+
try:
|
|
179
|
+
self.logger.info(
|
|
180
|
+
'Draining Worker tasks: pending=%s',
|
|
181
|
+
self.pending,
|
|
182
|
+
)
|
|
183
|
+
self.pool.stop()
|
|
184
|
+
finally:
|
|
185
|
+
try:
|
|
186
|
+
self.cleanup()
|
|
187
|
+
finally:
|
|
188
|
+
try:
|
|
189
|
+
if self.consumer is not None:
|
|
190
|
+
self.consumer.close()
|
|
191
|
+
finally:
|
|
192
|
+
try:
|
|
193
|
+
self.broker.close()
|
|
194
|
+
finally:
|
|
195
|
+
if self.inspector is not None and self.inspector.worker is self:
|
|
196
|
+
self.inspector.stop()
|
|
197
|
+
self.logger.info('Worker shutdown completed')
|
|
198
|
+
|
|
199
|
+
def cleanup(self) -> None:
|
|
200
|
+
self.logger.debug(
|
|
201
|
+
'Starting Worker cleanup: hooks=%s',
|
|
202
|
+
len(self.cleanups),
|
|
203
|
+
)
|
|
204
|
+
run_cleanups(
|
|
205
|
+
self.cleanups,
|
|
206
|
+
self,
|
|
207
|
+
)
|
|
208
|
+
self.logger.debug('Worker cleanup completed')
|
|
209
|
+
|
|
210
|
+
def stop(self) -> None:
|
|
211
|
+
if not self.stopping.is_set():
|
|
212
|
+
self.logger.info(
|
|
213
|
+
'Worker shutdown requested: pending=%s',
|
|
214
|
+
self.pending,
|
|
215
|
+
)
|
|
216
|
+
self.stopping.set()
|
|
217
|
+
if self.consumer is not None and not self.finished.is_set():
|
|
218
|
+
self.consumer.call(self.try_stop)
|
|
219
|
+
|
|
220
|
+
def try_stop(self) -> None:
|
|
221
|
+
if self.stopping.is_set() and self.pending == 0 and self.consumer is not None:
|
|
222
|
+
self.logger.debug('No pending tasks; requesting Consumer stop')
|
|
223
|
+
self.consumer.stop()
|
|
224
|
+
|
|
225
|
+
def receive(
|
|
226
|
+
self,
|
|
227
|
+
run: Type[Run],
|
|
228
|
+
event: Event,
|
|
229
|
+
) -> None:
|
|
230
|
+
if self.stopping.is_set():
|
|
231
|
+
self.logger.debug(
|
|
232
|
+
'Requeuing message received during shutdown: task=%s.%s',
|
|
233
|
+
run.__module__,
|
|
234
|
+
run.__qualname__,
|
|
235
|
+
)
|
|
236
|
+
event.nack()
|
|
237
|
+
return
|
|
238
|
+
if not isinstance(
|
|
239
|
+
event.body,
|
|
240
|
+
Mapping,
|
|
241
|
+
):
|
|
242
|
+
self.failed(
|
|
243
|
+
run,
|
|
244
|
+
event,
|
|
245
|
+
TypeError('Run message body must be a mapping'),
|
|
246
|
+
)
|
|
247
|
+
return
|
|
248
|
+
self.pending += 1
|
|
249
|
+
consumer = self.consumer
|
|
250
|
+
if consumer is None:
|
|
251
|
+
self.pending -= 1
|
|
252
|
+
raise RuntimeError('Worker consumer is not initialized')
|
|
253
|
+
inspector = self.inspector
|
|
254
|
+
task = run.name
|
|
255
|
+
self.logger.debug(
|
|
256
|
+
'Task message received: task=%s pending=%s',
|
|
257
|
+
task,
|
|
258
|
+
self.pending,
|
|
259
|
+
)
|
|
260
|
+
contexts = inspector.begin(task) if inspector is not None else ()
|
|
261
|
+
|
|
262
|
+
def schedule(
|
|
263
|
+
error: Optional[BaseException] = None,
|
|
264
|
+
result: Any = None,
|
|
265
|
+
) -> None:
|
|
266
|
+
if inspector is not None:
|
|
267
|
+
inspector.end(
|
|
268
|
+
task,
|
|
269
|
+
contexts,
|
|
270
|
+
error is None,
|
|
271
|
+
)
|
|
272
|
+
if self.finished.is_set():
|
|
273
|
+
self.logger.error(
|
|
274
|
+
'Cannot confirm delivery after Consumer exit: task=%s',
|
|
275
|
+
task,
|
|
276
|
+
)
|
|
277
|
+
return
|
|
278
|
+
self.logger.debug(
|
|
279
|
+
'Scheduling task result callback: task=%s success=%s',
|
|
280
|
+
task,
|
|
281
|
+
error is None,
|
|
282
|
+
)
|
|
283
|
+
consumer.call(lambda: self.complete(
|
|
284
|
+
run,
|
|
285
|
+
event,
|
|
286
|
+
error,
|
|
287
|
+
result,
|
|
288
|
+
))
|
|
289
|
+
|
|
290
|
+
try:
|
|
291
|
+
self.logger.debug(
|
|
292
|
+
'Submitting task to Pool: task=%s',
|
|
293
|
+
task,
|
|
294
|
+
)
|
|
295
|
+
self.pool.submit(
|
|
296
|
+
run,
|
|
297
|
+
dict(event.body),
|
|
298
|
+
callback=lambda result: schedule(result=result),
|
|
299
|
+
error_callback=schedule,
|
|
300
|
+
)
|
|
301
|
+
self.logger.debug(
|
|
302
|
+
'Pool task submission accepted: task=%s',
|
|
303
|
+
task,
|
|
304
|
+
)
|
|
305
|
+
except PoolFull:
|
|
306
|
+
self.logger.debug(
|
|
307
|
+
'Requeuing message because Pool is full: task=%s',
|
|
308
|
+
task,
|
|
309
|
+
)
|
|
310
|
+
if inspector is not None:
|
|
311
|
+
inspector.end(
|
|
312
|
+
task,
|
|
313
|
+
contexts,
|
|
314
|
+
None,
|
|
315
|
+
)
|
|
316
|
+
self.pending -= 1
|
|
317
|
+
event.nack()
|
|
318
|
+
return
|
|
319
|
+
except Exception:
|
|
320
|
+
if inspector is not None:
|
|
321
|
+
inspector.end(
|
|
322
|
+
task,
|
|
323
|
+
contexts,
|
|
324
|
+
None,
|
|
325
|
+
)
|
|
326
|
+
self.pending -= 1
|
|
327
|
+
raise
|
|
328
|
+
|
|
329
|
+
def complete(
|
|
330
|
+
self,
|
|
331
|
+
run: Type[Run],
|
|
332
|
+
event: Event,
|
|
333
|
+
error: Optional[BaseException] = None,
|
|
334
|
+
result: Any = None,
|
|
335
|
+
) -> None:
|
|
336
|
+
try:
|
|
337
|
+
if error is not None:
|
|
338
|
+
self.failed(
|
|
339
|
+
run,
|
|
340
|
+
event,
|
|
341
|
+
error,
|
|
342
|
+
)
|
|
343
|
+
else:
|
|
344
|
+
if self.consumer is None:
|
|
345
|
+
raise RuntimeError('Worker consumer is not initialized')
|
|
346
|
+
self.consumer.complete(
|
|
347
|
+
event,
|
|
348
|
+
result,
|
|
349
|
+
)
|
|
350
|
+
self.logger.debug(
|
|
351
|
+
'Consumer completion returned: task=%s.%s',
|
|
352
|
+
run.__module__,
|
|
353
|
+
run.__qualname__,
|
|
354
|
+
)
|
|
355
|
+
finally:
|
|
356
|
+
self.pending -= 1
|
|
357
|
+
self.logger.debug(
|
|
358
|
+
'Task completion handling finished: pending=%s',
|
|
359
|
+
self.pending,
|
|
360
|
+
)
|
|
361
|
+
self.try_stop()
|
|
362
|
+
|
|
363
|
+
def failed(
|
|
364
|
+
self,
|
|
365
|
+
run: Type[Run],
|
|
366
|
+
event: Event,
|
|
367
|
+
error: BaseException,
|
|
368
|
+
) -> None:
|
|
369
|
+
if isinstance(
|
|
370
|
+
error,
|
|
371
|
+
TypeError,
|
|
372
|
+
) and not isinstance(
|
|
373
|
+
event.body,
|
|
374
|
+
Mapping,
|
|
375
|
+
):
|
|
376
|
+
self.logger.warning(
|
|
377
|
+
'Rejected a non-mapping task message: task=%s.%s',
|
|
378
|
+
run.__module__,
|
|
379
|
+
run.__qualname__,
|
|
380
|
+
)
|
|
381
|
+
else:
|
|
382
|
+
self.logger.error(
|
|
383
|
+
'Task failed: task=%s.%s',
|
|
384
|
+
run.__module__,
|
|
385
|
+
run.__qualname__,
|
|
386
|
+
exc_info=(
|
|
387
|
+
type(error),
|
|
388
|
+
error,
|
|
389
|
+
error.__traceback__,
|
|
390
|
+
),
|
|
391
|
+
)
|
|
392
|
+
if self.consumer is None:
|
|
393
|
+
raise RuntimeError('Worker consumer is not initialized')
|
|
394
|
+
self.consumer.fail(
|
|
395
|
+
event,
|
|
396
|
+
run.on_error,
|
|
397
|
+
)
|
|
398
|
+
self.logger.debug(
|
|
399
|
+
'Consumer failure policy returned: task=%s.%s',
|
|
400
|
+
run.__module__,
|
|
401
|
+
run.__qualname__,
|
|
402
|
+
)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
class Handler(EventHandler):
|
|
406
|
+
def __init__(
|
|
407
|
+
self,
|
|
408
|
+
worker: Worker,
|
|
409
|
+
run: Type[Run],
|
|
410
|
+
):
|
|
411
|
+
self.worker = worker
|
|
412
|
+
self.run = run
|
|
413
|
+
|
|
414
|
+
def __call__(
|
|
415
|
+
self,
|
|
416
|
+
event: Event,
|
|
417
|
+
) -> None:
|
|
418
|
+
self.worker.receive(
|
|
419
|
+
self.run,
|
|
420
|
+
event,
|
|
421
|
+
)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from .Worker import (
|
|
2
|
+
Worker,
|
|
3
|
+
WorkerSetup,
|
|
4
|
+
WorkerCleanup,
|
|
5
|
+
)
|
|
6
|
+
from .Message import (
|
|
7
|
+
declare,
|
|
8
|
+
emit,
|
|
9
|
+
request,
|
|
10
|
+
)
|
|
11
|
+
from .Run import (
|
|
12
|
+
Run,
|
|
13
|
+
Context,
|
|
14
|
+
RunSetup,
|
|
15
|
+
RunComplete,
|
|
16
|
+
RunError,
|
|
17
|
+
)
|
|
18
|
+
from .Specification import Specification
|
|
19
|
+
from .Pool import (
|
|
20
|
+
Pool,
|
|
21
|
+
PoolFull,
|
|
22
|
+
PoolSetup,
|
|
23
|
+
PoolCleanup,
|
|
24
|
+
ThreadPool,
|
|
25
|
+
ProcessPool,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
__all__ = (
|
|
29
|
+
'Worker',
|
|
30
|
+
'declare',
|
|
31
|
+
'emit',
|
|
32
|
+
'request',
|
|
33
|
+
'WorkerSetup',
|
|
34
|
+
'WorkerCleanup',
|
|
35
|
+
'Run',
|
|
36
|
+
'Specification',
|
|
37
|
+
'Pool',
|
|
38
|
+
'PoolFull',
|
|
39
|
+
'PoolSetup',
|
|
40
|
+
'PoolCleanup',
|
|
41
|
+
'ThreadPool',
|
|
42
|
+
'ProcessPool',
|
|
43
|
+
'Context',
|
|
44
|
+
'RunSetup',
|
|
45
|
+
'RunComplete',
|
|
46
|
+
'RunError',
|
|
47
|
+
)
|
DefStream/__init__.py
ADDED
|
File without changes
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: DefStream
|
|
3
|
+
Version: 0.1.19
|
|
4
|
+
Summary: Synchronous message processing with pluggable broker adapters
|
|
5
|
+
Home-page: https://github.com/yong5eon/DefStream
|
|
6
|
+
Author: Heo Yongseon
|
|
7
|
+
Project-URL: Source, https://github.com/yong5eon/DefStream
|
|
8
|
+
Project-URL: Issues, https://github.com/yong5eon/DefStream/issues
|
|
9
|
+
Project-URL: Releases, https://github.com/yong5eon/DefStream/releases
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Requires-Python: >=3.9
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Provides-Extra: rabbitmq
|
|
20
|
+
Requires-Dist: pika>=1.3.2; extra == "rabbitmq"
|
|
21
|
+
Provides-Extra: redis
|
|
22
|
+
Requires-Dist: redis>=5.1.1; extra == "redis"
|
|
23
|
+
Dynamic: author
|
|
24
|
+
Dynamic: classifier
|
|
25
|
+
Dynamic: description
|
|
26
|
+
Dynamic: description-content-type
|
|
27
|
+
Dynamic: home-page
|
|
28
|
+
Dynamic: license-file
|
|
29
|
+
Dynamic: project-url
|
|
30
|
+
Dynamic: provides-extra
|
|
31
|
+
Dynamic: requires-python
|
|
32
|
+
Dynamic: summary
|
|
33
|
+
|
|
34
|
+
# DefStream
|
|
35
|
+
|
|
36
|
+
Synchronous event/message processing with pluggable broker adapters
|
|
37
|
+
|
|
38
|
+
DefStream processes broker messages with ordinary Python task methods and can run them in thread or process pools. It requires Python 3.9 or later.
|
|
39
|
+
|
|
40
|
+
## Installation
|
|
41
|
+
|
|
42
|
+
Install the client extra for the broker you use. A broker server must be available separately.
|
|
43
|
+
|
|
44
|
+
```shell
|
|
45
|
+
python -m pip install "DefStream[RabbitMQ]"
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
For Redis Streams, install `DefStream[Redis]`. To install both clients, use `DefStream[RabbitMQ,Redis]`. The plain `DefStream` installation includes neither broker client.
|
|
49
|
+
|
|
50
|
+
## Quick start
|
|
51
|
+
|
|
52
|
+
The following worker listens to a local RabbitMQ server at `127.0.0.1:5672`. The example credentials are for local development only.
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from signal import SIGINT, signal
|
|
56
|
+
|
|
57
|
+
from DefStream.Broker.Implements.RabbitMQ import Configuration, Connection, Reject, Subscription
|
|
58
|
+
from DefStream.Worker import Run, ThreadPool, Worker, WorkerSetup, declare
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class Add(
|
|
62
|
+
Run,
|
|
63
|
+
parameters=Subscription(queue='example.add', error='example.add.error'),
|
|
64
|
+
on_error=Reject(),
|
|
65
|
+
):
|
|
66
|
+
def run(self, a: int, b: int):
|
|
67
|
+
print(a + b)
|
|
68
|
+
return a + b
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class Prepare(WorkerSetup):
|
|
72
|
+
def __call__(self, worker: Worker) -> None:
|
|
73
|
+
for task in worker.tasks:
|
|
74
|
+
declare(task=task, broker=worker.broker)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == '__main__':
|
|
78
|
+
worker = Worker(
|
|
79
|
+
Connection(Configuration(host='127.0.0.1', port=5672, username='guest', password='guest')),
|
|
80
|
+
ThreadPool(4),
|
|
81
|
+
setups=Prepare(),
|
|
82
|
+
)
|
|
83
|
+
worker.add(Add)
|
|
84
|
+
signal(SIGINT, lambda *_: worker.stop())
|
|
85
|
+
worker.run()
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Run the script after starting RabbitMQ. The worker declares its task queue and waits for messages. Store production credentials outside source code.
|
|
89
|
+
|
|
90
|
+
## License
|
|
91
|
+
|
|
92
|
+
See the `LICENSE` file included with the source distribution for the license terms.
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
DefStream/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
DefStream/Broker/Configuration.py,sha256=jSeiHGAb_TTUKInU8_4cprafBpX2JMf24frm8BnEu7Y,345
|
|
3
|
+
DefStream/Broker/Connection.py,sha256=KWJDcLq3zYYaytgrTdDZcCoZWGd7KU6B2trAPsen0GU,2125
|
|
4
|
+
DefStream/Broker/Consumer.py,sha256=y9nMjhls0I78tCSInLMuBf0JrggBGGadjEFfPaOLLgs,1271
|
|
5
|
+
DefStream/Broker/Errors.py,sha256=_l1ieD1vBbX8MLXEzTK8m_21-8o0Szix3Pg-DNY0aNQ,206
|
|
6
|
+
DefStream/Broker/Event.py,sha256=xLm3XBD91yzfwjNbxz4xoiXZdlnsrp020PyMFMmAIYs,482
|
|
7
|
+
DefStream/Broker/Priority.py,sha256=PKxjpMkQEOIOT_cZjG8J6IA7tdQBk3xqQVNG0a7xU7c,621
|
|
8
|
+
DefStream/Broker/Protocol.py,sha256=vYI-8E6BPUeKMw5aHx4FbWGpwgFdqWSj2B9uX6smkK8,573
|
|
9
|
+
DefStream/Broker/__init__.py,sha256=NxRw26QT28QuWB_324BREAguNM_KNR0_7_-kchv7TCQ,810
|
|
10
|
+
DefStream/Broker/Implements/__init__.py,sha256=iwhKnzeBJLKxpRVjvzwiRE63_zNpIBfaKLITauVph-0,24
|
|
11
|
+
DefStream/Broker/Implements/RabbitMQ/Configuration.py,sha256=sHysXc1nlJHbSjWD3BZxlXjKBAbz2AKDtr7ACVVVj0c,1094
|
|
12
|
+
DefStream/Broker/Implements/RabbitMQ/Connection.py,sha256=kk_a3jPh5sZBvFm1Jkc08iG2WCFTyRB8jwsr5LAwnVM,14188
|
|
13
|
+
DefStream/Broker/Implements/RabbitMQ/Consumer.py,sha256=sDUzHIE12Cx9zUq3MUAF29eagOivNqxW-w1euNgiCbU,9707
|
|
14
|
+
DefStream/Broker/Implements/RabbitMQ/Event.py,sha256=r4r7XT_-2N6XSPz8hBiGNS5di6OYe37q3INYxJAYQCk,3362
|
|
15
|
+
DefStream/Broker/Implements/RabbitMQ/Exchange.py,sha256=EblUpf1SKUfmESREWNargJwGpoSwA5v0Th5MHcwW4Xg,8444
|
|
16
|
+
DefStream/Broker/Implements/RabbitMQ/Queue.py,sha256=b9gb65IuW-TpWsSbF-cHMiJZLq8PLH1li6D5K1UCW5Y,9419
|
|
17
|
+
DefStream/Broker/Implements/RabbitMQ/Serializer.py,sha256=PbL_V89GHPjqh1zvOM-apExhD7Bgau2jinpVCsVUg00,1718
|
|
18
|
+
DefStream/Broker/Implements/RabbitMQ/Stream.py,sha256=vhfnIhpCe1gduT09VIfp6ZhpQtHyqz6bjWhlWAjNy9E,5698
|
|
19
|
+
DefStream/Broker/Implements/RabbitMQ/Subscription.py,sha256=A1sCwWgytNYOIJ-D9PKps_1blxJNp1lGK0a1CDHh7Jw,1849
|
|
20
|
+
DefStream/Broker/Implements/RabbitMQ/__init__.py,sha256=-0Cj2PyjAVhPSP4w1geyLzwRioAYOuFaGMrp1Le5z4I,647
|
|
21
|
+
DefStream/Broker/Implements/Redis/Configuration.py,sha256=A86E44Jo2z_d45xBHH92atR-PPyySlT7QWgxc3UX1Yc,1551
|
|
22
|
+
DefStream/Broker/Implements/Redis/Connection.py,sha256=dFxdV8Pv3P3x8b9cKr9376r1afOZj0gr2WUxhpO1IQk,5237
|
|
23
|
+
DefStream/Broker/Implements/Redis/Consumer.py,sha256=FsHQmzb2zsrAFqarHwkfTdLptPDz2kqZWtHW0Lg20aw,8745
|
|
24
|
+
DefStream/Broker/Implements/Redis/Event.py,sha256=j1g-nIPjq2FxFZ1XqBpjYgM0TSBBXjTkLPXtgS2K8a4,4641
|
|
25
|
+
DefStream/Broker/Implements/Redis/Queue.py,sha256=8cyOGC0gKHbfHN5qDmYFRjt20DqRatNNBu-_uU7qJjI,6123
|
|
26
|
+
DefStream/Broker/Implements/Redis/Serializer.py,sha256=NWGxpnf5kegR9i_kr8cxQk5JiXO1lMdvTo2sPmghHe0,635
|
|
27
|
+
DefStream/Broker/Implements/Redis/Subscription.py,sha256=l7YKlnlNvhDSnmaQ7dSsG7rCCz1aNcFZwDmq_U-BN9c,808
|
|
28
|
+
DefStream/Broker/Implements/Redis/__init__.py,sha256=K9kAja14MbJMQkUveGnw3o-8k3t6QmWp2lb9iwQ4GoQ,449
|
|
29
|
+
DefStream/Descriptor/DataObject.py,sha256=hq7YufF6_NCs41TFUozpwFv6qI5iv3Cx5SHHq2IPeLc,12018
|
|
30
|
+
DefStream/Descriptor/Types.py,sha256=tnC9ZjsCx8e_zK5pXjJymx-UxRJNQPBPMxrfMyG2fIE,5112
|
|
31
|
+
DefStream/Descriptor/Value.py,sha256=dSEVedEc9glv3sWC4idUdW0EXtstCW7It2WeZfTGYF0,3525
|
|
32
|
+
DefStream/Descriptor/__init__.py,sha256=xu_dDulMEItY6d7OjMZptsBb9_zdH--rqPTTnkz8ZGg,2544
|
|
33
|
+
DefStream/Inspector/Extension.py,sha256=4VqIVtsMpeLzsnlG1fWVif1mBv3BakC7L1fFVso4gvc,954
|
|
34
|
+
DefStream/Inspector/Inspector.py,sha256=K8L52T0krZIN_zEh5E7jrbXVCEdIaarRs0G6iqOycq8,7522
|
|
35
|
+
DefStream/Inspector/Response.py,sha256=QtXyorMAize6iM08so3k-WY1PdArjE6VK3Lixjz6WvA,204
|
|
36
|
+
DefStream/Inspector/__init__.py,sha256=O1rJuT-n92AY3Z3im6XJ13mqaI67wpMQmbPwLDBbOQU,153
|
|
37
|
+
DefStream/Inspector/Extensions/Metrics.py,sha256=UEY5hHoJWyUYyJFc7ASN-mBv62ABDXY-DQH56csvdEs,8337
|
|
38
|
+
DefStream/Inspector/Extensions/Ping.py,sha256=PTeAXbObuhjzJipRd9ilTR0H-yaALERVBaFtEoyssbo,2377
|
|
39
|
+
DefStream/Inspector/Extensions/Play.html,sha256=ukZOjJVA9pdihL2zV-DPMeWRK9Ed841jOnmiAegMDuA,21012
|
|
40
|
+
DefStream/Inspector/Extensions/Play.py,sha256=S4VEkCuWvPAGKu8dkUFjK5t0vVMGhg-s-2KgXFz1LMs,9951
|
|
41
|
+
DefStream/Inspector/Extensions/Specifications.py,sha256=c5CpAW3iimdPZjvwLyiK6KxFtv9uSDSCMhdxg9TfPVA,3827
|
|
42
|
+
DefStream/Inspector/Extensions/Status.html,sha256=PBYBVzV1pgwST89_CIX_jhXHGwKQrpoL-KEyrzuKkJE,12060
|
|
43
|
+
DefStream/Inspector/Extensions/Status.py,sha256=NmmycqHb1VKFGpmuuyHcZBcUMl_Mzb64jFkW0Liw_r0,2781
|
|
44
|
+
DefStream/Inspector/Extensions/__init__.py,sha256=iq-3ToQFZtqyISthJ37jeIPEbZL8qc48LguPWNvH26I,220
|
|
45
|
+
DefStream/Validator/Pattern.py,sha256=gRRlEzS908b9OCXRpFnEG4gujNKCSTlhPUGwWGi-Dqg,674
|
|
46
|
+
DefStream/Validator/Validate.py,sha256=2Edk1sWh5_4QdYTc1xakDLPXMAWNrrjso1TLi7ZQ4Gg,700
|
|
47
|
+
DefStream/Validator/Validator.py,sha256=lvJGzDKecKIBAQrfFuSHBRUh6PCxiHRsabul6cYxcdA,556
|
|
48
|
+
DefStream/Validator/__init__.py,sha256=juhR_lm3XpqU-BUnTjbaYbFNXa_zs9Xi9-x_9EyvJgE,172
|
|
49
|
+
DefStream/Validator/Patterns/Array.py,sha256=XPc88iPhU2k30TIyMsB6pdDqUO_wjv-5YP96mTHbApc,514
|
|
50
|
+
DefStream/Validator/Patterns/Common.py,sha256=JJIOAcPO6epnIINaVu5SuwmnQoK3rLQn6vYyZ3JBuX8,1817
|
|
51
|
+
DefStream/Validator/Patterns/Compare.py,sha256=5xYVvaiQydvIi8WYgIADltSk0tz_ki76k-c89vZsFcw,3726
|
|
52
|
+
DefStream/Validator/Patterns/Condition.py,sha256=V8iEG9onH6bxlUnqFY8jDdmm6BIx2SBeEJAluVHeEUM,1135
|
|
53
|
+
DefStream/Validator/Patterns/DataObject.py,sha256=TRjHW45obrGT-ja49ZAlnSAMIY7eL6bqBe9DjI1eeSU,2282
|
|
54
|
+
DefStream/Validator/Patterns/DateTime.py,sha256=2ejPyRxFbvWf_0FmUrtXTBTvMnp8nakfW_TyCsRY_h4,2515
|
|
55
|
+
DefStream/Validator/Patterns/Number.py,sha256=k59Ue1Sw-D1iWz8OywjQljPVXdZlTawSl9jBpPjJKKY,919
|
|
56
|
+
DefStream/Validator/Patterns/Object.py,sha256=U6IAXvJIh2Bl2T6FnhAS7dpNcSMWbV09xtlRjcfPBRE,3616
|
|
57
|
+
DefStream/Validator/Patterns/Size.py,sha256=n-PdMTYNCw0-KN2wqWqK7nN6smVvrGLO4YfwBLlGA9Y,2016
|
|
58
|
+
DefStream/Validator/Patterns/String.py,sha256=V4EmJqZfXyByiHi4wsv0E5JNLB-qyHEZiTuMsMOK8lk,1857
|
|
59
|
+
DefStream/Validator/Patterns/Type.py,sha256=dcGZNRYjtE3Z5Kal2ZnLR-0fRaDWBzI7SkGXC0D2AyU,8874
|
|
60
|
+
DefStream/Validator/Patterns/__init__.py,sha256=2Kkk6BrvcLl0C22I0WhGphp2dF55Ig2b3LY0DMDNWm8,1603
|
|
61
|
+
DefStream/Worker/Executor.py,sha256=L0y7LzQ3uc50thms_8d_rQr8Tmrkgv1IxBCHxsIcmY8,8021
|
|
62
|
+
DefStream/Worker/Message.py,sha256=JG2I-398YpfAlcCqzds0sFQ35UrqAPZbmdaQTmQ7RyE,3242
|
|
63
|
+
DefStream/Worker/Pool.py,sha256=-MphXSHRFGMNv2FVXhME0lqewJAvENwoExfvLHCeW2w,4821
|
|
64
|
+
DefStream/Worker/Run.py,sha256=pGevc8E10tl0BqoM_tDwL0CuPrINIjjRt_I5v-7Aj-Y,7397
|
|
65
|
+
DefStream/Worker/Specification.py,sha256=MRPpHtSKYaDAa6t0vitCJCiCwQRL9QmG1Y2zibv_UWU,1807
|
|
66
|
+
DefStream/Worker/Startup.py,sha256=vuGfGHzJcx3vCBZAEs3UxDyzHKKW7U9lWUEkeFTAyN4,3955
|
|
67
|
+
DefStream/Worker/Worker.py,sha256=HGCS8HXqd26-MrZ42P7C9XQ8UxhawuT0_K7hfGRUPVU,8775
|
|
68
|
+
DefStream/Worker/__init__.py,sha256=TG_Ma2HFLi5DKTPbqKZ5VWR28g-_WhsKcrf3iYLJxcw,578
|
|
69
|
+
defstream-0.1.19.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
|
|
70
|
+
defstream-0.1.19.dist-info/METADATA,sha256=MNU-5fhvSe16YqZ-DwUzb2rz6sKAaBUCR7pNAi7iFEE,2938
|
|
71
|
+
defstream-0.1.19.dist-info/WHEEL,sha256=I5WPLlK4KOtpwtryk4Cz5RI94DKEfAXnwu6OFIhAols,101
|
|
72
|
+
defstream-0.1.19.dist-info/top_level.txt,sha256=zsnisIu5wHL4b7VM8e9AggG9KTSdUU4morFqpb8YWFs,10
|
|
73
|
+
defstream-0.1.19.dist-info/RECORD,,
|