DefStream 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- defstream-0.1.2/DefStream.egg-info/PKG-INFO +1197 -0
- defstream-0.1.2/DefStream.egg-info/SOURCES.txt +70 -0
- defstream-0.1.2/DefStream.egg-info/dependency_links.txt +1 -0
- defstream-0.1.2/DefStream.egg-info/requires.txt +6 -0
- defstream-0.1.2/DefStream.egg-info/top_level.txt +1 -0
- defstream-0.1.2/LICENSE +674 -0
- defstream-0.1.2/MANIFEST.in +1 -0
- defstream-0.1.2/PKG-INFO +1197 -0
- defstream-0.1.2/README.md +1165 -0
- defstream-0.1.2/VERSION +1 -0
- defstream-0.1.2/setup.cfg +4 -0
- defstream-0.1.2/setup.py +60 -0
- defstream-0.1.2/src/Broker/Configuration.py +18 -0
- defstream-0.1.2/src/Broker/Connection.py +97 -0
- defstream-0.1.2/src/Broker/Consumer.py +59 -0
- defstream-0.1.2/src/Broker/Errors.py +14 -0
- defstream-0.1.2/src/Broker/Event.py +35 -0
- defstream-0.1.2/src/Broker/Implements/__init__.py +1 -0
- defstream-0.1.2/src/Broker/Policies.py +25 -0
- defstream-0.1.2/src/Broker/Protocol.py +45 -0
- defstream-0.1.2/src/Broker/__init__.py +47 -0
- defstream-0.1.2/src/Descriptor/DataObject.py +384 -0
- defstream-0.1.2/src/Descriptor/Types.py +229 -0
- defstream-0.1.2/src/Descriptor/Value.py +132 -0
- defstream-0.1.2/src/Descriptor/__init__.py +172 -0
- defstream-0.1.2/src/Inspector/Extension.py +69 -0
- defstream-0.1.2/src/Inspector/Extensions/Console.html +373 -0
- defstream-0.1.2/src/Inspector/Extensions/Console.py +396 -0
- defstream-0.1.2/src/Inspector/Extensions/Dashboard.html +206 -0
- defstream-0.1.2/src/Inspector/Extensions/Dashboard.py +114 -0
- defstream-0.1.2/src/Inspector/Extensions/HealthCheck.py +92 -0
- defstream-0.1.2/src/Inspector/Extensions/Metrics.py +281 -0
- defstream-0.1.2/src/Inspector/Extensions/Specifications.py +160 -0
- defstream-0.1.2/src/Inspector/Extensions/__init__.py +13 -0
- defstream-0.1.2/src/Inspector/Inspector.py +216 -0
- defstream-0.1.2/src/Inspector/Response.py +14 -0
- defstream-0.1.2/src/Inspector/__init__.py +9 -0
- defstream-0.1.2/src/Validator/Pattern.py +36 -0
- defstream-0.1.2/src/Validator/Patterns/Array.py +30 -0
- defstream-0.1.2/src/Validator/Patterns/Common.py +93 -0
- defstream-0.1.2/src/Validator/Patterns/Compare.py +180 -0
- defstream-0.1.2/src/Validator/Patterns/Condition.py +60 -0
- defstream-0.1.2/src/Validator/Patterns/DataObject.py +85 -0
- defstream-0.1.2/src/Validator/Patterns/DateTime.py +111 -0
- defstream-0.1.2/src/Validator/Patterns/Number.py +43 -0
- defstream-0.1.2/src/Validator/Patterns/Object.py +150 -0
- defstream-0.1.2/src/Validator/Patterns/Size.py +110 -0
- defstream-0.1.2/src/Validator/Patterns/String.py +85 -0
- defstream-0.1.2/src/Validator/Patterns/Type.py +350 -0
- defstream-0.1.2/src/Validator/Patterns/__init__.py +137 -0
- defstream-0.1.2/src/Validator/Validate.py +29 -0
- defstream-0.1.2/src/Validator/Validator.py +30 -0
- defstream-0.1.2/src/Validator/__init__.py +11 -0
- defstream-0.1.2/src/Worker/Executor.py +298 -0
- defstream-0.1.2/src/Worker/Message.py +103 -0
- defstream-0.1.2/src/Worker/Pool.py +203 -0
- defstream-0.1.2/src/Worker/Run.py +187 -0
- defstream-0.1.2/src/Worker/Specification.py +91 -0
- defstream-0.1.2/src/Worker/Worker.py +321 -0
- defstream-0.1.2/src/Worker/__init__.py +49 -0
- defstream-0.1.2/src/__init__.py +0 -0
- defstream-0.1.2/test/test_dashboard.py +434 -0
- defstream-0.1.2/test/test_inspector.py +825 -0
- defstream-0.1.2/test/test_inspector_console.py +865 -0
- defstream-0.1.2/test/test_lifecycle_hooks.py +446 -0
- defstream-0.1.2/test/test_pool_callbacks.py +123 -0
- defstream-0.1.2/test/test_process_signals.py +188 -0
- defstream-0.1.2/test/test_protocol_contracts.py +287 -0
- defstream-0.1.2/test/test_runner_hooks.py +216 -0
- defstream-0.1.2/test/test_specifications.py +871 -0
- defstream-0.1.2/test/test_task_messages.py +597 -0
- defstream-0.1.2/test/test_worker.py +444 -0
|
@@ -0,0 +1,1197 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: DefStream
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Synchronous message processing with pluggable broker adapters
|
|
5
|
+
Home-page: https://github.com/yong5eon/DefStream
|
|
6
|
+
Author: Heo Yongseon
|
|
7
|
+
Project-URL: Source, https://github.com/yong5eon/DefStream
|
|
8
|
+
Project-URL: Issues, https://github.com/yong5eon/DefStream/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Provides-Extra: rabbitmq
|
|
19
|
+
Requires-Dist: DefStream-RabbitMQ>=0.1.0; extra == "rabbitmq"
|
|
20
|
+
Provides-Extra: redis
|
|
21
|
+
Requires-Dist: DefStream-Redis>=0.1.0; extra == "redis"
|
|
22
|
+
Dynamic: author
|
|
23
|
+
Dynamic: classifier
|
|
24
|
+
Dynamic: description
|
|
25
|
+
Dynamic: description-content-type
|
|
26
|
+
Dynamic: home-page
|
|
27
|
+
Dynamic: license-file
|
|
28
|
+
Dynamic: project-url
|
|
29
|
+
Dynamic: provides-extra
|
|
30
|
+
Dynamic: requires-python
|
|
31
|
+
Dynamic: summary
|
|
32
|
+
|
|
33
|
+
# DefStream
|
|
34
|
+
|
|
35
|
+
## CI and PyPI releases
|
|
36
|
+
|
|
37
|
+
The `CI` workflow runs on pull requests targeting `master` and pushes to
|
|
38
|
+
`master`. Python 3.9 through 3.14 build the distributions, install the wheels in
|
|
39
|
+
a clean virtual environment, and run this repository's tests against RabbitMQ
|
|
40
|
+
4.1 and Redis 7.4 service containers. The core JavaScript UI tests also run.
|
|
41
|
+
Only a successful `master` push can publish to PyPI.
|
|
42
|
+
|
|
43
|
+
The workflow checks out the current repository's event commit and the other
|
|
44
|
+
two `yong5eon/DefStream*` repositories at `master`, recording all commit IDs in
|
|
45
|
+
the log. Those repositories must be accessible and have a `master` branch.
|
|
46
|
+
All three packages are built locally, so the first release does not depend on
|
|
47
|
+
existing PyPI releases. This tests the source combination, not compatibility
|
|
48
|
+
with previously published dependency versions.
|
|
49
|
+
|
|
50
|
+
For a push, `VERSION` must contain `major.minor.patch`; the build replaces its
|
|
51
|
+
patch component with `github.run_number` (for example, `0.1.0` becomes `0.1.42`
|
|
52
|
+
on run 42). PR builds retain the checked-in version. No version commit, PR, or
|
|
53
|
+
Git tag is created. Change the major/minor in `VERSION` when starting a new
|
|
54
|
+
release series. Run numbers can have gaps; renaming/replacing this workflow or
|
|
55
|
+
switching an existing package to this scheme requires checking for collisions.
|
|
56
|
+
|
|
57
|
+
After every Python matrix entry passes, the publish job downloads the exact
|
|
58
|
+
Python 3.13 distributions already built and tested. A fully published version
|
|
59
|
+
is skipped only on a rerun; first-attempt collisions and partially published
|
|
60
|
+
versions fail explicitly. Network errors are not treated as missing releases.
|
|
61
|
+
Concurrent runs are serialized without cancelling the active run; GitHub may
|
|
62
|
+
supersede an older pending run when more pushes arrive.
|
|
63
|
+
|
|
64
|
+
Before enabling automatic releases:
|
|
65
|
+
|
|
66
|
+
1. In each repository, add `PYPI_API_TOKEN` under **Settings > Secrets and
|
|
67
|
+
variables > Actions**, or as a secret in the `pypi` environment. Use the
|
|
68
|
+
appropriate PyPI project scope once the project exists. Never commit tokens.
|
|
69
|
+
2. Configure the `pypi` environment to allow deployments only from `master`.
|
|
70
|
+
Do not require manual approval if fully automatic deployment is desired.
|
|
71
|
+
3. Protect `master`, require PRs, and require all six `Test Python 3.x` checks.
|
|
72
|
+
Do not require `Publish PyPI`, which intentionally does not run on PRs.
|
|
73
|
+
4. Keep the workflow filename/name stable and review changes to workflows and
|
|
74
|
+
packaging carefully: merging to `master` authorizes a public release.
|
|
75
|
+
|
|
76
|
+
Dependencies still need to be published before end users can install adapters
|
|
77
|
+
from PyPI. For the initial release, publish DefStream before its adapters.
|
|
78
|
+
|
|
79
|
+
**Just def.**<br>
|
|
80
|
+
**Let workers do the work.**<br>
|
|
81
|
+
**You don't need to await this.**
|
|
82
|
+
|
|
83
|
+
## Installation
|
|
84
|
+
|
|
85
|
+
```shell
|
|
86
|
+
pip install DefStream
|
|
87
|
+
pip install "DefStream[RabbitMQ]"
|
|
88
|
+
pip install "DefStream[Redis]"
|
|
89
|
+
pip install "DefStream[RabbitMQ,Redis]"
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
The base package has no broker client dependencies and does not include adapter
|
|
93
|
+
source files. Extras install the separate `DefStream-RabbitMQ` and/or
|
|
94
|
+
`DefStream-Redis` distributions, which depend on `pika` and `redis`, respectively.
|
|
95
|
+
You can also install the adapters directly:
|
|
96
|
+
|
|
97
|
+
```shell
|
|
98
|
+
pip install DefStream-RabbitMQ
|
|
99
|
+
pip install DefStream-Redis
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
The Python import paths remain `DefStream.Broker.Implements.RabbitMQ` and
|
|
103
|
+
`DefStream.Broker.Implements.Redis`. For local development of all three projects,
|
|
104
|
+
see the editable installation instructions below.
|
|
105
|
+
|
|
106
|
+
The project is now named DefStream. Distribution/import names use `DefStream`,
|
|
107
|
+
Prometheus metric names use `defstream_`, and integration-test environment
|
|
108
|
+
variables use `DEFSTREAM_`. Redis's default consumer group is now `DefStream`;
|
|
109
|
+
when continuing an existing deployment, explicitly keep its previous group name
|
|
110
|
+
to preserve consumer-group position and pending messages. Queue names explicitly
|
|
111
|
+
configured by applications are unchanged.
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from DefStream.Worker import Worker
|
|
115
|
+
from DefStream.Broker.Implements.RabbitMQ import Connection as RabbitConnection
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
There is no combined `DefStream.Broker.Broker` interface. Implementations expose
|
|
119
|
+
a concrete `Connection` class without a `Broker` compatibility alias.
|
|
120
|
+
|
|
121
|
+
Create broker connections directly with the implementation's configuration and
|
|
122
|
+
call `connect()` before use. Call `close()` when finished. Connection registries
|
|
123
|
+
or helpers, if needed, belong in application code.
|
|
124
|
+
|
|
125
|
+
RabbitMQ resource methods use snake_case: `create_exchange`, `delete_exchange`,
|
|
126
|
+
`bind_exchange_to_exchange`, `unbind_exchange_to_exchange`, `create_queue`,
|
|
127
|
+
`delete_queue`, `bind_exchange_to_queue`, `unbind_exchange_to_queue`,
|
|
128
|
+
`create_stream`, and `delete_stream`. The former camelCase names are removed.
|
|
129
|
+
|
|
130
|
+
## Connection capabilities
|
|
131
|
+
|
|
132
|
+
`DefStream.Broker.Connection` defines independent `typing.Protocol` contracts,
|
|
133
|
+
also exported from `DefStream.Broker`. Implementations explicitly inherit only
|
|
134
|
+
the capabilities they provide:
|
|
135
|
+
|
|
136
|
+
| Protocol | Contract | RabbitMQ | Redis |
|
|
137
|
+
| --- | --- | --- | --- |
|
|
138
|
+
| `Connection` | `connect()`, `close()` | Yes | Yes |
|
|
139
|
+
| `GetExchange` | `exchange(name) -> Sendable` | Yes | No |
|
|
140
|
+
| `GetQueue` | `queue(name) -> Sendable` | Yes | Yes |
|
|
141
|
+
| `GetStream` | `stream(name) -> Sendable` | Yes | Yes |
|
|
142
|
+
| `GetConsumer` | `consumer(handler=None) -> Consumer` | Yes | Yes |
|
|
143
|
+
|
|
144
|
+
For example, RabbitMQ's Connection inherits `Connection`,
|
|
145
|
+
`GetExchange`, `GetQueue`, `GetStream`, and
|
|
146
|
+
`GetConsumer`. Redis omits GetExchange and uses its Queue for both
|
|
147
|
+
GetQueue and GetStream. Unsupported methods are absent rather than stubs raising
|
|
148
|
+
NotImplementedError.
|
|
149
|
+
|
|
150
|
+
These are plain Protocols without generic type parameters. Resource factories
|
|
151
|
+
promise the shared Sendable contract; concrete adapter methods retain their
|
|
152
|
+
specific return annotations for adapter-specific operations.
|
|
153
|
+
The common resource-name argument is positional-only to accommodate existing
|
|
154
|
+
adapter argument names; call `provider.queue('name')` through a generic contract.
|
|
155
|
+
Concrete implementations retain their existing keyword arguments and can add
|
|
156
|
+
optional settings. The contracts intentionally do not promise arbitrary
|
|
157
|
+
`**kwargs`. Missing connection/factory implementations prevent instantiation of
|
|
158
|
+
explicit subclasses through `@abstractmethod`.
|
|
159
|
+
|
|
160
|
+
`DefStream.Worker.WorkerConnection` composes only Connection and
|
|
161
|
+
GetConsumer. Worker does not require queues, streams or exchanges.
|
|
162
|
+
Application connections can inherit these protocols explicitly or satisfy them
|
|
163
|
+
structurally without inheritance. Protocols are for static checking, not
|
|
164
|
+
`isinstance` capability checks.
|
|
165
|
+
|
|
166
|
+
Events (including the RPC failed-request wrapper), error policies and Redis JSON
|
|
167
|
+
codecs also explicitly inherit their existing Event, EventError, Encoder and
|
|
168
|
+
Decoder protocols. Message objects explicitly inherit Sendable, Callable,
|
|
169
|
+
Poppable and/or Readable as appropriate. Consumer, Run, configuration and
|
|
170
|
+
lifecycle-hook ABCs are retained: this change does not replace implementation
|
|
171
|
+
base classes or introduce new contracts for every callable.
|
|
172
|
+
|
|
173
|
+
## Message capabilities
|
|
174
|
+
|
|
175
|
+
Common message I/O contracts now live in `DefStream.Broker.Protocol` and are
|
|
176
|
+
also exported from `DefStream.Broker`:
|
|
177
|
+
|
|
178
|
+
| Protocol | Operation | Timeout |
|
|
179
|
+
| --- | --- | --- |
|
|
180
|
+
| `Sendable` | `send(body)` | Implementation-specific |
|
|
181
|
+
| `Callable` | `call(body, timeout=5)` | Seconds |
|
|
182
|
+
| `Poppable` | `pop(timeout=None)` | Milliseconds, preserving the existing API |
|
|
183
|
+
| `Readable` | `read(offset=0, timeout=None)` | Milliseconds, preserving the existing API |
|
|
184
|
+
|
|
185
|
+
These are structural `typing.Protocol` contracts, not runtime capability
|
|
186
|
+
checks. Implementations can expose additional optional broker-specific
|
|
187
|
+
arguments. The former common `Queue`, `Stream`, and `Exchange` base classes and
|
|
188
|
+
their modules have been removed. Concrete RabbitMQ classes retain their names:
|
|
189
|
+
Exchange is Sendable/Callable, Queue is Sendable/Callable/Poppable, and Stream is
|
|
190
|
+
Sendable/Readable. If `typing.Callable` is needed alongside the message
|
|
191
|
+
`Callable`, import the typing symbol as `PyCallable`.
|
|
192
|
+
`Event`, `EventHandler`, and `EventError` live together in `Broker.Event`
|
|
193
|
+
and are exported from `DefStream.Broker`. EventHandler is a Protocol with
|
|
194
|
+
`__call__(event: Event) -> None`; explicitly inheriting handlers must implement
|
|
195
|
+
it. The former standalone `Broker.EventHandler` module has been removed.
|
|
196
|
+
`EventError` is a failure-policy contract, not an I/O capability.
|
|
197
|
+
|
|
198
|
+
## RabbitMQ RPC
|
|
199
|
+
|
|
200
|
+
Both RabbitMQ Queue and Exchange explicitly inherit `Callable` and implement
|
|
201
|
+
`call()` directly, including request publication, reply matching, and timeout
|
|
202
|
+
handling; there is no shared RabbitMQ Rpc module.
|
|
203
|
+
Obtain them through `broker.queue()` or `broker.exchange()` so the
|
|
204
|
+
decoder and dedicated RPC connection parameters are available.
|
|
205
|
+
|
|
206
|
+
For Exchange RPC, `event` selects the routing key, just as with `send()`:
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
result = broker.exchange('operations').call(
|
|
210
|
+
{
|
|
211
|
+
'a': 10,
|
|
212
|
+
'b': 20,
|
|
213
|
+
},
|
|
214
|
+
timeout=5,
|
|
215
|
+
event='add',
|
|
216
|
+
)
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
Bind the exchange to the server queue before calling. An omitted event uses
|
|
220
|
+
the empty routing key. If the exchange routes to multiple queues, each may
|
|
221
|
+
execute the request; the client returns the first matching response, not an
|
|
222
|
+
aggregate. Use a single destination for ordinary RPC to avoid duplicate
|
|
223
|
+
side effects. Unroutable requests raise UnroutableError.
|
|
224
|
+
|
|
225
|
+
Use a queue obtained from `broker.queue(name)` for synchronous request/reply:
|
|
226
|
+
|
|
227
|
+
```python
|
|
228
|
+
result = broker.queue('add').call(
|
|
229
|
+
{
|
|
230
|
+
'a': 10,
|
|
231
|
+
'b': 20,
|
|
232
|
+
},
|
|
233
|
+
timeout=5,
|
|
234
|
+
)
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
An ordinary Run serves both one-way messages and RPC requests. For RPC, its
|
|
238
|
+
return value (including None) is published after RunComplete hooks succeed,
|
|
239
|
+
then the request is ACKed. Run/hooks still execute in the selected pool;
|
|
240
|
+
response publication and message settlement execute on the connection owner
|
|
241
|
+
thread. No connection or event object is passed to ProcessPool children.
|
|
242
|
+
|
|
243
|
+
Each call opens its own connection and an exclusive, auto-delete response
|
|
244
|
+
queue, subscribes before publishing, and matches responses by correlation ID.
|
|
245
|
+
It closes that connection and response queue on success, timeout, or failure.
|
|
246
|
+
This initial implementation favors simple ownership over connection reuse:
|
|
247
|
+
there is no persistent Rpc client or asynchronous/multiplexed call API yet.
|
|
248
|
+
The queue object's existing connection/channel must still only be used by its
|
|
249
|
+
owner thread. Do not block the Worker consumer thread waiting for a call to
|
|
250
|
+
itself; invoke RPC from a separate client.
|
|
251
|
+
|
|
252
|
+
Requests use reply_to, correlation_id, and a TTL based on the remaining
|
|
253
|
+
timeout. Calls raise TimeoutError if no response arrives before the deadline;
|
|
254
|
+
this does not cancel work already delivered. Connection establishment uses
|
|
255
|
+
bounded socket/stack timeouts. Blocking broker operations and connection
|
|
256
|
+
cleanup mean the timeout is not a hard wall-clock limit on the entire call.
|
|
257
|
+
Both request and response publishers use confirms and mandatory routing.
|
|
258
|
+
An unavailable request queue raises Pika's UnroutableError instead of waiting.
|
|
259
|
+
|
|
260
|
+
Requeue sends no final response, allowing retry. When the selected EventError
|
|
261
|
+
calls reject(), the server sends an error envelope before rejecting the request
|
|
262
|
+
(and preserving configured dead-letter routing). The client raises
|
|
263
|
+
`DefStream.Broker.RemoteError` with code `ProcessingError` and a generic message;
|
|
264
|
+
server exception details stay in server logs. Custom EventError implementations
|
|
265
|
+
must settle via the supplied event's ack/nack/reject methods.
|
|
266
|
+
|
|
267
|
+
A response destination may disappear when a caller times out. An unroutable
|
|
268
|
+
response is logged as a warning and the original request is settled normally;
|
|
269
|
+
the Worker keeps running. Other response publication/serialization failures
|
|
270
|
+
propagate without ACK/Reject, close the Worker, and leave unconfirmed requests
|
|
271
|
+
for broker redelivery. Response publication and request ACK are not atomic:
|
|
272
|
+
crashes can cause duplicate execution/replies, so side-effecting Runs must be
|
|
273
|
+
idempotent. RPC does not provide exactly-once delivery.
|
|
274
|
+
|
|
275
|
+
The RabbitMQ worker sample shares its Add and Divide tasks in
|
|
276
|
+
`sample/Worker/RabbitMQ/Tasks.py`. To try RPC, start
|
|
277
|
+
`sample/Worker/RabbitMQ/Worker.py`, then run
|
|
278
|
+
`sample/Worker/RabbitMQ/Client.py` in another terminal. The client uses
|
|
279
|
+
`request(body, task=..., broker=...)` with these shared tasks and shows
|
|
280
|
+
successful addition/division and a rejected division by zero.
|
|
281
|
+
For a standalone broker-only example, run
|
|
282
|
+
`sample/Broker/Implements/RabbitMQ/Sample.RPC.py`. It starts a responder on its
|
|
283
|
+
own connection/thread, demonstrates Queue and Exchange RPC plus RemoteError,
|
|
284
|
+
then removes only its uniquely named queue and exchange. A local RabbitMQ
|
|
285
|
+
server is required, but no separate Worker process is needed.
|
|
286
|
+
|
|
287
|
+
## Redis Streams
|
|
288
|
+
|
|
289
|
+
The Redis adapter uses Redis Streams, not Pub/Sub or lists. Install the Redis
|
|
290
|
+
extra and use Redis 6.2 or newer (including `XAUTOCLAIM` support):
|
|
291
|
+
|
|
292
|
+
```python
|
|
293
|
+
from DefStream.Broker.Implements.Redis import (
|
|
294
|
+
Connection,
|
|
295
|
+
Configuration,
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
broker = Connection(Configuration(
|
|
299
|
+
host='127.0.0.1',
|
|
300
|
+
port=6379,
|
|
301
|
+
))
|
|
302
|
+
broker.connect()
|
|
303
|
+
try:
|
|
304
|
+
queue = broker.queue('add')
|
|
305
|
+
queue.send({
|
|
306
|
+
'a': 10,
|
|
307
|
+
'b': 20,
|
|
308
|
+
})
|
|
309
|
+
event = queue.pop(timeout=1000)
|
|
310
|
+
if event is not None:
|
|
311
|
+
print(event.id, event.body)
|
|
312
|
+
event.ack()
|
|
313
|
+
finally:
|
|
314
|
+
broker.close()
|
|
315
|
+
```
|
|
316
|
+
|
|
317
|
+
`broker.queue()` and `broker.stream()` return the same `Queue`, implementing
|
|
318
|
+
Sendable, Poppable and Readable. Resource methods are `create_queue`,
|
|
319
|
+
`delete_queue`, `create_stream`, and `delete_stream`. Creating a queue creates
|
|
320
|
+
the stream and consumer group; sending or subscribing can create them lazily.
|
|
321
|
+
Group creation starts at `0-0`, so messages published before subscription are
|
|
322
|
+
not lost. Use the concrete `Connection` class directly. Exchanges and RPC are not
|
|
323
|
+
supported; Redis Connection neither inherits GetExchange nor exposes `exchange()`.
|
|
324
|
+
|
|
325
|
+
Configuration supports username/password (including password-only auth),
|
|
326
|
+
`database`, TLS (`secure=True`), connection limit (`max`, default 10), and socket
|
|
327
|
+
timeout (`timeout`, seconds, default 5). The default `ConnectionType.Pool`
|
|
328
|
+
uses redis-py's managed connection pool; `ConnectionType.Cluster` uses
|
|
329
|
+
RedisCluster and requires database 0. Cluster mode is implemented but the local
|
|
330
|
+
integration suite exercises standalone Redis only. Connection failures propagate
|
|
331
|
+
from `connect()`, which verifies connectivity with PING.
|
|
332
|
+
|
|
333
|
+
Bodies use a single stream field named `body` containing UTF-8 JSON, preserving
|
|
334
|
+
nested mappings, lists, numbers, booleans and null. Pass callable `encode` and
|
|
335
|
+
`decode` configuration values for a different byte codec. Legacy field-per-value
|
|
336
|
+
streams are not wire-compatible and are not decoded using `eval`.
|
|
337
|
+
|
|
338
|
+
`pop(timeout=...)` waits for one group delivery; timeout is in milliseconds.
|
|
339
|
+
`None` or zero waits indefinitely, matching the existing RabbitMQ convention.
|
|
340
|
+
Positive timeouts return None when no delivery arrives.
|
|
341
|
+
`read(offset=0, timeout=...)` yields history followed by new events until its total
|
|
342
|
+
millisecond timeout; None or zero tails indefinitely. Redis offsets may be
|
|
343
|
+
stream-ID strings, such as `'1720000000000-0'`; the offset is exclusive.
|
|
344
|
+
History reads do not participate in consumer groups, so their events cannot be
|
|
345
|
+
acknowledged/requeued. Closing the iterator stops reading.
|
|
346
|
+
|
|
347
|
+
Workers accept `Subscription(queue='add', group='DefStream')` from the Redis
|
|
348
|
+
adapter; a queue-name string or equivalent mapping also works. Consumers within
|
|
349
|
+
one group compete for messages; separate groups each receive the stream.
|
|
350
|
+
`broker.consumer(qos=1)` limits total unacknowledged local deliveries across all
|
|
351
|
+
subscriptions. Owner-thread callbacks handle pool completion and shutdown,
|
|
352
|
+
including ThreadPool and ProcessPool workers. Redis Run return values are
|
|
353
|
+
acknowledged normally but are not sent back as RPC responses.
|
|
354
|
+
|
|
355
|
+
- `ack()` and `reject()` use XACK without deleting history. Reject has no built-in
|
|
356
|
+
dead-letter destination.
|
|
357
|
+
- Default failure handling uses Requeue. `nack()` atomically appends a copy to the
|
|
358
|
+
stream and acknowledges the original pending entry using a single-key Lua
|
|
359
|
+
script. The retry has a new stream ID and is visible to **all groups**, not only
|
|
360
|
+
the failing group. Separate retry streams are needed for group-isolated retry.
|
|
361
|
+
- Consumers reclaim abandoned pending entries with XAUTOCLAIM after 60 seconds
|
|
362
|
+
of idle time by default. Set `reclaim_idle` on `broker.consumer()` to a larger
|
|
363
|
+
millisecond value for long tasks, or None to disable automatic recovery.
|
|
364
|
+
Long-running active tasks can otherwise be reclaimed by another consumer.
|
|
365
|
+
The default Worker uses the default consumer settings.
|
|
366
|
+
- Shutdown does not acknowledge unfinished work. Connection failures and
|
|
367
|
+
decoding/handler/callback errors propagate; there is no automatic reconnect.
|
|
368
|
+
Unconfirmed entries remain pending for later reclamation.
|
|
369
|
+
|
|
370
|
+
Delivery is at least once: make side effects idempotent. History is retained
|
|
371
|
+
after settlement and retries add entries; configure Redis retention externally
|
|
372
|
+
without trimming pending entries. The adapter does not silently trim/delete
|
|
373
|
+
messages or destroy groups.
|
|
374
|
+
|
|
375
|
+
Run `sample/Broker/Redis/Sample.py` for a finite send/pop/history demo.
|
|
376
|
+
`sample/Broker/Redis/Sample.Stream.py` uses `broker.stream()` to publish three
|
|
377
|
+
events, read from the beginning, and resume after the first event's stream ID.
|
|
378
|
+
These history reads do not require ACK. Each read uses a 1000 ms timeout, and
|
|
379
|
+
the sample deletes only its uniquely named stream and closes the connection.
|
|
380
|
+
`sample/Worker/Redis/Sample.py` uses Prepare to create each registered queue and
|
|
381
|
+
consumer group and publish five messages per Run before consuming `add` and
|
|
382
|
+
`divide`. Operands range from 0 to 99; the first Divide message always uses a
|
|
383
|
+
zero divisor to demonstrate logged ZeroDivisionError and Reject without endless
|
|
384
|
+
requeue. Other divisors are random and may also be zero. Successful messages
|
|
385
|
+
continue processing. It runs until Ctrl+C, then drains work and runs Cleanup.
|
|
386
|
+
Cleanup logs completion without deleting shared queues or stream history.
|
|
387
|
+
You can publish more `{'a': 10, 'b': 20}` messages from another process.
|
|
388
|
+
Enable the real Redis integration tests with `DEFSTREAM_REDIS_TESTS=1`; they use
|
|
389
|
+
localhost:6379 and create/delete only uniquely named test streams.
|
|
390
|
+
|
|
391
|
+
## Workers
|
|
392
|
+
|
|
393
|
+
### Inspector: HTTP health checks and metrics
|
|
394
|
+
|
|
395
|
+
Inspector uses only the Python standard library: a threaded WSGI HTTP server
|
|
396
|
+
managed by the Worker. It starts before broker connection/setup, and closes
|
|
397
|
+
after pool draining, cleanup and broker closure, including startup failures.
|
|
398
|
+
An occupied HTTP port fails Worker startup explicitly. Inspector is optional;
|
|
399
|
+
workers without one keep their existing behavior.
|
|
400
|
+
|
|
401
|
+
```python
|
|
402
|
+
from DefStream.Inspector import Inspector
|
|
403
|
+
from DefStream.Inspector.Extensions import (
|
|
404
|
+
HealthCheck,
|
|
405
|
+
Metrics,
|
|
406
|
+
)
|
|
407
|
+
from DefStream.Worker import (
|
|
408
|
+
ThreadPool,
|
|
409
|
+
Worker,
|
|
410
|
+
)
|
|
411
|
+
|
|
412
|
+
worker = Worker(
|
|
413
|
+
connection,
|
|
414
|
+
ThreadPool(10),
|
|
415
|
+
inspector=Inspector(
|
|
416
|
+
host='127.0.0.1',
|
|
417
|
+
port=9999,
|
|
418
|
+
exts=(
|
|
419
|
+
HealthCheck('/status'),
|
|
420
|
+
Metrics('/metrics'),
|
|
421
|
+
),
|
|
422
|
+
),
|
|
423
|
+
)
|
|
424
|
+
worker.add(Sample)
|
|
425
|
+
worker.run()
|
|
426
|
+
```
|
|
427
|
+
|
|
428
|
+
- `GET /status` returns JSON with `status`. It returns 200 only
|
|
429
|
+
after subscriptions are ready and the consumer loop answers a thread-safe
|
|
430
|
+
callback within `HealthCheck(timeout=1)` seconds (default 1). Idle workers are healthy.
|
|
431
|
+
Starting, stopping, stopped and unresponsive workers return 503 while HTTP
|
|
432
|
+
remains open. A terminated process/server fails the HTTP connection instead.
|
|
433
|
+
The probe checks consumer-loop responsiveness, not every dependency or a
|
|
434
|
+
hung individual task; choose a timeout above normal consumer-loop latency.
|
|
435
|
+
Repeated/concurrent checks share an outstanding probe rather than filling
|
|
436
|
+
a stalled consumer's callback queue.
|
|
437
|
+
- `GET /metrics` returns Prometheus text format, **200 even when not ready**,
|
|
438
|
+
so failures and shutdown can still be observed. Metrics include
|
|
439
|
+
`defstream_worker_running`, `defstream_worker_uptime_seconds`,
|
|
440
|
+
`defstream_tasks_active`, `defstream_tasks_total` (success/failure), and
|
|
441
|
+
`defstream_task_duration_seconds` (summary count/sum). The running gauge
|
|
442
|
+
describes lifecycle state; use `/status` for the responsiveness probe.
|
|
443
|
+
Task labels use module-qualified class names; no message body/ID is exposed.
|
|
444
|
+
Counters describe completed execution attempts, including retries and hooks,
|
|
445
|
+
not confirmed ACKs or unique messages. Invalid bodies, pool-full requeues and
|
|
446
|
+
failed submissions do not count as task executions. Duration includes pool
|
|
447
|
+
wait and hooks. Metrics are local to one Worker process and reset on restart.
|
|
448
|
+
- Paths are explicit extensions; there are no default routes. Both endpoints
|
|
449
|
+
support HEAD. Unknown paths return 404 and unsupported methods return 405.
|
|
450
|
+
Extensions default to GET/HEAD; request-aware extensions can declare `methods`
|
|
451
|
+
and override `handle(inspector, environ)` without changing existing endpoints.
|
|
452
|
+
Additional endpoints can inherit `Extension` and return `Response`, both
|
|
453
|
+
exported from `DefStream.Inspector`. Built-in extensions are exported from
|
|
454
|
+
`DefStream.Inspector.Extensions`.
|
|
455
|
+
- Each extension owns its behavior and state: HealthCheck owns the consumer
|
|
456
|
+
probe, lock and timeout; Metrics owns counters, execution timing and output.
|
|
457
|
+
Inspector owns only HTTP serving, Worker attachment, routing and event
|
|
458
|
+
dispatch. Unregistered extensions do not collect data or create probe state.
|
|
459
|
+
`Extension.start(inspector)` initializes an extension before HTTP startup.
|
|
460
|
+
`begin(task)` returns an extension-owned context; `end(task, context, success)`
|
|
461
|
+
receives it on completion. `success=None` means submission was rejected and
|
|
462
|
+
must not count as a completed attempt. These hooks execute in the parent
|
|
463
|
+
process and may run on different threads; extensions must synchronize state.
|
|
464
|
+
- Inspector is attached to one Worker and cannot be restarted or shared.
|
|
465
|
+
Task execution remains in the existing ThreadPool/ProcessPool; the HTTP
|
|
466
|
+
server and metric aggregation stay in the parent process.
|
|
467
|
+
- This is a small operational HTTP interface, not a public web server. No
|
|
468
|
+
TLS is provided. Health, metrics and Specifications routes are unauthenticated;
|
|
469
|
+
the optional Console extension supports an optional Bearer token for task data
|
|
470
|
+
and execution. Keep Inspector on loopback or a trusted management network.
|
|
471
|
+
|
|
472
|
+
The RabbitMQ and Redis Worker samples enable `/status` and `/metrics` on
|
|
473
|
+
`127.0.0.1:9999`; use different ports if running both at once. For Docker,
|
|
474
|
+
use `host='0.0.0.0'` to expose it outside the container (restrict network access).
|
|
475
|
+
A health command inside the container needs no curl dependency:
|
|
476
|
+
|
|
477
|
+
```dockerfile
|
|
478
|
+
HEALTHCHECK --interval=10s --timeout=3s --start-period=15s --retries=3 CMD python -c "from urllib.request import urlopen; urlopen('http://127.0.0.1:9999/status', timeout=2).close()"
|
|
479
|
+
```
|
|
480
|
+
|
|
481
|
+
### Live dashboard
|
|
482
|
+
|
|
483
|
+
Register `Dashboard` together with exactly one `Metrics` extension:
|
|
484
|
+
|
|
485
|
+
```python
|
|
486
|
+
from DefStream.Inspector.Extensions import (
|
|
487
|
+
Dashboard,
|
|
488
|
+
Metrics,
|
|
489
|
+
)
|
|
490
|
+
|
|
491
|
+
inspector = Inspector(exts=(
|
|
492
|
+
Metrics('/metrics'),
|
|
493
|
+
Dashboard(
|
|
494
|
+
'/dashboard',
|
|
495
|
+
title='DefStream',
|
|
496
|
+
description='Worker activity',
|
|
497
|
+
),
|
|
498
|
+
))
|
|
499
|
+
```
|
|
500
|
+
|
|
501
|
+
Open `/dashboard` for a lightweight, dependency-free status page. Both Worker
|
|
502
|
+
samples register this route. The page reads the existing Metrics collector;
|
|
503
|
+
there is no second collector, broker connection or server-side history.
|
|
504
|
+
`title` defaults to `DefStream`; optional `description` is the subtitle.
|
|
505
|
+
HTML changes take effect on refresh without restarting.
|
|
506
|
+
|
|
507
|
+
- Shows worker lifecycle, uptime, in-flight submissions, cumulative success
|
|
508
|
+
and failure counts, total throughput, and per-task throughput and average time.
|
|
509
|
+
- Select **Last second** or **Last minute** for real server-side 1/60-second
|
|
510
|
+
completion windows, not a per-second value multiplied by 60. Windows end at
|
|
511
|
+
the last fully elapsed monotonic second (less than one second behind live
|
|
512
|
+
totals). The minute window advances every second, not once per calendar minute.
|
|
513
|
+
Startup windows include only data since startup, without extrapolation.
|
|
514
|
+
History is available immediately on page load and after reconnect.
|
|
515
|
+
- Selected-window metrics include completed success/failure counts, failure
|
|
516
|
+
percentage, average and maximum duration, both overall and per task. An empty
|
|
517
|
+
window shows zero completions and `--` for failure percentage/average/maximum.
|
|
518
|
+
Oldest in-flight age is measured from submission, including pool wait.
|
|
519
|
+
- Averages use the duration of attempts completed in the selected window,
|
|
520
|
+
including pool wait and hooks; these are not p95 latency. Retries count
|
|
521
|
+
as separate attempts; pool-full/rejected submissions do not count as completed.
|
|
522
|
+
In flight includes submitted work waiting in the pool, not broker queue depth.
|
|
523
|
+
- Lifecycle is not consumer responsiveness. Use `HealthCheck` for health probes.
|
|
524
|
+
Requests time out after five seconds; failed polls mark totals as stale and
|
|
525
|
+
clear current rates. Pause/resume is available; hidden tabs stop polling.
|
|
526
|
+
- `GET /dashboard` with `Accept: application/json` returns a detached snapshot
|
|
527
|
+
(`instance`, `sampled_at`, `uptime`, `running`, `tasks`). Tasks are keyed by
|
|
528
|
+
module-qualified name with cumulative `active`, `success`, `failure`, `duration`
|
|
529
|
+
(seconds), `oldest_active` (seconds), and `windows` keyed by `"1"`/`"60"` with
|
|
530
|
+
`success`, `failure`, `duration`, `max_duration`. `window_end` is the monotonic
|
|
531
|
+
exclusive end of both windows. Metrics keeps at most 61 one-second buckets per
|
|
532
|
+
task (including the current incomplete second), not individual completion
|
|
533
|
+
records. Pending submission timestamps are retained only until completion.
|
|
534
|
+
Prometheus also exposes `defstream_tasks_window` by `window_seconds`/`outcome`,
|
|
535
|
+
`defstream_task_window_duration_seconds_sum`,
|
|
536
|
+
`defstream_task_window_duration_seconds_max`, and
|
|
537
|
+
`defstream_task_oldest_active_seconds`. No p95 is collected.
|
|
538
|
+
Prometheus and Dashboard use the same lock-protected snapshot.
|
|
539
|
+
- Like Metrics, Dashboard is unauthenticated. Restrict access at the application,
|
|
540
|
+
network or proxy layer. It shows one Worker only, resets on restart, and is
|
|
541
|
+
not a replacement for Prometheus/Grafana history or alerting.
|
|
542
|
+
|
|
543
|
+
### Request/response specifications
|
|
544
|
+
|
|
545
|
+
`DefStream.Descriptor` and `DefStream.Validator` reuse the local Liquirizia
|
|
546
|
+
Description/Validator APIs under the repository's GPLv3 license. Descriptor has
|
|
547
|
+
**no DataModel dependency**: manual schemas and dataclass `ToSchema`/`ToObject`
|
|
548
|
+
remain available. The original temporary sources are unchanged. Existing names
|
|
549
|
+
and implementation style are retained in the ported modules to minimize drift.
|
|
550
|
+
DataModel-only conversions and `IsDataModel` are not included.
|
|
551
|
+
|
|
552
|
+
```python
|
|
553
|
+
from DefStream.Descriptor import (
|
|
554
|
+
Integer,
|
|
555
|
+
Object,
|
|
556
|
+
Properties,
|
|
557
|
+
)
|
|
558
|
+
from DefStream.Validator import Validator
|
|
559
|
+
from DefStream.Validator.Patterns import (
|
|
560
|
+
IsInteger,
|
|
561
|
+
IsObject,
|
|
562
|
+
)
|
|
563
|
+
from DefStream.Validator.Patterns.Object import (
|
|
564
|
+
IsMappingOf,
|
|
565
|
+
IsRequiredIn,
|
|
566
|
+
)
|
|
567
|
+
from DefStream.Worker import (
|
|
568
|
+
Run,
|
|
569
|
+
Specification,
|
|
570
|
+
)
|
|
571
|
+
|
|
572
|
+
class Add(
|
|
573
|
+
Run,
|
|
574
|
+
parameters={'queue': 'add'},
|
|
575
|
+
request=Specification(
|
|
576
|
+
schema=Object(
|
|
577
|
+
properties=Properties(
|
|
578
|
+
a=Integer('First operand'),
|
|
579
|
+
b=Integer('Second operand'),
|
|
580
|
+
),
|
|
581
|
+
),
|
|
582
|
+
validator=Validator(IsObject(
|
|
583
|
+
IsRequiredIn(
|
|
584
|
+
'a',
|
|
585
|
+
'b',
|
|
586
|
+
),
|
|
587
|
+
IsMappingOf({
|
|
588
|
+
'a': IsInteger(),
|
|
589
|
+
'b': IsInteger(),
|
|
590
|
+
}),
|
|
591
|
+
)),
|
|
592
|
+
example={
|
|
593
|
+
'a': 10,
|
|
594
|
+
'b': 20,
|
|
595
|
+
},
|
|
596
|
+
),
|
|
597
|
+
response=Specification(
|
|
598
|
+
schema=Integer('Sum'),
|
|
599
|
+
validator=Validator(IsInteger()),
|
|
600
|
+
example=30,
|
|
601
|
+
),
|
|
602
|
+
):
|
|
603
|
+
def run(
|
|
604
|
+
self,
|
|
605
|
+
a: int,
|
|
606
|
+
b: int,
|
|
607
|
+
):
|
|
608
|
+
return a + b
|
|
609
|
+
```
|
|
610
|
+
|
|
611
|
+
`Specification` accepts `schema`, `validator`, `example`, `description`, and
|
|
612
|
+
`format` (default `application/json`).
|
|
613
|
+
For example, `format='text/plain'` sets
|
|
614
|
+
the message's AsyncAPI `contentType`. Request and response can declare different
|
|
615
|
+
media types. This is contract metadata; it does not change broker codecs or the
|
|
616
|
+
JSON content type of the specifications HTTP endpoint. Configure compatible
|
|
617
|
+
broker encoders/decoders separately when using a different wire format.
|
|
618
|
+
`schema` can be a Descriptor value, Schema/condition, JSON-schema mapping, or
|
|
619
|
+
dataclass type. It describes the wire data; **it does not automatically validate
|
|
620
|
+
or instantiate a model**. Use Validator for runtime rules and transformations.
|
|
621
|
+
Custom validator rules are not reverse-engineered into JSON Schema. Examples
|
|
622
|
+
are documentation values, not automatically executed or validated.
|
|
623
|
+
|
|
624
|
+
Execution order is setup hooks, request validation/transformation, `run`,
|
|
625
|
+
response validation/transformation, completion hooks, then RPC response/ACK.
|
|
626
|
+
Request validation must return a mapping with string keys; specified tasks
|
|
627
|
+
accept keyword arguments. Output can be any existing broker-serializable value.
|
|
628
|
+
The returned validator value is used, not discarded. Validation failures invoke
|
|
629
|
+
RunError hooks and the existing `on_error` policy; configure Reject where
|
|
630
|
+
permanent invalid messages must not be requeued indefinitely. Request and
|
|
631
|
+
response contracts are inherited. Tasks without contracts retain their behavior.
|
|
632
|
+
The send/request helpers do not run the validators again.
|
|
633
|
+
|
|
634
|
+
Necessary changes from the source: `AllOf` dumps as `allOf`; string `min`/`max`
|
|
635
|
+
dump as `minLength`/`maxLength`; unsupported schema values/references fail
|
|
636
|
+
explicitly. Conversion patterns use `ast.literal_eval`, not `eval`: numeric
|
|
637
|
+
literal strings still work but executable expressions are rejected. `Validate`
|
|
638
|
+
supports functions without defaults and positional-only arguments.
|
|
639
|
+
|
|
640
|
+
### AsyncAPI JSON
|
|
641
|
+
|
|
642
|
+
```python
|
|
643
|
+
from DefStream.Inspector.Extensions import Specifications
|
|
644
|
+
|
|
645
|
+
# Add to Inspector(exts=(...)).
|
|
646
|
+
specifications = Specifications(
|
|
647
|
+
'/asyncapi.json',
|
|
648
|
+
version='1.0.0',
|
|
649
|
+
description='Worker task contracts',
|
|
650
|
+
format='application/json',
|
|
651
|
+
)
|
|
652
|
+
```
|
|
653
|
+
|
|
654
|
+
`Specifications.format` sets the document's `defaultContentType` and defaults to
|
|
655
|
+
`application/json`. Messages without a request Specification inherit this default;
|
|
656
|
+
each declared Specification's `format` remains an explicit message-level override
|
|
657
|
+
(it also defaults to JSON). This does not change broker codecs or the JSON HTTP
|
|
658
|
+
response format of this endpoint.
|
|
659
|
+
|
|
660
|
+
`GET /asyncapi.json` serves an AsyncAPI **3.1.0** document generated from that
|
|
661
|
+
Worker's registered tasks, without a global registry or broker connection.
|
|
662
|
+
The document is available during startup/shutdown while HTTP remains open.
|
|
663
|
+
It describes Worker-side **receive** operations, queue addresses, input message
|
|
664
|
+
schemas/examples. Operation descriptions come from `request.description`;
|
|
665
|
+
when it is None, the field is omitted without falling back to task docstrings.
|
|
666
|
+
Unspecified inputs have an object payload
|
|
667
|
+
with no additional constraints. Queue names can come from strings, subscription
|
|
668
|
+
mappings or subscription objects; missing addresses fail explicitly.
|
|
669
|
+
Operation keys are task class names (without modules). Receive channel keys are
|
|
670
|
+
the actual queue names, preserved without replacement; JSON Pointer references
|
|
671
|
+
escape `/` and `~`. Request messages are defined directly under each channel,
|
|
672
|
+
referenced as `#/channels/<queue>/messages/request`. Specification has no
|
|
673
|
+
`channel` option; subscription parameters determine the queue.
|
|
674
|
+
Output contracts generate a standard operation **`reply`**, referencing the same
|
|
675
|
+
channel and its `response` message at `#/channels/<queue>/messages/response`.
|
|
676
|
+
Request and response definitions live together under the channel's `messages`;
|
|
677
|
+
the receive operation selects only `request`, and its reply selects only
|
|
678
|
+
`response`. No separate `.completion` channel is generated. Duplicate operation
|
|
679
|
+
or queue channel keys fail explicitly.
|
|
680
|
+
This describes the declared request/reply contract, not broker capabilities:
|
|
681
|
+
declaring an output does not turn Redis into RPC, and ordinary sends do not
|
|
682
|
+
request replies. Dynamic RabbitMQ reply address extraction, correlation and
|
|
683
|
+
transport bindings are not modeled yet. No server credentials are exposed. This extension
|
|
684
|
+
serves JSON only; the optional Console extension provides the interactive task UI.
|
|
685
|
+
|
|
686
|
+
The RabbitMQ Worker sample declares contracts for Add and Divide and serves
|
|
687
|
+
them at `/asyncapi.json`. The Redis sample exposes the same endpoint, with
|
|
688
|
+
unconstrained inputs for tasks that have no contract yet.
|
|
689
|
+
|
|
690
|
+
### Interactive task console
|
|
691
|
+
|
|
692
|
+
`Console` provides a self-contained Swagger-style screen: registered tasks,
|
|
693
|
+
descriptions, request/response schemas, editable JSON examples and execution
|
|
694
|
+
results. It has no CDN or JavaScript package dependencies. It is not Swagger UI
|
|
695
|
+
(which consumes OpenAPI); it displays the same generated AsyncAPI contracts.
|
|
696
|
+
Schemas are displayed as expandable field trees with types, required markers,
|
|
697
|
+
descriptions and constraints. Nested objects, array items and composition
|
|
698
|
+
branches are shown separately; the complete raw JSON schema remains available.
|
|
699
|
+
Local schema references are expanded with recursion detection; external
|
|
700
|
+
references are labeled without fetching them. Deep models are limited to
|
|
701
|
+
12 nesting levels in the tree, with the full schema preserved in the raw view.
|
|
702
|
+
The bundled HTML is read on each page request with caching disabled. Editing
|
|
703
|
+
`Console.html` takes effect on browser refresh without restarting the Worker;
|
|
704
|
+
Python code and task contract changes still require a restart.
|
|
705
|
+
|
|
706
|
+
```python
|
|
707
|
+
from DefStream.Inspector.Extensions import Console
|
|
708
|
+
from DefStream.Broker.Implements.RabbitMQ import (
|
|
709
|
+
Connection,
|
|
710
|
+
Configuration,
|
|
711
|
+
)
|
|
712
|
+
|
|
713
|
+
def create_connection() -> Connection:
|
|
714
|
+
return Connection(Configuration(
|
|
715
|
+
host='127.0.0.1',
|
|
716
|
+
port=5672,
|
|
717
|
+
username='guest',
|
|
718
|
+
password='guest',
|
|
719
|
+
))
|
|
720
|
+
|
|
721
|
+
# Add to Inspector(exts=(...)).
|
|
722
|
+
console = Console(
|
|
723
|
+
'/console',
|
|
724
|
+
broker=create_connection,
|
|
725
|
+
title='DefStream',
|
|
726
|
+
description='Task contracts & interactive messaging',
|
|
727
|
+
timeout=5,
|
|
728
|
+
)
|
|
729
|
+
```
|
|
730
|
+
|
|
731
|
+
`Console.title` defaults to `DefStream` and sets both the HTML document title
|
|
732
|
+
and header name without a suffix. Optional `description` is the subtitle;
|
|
733
|
+
omitting it hides the subtitle. Both values are escaped as plain text.
|
|
734
|
+
After loading tasks, the document version is displayed beneath the subtitle.
|
|
735
|
+
Console uses the first registered `Specifications` extension, including its
|
|
736
|
+
version, description and media type settings. Without one, it generates a
|
|
737
|
+
default document with version `1.0.0`.
|
|
738
|
+
Inspector has no `name` option: health responses contain status only, metrics
|
|
739
|
+
have no worker-name label, and Specifications uses `DefStream` as its info title.
|
|
740
|
+
|
|
741
|
+
The factory must return a **new, independently owned producer connection**
|
|
742
|
+
on every call, never the Worker's consumer connection or a shared singleton.
|
|
743
|
+
The extension connects for execution and closes in finally, including failed
|
|
744
|
+
connect/publish/RPC. Catalog loading inspects a separate, unconnected producer's
|
|
745
|
+
RPC capability and closes it. It never declares/deletes queues or calls Run
|
|
746
|
+
directly. Validators run on the actual Worker, not again in the HTTP thread.
|
|
747
|
+
|
|
748
|
+
- Open `/console`. With `token` omitted or `None`, no authentication is required:
|
|
749
|
+
tasks load automatically and the token controls are hidden. To enable Bearer
|
|
750
|
+
authentication, pass `token='your-random-token-at-least-16-characters'`,
|
|
751
|
+
enter it in the page, then choose **Load tasks**. HTML is a public,
|
|
752
|
+
static shell with no embedded token or task data; when authentication is
|
|
753
|
+
enabled, task metadata and execution
|
|
754
|
+
both require `Authorization: Bearer <token>`. Tokens must have at least 16
|
|
755
|
+
characters; use a randomly generated strong token. The page keeps the token
|
|
756
|
+
only in memory, never browser storage; Clear token removes it.
|
|
757
|
+
- **Send** invokes the existing task `send` helper and displays the message ID.
|
|
758
|
+
**RPC** invokes `request` and displays the result. RPC is enabled only when
|
|
759
|
+
the producer supports Requestable (currently RabbitMQ, not Redis); it does
|
|
760
|
+
not depend on a response contract. Redis still supports Send.
|
|
761
|
+
- `GET /console` with `Accept: application/json` returns the catalog (authenticated
|
|
762
|
+
only when a token is configured).
|
|
763
|
+
`POST /console` accepts JSON
|
|
764
|
+
`{"task":"Add","mode":"send","body":{"a":10,"b":20}}`;
|
|
765
|
+
use `"mode":"request"` for RPC. Only registered, unambiguous task names are
|
|
766
|
+
accepted. Body must be a JSON object; arbitrary queues cannot be selected.
|
|
767
|
+
- Unknown/invalid inputs return 400, unauthorized calls 401, cross-origin
|
|
768
|
+
requests 403, bodies above 64 KiB 413, non-JSON requests 415, excess concurrent
|
|
769
|
+
calls 429 (four slots), stopped/not-ready Workers 503, unsupported RPC 501,
|
|
770
|
+
remote task errors 502 and RPC timeouts 504. Infrastructure failures are
|
|
771
|
+
logged and return a generic 500 without exposing connection details.
|
|
772
|
+
- Timeouts **do not cancel already delivered work**. Send/RPC are not retried
|
|
773
|
+
by the UI. These operations can produce real side effects; use test queues
|
|
774
|
+
and idempotent tasks. Configure broker connection/socket timeouts as well as
|
|
775
|
+
RPC timeout; metadata `Specification.format` does not change producer codecs.
|
|
776
|
+
- No CORS is enabled. Calls with an Origin must match the HTTP Host. Remote
|
|
777
|
+
access must use HTTPS through a trusted reverse proxy to protect the token;
|
|
778
|
+
do not expose the standard-library WSGI server directly to the Internet.
|
|
779
|
+
Other Inspector extensions remain unauthenticated.
|
|
780
|
+
|
|
781
|
+
Both Worker samples enable `/console` without a token. Authentication, access
|
|
782
|
+
control, token configuration and secret management are the application's
|
|
783
|
+
responsibility.
|
|
784
|
+
|
|
785
|
+
### Task-directed messages and declarations
|
|
786
|
+
|
|
787
|
+
`declare`, `send` and `request` are exported from `DefStream.Worker`. They resolve the
|
|
788
|
+
destination from a concrete Run class's `parameters`; the sending application
|
|
789
|
+
does not need a Worker or global task registry. Bodies must be mappings with
|
|
790
|
+
string keys, matching `run(**body)`. A scalar Run result must be wrapped in a
|
|
791
|
+
mapping before forwarding to another task.
|
|
792
|
+
|
|
793
|
+
Connections are **borrowed**: callers must connect and close them. Neither helper
|
|
794
|
+
connects, declares destinations, nor closes the passed connection, including on
|
|
795
|
+
failure. One connection can publish many records:
|
|
796
|
+
|
|
797
|
+
```python
|
|
798
|
+
from DefStream.Worker import (
|
|
799
|
+
declare,
|
|
800
|
+
send,
|
|
801
|
+
request,
|
|
802
|
+
)
|
|
803
|
+
from DefStream.Broker.Implements.RabbitMQ import (
|
|
804
|
+
Connection,
|
|
805
|
+
Configuration,
|
|
806
|
+
)
|
|
807
|
+
|
|
808
|
+
connection = Connection(Configuration(
|
|
809
|
+
host='127.0.0.1',
|
|
810
|
+
port=5672,
|
|
811
|
+
username='guest',
|
|
812
|
+
password='guest',
|
|
813
|
+
))
|
|
814
|
+
connection.connect()
|
|
815
|
+
try:
|
|
816
|
+
# Sample is an application Run class with subscription parameters.
|
|
817
|
+
declare(
|
|
818
|
+
task=Sample,
|
|
819
|
+
broker=connection,
|
|
820
|
+
)
|
|
821
|
+
for index in range(1000):
|
|
822
|
+
send(
|
|
823
|
+
{
|
|
824
|
+
'a': index,
|
|
825
|
+
'b': 1,
|
|
826
|
+
},
|
|
827
|
+
task=Sample,
|
|
828
|
+
broker=connection,
|
|
829
|
+
)
|
|
830
|
+
response = request(
|
|
831
|
+
{
|
|
832
|
+
'a': 10,
|
|
833
|
+
'b': 20,
|
|
834
|
+
},
|
|
835
|
+
task=Sample,
|
|
836
|
+
broker=connection,
|
|
837
|
+
timeout=5,
|
|
838
|
+
)
|
|
839
|
+
finally:
|
|
840
|
+
connection.close()
|
|
841
|
+
```
|
|
842
|
+
|
|
843
|
+
`send` returns the published message ID. RabbitMQ uses one reusable publisher
|
|
844
|
+
channel per connection, publisher confirms and mandatory routing; an absent
|
|
845
|
+
destination raises rather than silently dropping a task. Redis uses XADD.
|
|
846
|
+
The former task-directed `emit` name has been removed without an alias.
|
|
847
|
+
`emit` is reserved for a future exchange/event publishing API, not implemented yet.
|
|
848
|
+
RabbitMQ `request` delegates to Queue RPC, returning its result or propagating
|
|
849
|
+
TimeoutError/RemoteError/publication errors. Redis `request` raises
|
|
850
|
+
NotImplementedError. RPC still opens a dedicated request/reply connection per
|
|
851
|
+
call internally; the caller-owned connection remains open. This is not pooled
|
|
852
|
+
or multiplexed RPC.
|
|
853
|
+
|
|
854
|
+
`worker.tasks` replaces `worker.runs` and contains registered Run classes.
|
|
855
|
+
Declarations remain explicit, not a side effect of subscribing or sending:
|
|
856
|
+
|
|
857
|
+
```python
|
|
858
|
+
class Declare(WorkerSetup):
|
|
859
|
+
def __call__(
|
|
860
|
+
self,
|
|
861
|
+
worker: Worker,
|
|
862
|
+
) -> None:
|
|
863
|
+
for task in worker.tasks:
|
|
864
|
+
declare(
|
|
865
|
+
task=task,
|
|
866
|
+
broker=worker.broker,
|
|
867
|
+
)
|
|
868
|
+
```
|
|
869
|
+
|
|
870
|
+
Pass `setups=Declare()` to Worker. Its broker is already connected before setup
|
|
871
|
+
runs. `declare(task=task, broker=connection)` delegates to
|
|
872
|
+
`connection.declare(task.parameters)` without opening or closing the connection.
|
|
873
|
+
It can be used without a Worker; there is no `worker.declare()` method.
|
|
874
|
+
RabbitMQ declares durable queues and, when `error` is provided, the durable DLQ
|
|
875
|
+
and dead-letter routing through the default exchange. Redis creates the stream
|
|
876
|
+
and named consumer group. Existing incompatible RabbitMQ declarations fail
|
|
877
|
+
explicitly; no queue is deleted/recreated automatically. This initial task
|
|
878
|
+
declaration API provisions ordinary queues, not RabbitMQ stream queues or
|
|
879
|
+
exchange bindings.
|
|
880
|
+
|
|
881
|
+
Adapters explicitly implement the optional Declarable, TaskSender and Requestable
|
|
882
|
+
protocols in `Broker.Connection` (also exported from `Broker`). These three
|
|
883
|
+
protocols support runtime capability checks so helpers can report unsupported
|
|
884
|
+
operations. Other connection protocols remain static contracts. A normal Worker
|
|
885
|
+
still requires only connect/close/consumer; declaration is optional.
|
|
886
|
+
|
|
887
|
+
Completion hooks can forward work:
|
|
888
|
+
|
|
889
|
+
```python
|
|
890
|
+
class SendNext(RunComplete):
|
|
891
|
+
def __call__(
|
|
892
|
+
self,
|
|
893
|
+
context,
|
|
894
|
+
completion,
|
|
895
|
+
) -> None:
|
|
896
|
+
connection = Connection(Configuration(
|
|
897
|
+
host='127.0.0.1',
|
|
898
|
+
port=5672,
|
|
899
|
+
username='guest',
|
|
900
|
+
password='guest',
|
|
901
|
+
))
|
|
902
|
+
connection.connect()
|
|
903
|
+
try:
|
|
904
|
+
for record in completion:
|
|
905
|
+
send(
|
|
906
|
+
record,
|
|
907
|
+
task=Sample2,
|
|
908
|
+
broker=connection,
|
|
909
|
+
)
|
|
910
|
+
finally:
|
|
911
|
+
connection.close()
|
|
912
|
+
```
|
|
913
|
+
|
|
914
|
+
Here completion is an iterable of mappings. For one mapping, call send once
|
|
915
|
+
instead. Use `completes=SendNext()` on the source Run. Both destinations must
|
|
916
|
+
already be declared. Hooks execute inside the pool thread/process: open their
|
|
917
|
+
producer connection there, never share the consumer connection or a live
|
|
918
|
+
connection across processes/threads. For ProcessPool, hook and Run classes must
|
|
919
|
+
be defined at module scope. Connections can be reused within a batch; reuse
|
|
920
|
+
across multiple Runs can be managed by application PoolSetup/PoolCleanup hooks.
|
|
921
|
+
Keep RabbitMQ heartbeats serviced during long idle/compute periods.
|
|
922
|
+
|
|
923
|
+
Hook publication failures propagate as task failures and prevent success ACK.
|
|
924
|
+
Earlier publications in a partially completed batch are not rolled back.
|
|
925
|
+
Publication and source ACK are not atomic: retries/crashes may duplicate
|
|
926
|
+
downstream messages, so recipients must be idempotent. Blocking requests from a
|
|
927
|
+
hook to the same exhausted pool can deadlock until timeout; use separate
|
|
928
|
+
capacity/workers for such dependencies.
|
|
929
|
+
|
|
930
|
+
Workers receive a broker and a bounded execution pool. Register Run classes
|
|
931
|
+
explicitly; importing or defining a Run does not register it globally.
|
|
932
|
+
`EventContext`, `EventProperties`, `EventRunner`, worker `Factory`, `Invoker`,
|
|
933
|
+
and automatic module loading have been removed.
|
|
934
|
+
|
|
935
|
+
```python
|
|
936
|
+
from DefStream.Worker import (
|
|
937
|
+
declare,
|
|
938
|
+
Run,
|
|
939
|
+
ThreadPool,
|
|
940
|
+
Worker,
|
|
941
|
+
WorkerSetup,
|
|
942
|
+
)
|
|
943
|
+
from DefStream.Broker.Implements.RabbitMQ import (
|
|
944
|
+
Connection,
|
|
945
|
+
Configuration,
|
|
946
|
+
Subscription,
|
|
947
|
+
Reject,
|
|
948
|
+
)
|
|
949
|
+
|
|
950
|
+
class Add(
|
|
951
|
+
Run,
|
|
952
|
+
parameters=Subscription(
|
|
953
|
+
queue='add',
|
|
954
|
+
),
|
|
955
|
+
on_error=Reject(),
|
|
956
|
+
):
|
|
957
|
+
def run(
|
|
958
|
+
self,
|
|
959
|
+
a: int,
|
|
960
|
+
b: int,
|
|
961
|
+
):
|
|
962
|
+
print(a + b)
|
|
963
|
+
|
|
964
|
+
class DeclareQueues(WorkerSetup):
|
|
965
|
+
def __call__(
|
|
966
|
+
self,
|
|
967
|
+
worker: Worker,
|
|
968
|
+
) -> None:
|
|
969
|
+
broker = worker.broker
|
|
970
|
+
if not isinstance(
|
|
971
|
+
broker,
|
|
972
|
+
Connection,
|
|
973
|
+
):
|
|
974
|
+
raise TypeError('A RabbitMQ broker is required')
|
|
975
|
+
for task in worker.tasks:
|
|
976
|
+
declare(
|
|
977
|
+
task=task,
|
|
978
|
+
broker=broker,
|
|
979
|
+
)
|
|
980
|
+
|
|
981
|
+
broker = Connection(Configuration(
|
|
982
|
+
host='127.0.0.1',
|
|
983
|
+
port=5672,
|
|
984
|
+
username='guest',
|
|
985
|
+
password='guest',
|
|
986
|
+
))
|
|
987
|
+
worker = Worker(
|
|
988
|
+
broker,
|
|
989
|
+
ThreadPool(10),
|
|
990
|
+
setups=DeclareQueues(),
|
|
991
|
+
)
|
|
992
|
+
worker.add(Add)
|
|
993
|
+
worker.run()
|
|
994
|
+
```
|
|
995
|
+
|
|
996
|
+
Provision queues in `WorkerSetup` or separately before starting. Worker connects
|
|
997
|
+
the injected broker, runs its setup hooks, creates its consumer, and subscribes
|
|
998
|
+
only registered Runs. It owns that broker
|
|
999
|
+
connection and pool for one run and closes both on exit; do not share them with
|
|
1000
|
+
another Worker. Call `worker.stop()` to stop accepting work, finish outstanding
|
|
1001
|
+
tasks and message confirmations, then close resources. Shutdown currently waits
|
|
1002
|
+
for running tasks without a timeout.
|
|
1003
|
+
`Pool.submit(RunClass, body)` replaces event-name-based submission.
|
|
1004
|
+
It accepts `callback(result)` and `error_callback(exception)` keyword arguments,
|
|
1005
|
+
like `apply_async`, while retaining the executor implementation and returned
|
|
1006
|
+
Future. Only one outcome callback runs, after the Run and its hooks finish.
|
|
1007
|
+
These callbacks stay in the parent process, including with ProcessPool; they
|
|
1008
|
+
must schedule broker operations through `Consumer.call`, not execute them
|
|
1009
|
+
directly. Worker uses these callbacks for ACK and the Run's failure policy.
|
|
1010
|
+
Both `Pool.stop()` and `Pool.shutdown()` wait for tasks; forced termination is
|
|
1011
|
+
not provided by this executor-based implementation.
|
|
1012
|
+
Worker is not an event handler or callable. It creates the consumer without a
|
|
1013
|
+
default handler and supplies a Run-specific handler for each subscription.
|
|
1014
|
+
For standalone broker consumers, `consumer(handler)` and `subs(queue)` remain
|
|
1015
|
+
available for standalone consumption.
|
|
1016
|
+
There is one subscription method: `subs(queue_or_settings, handler=None)`.
|
|
1017
|
+
It accepts a queue name, a RabbitMQ `Subscription`, or a settings dict.
|
|
1018
|
+
An explicit handler overrides the consumer's default handler; one of them is
|
|
1019
|
+
required. Queue-name calls retain the consumer's default offset and QoS.
|
|
1020
|
+
The separate `subscribe` method has been removed.
|
|
1021
|
+
|
|
1022
|
+
RabbitMQ subscription settings can be a
|
|
1023
|
+
`Subscription(queue=..., offset=..., qos=..., error=None)`
|
|
1024
|
+
|
|
1025
|
+
`error` optionally records a non-empty dead-letter queue name. Subscription
|
|
1026
|
+
itself does not provision resources. Explicit `declare(task=task, broker=connection)` (or
|
|
1027
|
+
`connection.declare(task.parameters)`) creates the DLQ and configures the source
|
|
1028
|
+
queue to dead-letter to it. `consumer.subs()` still never declares resources.
|
|
1029
|
+
Reject sends to the DLQ; Requeue does not. Existing queues with conflicting
|
|
1030
|
+
declaration arguments raise the broker error rather than being silently changed.
|
|
1031
|
+
or a dict with those fields. Unknown settings are rejected, not ignored.
|
|
1032
|
+
Set `qos` in each Run's subscription (mapped to RabbitMQ's `prefetch_count`) to limit its unacknowledged
|
|
1033
|
+
deliveries independently (`global_qos=False`). Omitted/None uses the Consumer's
|
|
1034
|
+
`qos` default (1); 0 means unlimited, and valid values are 0 through 65535.
|
|
1035
|
+
The limit includes execution and hooks until ACK or failure settlement.
|
|
1036
|
+
It is not a per-Run execution pool size: Runs share the Worker pool.
|
|
1037
|
+
Redis Streams `COUNT` and Kafka `max.poll.records` are batch-return limits,
|
|
1038
|
+
not equivalent broker-enforced unacknowledged-message limits; they are not
|
|
1039
|
+
exposed as a universal Run QoS setting.
|
|
1040
|
+
Duplicate queue registrations are rejected: this version associates one queue
|
|
1041
|
+
with one Run, not message-type-based routing. `queue.error` is not implemented;
|
|
1042
|
+
configure dead-letter routing on the broker separately.
|
|
1043
|
+
|
|
1044
|
+
Each delivery creates a Run instance and calls it with `instance(**body)`;
|
|
1045
|
+
`Run.__call__` manages hooks and delegates to `run`. Bodies must be
|
|
1046
|
+
mappings with string keys. Pool capacity includes submitted/running tasks and
|
|
1047
|
+
has no unbounded waiting queue. The default pool size is the CPU count (or 1).
|
|
1048
|
+
A full pool causes NACK with `requeue=True`. Successful tasks are ACKed only
|
|
1049
|
+
after completion. Failures are logged and use the Run's `on_error` policy:
|
|
1050
|
+
RabbitMQ defaults to `Requeue()`, or use `Reject()` for `requeue=False`.
|
|
1051
|
+
Requeue does not guarantee placement at the tail and may cause rapid repeated
|
|
1052
|
+
delivery. Reject uses the broker's dead-letter routing if configured, otherwise
|
|
1053
|
+
the message is discarded. Delivery is at-least-once; handlers must tolerate
|
|
1054
|
+
duplicates. `NoAck` is not a failure policy.
|
|
1055
|
+
|
|
1056
|
+
Run classes also accept `setups`, `completes`, and `errors`, each as a single
|
|
1057
|
+
hook or an ordered sequence. The `RunSetup`, `RunComplete`, `RunError`,
|
|
1058
|
+
and `Context` interfaces are exported from `DefStream.Worker`.
|
|
1059
|
+
`errors` contains execution-error hooks (`RunError`), while `on_error` selects
|
|
1060
|
+
the failed-event policy (`EventError`, exported from `DefStream.Broker`).
|
|
1061
|
+
These are separate responsibilities: error hooks run in the pool, then the
|
|
1062
|
+
event policy runs on the connection owner thread. The former `FailurePolicy`
|
|
1063
|
+
and `error_policy` names have been replaced by `EventError` and `on_error`.
|
|
1064
|
+
RunSetup hooks receive `Context` (`args` and `kwargs`), completion hooks
|
|
1065
|
+
receive that context and the result, and error hooks receive context
|
|
1066
|
+
and the exception. Setups run before `run`; completes
|
|
1067
|
+
run after success; errors run after `run` failure. RunError hooks
|
|
1068
|
+
do not turn failure into success. If any hook raises, remaining hooks stop
|
|
1069
|
+
immediately and the failure policy applies (no recursive error-hook execution).
|
|
1070
|
+
Run instances are constructed at submission, after capacity is acquired.
|
|
1071
|
+
Constructor exceptions are submission errors, not execution errors.
|
|
1072
|
+
The hook/execution lifecycle runs in the pool, and only then is ACK or the failure
|
|
1073
|
+
policy applied on the connection thread. Pool capacity remains occupied until
|
|
1074
|
+
hooks finish. Hooks used with ProcessPool must also be picklable.
|
|
1075
|
+
### Lifecycle hooks
|
|
1076
|
+
|
|
1077
|
+
All hook interfaces are exported from `DefStream.Worker`:
|
|
1078
|
+
|
|
1079
|
+
| Scope | Hooks | Arguments |
|
|
1080
|
+
| --- | --- | --- |
|
|
1081
|
+
| Run | `RunSetup`, `RunComplete`, `RunError` | Context; context and result; context and exception |
|
|
1082
|
+
| Pool | `PoolSetup`, `PoolCleanup` | No arguments; executed inside each pool thread/process |
|
|
1083
|
+
| Worker | `WorkerSetup`, `WorkerCleanup` | The Worker instance |
|
|
1084
|
+
|
|
1085
|
+
`Runner` has been renamed to `Run`, in the `DefStream.Worker.Run` module.
|
|
1086
|
+
The old `Setup`, `Complete`, `Error`, and `Initializer` names have been replaced
|
|
1087
|
+
by the scoped names above; there are no compatibility aliases.
|
|
1088
|
+
|
|
1089
|
+
Pass `setups=` and `cleanups=` to Worker, ThreadPool, or ProcessPool. Each accepts
|
|
1090
|
+
a single hook or an ordered sequence. Worker setup hooks run after broker
|
|
1091
|
+
connection and before subscriptions; `worker.tasks` contains the registered Run
|
|
1092
|
+
classes. Use these hooks to declare queues/DLQs or check application dependencies.
|
|
1093
|
+
Worker cleanup hooks run after the pool has shut down and before the consumer
|
|
1094
|
+
and broker are closed, including when connection, setup, subscription, or
|
|
1095
|
+
consumption fails. They must tolerate partially initialized resources.
|
|
1096
|
+
|
|
1097
|
+
Pool execution environments start lazily on the first submission. Each runs its
|
|
1098
|
+
setup hooks once, processes tasks, and runs its cleanup hooks in the same
|
|
1099
|
+
thread/process on graceful shutdown. An unused pool runs neither hook.
|
|
1100
|
+
Cleanup runs only after all setup hooks in that environment succeed, including
|
|
1101
|
+
when a task fails; a failing setup must release its own partially acquired
|
|
1102
|
+
resources. Initialize actual task DB connections here, not in the parent process.
|
|
1103
|
+
ProcessPool uses spawned processes; hooks and their configuration must be
|
|
1104
|
+
picklable. Pool setup and cleanup hooks do not receive the Worker or broker.
|
|
1105
|
+
|
|
1106
|
+
Persistent execution loops run on the standard executors so that cleanup executes
|
|
1107
|
+
in `finally` inside the owning environment, rather than submitting arbitrary
|
|
1108
|
+
cleanup tasks that might all land on one worker. Futures, bounded capacity, and
|
|
1109
|
+
parent-side completion callbacks are retained. A ProcessPool also owns a
|
|
1110
|
+
management process for task/result transport, closed during shutdown.
|
|
1111
|
+
|
|
1112
|
+
Cleanup hooks run in registration order. A cleanup failure does not skip later
|
|
1113
|
+
cleanup hooks or resource closure; failures propagate with exception chaining.
|
|
1114
|
+
Pool environment failures fail outstanding Futures and are also raised by
|
|
1115
|
+
`stop()`/`shutdown()`. Ordinary Run failures do not break the pool.
|
|
1116
|
+
Cleanup cannot be guaranteed after forced termination or a process crash.
|
|
1117
|
+
|
|
1118
|
+
Broker confirmations run on the connection owner thread, not pool threads.
|
|
1119
|
+
`DefStream.Broker.Event` is the common event contract (`body`, `ack`, `nack`,
|
|
1120
|
+
`reject`); RabbitMQ's `Event` implements it and retains its broker metadata.
|
|
1121
|
+
Worker and failure policies use this contract; there is no separate `Delivery`.
|
|
1122
|
+
Only the Run instance and body are sent to ProcessPool children; Run classes
|
|
1123
|
+
must be importable at module scope and instances/inputs/results must be picklable.
|
|
1124
|
+
Use the `if __name__ == '__main__':` guard with ProcessPool.
|
|
1125
|
+
For graceful Ctrl+C shutdown, register a parent-side `SIGINT` handler that calls
|
|
1126
|
+
`worker.stop()` (as in the RabbitMQ sample). Pool child processes ignore SIGINT
|
|
1127
|
+
so they can finish active work and run PoolCleanup instead of raising
|
|
1128
|
+
KeyboardInterrupt. The parent drains message confirmations, stops the pool,
|
|
1129
|
+
runs WorkerCleanup, and closes broker resources. The library does not replace
|
|
1130
|
+
the application's parent-side signal handler.
|
|
1131
|
+
|
|
1132
|
+
Install the core and both adapters before running the test suite. Tests use the
|
|
1133
|
+
installed `DefStream` package rather than importing the source directory as `src`.
|
|
1134
|
+
Run unit tests with `python -m unittest discover -s test -p "test_*.py"`.
|
|
1135
|
+
The core suite retains shared Worker, Inspector, and cross-adapter contract
|
|
1136
|
+
tests. Adapter-specific unit and integration tests live in each adapter's
|
|
1137
|
+
`test` directory. From the common parent directory, run all three suites:
|
|
1138
|
+
|
|
1139
|
+
```powershell
|
|
1140
|
+
python -m unittest discover -s .\DefStream\test -p "test_*.py"
|
|
1141
|
+
python -m unittest discover -s .\DefStream-RabbitMQ\test -p "test_*.py"
|
|
1142
|
+
python -m unittest discover -s .\DefStream-Redis\test -p "test_*.py"
|
|
1143
|
+
```
|
|
1144
|
+
|
|
1145
|
+
To include local RabbitMQ integration tests, set `DEFSTREAM_RABBITMQ_TESTS=1`
|
|
1146
|
+
first. They use `127.0.0.1:5672` with guest credentials and create/delete only
|
|
1147
|
+
uniquely named test queues. Redis integration tests are enabled separately with
|
|
1148
|
+
`DEFSTREAM_REDIS_TESTS=1`.
|
|
1149
|
+
|
|
1150
|
+
Examples are organized under:
|
|
1151
|
+
|
|
1152
|
+
- `sample/Worker/Sample.py` for a self-contained in-memory broker and worker.
|
|
1153
|
+
It declares a queue and publishes two messages in `WorkerSetup`, processes
|
|
1154
|
+
them with a ThreadPool, confirms deliveries on the consumer thread, then
|
|
1155
|
+
exits automatically after draining the queue and runs `WorkerCleanup`.
|
|
1156
|
+
No RabbitMQ server is required. This finite sample consumes one message at
|
|
1157
|
+
a time and rejects failed messages by default; it is not a persistent broker.
|
|
1158
|
+
- `sample/Worker/RabbitMQ` for RabbitMQ-backed workers
|
|
1159
|
+
- `sample/Broker/Implements/RabbitMQ` for RabbitMQ queues, exchanges, and streams
|
|
1160
|
+
- `sample/Broker` for a small in-memory broker
|
|
1161
|
+
|
|
1162
|
+
RabbitMQ examples require a reachable RabbitMQ server and pre-existing queues
|
|
1163
|
+
unless a Worker setup provisions them.
|
|
1164
|
+
|
|
1165
|
+
## Development in VS Code
|
|
1166
|
+
|
|
1167
|
+
The core sources remain in `src/Worker` and `src/Broker`. RabbitMQ and Redis
|
|
1168
|
+
implementation files live directly under `src` in the sibling
|
|
1169
|
+
`DefStream-RabbitMQ` and `DefStream-Redis` projects. Their distributions provide
|
|
1170
|
+
only their adapter packages, not the parent packages owned by the core.
|
|
1171
|
+
|
|
1172
|
+
From the common parent directory, install all three projects into the workspace
|
|
1173
|
+
virtual environment:
|
|
1174
|
+
|
|
1175
|
+
```powershell
|
|
1176
|
+
.\.venv\Scripts\python.exe -m pip install `
|
|
1177
|
+
--config-settings editable_mode=strict -e .\DefStream
|
|
1178
|
+
.\.venv\Scripts\python.exe -m pip install `
|
|
1179
|
+
-e .\DefStream-RabbitMQ `
|
|
1180
|
+
-e .\DefStream-Redis
|
|
1181
|
+
```
|
|
1182
|
+
|
|
1183
|
+
Select that virtual environment as VS Code's Python interpreter. No manual
|
|
1184
|
+
`site-packages/DefStream` symbolic link or junction is needed. Remove an existing
|
|
1185
|
+
link before installing the split projects: otherwise installing adapter files
|
|
1186
|
+
under that link can write into the core source directory.
|
|
1187
|
+
|
|
1188
|
+
Use strict editable mode for the core and default editable mode for the
|
|
1189
|
+
adapters. The default adapter finders provide parent namespace entries, which
|
|
1190
|
+
can otherwise mask the core's default editable finder. Strict mode exposes the
|
|
1191
|
+
core as regular packages through a link tree under its `build` directory.
|
|
1192
|
+
Keep that link tree while using the editable installation.
|
|
1193
|
+
|
|
1194
|
+
Edits to existing source files are reflected without reinstalling. Reinstall
|
|
1195
|
+
the core after adding or removing its modules. Reinstall the relevant project
|
|
1196
|
+
after changing package metadata or dependencies, recreating the virtual
|
|
1197
|
+
environment, or moving a project directory.
|