deegzlibs-command-bus 2.2.0__tar.gz → 2.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/PKG-INFO +58 -33
- deegzlibs_command_bus-2.3.0/README.md +107 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/pyproject.toml +3 -3
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/setup.py +1 -1
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/__init__.py +2 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/bus.py +14 -6
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/cli.py +74 -16
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/event_bus.py +16 -7
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/interfaces.py +2 -2
- deegzlibs_command_bus-2.3.0/src/command_bus/worker_app.py +105 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/deegzlibs_command_bus.egg-info/PKG-INFO +58 -33
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/deegzlibs_command_bus.egg-info/SOURCES.txt +3 -1
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_cli.py +43 -4
- deegzlibs_command_bus-2.3.0/tests/test_worker_app.py +87 -0
- deegzlibs_command_bus-2.2.0/README.md +0 -82
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/setup.cfg +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/__init__.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/queue/__init__.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/queue/file.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/queue/in_memory.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/queue/in_memory_pubsub.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/queue/rabbitmq.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/queue/rabbitmq_fanout.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/queue/redis.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/queue/redis_pubsub.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/queue/sns_pubsub.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/queue/sqs.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/response/__init__.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/response/file.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/response/in_memory.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/adapters/response/redis.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/command_bus_group.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/parsers/__init__.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/parsers/base.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/parsers/base64_parser.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/parsers/codec.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/parsers/json_parser.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/parsers/repr_parser.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/registry.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/command_bus/utils.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/deegzlibs_command_bus.egg-info/dependency_links.txt +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/deegzlibs_command_bus.egg-info/entry_points.txt +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/deegzlibs_command_bus.egg-info/requires.txt +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/src/deegzlibs_command_bus.egg-info/top_level.txt +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_base64_parser.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_bus_event_decorator.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_command_bus_group.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_event_bus_pubsub.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_file_queue_adapter.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_file_response_store.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_handler_async.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_in_memory_adapter.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_interfaces.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_json_parser.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_message_codec.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_message_parser_base.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_parser.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_rabbitmq_adapter.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_rabbitmq_fanout_adapter.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_redis_pubsub_adapter.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_redis_queue_adapter.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_registry.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_response_store.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_sns_pubsub_adapter.py +0 -0
- {deegzlibs_command_bus-2.2.0 → deegzlibs_command_bus-2.3.0}/tests/test_sqs_adapter.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: deegzlibs-command-bus
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.3.0
|
|
4
4
|
Summary: A small command bus with pluggable queue adapters and response stores (e.g. SQS, RabbitMQ, Redis, etc.)
|
|
5
5
|
Home-page: https://github.com/deegzlibs/deegzlibs
|
|
6
6
|
Author: Diego Alejos
|
|
@@ -44,7 +44,7 @@ Dynamic: requires-python
|
|
|
44
44
|
|
|
45
45
|
# DeegzLibs CommandBus (Python)
|
|
46
46
|
|
|
47
|
-
A small command and event bus with pluggable queue adapters.
|
|
47
|
+
A small command and event bus with pluggable queue adapters. Register handlers, send commands or publish events, and process them in-process or through a queue (SQS, SNS, RabbitMQ, Redis, and more).
|
|
48
48
|
|
|
49
49
|
## Installation
|
|
50
50
|
|
|
@@ -54,72 +54,97 @@ pip install deegzlibs-command-bus
|
|
|
54
54
|
|
|
55
55
|
Optional extras: **`[sqs]`**, **`[sns]`**, **`[boto3]`**, **`[redis]`**, **`[rabbitmq]`**. See [Installation](docs/installation.md).
|
|
56
56
|
|
|
57
|
-
## Quick start
|
|
57
|
+
## Quick start — commands (WorkerApp)
|
|
58
58
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
One message is handled by **one worker** (competing consumers):
|
|
59
|
+
**WorkerApp** wraps your handlers and a **queue adapter**. The adapter is the queue itself — `execute()` writes to it, `work()` reads from it. Producer and worker must use the **same adapter** (same backend and queue name):
|
|
62
60
|
|
|
63
61
|
```python
|
|
64
|
-
|
|
62
|
+
import asyncio
|
|
63
|
+
from command_bus import WorkerApp
|
|
65
64
|
from command_bus.adapters import InMemoryQueueAdapter
|
|
66
65
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
# Where commands wait between send and process.
|
|
67
|
+
# Swap for SqsQueueAdapter, RedisQueueAdapter, etc. in production.
|
|
68
|
+
queue = InMemoryQueueAdapter(queue_name="orders")
|
|
69
|
+
|
|
70
|
+
app = WorkerApp(queue_adapter=queue)
|
|
70
71
|
|
|
71
|
-
@
|
|
72
|
+
@app.command()
|
|
72
73
|
def on_order_created(order_id: str, amount_cents: int):
|
|
73
74
|
print(f"Order {order_id}: {amount_cents} cents")
|
|
74
75
|
|
|
75
|
-
|
|
76
|
-
|
|
76
|
+
async def main():
|
|
77
|
+
# Producer: enqueue via the adapter
|
|
78
|
+
await app.execute(on_order_created(order_id="ord-1", amount_cents=1999), wait=False)
|
|
79
|
+
# Worker: dequeue via the same adapter
|
|
80
|
+
await app.work()
|
|
77
81
|
|
|
78
|
-
|
|
79
|
-
|
|
82
|
+
if __name__ == "__main__":
|
|
83
|
+
asyncio.run(main())
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
In production, split this into two modules that share the same queue setup — see [Client and worker](docs/client-and-worker.md).
|
|
87
|
+
|
|
88
|
+
Run the worker as a separate process (like `uvicorn` for web apps):
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
command-bus-worker myapp.worker:app --workers 4
|
|
80
92
|
```
|
|
81
93
|
|
|
82
|
-
|
|
94
|
+
The CLI imports your module and runs `app.work()` in a loop — it does **not** call `main()`. Your worker module defines `app` (with its adapter and handlers); the client module sends commands using the same queue configuration.
|
|
95
|
+
|
|
96
|
+
- **`--workers`** — how many worker processes share the queue
|
|
97
|
+
- **`--concurrency`** — how many messages each process handles at once
|
|
83
98
|
|
|
84
|
-
|
|
99
|
+
If you have the repo locally, try the in-memory demo:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
PYTHONPATH=src python examples/worker_app_demo.py
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
More on WorkerApp (response stores, production queues, concurrency): [WorkerApp docs](docs/worker-app.md).
|
|
106
|
+
|
|
107
|
+
Need the lower-level API (`CommandBus`, `Router`, message classes)? See the [full quick start](docs/quickstart.md).
|
|
108
|
+
|
|
109
|
+
## Quick start — events
|
|
110
|
+
|
|
111
|
+
Each event is delivered to **every subscriber**:
|
|
85
112
|
|
|
86
113
|
```python
|
|
87
114
|
from command_bus import EventBus, Router
|
|
88
115
|
from command_bus.adapters import InMemoryPubSubAdapter
|
|
89
116
|
|
|
90
117
|
router = Router()
|
|
91
|
-
|
|
92
|
-
|
|
118
|
+
bus = EventBus(
|
|
119
|
+
queue_adapter=InMemoryPubSubAdapter(queue_name="order-events"),
|
|
120
|
+
command_router=router,
|
|
121
|
+
)
|
|
93
122
|
|
|
94
123
|
@router.event()
|
|
95
124
|
def on_order_created(order_id: str, amount_cents: int):
|
|
96
125
|
print(f"Order {order_id}: {amount_cents} cents")
|
|
97
126
|
|
|
98
|
-
# Producer (no local handler required)
|
|
99
127
|
await bus.publish(on_order_created(order_id="ord-1", amount_cents=1999))
|
|
100
|
-
|
|
101
|
-
# Each worker with its own adapter on the same topic receives the event
|
|
102
128
|
await bus.work()
|
|
103
129
|
```
|
|
104
130
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
For full examples (messages, handlers, SQS/Redis/SNS, execute-and-wait), see the [documentation](docs/index.md).
|
|
131
|
+
See [Pub/sub events](docs/pubsub-events.md) for fan-out adapters and multi-worker setups.
|
|
108
132
|
|
|
109
133
|
## Documentation
|
|
110
134
|
|
|
111
135
|
| Topic | Description |
|
|
112
136
|
|-------|-------------|
|
|
137
|
+
| [Quick start](docs/quickstart.md) | WorkerApp plus step-by-step wiring. |
|
|
138
|
+
| [WorkerApp](docs/worker-app.md) | Handlers, concurrency, in-memory and production setup. |
|
|
139
|
+
| [Worker CLI](docs/cli.md) | `command-bus-worker module[:attr]`. |
|
|
113
140
|
| [Installation](docs/installation.md) | Package and extras. |
|
|
114
|
-
| [
|
|
115
|
-
| [
|
|
116
|
-
| [Message formats and parsers](docs/message-formats-and-parsers.md) | Repr, JSON, Base64, custom parser. |
|
|
117
|
-
| [Client and worker](docs/client-and-worker.md) | Shared module, producer, consumer. |
|
|
118
|
-
| [Worker CLI](docs/cli.md) | `command-bus-worker module[:attr]` — `CommandBus`, `EventBus`, or `BusGroup`. |
|
|
141
|
+
| [Handler decorator](docs/handler-decorator.md) | `@app.command()` / `@router.command()`. |
|
|
142
|
+
| [Client and worker](docs/client-and-worker.md) | Split producer and consumer in production. |
|
|
119
143
|
| [Queue adapters](docs/queue-adapters.md) | In-memory, SQS, RabbitMQ, Redis. |
|
|
120
|
-
| [
|
|
121
|
-
| [
|
|
122
|
-
| [
|
|
144
|
+
| [Execute and wait](docs/execute-and-wait.md) | Send a command and get the result back. |
|
|
145
|
+
| [Pub/sub events](docs/pubsub-events.md) | EventBus and fan-out. |
|
|
146
|
+
| [Message formats and parsers](docs/message-formats-and-parsers.md) | Repr, JSON, Base64. |
|
|
147
|
+
| [API reference](docs/api-reference.md) | Types and methods. |
|
|
123
148
|
|
|
124
149
|
## License
|
|
125
150
|
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# DeegzLibs CommandBus (Python)
|
|
2
|
+
|
|
3
|
+
A small command and event bus with pluggable queue adapters. Register handlers, send commands or publish events, and process them in-process or through a queue (SQS, SNS, RabbitMQ, Redis, and more).
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install deegzlibs-command-bus
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Optional extras: **`[sqs]`**, **`[sns]`**, **`[boto3]`**, **`[redis]`**, **`[rabbitmq]`**. See [Installation](docs/installation.md).
|
|
12
|
+
|
|
13
|
+
## Quick start — commands (WorkerApp)
|
|
14
|
+
|
|
15
|
+
**WorkerApp** wraps your handlers and a **queue adapter**. The adapter is the queue itself — `execute()` writes to it, `work()` reads from it. Producer and worker must use the **same adapter** (same backend and queue name):
|
|
16
|
+
|
|
17
|
+
```python
|
|
18
|
+
import asyncio
|
|
19
|
+
from command_bus import WorkerApp
|
|
20
|
+
from command_bus.adapters import InMemoryQueueAdapter
|
|
21
|
+
|
|
22
|
+
# Where commands wait between send and process.
|
|
23
|
+
# Swap for SqsQueueAdapter, RedisQueueAdapter, etc. in production.
|
|
24
|
+
queue = InMemoryQueueAdapter(queue_name="orders")
|
|
25
|
+
|
|
26
|
+
app = WorkerApp(queue_adapter=queue)
|
|
27
|
+
|
|
28
|
+
@app.command()
|
|
29
|
+
def on_order_created(order_id: str, amount_cents: int):
|
|
30
|
+
print(f"Order {order_id}: {amount_cents} cents")
|
|
31
|
+
|
|
32
|
+
async def main():
|
|
33
|
+
# Producer: enqueue via the adapter
|
|
34
|
+
await app.execute(on_order_created(order_id="ord-1", amount_cents=1999), wait=False)
|
|
35
|
+
# Worker: dequeue via the same adapter
|
|
36
|
+
await app.work()
|
|
37
|
+
|
|
38
|
+
if __name__ == "__main__":
|
|
39
|
+
asyncio.run(main())
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
In production, split this into two modules that share the same queue setup — see [Client and worker](docs/client-and-worker.md).
|
|
43
|
+
|
|
44
|
+
Run the worker as a separate process (like `uvicorn` for web apps):
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
command-bus-worker myapp.worker:app --workers 4
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
The CLI imports your module and runs `app.work()` in a loop — it does **not** call `main()`. Your worker module defines `app` (with its adapter and handlers); the client module sends commands using the same queue configuration.
|
|
51
|
+
|
|
52
|
+
- **`--workers`** — how many worker processes share the queue
|
|
53
|
+
- **`--concurrency`** — how many messages each process handles at once
|
|
54
|
+
|
|
55
|
+
If you have the repo locally, try the in-memory demo:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
PYTHONPATH=src python examples/worker_app_demo.py
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
More on WorkerApp (response stores, production queues, concurrency): [WorkerApp docs](docs/worker-app.md).
|
|
62
|
+
|
|
63
|
+
Need the lower-level API (`CommandBus`, `Router`, message classes)? See the [full quick start](docs/quickstart.md).
|
|
64
|
+
|
|
65
|
+
## Quick start — events
|
|
66
|
+
|
|
67
|
+
Each event is delivered to **every subscriber**:
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
from command_bus import EventBus, Router
|
|
71
|
+
from command_bus.adapters import InMemoryPubSubAdapter
|
|
72
|
+
|
|
73
|
+
router = Router()
|
|
74
|
+
bus = EventBus(
|
|
75
|
+
queue_adapter=InMemoryPubSubAdapter(queue_name="order-events"),
|
|
76
|
+
command_router=router,
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
@router.event()
|
|
80
|
+
def on_order_created(order_id: str, amount_cents: int):
|
|
81
|
+
print(f"Order {order_id}: {amount_cents} cents")
|
|
82
|
+
|
|
83
|
+
await bus.publish(on_order_created(order_id="ord-1", amount_cents=1999))
|
|
84
|
+
await bus.work()
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
See [Pub/sub events](docs/pubsub-events.md) for fan-out adapters and multi-worker setups.
|
|
88
|
+
|
|
89
|
+
## Documentation
|
|
90
|
+
|
|
91
|
+
| Topic | Description |
|
|
92
|
+
|-------|-------------|
|
|
93
|
+
| [Quick start](docs/quickstart.md) | WorkerApp plus step-by-step wiring. |
|
|
94
|
+
| [WorkerApp](docs/worker-app.md) | Handlers, concurrency, in-memory and production setup. |
|
|
95
|
+
| [Worker CLI](docs/cli.md) | `command-bus-worker module[:attr]`. |
|
|
96
|
+
| [Installation](docs/installation.md) | Package and extras. |
|
|
97
|
+
| [Handler decorator](docs/handler-decorator.md) | `@app.command()` / `@router.command()`. |
|
|
98
|
+
| [Client and worker](docs/client-and-worker.md) | Split producer and consumer in production. |
|
|
99
|
+
| [Queue adapters](docs/queue-adapters.md) | In-memory, SQS, RabbitMQ, Redis. |
|
|
100
|
+
| [Execute and wait](docs/execute-and-wait.md) | Send a command and get the result back. |
|
|
101
|
+
| [Pub/sub events](docs/pubsub-events.md) | EventBus and fan-out. |
|
|
102
|
+
| [Message formats and parsers](docs/message-formats-and-parsers.md) | Repr, JSON, Base64. |
|
|
103
|
+
| [API reference](docs/api-reference.md) | Types and methods. |
|
|
104
|
+
|
|
105
|
+
## License
|
|
106
|
+
|
|
107
|
+
MIT
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
6
|
name = "deegzlibs-command-bus"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.3.0"
|
|
8
8
|
description = "A small command bus with pluggable queue adapters (e.g. AWS SQS)"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = ["Diego Alejos <lego.admin@gmail.com>"]
|
|
@@ -33,7 +33,7 @@ redis = ["redis"]
|
|
|
33
33
|
|
|
34
34
|
[project]
|
|
35
35
|
name = "deegzlibs-command-bus"
|
|
36
|
-
version = "2.
|
|
36
|
+
version = "2.3.0"
|
|
37
37
|
description = "A small command bus with pluggable queue adapters and response stores (e.g. SQS, RabbitMQ, Redis, etc.)"
|
|
38
38
|
readme = "README.md"
|
|
39
39
|
requires-python = ">=3.8"
|
|
@@ -80,7 +80,7 @@ where = ["src"]
|
|
|
80
80
|
|
|
81
81
|
[tool.commitizen]
|
|
82
82
|
name = "cz_conventional_commits"
|
|
83
|
-
version = "2.
|
|
83
|
+
version = "2.3.0"
|
|
84
84
|
tag_format = "v$version"
|
|
85
85
|
version_files = [
|
|
86
86
|
"pyproject.toml:version",
|
|
@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
|
|
|
5
5
|
|
|
6
6
|
setup(
|
|
7
7
|
name="deegzlibs-command-bus",
|
|
8
|
-
version="2.
|
|
8
|
+
version="2.3.0",
|
|
9
9
|
author="Diego Alejos",
|
|
10
10
|
author_email="lego.admin@gmail.com",
|
|
11
11
|
description="A small command bus with pluggable queue adapters (e.g. AWS SQS)",
|
|
@@ -40,6 +40,7 @@ from .registry import (
|
|
|
40
40
|
RouterEntry,
|
|
41
41
|
get_qual_name,
|
|
42
42
|
)
|
|
43
|
+
from .worker_app import WorkerApp
|
|
43
44
|
|
|
44
45
|
__all__ = [
|
|
45
46
|
"Base64MessageCodec",
|
|
@@ -74,6 +75,7 @@ __all__ = [
|
|
|
74
75
|
"RouterInterface",
|
|
75
76
|
"TransmissibleBaseModel",
|
|
76
77
|
"WorkerConfig",
|
|
78
|
+
"WorkerApp",
|
|
77
79
|
"configure_json_parser",
|
|
78
80
|
"resolve_bus_attr_on_module",
|
|
79
81
|
"get_qual_name",
|
|
@@ -151,9 +151,17 @@ class CommandBus(CommandBusInterface):
|
|
|
151
151
|
ttl_seconds=self.response_ttl_seconds,
|
|
152
152
|
)
|
|
153
153
|
|
|
154
|
-
async def work(self) ->
|
|
155
|
-
"""Poll the queue and dispatch
|
|
156
|
-
messages = self.queue_adapter.get_messages()
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
154
|
+
async def work(self, *, concurrency: int = 1) -> int:
|
|
155
|
+
"""Poll the queue and dispatch messages to handlers (up to ``concurrency`` in parallel)."""
|
|
156
|
+
messages = self.queue_adapter.get_messages(max_messages=max(1, concurrency))
|
|
157
|
+
if not messages:
|
|
158
|
+
return 0
|
|
159
|
+
|
|
160
|
+
async def _handle(message: Any) -> None:
|
|
161
|
+
try:
|
|
162
|
+
await self.dispatch(message.body)
|
|
163
|
+
finally:
|
|
164
|
+
self.queue_adapter.dequeue(message)
|
|
165
|
+
|
|
166
|
+
await asyncio.gather(*(_handle(m) for m in messages))
|
|
167
|
+
return len(messages)
|
|
@@ -10,16 +10,19 @@ import multiprocessing
|
|
|
10
10
|
import signal
|
|
11
11
|
import sys
|
|
12
12
|
import time
|
|
13
|
-
from typing import Any, List, Optional, Sequence, Tuple
|
|
13
|
+
from typing import Any, List, Optional, Sequence, Tuple, Union
|
|
14
14
|
|
|
15
|
+
from .adapters.queue.in_memory import InMemoryQueueAdapter
|
|
15
16
|
from .bus import CommandBus
|
|
16
17
|
from .command_bus_group import BusGroup, CommandBusGroup, resolve_bus_attr_on_module
|
|
17
18
|
from .event_bus import EventBus
|
|
19
|
+
from .worker_app import WorkerApp
|
|
18
20
|
|
|
19
21
|
logger = logging.getLogger(__name__)
|
|
20
22
|
|
|
21
23
|
_DEFAULT_BUS_ATTR = "bus"
|
|
22
24
|
_BUS_TYPES = (CommandBus, EventBus)
|
|
25
|
+
_WORKER_TARGET_TYPES = (CommandBus, EventBus, WorkerApp)
|
|
23
26
|
|
|
24
27
|
|
|
25
28
|
def _parse_target(spec: str) -> Tuple[str, str]:
|
|
@@ -80,12 +83,28 @@ def _get_multiprocessing_context() -> multiprocessing.context.BaseContext:
|
|
|
80
83
|
return multiprocessing.get_context("fork")
|
|
81
84
|
|
|
82
85
|
|
|
86
|
+
def _queue_name_for_target(target: Union[CommandBus, EventBus, WorkerApp], attr: str) -> str:
|
|
87
|
+
if isinstance(target, WorkerApp):
|
|
88
|
+
return getattr(target.queue_adapter, "queue_name", attr)
|
|
89
|
+
return getattr(target.queue_adapter, "queue_name", attr)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
async def _do_work(
|
|
93
|
+
target: Union[CommandBus, EventBus, WorkerApp],
|
|
94
|
+
concurrency: int,
|
|
95
|
+
) -> int:
|
|
96
|
+
if isinstance(target, WorkerApp):
|
|
97
|
+
return await target.work(concurrency=concurrency)
|
|
98
|
+
return await target.work(concurrency=concurrency)
|
|
99
|
+
|
|
100
|
+
|
|
83
101
|
def _process_worker_main(
|
|
84
102
|
module_name: str,
|
|
85
103
|
bus_attr: str,
|
|
86
104
|
worker_num: int,
|
|
87
105
|
poll_interval: float,
|
|
88
106
|
log_level: int,
|
|
107
|
+
concurrency: int,
|
|
89
108
|
) -> None:
|
|
90
109
|
"""Entry point for each worker process (import module, run work loop)."""
|
|
91
110
|
# Foreground process group delivers SIGINT to all workers; parent coordinates shutdown.
|
|
@@ -117,18 +136,27 @@ def _process_worker_main(
|
|
|
117
136
|
wlog.error("Failed to import %r: %s", module_name, e)
|
|
118
137
|
sys.exit(1)
|
|
119
138
|
|
|
120
|
-
|
|
121
|
-
if not isinstance(
|
|
122
|
-
wlog.error(
|
|
139
|
+
target = getattr(mod, bus_attr, None)
|
|
140
|
+
if not isinstance(target, _WORKER_TARGET_TYPES):
|
|
141
|
+
wlog.error(
|
|
142
|
+
"Attribute %r is not a CommandBus, EventBus, or WorkerApp",
|
|
143
|
+
bus_attr,
|
|
144
|
+
)
|
|
123
145
|
sys.exit(1)
|
|
124
146
|
|
|
125
147
|
name = f"{bus_attr}:{worker_num}"
|
|
126
|
-
|
|
148
|
+
qn = _queue_name_for_target(target, bus_attr)
|
|
149
|
+
wlog.info(
|
|
150
|
+
"Worker started for %r (queue_name=%s, concurrency=%d)",
|
|
151
|
+
bus_attr,
|
|
152
|
+
qn,
|
|
153
|
+
concurrency,
|
|
154
|
+
)
|
|
127
155
|
|
|
128
156
|
async def _loop() -> None:
|
|
129
157
|
while not stop:
|
|
130
158
|
try:
|
|
131
|
-
await
|
|
159
|
+
await _do_work(target, concurrency)
|
|
132
160
|
except asyncio.CancelledError:
|
|
133
161
|
raise
|
|
134
162
|
except Exception:
|
|
@@ -216,6 +244,7 @@ def _run_worker_processes(
|
|
|
216
244
|
jobs: List[Tuple[str, int]],
|
|
217
245
|
poll_interval: float,
|
|
218
246
|
log_level: int,
|
|
247
|
+
concurrency: int,
|
|
219
248
|
) -> None:
|
|
220
249
|
ctx = _get_multiprocessing_context()
|
|
221
250
|
method = ctx.get_start_method()
|
|
@@ -225,7 +254,7 @@ def _run_worker_processes(
|
|
|
225
254
|
for bus_attr, worker_num in jobs:
|
|
226
255
|
proc = ctx.Process(
|
|
227
256
|
target=_process_worker_main,
|
|
228
|
-
args=(module_name, bus_attr, worker_num, poll_interval, log_level),
|
|
257
|
+
args=(module_name, bus_attr, worker_num, poll_interval, log_level, concurrency),
|
|
229
258
|
name=f"command-bus-{bus_attr}-{worker_num}",
|
|
230
259
|
)
|
|
231
260
|
proc.start()
|
|
@@ -239,10 +268,11 @@ def main(argv: Optional[Sequence[str]] = None) -> None:
|
|
|
239
268
|
|
|
240
269
|
parser = argparse.ArgumentParser(
|
|
241
270
|
description=(
|
|
242
|
-
"Run CommandBus/EventBus.work() in worker processes "
|
|
271
|
+
"Run CommandBus/EventBus/WorkerApp.work() in worker processes "
|
|
243
272
|
"(fork on POSIX, spawn on Windows). "
|
|
244
|
-
"Target is module:attribute like uvicorn — e.g. myapp.worker:
|
|
245
|
-
"myapp.worker:
|
|
273
|
+
"Target is module:attribute like uvicorn — e.g. myapp.worker:app, "
|
|
274
|
+
"myapp.worker:bus, or myapp.worker:command_bus_group. "
|
|
275
|
+
"If :attribute is omitted, :bus is assumed."
|
|
246
276
|
)
|
|
247
277
|
)
|
|
248
278
|
parser.add_argument(
|
|
@@ -250,7 +280,7 @@ def main(argv: Optional[Sequence[str]] = None) -> None:
|
|
|
250
280
|
metavar="TARGET",
|
|
251
281
|
help=(
|
|
252
282
|
"Import path and object: dotted.module:attribute. "
|
|
253
|
-
"Attribute must be a CommandBus, EventBus, or BusGroup. "
|
|
283
|
+
"Attribute must be a CommandBus, EventBus, WorkerApp, or BusGroup. "
|
|
254
284
|
"Omit :attribute to use attribute name 'bus' (e.g. myapp.worker is myapp.worker:bus)."
|
|
255
285
|
),
|
|
256
286
|
)
|
|
@@ -260,10 +290,20 @@ def main(argv: Optional[Sequence[str]] = None) -> None:
|
|
|
260
290
|
default=1,
|
|
261
291
|
metavar="N",
|
|
262
292
|
help=(
|
|
263
|
-
"Process count for a single CommandBus or
|
|
293
|
+
"Process count for a single CommandBus, EventBus, or WorkerApp. "
|
|
264
294
|
"For BusGroup, default count when WorkerConfig.workers is omitted (default: 1)"
|
|
265
295
|
),
|
|
266
296
|
)
|
|
297
|
+
parser.add_argument(
|
|
298
|
+
"--concurrency",
|
|
299
|
+
type=int,
|
|
300
|
+
default=None,
|
|
301
|
+
metavar="N",
|
|
302
|
+
help=(
|
|
303
|
+
"In-process parallel message dispatch per worker process. "
|
|
304
|
+
"For WorkerApp, defaults to the app's concurrency when omitted (default: 1 otherwise)"
|
|
305
|
+
),
|
|
306
|
+
)
|
|
267
307
|
parser.add_argument(
|
|
268
308
|
"--poll-interval",
|
|
269
309
|
type=float,
|
|
@@ -292,6 +332,8 @@ def main(argv: Optional[Sequence[str]] = None) -> None:
|
|
|
292
332
|
|
|
293
333
|
if args.workers < 1:
|
|
294
334
|
parser.error("--workers must be >= 1")
|
|
335
|
+
if args.concurrency is not None and args.concurrency < 1:
|
|
336
|
+
parser.error("--concurrency must be >= 1")
|
|
295
337
|
|
|
296
338
|
module_name, attr_name = _parse_target(args.target)
|
|
297
339
|
|
|
@@ -304,6 +346,20 @@ def main(argv: Optional[Sequence[str]] = None) -> None:
|
|
|
304
346
|
if obj is None:
|
|
305
347
|
raise SystemExit(f"Module {module_name!r} has no attribute {attr_name!r}")
|
|
306
348
|
|
|
349
|
+
if isinstance(obj, WorkerApp):
|
|
350
|
+
concurrency = args.concurrency if args.concurrency is not None else obj.concurrency
|
|
351
|
+
if isinstance(obj.queue_adapter, InMemoryQueueAdapter) and args.workers > 1:
|
|
352
|
+
logger.warning(
|
|
353
|
+
"InMemoryQueueAdapter is process-local; each of %d worker processes "
|
|
354
|
+
"will have its own queue. Use a shared backend (SQS, Redis, RabbitMQ) "
|
|
355
|
+
"for multi-process workers, or run with --workers 1.",
|
|
356
|
+
args.workers,
|
|
357
|
+
)
|
|
358
|
+
elif args.concurrency is not None:
|
|
359
|
+
concurrency = args.concurrency
|
|
360
|
+
else:
|
|
361
|
+
concurrency = 1
|
|
362
|
+
|
|
307
363
|
if isinstance(obj, (BusGroup, CommandBusGroup)):
|
|
308
364
|
obj.validate(mod)
|
|
309
365
|
jobs = obj.iter_jobs(mod, args.workers)
|
|
@@ -318,19 +374,20 @@ def main(argv: Optional[Sequence[str]] = None) -> None:
|
|
|
318
374
|
attr,
|
|
319
375
|
qn,
|
|
320
376
|
)
|
|
321
|
-
elif isinstance(obj,
|
|
377
|
+
elif isinstance(obj, _WORKER_TARGET_TYPES):
|
|
322
378
|
jobs = _worker_jobs(attr_name, args.workers)
|
|
323
|
-
qn =
|
|
379
|
+
qn = _queue_name_for_target(obj, attr_name)
|
|
324
380
|
logger.info(
|
|
325
|
-
"Launching %d process(es) for %s:%s (queue_name=%s)",
|
|
381
|
+
"Launching %d process(es) for %s:%s (queue_name=%s, concurrency=%d)",
|
|
326
382
|
args.workers,
|
|
327
383
|
module_name,
|
|
328
384
|
attr_name,
|
|
329
385
|
qn,
|
|
386
|
+
concurrency,
|
|
330
387
|
)
|
|
331
388
|
else:
|
|
332
389
|
raise SystemExit(
|
|
333
|
-
f"Attribute {attr_name!r} must be a CommandBus, EventBus, or BusGroup "
|
|
390
|
+
f"Attribute {attr_name!r} must be a CommandBus, EventBus, WorkerApp, or BusGroup "
|
|
334
391
|
f"(got {type(obj).__name__})"
|
|
335
392
|
)
|
|
336
393
|
|
|
@@ -340,6 +397,7 @@ def main(argv: Optional[Sequence[str]] = None) -> None:
|
|
|
340
397
|
jobs,
|
|
341
398
|
max(0.0, args.poll_interval),
|
|
342
399
|
log_level,
|
|
400
|
+
concurrency,
|
|
343
401
|
)
|
|
344
402
|
except KeyboardInterrupt:
|
|
345
403
|
sys.exit(130)
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"""Pub/sub event bus: publish fans out to all subscribers."""
|
|
2
2
|
|
|
3
|
+
import asyncio
|
|
3
4
|
import logging
|
|
4
|
-
from typing import Optional, Type
|
|
5
|
+
from typing import Any, Optional, Type
|
|
5
6
|
|
|
6
7
|
from .interfaces import (
|
|
7
8
|
EventBusInterface,
|
|
@@ -66,9 +67,17 @@ class EventBus(EventBusInterface):
|
|
|
66
67
|
entry.handler_class,
|
|
67
68
|
)
|
|
68
69
|
|
|
69
|
-
async def work(self) ->
|
|
70
|
-
"""Poll the subscription and dispatch
|
|
71
|
-
messages = self.queue_adapter.get_messages()
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
70
|
+
async def work(self, *, concurrency: int = 1) -> int:
|
|
71
|
+
"""Poll the subscription and dispatch messages (up to ``concurrency`` in parallel)."""
|
|
72
|
+
messages = self.queue_adapter.get_messages(max_messages=max(1, concurrency))
|
|
73
|
+
if not messages:
|
|
74
|
+
return 0
|
|
75
|
+
|
|
76
|
+
async def _handle(message: Any) -> None:
|
|
77
|
+
try:
|
|
78
|
+
await self.dispatch(message.body)
|
|
79
|
+
finally:
|
|
80
|
+
self.queue_adapter.dequeue(message)
|
|
81
|
+
|
|
82
|
+
await asyncio.gather(*(_handle(m) for m in messages))
|
|
83
|
+
return len(messages)
|
|
@@ -128,7 +128,7 @@ class CommandBusInterface(ABC):
|
|
|
128
128
|
pass
|
|
129
129
|
|
|
130
130
|
@abstractmethod
|
|
131
|
-
async def work(self) ->
|
|
131
|
+
async def work(self, *, concurrency: int = 1) -> int:
|
|
132
132
|
pass
|
|
133
133
|
|
|
134
134
|
|
|
@@ -151,5 +151,5 @@ class EventBusInterface(ABC):
|
|
|
151
151
|
pass
|
|
152
152
|
|
|
153
153
|
@abstractmethod
|
|
154
|
-
async def work(self) ->
|
|
154
|
+
async def work(self, *, concurrency: int = 1) -> int:
|
|
155
155
|
pass
|