queue-func 0.1.2__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- queue_func-0.2.2/PKG-INFO +262 -0
- queue_func-0.2.2/README.md +235 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/pyproject.toml +6 -1
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/broker/base.py +4 -24
- queue_func-0.2.2/queue_func/broker/queue.py +34 -0
- queue_func-0.2.2/queue_func/broker/rabbit_aio_pika.py +47 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/config/general.py +6 -6
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/config/queue.py +1 -1
- queue_func-0.2.2/queue_func/config/rabbit_aio_pika.py +55 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/consumer/base.py +38 -22
- queue_func-0.2.2/queue_func/consumer/queue.py +32 -0
- queue_func-0.2.2/queue_func/consumer/rabbit_aio_pika.py +99 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/factory/backend.py +1 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/factory/builders/__init__.py +2 -0
- queue_func-0.2.2/queue_func/factory/builders/rabbit_aio_pika.py +26 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/factory/deco.py +30 -0
- queue_func-0.2.2/queue_func/producer/base.py +69 -0
- queue_func-0.2.2/queue_func/producer/queue.py +21 -0
- queue_func-0.2.2/queue_func/producer/rabbit_aio_pika.py +83 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/serializer/base.py +1 -1
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/serializer/orjson_ser.py +2 -2
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/serializer/raw.py +2 -2
- queue_func-0.2.2/queue_func/shared_resource/base.py +25 -0
- queue_func-0.2.2/queue_func/shared_resource/queue.py +23 -0
- queue_func-0.2.2/queue_func/shared_resource/rabbit_aio_pika.py +111 -0
- queue_func-0.2.2/queue_func/task/__init__.py +1 -0
- queue_func-0.2.2/queue_func/utils/__init__.py +1 -0
- queue_func-0.2.2/queue_func/utils/deco.py +36 -0
- queue_func-0.2.2/queue_func/utils/hook.py +50 -0
- queue_func-0.2.2/queue_func/utils/waiter.py +35 -0
- queue_func-0.1.2/PKG-INFO +0 -183
- queue_func-0.1.2/README.md +0 -158
- queue_func-0.1.2/queue_func/broker/queue.py +0 -44
- queue_func-0.1.2/queue_func/consumer/queue.py +0 -55
- queue_func-0.1.2/queue_func/producer/base.py +0 -44
- queue_func-0.1.2/queue_func/producer/queue.py +0 -32
- {queue_func-0.1.2 → queue_func-0.2.2}/LICENSE +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/__init__.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/broker/__init__.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/config/__init__.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/config/utils.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/consumer/__init__.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/errors.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/factory/__init__.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/factory/builders/queue.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/factory/registry.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/factory/runtime.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/pool_executor/__init__.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/pool_executor/base.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/pool_executor/general.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/pool_executor/utils.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/producer/__init__.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/serializer/__init__.py +0 -0
- {queue_func-0.1.2/queue_func/task → queue_func-0.2.2/queue_func/shared_resource}/__init__.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/task/base.py +0 -0
- {queue_func-0.1.2 → queue_func-0.2.2}/queue_func/task/thread.py +0 -0
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: queue-func
|
|
3
|
+
Version: 0.2.2
|
|
4
|
+
Summary: Async coroutine functions as queue-backed tasks (QueueFunc).
|
|
5
|
+
License: Apache-2.0
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Author: liuseki
|
|
8
|
+
Author-email: liuseki@163.com
|
|
9
|
+
Requires-Python: >=3.12,<4.0
|
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
16
|
+
Provides-Extra: rabbit
|
|
17
|
+
Requires-Dist: aio-pika (>=9.6,<10.0) ; extra == "rabbit"
|
|
18
|
+
Requires-Dist: orjson
|
|
19
|
+
Requires-Dist: pydantic (>=2.14.1,<3.0.0)
|
|
20
|
+
Project-URL: Changelog, https://github.com/sekil9529/queue-func/blob/master/Changelog.rst
|
|
21
|
+
Project-URL: Documentation, https://github.com/sekil9529/queue-func#readme
|
|
22
|
+
Project-URL: Homepage, https://github.com/sekil9529/queue-func
|
|
23
|
+
Project-URL: Issues, https://github.com/sekil9529/queue-func/issues
|
|
24
|
+
Project-URL: Repository, https://github.com/sekil9529/queue-func
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# queue-func
|
|
28
|
+
|
|
29
|
+
[](https://coveralls.io/github/sekil9529/queue-func)
|
|
30
|
+
[](https://github.com/sekil9529/queue-func/actions?query=workflow%3Atests)
|
|
31
|
+
[](https://pypi.org/project/queue-func/)
|
|
32
|
+
[](https://pypi.org/project/queue-func/)
|
|
33
|
+
[](https://pypi.org/project/queue-func/)
|
|
34
|
+
[](https://pypi.org/project/queue-func/)
|
|
35
|
+
|
|
36
|
+
把 **`async def` 协程** 变成可 **`push`** 的队列任务:在 asyncio 里投递、消费,并按通道控制并发与背压。
|
|
37
|
+
|
|
38
|
+
仓库:[github.com/sekil9529/queue-func](https://github.com/sekil9529/queue-func)。版本记录见 [Changelog.rst](Changelog.rst)。
|
|
39
|
+
|
|
40
|
+
> **需要 Python 3.12 及以上。** 任务入口必须是协程函数,普通函数不能作为任务。
|
|
41
|
+
>
|
|
42
|
+
> RabbitMQ 后端需要另装可选依赖,底层使用 [aio-pika](https://github.com/mosquito/aio-pika) `^9.6`。
|
|
43
|
+
|
|
44
|
+
## 特点
|
|
45
|
+
|
|
46
|
+
- 完全在 asyncio 里运行。消费者在同一事件循环里按上限并发执行任务,可以和 FastAPI / Starlette 放在一个进程里。
|
|
47
|
+
- 一条链路是序列化、投递、反序列化、执行。启停由通道或 `QueueFuncRuntime` 显式管理。
|
|
48
|
+
- 每个 `QueueFunc` 有自己的队列和消费并发上限。`QueueFuncRuntime` 可以同时跑多段流水线,各段互不影响。
|
|
49
|
+
- 序列化器的 `pack` / `unpack` 决定队列里的数据结构。
|
|
50
|
+
- 通过 `register_backend` 接入新的队列后端。
|
|
51
|
+
|
|
52
|
+
## 当前后端
|
|
53
|
+
|
|
54
|
+
| 后端 | 说明 |
|
|
55
|
+
|------|------|
|
|
56
|
+
| `Backend.QUEUE` | 进程内 `asyncio.Queue` |
|
|
57
|
+
| `Backend.RABBIT_AIO_PIKA` | RabbitMQ,安装可选依赖 `rabbit` |
|
|
58
|
+
|
|
59
|
+
内存队列的消息留在进程里,序列化默认是 `raw`(队列中仍是 `(args, kwargs)`)。RabbitMQ 默认用 orjson,队列中是 `bytes`。
|
|
60
|
+
|
|
61
|
+
## 安装
|
|
62
|
+
|
|
63
|
+
```shell
|
|
64
|
+
pip install queue-func
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
使用 RabbitMQ:
|
|
68
|
+
|
|
69
|
+
```shell
|
|
70
|
+
pip install "queue-func[rabbit]"
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
从仓库安装(开发):
|
|
74
|
+
|
|
75
|
+
```shell
|
|
76
|
+
git clone https://github.com/sekil9529/queue-func.git
|
|
77
|
+
cd queue-func
|
|
78
|
+
poetry install -E rabbit
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## 用法
|
|
82
|
+
|
|
83
|
+
### 进程内队列
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
import asyncio
|
|
87
|
+
|
|
88
|
+
from queue_func.factory import Backend, queue
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@queue(Backend.QUEUE, consume_timeout=0.5, maxsize=1000)
|
|
92
|
+
async def process_order(order_id: int) -> None:
|
|
93
|
+
print(order_id)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def main() -> None:
|
|
97
|
+
await process_order.broker.start_consumer()
|
|
98
|
+
await process_order.push(args=(42,))
|
|
99
|
+
await process_order.broker.stop_consumer()
|
|
100
|
+
await process_order.broker.join_consumer()
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
asyncio.run(main())
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`push` 就是 `broker.publish`。不传 `timeout` 时用配置里的 `publish_timeout`;传入数字只作用于这一次。`None` 表示不限时。超时抛出 `queue_func.PublishTimeoutError`。
|
|
107
|
+
|
|
108
|
+
装饰之后仍可直接调用,不经过队列:
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
await process_order(1)
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### 多段流水线
|
|
115
|
+
|
|
116
|
+
`QueueFuncRuntime.start()` 按列表反序启动消费者,下游先就绪。`stop()` 先逐个停止消费者,再停止全部生产者。
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
import asyncio
|
|
120
|
+
|
|
121
|
+
from queue_func.factory import Backend, QueueFuncRuntime, queue
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@queue(Backend.QUEUE, consumer_pool_maxsize=4)
|
|
125
|
+
async def task_b(job: str) -> None:
|
|
126
|
+
print(job)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@queue(Backend.QUEUE, consumer_pool_maxsize=2)
|
|
130
|
+
async def task_a(job: str) -> None:
|
|
131
|
+
await task_b.push(args=(job,))
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
async def main() -> None:
|
|
135
|
+
runtime = QueueFuncRuntime([task_a, task_b])
|
|
136
|
+
await runtime.start()
|
|
137
|
+
await task_a.push(args=("job-1",))
|
|
138
|
+
await runtime.stop()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
asyncio.run(main())
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
三段链式示例(含日志):
|
|
145
|
+
|
|
146
|
+
```shell
|
|
147
|
+
poetry run python examples/demo_queue.py
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### RabbitMQ
|
|
151
|
+
|
|
152
|
+
`url` 和 `queue_name` 必填。`enable_declare_queue=True` 时,队列不存在会创建。默认交换机名是 `""`,路由键默认用队列名。`push` 会等待发布确认;`publish_timeout` 是这次确认的超时。
|
|
153
|
+
|
|
154
|
+
消息交给消费池之后就确认。处理函数出错不会把消息退回队列。`stop` 会设上停止标志,当前这次拉取结束(空闲时最多再等一个 `consume_timeout`)后退出;还没取走的消息留在 RabbitMQ 上。
|
|
155
|
+
|
|
156
|
+
同一个 `QueueFunc` 的生产者和消费者可以分进程运行。只投递的进程把 `enable_consumer` 设为 `False`。
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
import asyncio
|
|
160
|
+
|
|
161
|
+
from queue_func.factory import Backend, queue
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@queue(
|
|
165
|
+
Backend.RABBIT_AIO_PIKA,
|
|
166
|
+
url="amqp://guest:guest@127.0.0.1/",
|
|
167
|
+
queue_name="process_order",
|
|
168
|
+
enable_declare_queue=True,
|
|
169
|
+
)
|
|
170
|
+
async def process_order(order_id: int) -> None:
|
|
171
|
+
print(order_id)
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
async def main() -> None:
|
|
175
|
+
await process_order.broker.start_consumer()
|
|
176
|
+
await process_order.push(args=(42,))
|
|
177
|
+
await process_order.broker.stop_consumer()
|
|
178
|
+
await process_order.broker.join_consumer()
|
|
179
|
+
await process_order.broker.stop_producer()
|
|
180
|
+
await process_order.broker.join_producer()
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
asyncio.run(main())
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
本地 RabbitMQ(`127.0.0.1:5672`,`guest` / `guest`)上的链式示例:
|
|
187
|
+
|
|
188
|
+
```shell
|
|
189
|
+
poetry run python examples/demo_rabbit_aio_pika.py
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
常用参数:
|
|
193
|
+
|
|
194
|
+
| 参数 | 作用 |
|
|
195
|
+
|------|------|
|
|
196
|
+
| `consumer_pool_maxsize` | 同时执行的协程数,默认 200 |
|
|
197
|
+
| `consume_timeout` | 一次拉取的等待时间,默认 2 秒 |
|
|
198
|
+
| `publish_timeout` | 投递等待上限,默认不限 |
|
|
199
|
+
| `maxsize` | 仅内存队列,`asyncio.Queue` 容量,默认 200 |
|
|
200
|
+
| `serializer_cls` | 队列中的数据结构 |
|
|
201
|
+
| `prefetch_count` | 仅 RabbitMQ,未确认消息窗口 |
|
|
202
|
+
| `delivery_mode` | 仅 RabbitMQ,默认持久化 |
|
|
203
|
+
| `exchange_name` / `routing_key` | 仅 RabbitMQ |
|
|
204
|
+
|
|
205
|
+
其余字段见 `queue_func.config.queue.Config` 和 `queue_func.config.rabbit_aio_pika.Config`。
|
|
206
|
+
|
|
207
|
+
## 架构
|
|
208
|
+
|
|
209
|
+
一个 `@queue` 装饰的协程就是一条队列。`push` 投到它自己这条队列上。
|
|
210
|
+
|
|
211
|
+
- **Broker** 组装生产者、消费者和双方共用的资源,并提供启停与投递。
|
|
212
|
+
- **Producer** 序列化后把数据写入队列。
|
|
213
|
+
- **Consumer** 取出数据,反序列化后交给消费池。
|
|
214
|
+
- **PoolExecutor** 按并发上限执行业务协程。
|
|
215
|
+
|
|
216
|
+
```mermaid
|
|
217
|
+
flowchart TB
|
|
218
|
+
PUSH["投递"] --> B["通道"]
|
|
219
|
+
B --> PRO["生产者"]
|
|
220
|
+
B -.->|启停| CON["消费者"]
|
|
221
|
+
|
|
222
|
+
PRO -->|序列化| Q[("队列")]
|
|
223
|
+
Q -->|反序列化| CON
|
|
224
|
+
CON --> POOL["消费池"]
|
|
225
|
+
POOL --> FN["业务协程"]
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
## 项目结构
|
|
229
|
+
|
|
230
|
+
```
|
|
231
|
+
queue_func/
|
|
232
|
+
├── broker/
|
|
233
|
+
├── producer/
|
|
234
|
+
├── consumer/
|
|
235
|
+
├── serializer/ # raw、orjson
|
|
236
|
+
├── shared_resource/
|
|
237
|
+
├── config/
|
|
238
|
+
├── factory/ # @queue、QueueFunc、QueueFuncRuntime
|
|
239
|
+
├── pool_executor/
|
|
240
|
+
└── utils/
|
|
241
|
+
examples/
|
|
242
|
+
├── demo_queue.py
|
|
243
|
+
└── demo_rabbit_aio_pika.py
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
## 开发与测试
|
|
247
|
+
|
|
248
|
+
```shell
|
|
249
|
+
poetry install -E rabbit
|
|
250
|
+
pytest
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
`pytest.ini` 会生成覆盖率页面(`htmlcov/`)和 `report.html`。
|
|
254
|
+
|
|
255
|
+
## 版本
|
|
256
|
+
|
|
257
|
+
遵循[语义化版本](https://semver.org/lang/zh-CN/)。变更记录在 [Changelog.rst](Changelog.rst)。
|
|
258
|
+
|
|
259
|
+
## 许可证
|
|
260
|
+
|
|
261
|
+
Apache-2.0
|
|
262
|
+
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
# queue-func
|
|
2
|
+
|
|
3
|
+
[](https://coveralls.io/github/sekil9529/queue-func)
|
|
4
|
+
[](https://github.com/sekil9529/queue-func/actions?query=workflow%3Atests)
|
|
5
|
+
[](https://pypi.org/project/queue-func/)
|
|
6
|
+
[](https://pypi.org/project/queue-func/)
|
|
7
|
+
[](https://pypi.org/project/queue-func/)
|
|
8
|
+
[](https://pypi.org/project/queue-func/)
|
|
9
|
+
|
|
10
|
+
把 **`async def` 协程** 变成可 **`push`** 的队列任务:在 asyncio 里投递、消费,并按通道控制并发与背压。
|
|
11
|
+
|
|
12
|
+
仓库:[github.com/sekil9529/queue-func](https://github.com/sekil9529/queue-func)。版本记录见 [Changelog.rst](Changelog.rst)。
|
|
13
|
+
|
|
14
|
+
> **需要 Python 3.12 及以上。** 任务入口必须是协程函数,普通函数不能作为任务。
|
|
15
|
+
>
|
|
16
|
+
> RabbitMQ 后端需要另装可选依赖,底层使用 [aio-pika](https://github.com/mosquito/aio-pika) `^9.6`。
|
|
17
|
+
|
|
18
|
+
## 特点
|
|
19
|
+
|
|
20
|
+
- 完全在 asyncio 里运行。消费者在同一事件循环里按上限并发执行任务,可以和 FastAPI / Starlette 放在一个进程里。
|
|
21
|
+
- 一条链路是序列化、投递、反序列化、执行。启停由通道或 `QueueFuncRuntime` 显式管理。
|
|
22
|
+
- 每个 `QueueFunc` 有自己的队列和消费并发上限。`QueueFuncRuntime` 可以同时跑多段流水线,各段互不影响。
|
|
23
|
+
- 序列化器的 `pack` / `unpack` 决定队列里的数据结构。
|
|
24
|
+
- 通过 `register_backend` 接入新的队列后端。
|
|
25
|
+
|
|
26
|
+
## 当前后端
|
|
27
|
+
|
|
28
|
+
| 后端 | 说明 |
|
|
29
|
+
|------|------|
|
|
30
|
+
| `Backend.QUEUE` | 进程内 `asyncio.Queue` |
|
|
31
|
+
| `Backend.RABBIT_AIO_PIKA` | RabbitMQ,安装可选依赖 `rabbit` |
|
|
32
|
+
|
|
33
|
+
内存队列的消息留在进程里,序列化默认是 `raw`(队列中仍是 `(args, kwargs)`)。RabbitMQ 默认用 orjson,队列中是 `bytes`。
|
|
34
|
+
|
|
35
|
+
## 安装
|
|
36
|
+
|
|
37
|
+
```shell
|
|
38
|
+
pip install queue-func
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
使用 RabbitMQ:
|
|
42
|
+
|
|
43
|
+
```shell
|
|
44
|
+
pip install "queue-func[rabbit]"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
从仓库安装(开发):
|
|
48
|
+
|
|
49
|
+
```shell
|
|
50
|
+
git clone https://github.com/sekil9529/queue-func.git
|
|
51
|
+
cd queue-func
|
|
52
|
+
poetry install -E rabbit
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## 用法
|
|
56
|
+
|
|
57
|
+
### 进程内队列
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
import asyncio
|
|
61
|
+
|
|
62
|
+
from queue_func.factory import Backend, queue
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@queue(Backend.QUEUE, consume_timeout=0.5, maxsize=1000)
|
|
66
|
+
async def process_order(order_id: int) -> None:
|
|
67
|
+
print(order_id)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
async def main() -> None:
|
|
71
|
+
await process_order.broker.start_consumer()
|
|
72
|
+
await process_order.push(args=(42,))
|
|
73
|
+
await process_order.broker.stop_consumer()
|
|
74
|
+
await process_order.broker.join_consumer()
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
asyncio.run(main())
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`push` 就是 `broker.publish`。不传 `timeout` 时用配置里的 `publish_timeout`;传入数字只作用于这一次。`None` 表示不限时。超时抛出 `queue_func.PublishTimeoutError`。
|
|
81
|
+
|
|
82
|
+
装饰之后仍可直接调用,不经过队列:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
await process_order(1)
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### 多段流水线
|
|
89
|
+
|
|
90
|
+
`QueueFuncRuntime.start()` 按列表反序启动消费者,下游先就绪。`stop()` 先逐个停止消费者,再停止全部生产者。
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
import asyncio
|
|
94
|
+
|
|
95
|
+
from queue_func.factory import Backend, QueueFuncRuntime, queue
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@queue(Backend.QUEUE, consumer_pool_maxsize=4)
|
|
99
|
+
async def task_b(job: str) -> None:
|
|
100
|
+
print(job)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@queue(Backend.QUEUE, consumer_pool_maxsize=2)
|
|
104
|
+
async def task_a(job: str) -> None:
|
|
105
|
+
await task_b.push(args=(job,))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
async def main() -> None:
|
|
109
|
+
runtime = QueueFuncRuntime([task_a, task_b])
|
|
110
|
+
await runtime.start()
|
|
111
|
+
await task_a.push(args=("job-1",))
|
|
112
|
+
await runtime.stop()
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
asyncio.run(main())
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
三段链式示例(含日志):
|
|
119
|
+
|
|
120
|
+
```shell
|
|
121
|
+
poetry run python examples/demo_queue.py
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### RabbitMQ
|
|
125
|
+
|
|
126
|
+
`url` 和 `queue_name` 必填。`enable_declare_queue=True` 时,队列不存在会创建。默认交换机名是 `""`,路由键默认用队列名。`push` 会等待发布确认;`publish_timeout` 是这次确认的超时。
|
|
127
|
+
|
|
128
|
+
消息交给消费池之后就确认。处理函数出错不会把消息退回队列。`stop` 会设上停止标志,当前这次拉取结束(空闲时最多再等一个 `consume_timeout`)后退出;还没取走的消息留在 RabbitMQ 上。
|
|
129
|
+
|
|
130
|
+
同一个 `QueueFunc` 的生产者和消费者可以分进程运行。只投递的进程把 `enable_consumer` 设为 `False`。
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
import asyncio
|
|
134
|
+
|
|
135
|
+
from queue_func.factory import Backend, queue
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@queue(
|
|
139
|
+
Backend.RABBIT_AIO_PIKA,
|
|
140
|
+
url="amqp://guest:guest@127.0.0.1/",
|
|
141
|
+
queue_name="process_order",
|
|
142
|
+
enable_declare_queue=True,
|
|
143
|
+
)
|
|
144
|
+
async def process_order(order_id: int) -> None:
|
|
145
|
+
print(order_id)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
async def main() -> None:
|
|
149
|
+
await process_order.broker.start_consumer()
|
|
150
|
+
await process_order.push(args=(42,))
|
|
151
|
+
await process_order.broker.stop_consumer()
|
|
152
|
+
await process_order.broker.join_consumer()
|
|
153
|
+
await process_order.broker.stop_producer()
|
|
154
|
+
await process_order.broker.join_producer()
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
asyncio.run(main())
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
本地 RabbitMQ(`127.0.0.1:5672`,`guest` / `guest`)上的链式示例:
|
|
161
|
+
|
|
162
|
+
```shell
|
|
163
|
+
poetry run python examples/demo_rabbit_aio_pika.py
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
常用参数:
|
|
167
|
+
|
|
168
|
+
| 参数 | 作用 |
|
|
169
|
+
|------|------|
|
|
170
|
+
| `consumer_pool_maxsize` | 同时执行的协程数,默认 200 |
|
|
171
|
+
| `consume_timeout` | 一次拉取的等待时间,默认 2 秒 |
|
|
172
|
+
| `publish_timeout` | 投递等待上限,默认不限 |
|
|
173
|
+
| `maxsize` | 仅内存队列,`asyncio.Queue` 容量,默认 200 |
|
|
174
|
+
| `serializer_cls` | 队列中的数据结构 |
|
|
175
|
+
| `prefetch_count` | 仅 RabbitMQ,未确认消息窗口 |
|
|
176
|
+
| `delivery_mode` | 仅 RabbitMQ,默认持久化 |
|
|
177
|
+
| `exchange_name` / `routing_key` | 仅 RabbitMQ |
|
|
178
|
+
|
|
179
|
+
其余字段见 `queue_func.config.queue.Config` 和 `queue_func.config.rabbit_aio_pika.Config`。
|
|
180
|
+
|
|
181
|
+
## 架构
|
|
182
|
+
|
|
183
|
+
一个 `@queue` 装饰的协程就是一条队列。`push` 投到它自己这条队列上。
|
|
184
|
+
|
|
185
|
+
- **Broker** 组装生产者、消费者和双方共用的资源,并提供启停与投递。
|
|
186
|
+
- **Producer** 序列化后把数据写入队列。
|
|
187
|
+
- **Consumer** 取出数据,反序列化后交给消费池。
|
|
188
|
+
- **PoolExecutor** 按并发上限执行业务协程。
|
|
189
|
+
|
|
190
|
+
```mermaid
|
|
191
|
+
flowchart TB
|
|
192
|
+
PUSH["投递"] --> B["通道"]
|
|
193
|
+
B --> PRO["生产者"]
|
|
194
|
+
B -.->|启停| CON["消费者"]
|
|
195
|
+
|
|
196
|
+
PRO -->|序列化| Q[("队列")]
|
|
197
|
+
Q -->|反序列化| CON
|
|
198
|
+
CON --> POOL["消费池"]
|
|
199
|
+
POOL --> FN["业务协程"]
|
|
200
|
+
```
|
|
201
|
+
|
|
202
|
+
## 项目结构
|
|
203
|
+
|
|
204
|
+
```
|
|
205
|
+
queue_func/
|
|
206
|
+
├── broker/
|
|
207
|
+
├── producer/
|
|
208
|
+
├── consumer/
|
|
209
|
+
├── serializer/ # raw、orjson
|
|
210
|
+
├── shared_resource/
|
|
211
|
+
├── config/
|
|
212
|
+
├── factory/ # @queue、QueueFunc、QueueFuncRuntime
|
|
213
|
+
├── pool_executor/
|
|
214
|
+
└── utils/
|
|
215
|
+
examples/
|
|
216
|
+
├── demo_queue.py
|
|
217
|
+
└── demo_rabbit_aio_pika.py
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
## 开发与测试
|
|
221
|
+
|
|
222
|
+
```shell
|
|
223
|
+
poetry install -E rabbit
|
|
224
|
+
pytest
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
`pytest.ini` 会生成覆盖率页面(`htmlcov/`)和 `report.html`。
|
|
228
|
+
|
|
229
|
+
## 版本
|
|
230
|
+
|
|
231
|
+
遵循[语义化版本](https://semver.org/lang/zh-CN/)。变更记录在 [Changelog.rst](Changelog.rst)。
|
|
232
|
+
|
|
233
|
+
## 许可证
|
|
234
|
+
|
|
235
|
+
Apache-2.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "queue-func"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.2"
|
|
4
4
|
description = "Async coroutine functions as queue-backed tasks (QueueFunc)."
|
|
5
5
|
authors = ["liuseki <liuseki@163.com>"]
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -18,6 +18,10 @@ Issues = "https://github.com/sekil9529/queue-func/issues"
|
|
|
18
18
|
python = "^3.12"
|
|
19
19
|
orjson = "*"
|
|
20
20
|
pydantic = "^2.14.1"
|
|
21
|
+
aio-pika = { version = "^9.6", optional = true }
|
|
22
|
+
|
|
23
|
+
[tool.poetry.extras]
|
|
24
|
+
rabbit = ["aio-pika"]
|
|
21
25
|
|
|
22
26
|
[[tool.poetry.source]]
|
|
23
27
|
name = "aliyun"
|
|
@@ -28,6 +32,7 @@ priority = "primary"
|
|
|
28
32
|
pytest = "*"
|
|
29
33
|
pytest-cov = "*"
|
|
30
34
|
pytest-html = "*"
|
|
35
|
+
aio-pika = "^9.6"
|
|
31
36
|
|
|
32
37
|
[tool.pytest.ini_options]
|
|
33
38
|
testpaths = ["tests"]
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
import abc
|
|
4
4
|
import typing as t
|
|
5
5
|
|
|
6
|
-
from queue_func.serializer.base import
|
|
6
|
+
from queue_func.serializer.base import Message
|
|
7
|
+
from queue_func.shared_resource.base import Resource
|
|
7
8
|
|
|
8
9
|
if t.TYPE_CHECKING:
|
|
9
10
|
from queue_func.config.general import Config
|
|
@@ -22,33 +23,12 @@ class BaseBroker(t.Generic[ConfigType, Message], metaclass=abc.ABCMeta):
|
|
|
22
23
|
"""
|
|
23
24
|
|
|
24
25
|
__slots__ = (
|
|
25
|
-
"_config",
|
|
26
|
-
"_serializer",
|
|
27
26
|
"_producer",
|
|
28
27
|
"_consumer",
|
|
29
28
|
)
|
|
30
29
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
self._config = config
|
|
34
|
-
self._serializer: type[BaseSerializer[Message]] = self._config.serializer_cls
|
|
35
|
-
self._producer = self._create_producer()
|
|
36
|
-
self._consumer = self._create_consumer()
|
|
37
|
-
|
|
38
|
-
@abc.abstractmethod
|
|
39
|
-
def _create_producer(self) -> "BaseProducer[Message]":
|
|
40
|
-
|
|
41
|
-
pass
|
|
42
|
-
|
|
43
|
-
@abc.abstractmethod
|
|
44
|
-
def _create_consumer(self) -> "BaseConsumer[Message]":
|
|
45
|
-
|
|
46
|
-
pass
|
|
47
|
-
|
|
48
|
-
@property
|
|
49
|
-
def serializer(self) -> type[BaseSerializer[Message]]:
|
|
50
|
-
|
|
51
|
-
return self._serializer
|
|
30
|
+
_producer: "BaseProducer"
|
|
31
|
+
_consumer: "BaseConsumer"
|
|
52
32
|
|
|
53
33
|
async def publish(
|
|
54
34
|
self,
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import typing as t
|
|
5
|
+
|
|
6
|
+
from queue_func.serializer.base import Message
|
|
7
|
+
from queue_func.shared_resource.queue import Resource
|
|
8
|
+
from queue_func.broker.base import BaseBroker as _BaseBroker
|
|
9
|
+
from queue_func.producer.queue import Producer
|
|
10
|
+
from queue_func.consumer.queue import Consumer
|
|
11
|
+
|
|
12
|
+
if t.TYPE_CHECKING:
|
|
13
|
+
from queue_func.config.queue import Config
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Broker(_BaseBroker["Config", Message]):
|
|
17
|
+
"""内存队列通道。队列由这里创建,再交给 Producer 和 Consumer。"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, config: "Config"):
|
|
20
|
+
resource = Resource(
|
|
21
|
+
serializer_cls=config.serializer_cls,
|
|
22
|
+
maxsize=config.maxsize)
|
|
23
|
+
self._producer = Producer(
|
|
24
|
+
resource=resource,
|
|
25
|
+
publish_timeout=config.publish_timeout,
|
|
26
|
+
)
|
|
27
|
+
self._consumer = Consumer(
|
|
28
|
+
resource=resource,
|
|
29
|
+
coro=config.coro,
|
|
30
|
+
enable_consumer=config.enable_consumer,
|
|
31
|
+
consume_timeout=config.consume_timeout,
|
|
32
|
+
pool_executor_cls=config.consumer_pool_executor_cls,
|
|
33
|
+
pool_maxsize=config.consumer_pool_maxsize,
|
|
34
|
+
)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
|
|
3
|
+
import typing as t
|
|
4
|
+
|
|
5
|
+
from queue_func.broker.base import BaseBroker as _BaseBroker
|
|
6
|
+
from queue_func.consumer.rabbit_aio_pika import Consumer
|
|
7
|
+
from queue_func.producer.rabbit_aio_pika import Producer
|
|
8
|
+
from queue_func.shared_resource.rabbit_aio_pika import Resource
|
|
9
|
+
|
|
10
|
+
if t.TYPE_CHECKING:
|
|
11
|
+
from queue_func.config.rabbit_aio_pika import Config
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Broker(_BaseBroker["Config", bytes]):
|
|
15
|
+
"""RabbitMQ 通道。一条连接上分开发布和消费,再交给 Producer 和 Consumer。"""
|
|
16
|
+
|
|
17
|
+
def __init__(self, config: "Config"):
|
|
18
|
+
|
|
19
|
+
resource = Resource(
|
|
20
|
+
serializer_cls=config.serializer_cls,
|
|
21
|
+
url=config.url,
|
|
22
|
+
queue_name=config.queue_name,
|
|
23
|
+
enable_declare_queue=config.enable_declare_queue,
|
|
24
|
+
queue_durable=config.queue_durable,
|
|
25
|
+
queue_exclusive=config.queue_exclusive,
|
|
26
|
+
queue_passive=config.queue_passive,
|
|
27
|
+
queue_auto_delete=config.queue_auto_delete,
|
|
28
|
+
queue_arguments=config.queue_arguments,
|
|
29
|
+
)
|
|
30
|
+
self._producer = Producer(
|
|
31
|
+
resource=resource,
|
|
32
|
+
publish_timeout=config.publish_timeout,
|
|
33
|
+
exchange_name=config.exchange_name,
|
|
34
|
+
routing_key=config.routing_key,
|
|
35
|
+
delivery_mode=config.delivery_mode,
|
|
36
|
+
)
|
|
37
|
+
self._consumer = Consumer(
|
|
38
|
+
resource=resource,
|
|
39
|
+
prefetch_count=config.prefetch_count,
|
|
40
|
+
prefetch_size=config.prefetch_size,
|
|
41
|
+
global_qos=config.global_qos,
|
|
42
|
+
coro=config.coro,
|
|
43
|
+
enable_consumer=config.enable_consumer,
|
|
44
|
+
consume_timeout=config.consume_timeout,
|
|
45
|
+
pool_executor_cls=config.consumer_pool_executor_cls,
|
|
46
|
+
pool_maxsize=config.consumer_pool_maxsize,
|
|
47
|
+
)
|
|
@@ -11,18 +11,18 @@ from queue_func.pool_executor.base import BasePoolExecutor
|
|
|
11
11
|
from queue_func.pool_executor.general import PoolExecutor
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
@pyd_dataclass(frozen=True, config=ExtraIgnoreConfig)
|
|
14
|
+
@pyd_dataclass(frozen=True, kw_only=True, config=ExtraIgnoreConfig)
|
|
15
15
|
class Config:
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
## shared resource
|
|
18
|
+
serializer_cls: type[BaseSerializer] = Serializer
|
|
19
|
+
|
|
20
|
+
## consumer
|
|
18
21
|
coro: t.Callable[[...], t.Coroutine]
|
|
19
22
|
enable_consumer: bool = True
|
|
20
23
|
consume_timeout: float = 2.0
|
|
21
24
|
consumer_pool_executor_cls: type[BasePoolExecutor] = PoolExecutor
|
|
22
25
|
consumer_pool_maxsize: int = 200
|
|
23
26
|
|
|
24
|
-
|
|
25
|
-
serializer_cls: type[BaseSerializer] = Serializer
|
|
26
|
-
|
|
27
|
-
# producer
|
|
27
|
+
## producer
|
|
28
28
|
publish_timeout: float | None = None
|